| { |
| "status": "PASS", |
| "completed_at": "2026-08-04T19:03:50.202268+09:00", |
| "base_dir": "/home/saul9523/dgx_ai_factory/models/deepseek_r1_distill_llama_70b_hf", |
| "adapter_dir": "/home/saul9523/dgx_ai_factory/releases/deepseek70b_qlora_7816_conditional_stable_v1/adapter", |
| "output_dir": "/home/saul9523/dgx_ai_factory/models/deepseek70b_7816_merged_bf16", |
| "shard_count": 17, |
| "lora_module_count": 560, |
| "applied_module_count": 560, |
| "use_rslora": false, |
| "fan_in_fan_out": false, |
| "output_total_weight_bytes": 141107497648, |
| "applied": { |
| "model.layers.0.mlp.down_proj": { |
| "base_key": "model.layers.0.mlp.down_proj.weight", |
| "module": "model.layers.0.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0036352903116494417, |
| "delta_mean_abs": 3.136198574793525e-05 |
| }, |
| "model.layers.0.mlp.gate_proj": { |
| "base_key": "model.layers.0.mlp.gate_proj.weight", |
| "module": "model.layers.0.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0010508858831599355, |
| "delta_mean_abs": 5.487564703798853e-05 |
| }, |
| "model.layers.0.mlp.up_proj": { |
| "base_key": "model.layers.0.mlp.up_proj.weight", |
| "module": "model.layers.0.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0011367766419425607, |
| "delta_mean_abs": 5.960034468444064e-05 |
| }, |
| "model.layers.0.self_attn.k_proj": { |
| "base_key": "model.layers.0.self_attn.k_proj.weight", |
| "module": "model.layers.0.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.002157435519620776, |
| "delta_mean_abs": 9.706999844638631e-05 |
| }, |
| "model.layers.0.self_attn.o_proj": { |
| "base_key": "model.layers.0.self_attn.o_proj.weight", |
| "module": "model.layers.0.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.001382035086862743, |
| "delta_mean_abs": 5.239765960141085e-05 |
| }, |
| "model.layers.0.self_attn.q_proj": { |
| "base_key": "model.layers.0.self_attn.q_proj.weight", |
| "module": "model.layers.0.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.003898917231708765, |
| "delta_mean_abs": 0.0001100506997318007 |
| }, |
| "model.layers.0.self_attn.v_proj": { |
| "base_key": "model.layers.0.self_attn.v_proj.weight", |
| "module": "model.layers.0.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.009479931555688381, |
| "delta_mean_abs": 9.415674867341295e-05 |
| }, |
| "model.layers.1.mlp.down_proj": { |
| "base_key": "model.layers.1.mlp.down_proj.weight", |
| "module": "model.layers.1.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0007907061954028904, |
| "delta_mean_abs": 1.6571046216995455e-05 |
| }, |
| "model.layers.1.mlp.gate_proj": { |
| "base_key": "model.layers.1.mlp.gate_proj.weight", |
| "module": "model.layers.1.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006334871868602931, |
| "delta_mean_abs": 3.3563090255483985e-05 |
| }, |
| "model.layers.1.mlp.up_proj": { |
| "base_key": "model.layers.1.mlp.up_proj.weight", |
| "module": "model.layers.1.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.000692803761921823, |
| "delta_mean_abs": 3.019714858965017e-05 |
| }, |
| "model.layers.1.self_attn.k_proj": { |
| "base_key": "model.layers.1.self_attn.k_proj.weight", |
| "module": "model.layers.1.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0019693966023623943, |
| "delta_mean_abs": 0.00010614078200887889 |
| }, |
| "model.layers.1.self_attn.o_proj": { |
| "base_key": "model.layers.1.self_attn.o_proj.weight", |
| "module": "model.layers.1.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004677238466683775, |
| "delta_mean_abs": 2.7173442504135892e-05 |
| }, |
| "model.layers.1.self_attn.q_proj": { |
| "base_key": "model.layers.1.self_attn.q_proj.weight", |
| "module": "model.layers.1.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0030918612610548735, |
| "delta_mean_abs": 8.843185059959069e-05 |
| }, |
| "model.layers.1.self_attn.v_proj": { |
| "base_key": "model.layers.1.self_attn.v_proj.weight", |
| "module": "model.layers.1.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.002262373920530081, |
| "delta_mean_abs": 4.33517707278952e-05 |
| }, |
| "model.layers.2.mlp.down_proj": { |
| "base_key": "model.layers.2.mlp.down_proj.weight", |
| "module": "model.layers.2.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004056089965160936, |
| "delta_mean_abs": 1.764399348758161e-05 |
| }, |
| "model.layers.2.mlp.gate_proj": { |
| "base_key": "model.layers.2.mlp.gate_proj.weight", |
| "module": "model.layers.2.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.000763745978474617, |
| "delta_mean_abs": 3.423832094995305e-05 |
| }, |
| "model.layers.2.mlp.up_proj": { |
| "base_key": "model.layers.2.mlp.up_proj.weight", |
| "module": "model.layers.2.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006704325787723064, |
| "delta_mean_abs": 3.197841579094529e-05 |
| }, |
| "model.layers.2.self_attn.k_proj": { |
| "base_key": "model.layers.2.self_attn.k_proj.weight", |
| "module": "model.layers.2.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005617460119538009, |
| "delta_mean_abs": 3.201706203981303e-05 |
| }, |
| "model.layers.2.self_attn.o_proj": { |
| "base_key": "model.layers.2.self_attn.o_proj.weight", |
| "module": "model.layers.2.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005339219933375716, |
| "delta_mean_abs": 2.975089955725707e-05 |
| }, |
| "model.layers.2.self_attn.q_proj": { |
| "base_key": "model.layers.2.self_attn.q_proj.weight", |
| "module": "model.layers.2.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.000573021185118705, |
| "delta_mean_abs": 3.4839671570807695e-05 |
| }, |
| "model.layers.2.self_attn.v_proj": { |
| "base_key": "model.layers.2.self_attn.v_proj.weight", |
| "module": "model.layers.2.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0007892426801845431, |
| "delta_mean_abs": 3.2222313166130334e-05 |
| }, |
| "model.layers.3.mlp.down_proj": { |
| "base_key": "model.layers.3.mlp.down_proj.weight", |
| "module": "model.layers.3.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.003743372857570648, |
| "delta_mean_abs": 2.3273358237929642e-05 |
| }, |
| "model.layers.3.mlp.gate_proj": { |
| "base_key": "model.layers.3.mlp.gate_proj.weight", |
| "module": "model.layers.3.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0007301653386093676, |
| "delta_mean_abs": 3.1983588996808976e-05 |
| }, |
| "model.layers.3.mlp.up_proj": { |
| "base_key": "model.layers.3.mlp.up_proj.weight", |
| "module": "model.layers.3.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006075630662962794, |
| "delta_mean_abs": 3.1319137633545324e-05 |
| }, |
| "model.layers.3.self_attn.k_proj": { |
| "base_key": "model.layers.3.self_attn.k_proj.weight", |
| "module": "model.layers.3.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00039019607356749475, |
| "delta_mean_abs": 2.669027162482962e-05 |
| }, |
| "model.layers.3.self_attn.o_proj": { |
| "base_key": "model.layers.3.self_attn.o_proj.weight", |
| "module": "model.layers.3.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00043108317186124623, |
| "delta_mean_abs": 2.5481347620370798e-05 |
| }, |
| "model.layers.3.self_attn.q_proj": { |
| "base_key": "model.layers.3.self_attn.q_proj.weight", |
| "module": "model.layers.3.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005321042845025659, |
| "delta_mean_abs": 2.9094042474753223e-05 |
| }, |
| "model.layers.3.self_attn.v_proj": { |
| "base_key": "model.layers.3.self_attn.v_proj.weight", |
| "module": "model.layers.3.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004479637718759477, |
| "delta_mean_abs": 2.969752313219942e-05 |
| }, |
| "model.layers.4.mlp.down_proj": { |
| "base_key": "model.layers.4.mlp.down_proj.weight", |
| "module": "model.layers.4.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004106768174096942, |
| "delta_mean_abs": 1.8390141121926717e-05 |
| }, |
| "model.layers.4.mlp.gate_proj": { |
| "base_key": "model.layers.4.mlp.gate_proj.weight", |
| "module": "model.layers.4.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0007407861994579434, |
| "delta_mean_abs": 3.1805113394511864e-05 |
| }, |
| "model.layers.4.mlp.up_proj": { |
| "base_key": "model.layers.4.mlp.up_proj.weight", |
| "module": "model.layers.4.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0007738450076431036, |
| "delta_mean_abs": 3.074496635235846e-05 |
| }, |
| "model.layers.4.self_attn.k_proj": { |
| "base_key": "model.layers.4.self_attn.k_proj.weight", |
| "module": "model.layers.4.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00035501853562891483, |
| "delta_mean_abs": 2.444405254209414e-05 |
| }, |
| "model.layers.4.self_attn.o_proj": { |
| "base_key": "model.layers.4.self_attn.o_proj.weight", |
| "module": "model.layers.4.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005852633621543646, |
| "delta_mean_abs": 2.7335896447766572e-05 |
| }, |
| "model.layers.4.self_attn.q_proj": { |
| "base_key": "model.layers.4.self_attn.q_proj.weight", |
| "module": "model.layers.4.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.000560253334697336, |
| "delta_mean_abs": 2.587987000879366e-05 |
| }, |
| "model.layers.4.self_attn.v_proj": { |
| "base_key": "model.layers.4.self_attn.v_proj.weight", |
| "module": "model.layers.4.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0003859245916828513, |
| "delta_mean_abs": 2.7912094083148986e-05 |
| }, |
| "model.layers.5.mlp.down_proj": { |
| "base_key": "model.layers.5.mlp.down_proj.weight", |
| "module": "model.layers.5.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.000375310773961246, |
| "delta_mean_abs": 1.9491764760459773e-05 |
| }, |
| "model.layers.5.mlp.gate_proj": { |
| "base_key": "model.layers.5.mlp.gate_proj.weight", |
| "module": "model.layers.5.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0007695615058764815, |
| "delta_mean_abs": 3.2462496164953336e-05 |
| }, |
| "model.layers.5.mlp.up_proj": { |
| "base_key": "model.layers.5.mlp.up_proj.weight", |
| "module": "model.layers.5.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.000634767347946763, |
| "delta_mean_abs": 3.161425775033422e-05 |
| }, |
| "model.layers.5.self_attn.k_proj": { |
| "base_key": "model.layers.5.self_attn.k_proj.weight", |
| "module": "model.layers.5.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004232973442412913, |
| "delta_mean_abs": 2.9286964490893297e-05 |
| }, |
| "model.layers.5.self_attn.o_proj": { |
| "base_key": "model.layers.5.self_attn.o_proj.weight", |
| "module": "model.layers.5.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006159860058687627, |
| "delta_mean_abs": 2.822165515681263e-05 |
| }, |
| "model.layers.5.self_attn.q_proj": { |
| "base_key": "model.layers.5.self_attn.q_proj.weight", |
| "module": "model.layers.5.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006719998782500625, |
| "delta_mean_abs": 2.7760039301938377e-05 |
| }, |
| "model.layers.5.self_attn.v_proj": { |
| "base_key": "model.layers.5.self_attn.v_proj.weight", |
| "module": "model.layers.5.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00048304349184036255, |
| "delta_mean_abs": 2.9610157071147114e-05 |
| }, |
| "model.layers.6.mlp.down_proj": { |
| "base_key": "model.layers.6.mlp.down_proj.weight", |
| "module": "model.layers.6.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00045023843995295465, |
| "delta_mean_abs": 1.9406774299568497e-05 |
| }, |
| "model.layers.6.mlp.gate_proj": { |
| "base_key": "model.layers.6.mlp.gate_proj.weight", |
| "module": "model.layers.6.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0007773020770400763, |
| "delta_mean_abs": 3.3434360375395045e-05 |
| }, |
| "model.layers.6.mlp.up_proj": { |
| "base_key": "model.layers.6.mlp.up_proj.weight", |
| "module": "model.layers.6.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006371574709191918, |
| "delta_mean_abs": 3.128765820292756e-05 |
| }, |
| "model.layers.6.self_attn.k_proj": { |
| "base_key": "model.layers.6.self_attn.k_proj.weight", |
| "module": "model.layers.6.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00036013053613714874, |
| "delta_mean_abs": 2.4566277716076e-05 |
| }, |
| "model.layers.6.self_attn.o_proj": { |
| "base_key": "model.layers.6.self_attn.o_proj.weight", |
| "module": "model.layers.6.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005764392553828657, |
| "delta_mean_abs": 2.6879704819293693e-05 |
| }, |
| "model.layers.6.self_attn.q_proj": { |
| "base_key": "model.layers.6.self_attn.q_proj.weight", |
| "module": "model.layers.6.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004684393643401563, |
| "delta_mean_abs": 2.5988611014327034e-05 |
| }, |
| "model.layers.6.self_attn.v_proj": { |
| "base_key": "model.layers.6.self_attn.v_proj.weight", |
| "module": "model.layers.6.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004568297299556434, |
| "delta_mean_abs": 2.989911263284739e-05 |
| }, |
| "model.layers.7.mlp.down_proj": { |
| "base_key": "model.layers.7.mlp.down_proj.weight", |
| "module": "model.layers.7.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0002940115809906274, |
| "delta_mean_abs": 1.8584800272947177e-05 |
| }, |
| "model.layers.7.mlp.gate_proj": { |
| "base_key": "model.layers.7.mlp.gate_proj.weight", |
| "module": "model.layers.7.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0007729062926955521, |
| "delta_mean_abs": 3.5601809940999374e-05 |
| }, |
| "model.layers.7.mlp.up_proj": { |
| "base_key": "model.layers.7.mlp.up_proj.weight", |
| "module": "model.layers.7.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006676525226794183, |
| "delta_mean_abs": 3.286090577603318e-05 |
| }, |
| "model.layers.7.self_attn.k_proj": { |
| "base_key": "model.layers.7.self_attn.k_proj.weight", |
| "module": "model.layers.7.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004221448034513742, |
| "delta_mean_abs": 2.9217031624284573e-05 |
| }, |
| "model.layers.7.self_attn.o_proj": { |
| "base_key": "model.layers.7.self_attn.o_proj.weight", |
| "module": "model.layers.7.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006508776568807662, |
| "delta_mean_abs": 2.695505645533558e-05 |
| }, |
| "model.layers.7.self_attn.q_proj": { |
| "base_key": "model.layers.7.self_attn.q_proj.weight", |
| "module": "model.layers.7.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006043009925633669, |
| "delta_mean_abs": 2.6296636860934086e-05 |
| }, |
| "model.layers.7.self_attn.v_proj": { |
| "base_key": "model.layers.7.self_attn.v_proj.weight", |
| "module": "model.layers.7.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00044372607953846455, |
| "delta_mean_abs": 2.892112752306275e-05 |
| }, |
| "model.layers.8.mlp.down_proj": { |
| "base_key": "model.layers.8.mlp.down_proj.weight", |
| "module": "model.layers.8.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.000441599142504856, |
| "delta_mean_abs": 1.895054265332874e-05 |
| }, |
| "model.layers.8.mlp.gate_proj": { |
| "base_key": "model.layers.8.mlp.gate_proj.weight", |
| "module": "model.layers.8.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006955206044949591, |
| "delta_mean_abs": 3.6487523175310344e-05 |
| }, |
| "model.layers.8.mlp.up_proj": { |
| "base_key": "model.layers.8.mlp.up_proj.weight", |
| "module": "model.layers.8.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.000758365960791707, |
| "delta_mean_abs": 3.4612745366757736e-05 |
| }, |
| "model.layers.8.self_attn.k_proj": { |
| "base_key": "model.layers.8.self_attn.k_proj.weight", |
| "module": "model.layers.8.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004652384377550334, |
| "delta_mean_abs": 2.9981387342559174e-05 |
| }, |
| "model.layers.8.self_attn.o_proj": { |
| "base_key": "model.layers.8.self_attn.o_proj.weight", |
| "module": "model.layers.8.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00045202657929621637, |
| "delta_mean_abs": 2.738096191023942e-05 |
| }, |
| "model.layers.8.self_attn.q_proj": { |
| "base_key": "model.layers.8.self_attn.q_proj.weight", |
| "module": "model.layers.8.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006173077272251248, |
| "delta_mean_abs": 3.0444767617154866e-05 |
| }, |
| "model.layers.8.self_attn.v_proj": { |
| "base_key": "model.layers.8.self_attn.v_proj.weight", |
| "module": "model.layers.8.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.000500081165228039, |
| "delta_mean_abs": 2.864183989004232e-05 |
| }, |
| "model.layers.9.self_attn.q_proj": { |
| "base_key": "model.layers.9.self_attn.q_proj.weight", |
| "module": "model.layers.9.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005254327552393079, |
| "delta_mean_abs": 2.826799391186796e-05 |
| }, |
| "model.layers.9.mlp.down_proj": { |
| "base_key": "model.layers.9.mlp.down_proj.weight", |
| "module": "model.layers.9.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0003998197498731315, |
| "delta_mean_abs": 2.020290048676543e-05 |
| }, |
| "model.layers.9.mlp.gate_proj": { |
| "base_key": "model.layers.9.mlp.gate_proj.weight", |
| "module": "model.layers.9.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006650558789260685, |
| "delta_mean_abs": 3.5049022699240595e-05 |
| }, |
| "model.layers.9.mlp.up_proj": { |
| "base_key": "model.layers.9.mlp.up_proj.weight", |
| "module": "model.layers.9.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006629395065829158, |
| "delta_mean_abs": 3.3303804229944944e-05 |
| }, |
| "model.layers.9.self_attn.k_proj": { |
| "base_key": "model.layers.9.self_attn.k_proj.weight", |
| "module": "model.layers.9.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005030132015235722, |
| "delta_mean_abs": 2.930740447482094e-05 |
| }, |
| "model.layers.9.self_attn.o_proj": { |
| "base_key": "model.layers.9.self_attn.o_proj.weight", |
| "module": "model.layers.9.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006178875919431448, |
| "delta_mean_abs": 2.754388151515741e-05 |
| }, |
| "model.layers.9.self_attn.v_proj": { |
| "base_key": "model.layers.9.self_attn.v_proj.weight", |
| "module": "model.layers.9.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005247238441370428, |
| "delta_mean_abs": 2.8585352993104607e-05 |
| }, |
| "model.layers.10.mlp.down_proj": { |
| "base_key": "model.layers.10.mlp.down_proj.weight", |
| "module": "model.layers.10.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0003578914038371295, |
| "delta_mean_abs": 1.8644699594005942e-05 |
| }, |
| "model.layers.10.mlp.gate_proj": { |
| "base_key": "model.layers.10.mlp.gate_proj.weight", |
| "module": "model.layers.10.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0007022873614914715, |
| "delta_mean_abs": 3.4550026612123474e-05 |
| }, |
| "model.layers.10.mlp.up_proj": { |
| "base_key": "model.layers.10.mlp.up_proj.weight", |
| "module": "model.layers.10.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005343113443814218, |
| "delta_mean_abs": 3.248727080062963e-05 |
| }, |
| "model.layers.10.self_attn.k_proj": { |
| "base_key": "model.layers.10.self_attn.k_proj.weight", |
| "module": "model.layers.10.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00036571527016349137, |
| "delta_mean_abs": 2.6312834961572662e-05 |
| }, |
| "model.layers.10.self_attn.o_proj": { |
| "base_key": "model.layers.10.self_attn.o_proj.weight", |
| "module": "model.layers.10.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005304256919771433, |
| "delta_mean_abs": 2.6340872864238918e-05 |
| }, |
| "model.layers.10.self_attn.q_proj": { |
| "base_key": "model.layers.10.self_attn.q_proj.weight", |
| "module": "model.layers.10.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00047226346214301884, |
| "delta_mean_abs": 2.642719482537359e-05 |
| }, |
| "model.layers.10.self_attn.v_proj": { |
| "base_key": "model.layers.10.self_attn.v_proj.weight", |
| "module": "model.layers.10.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004641205014195293, |
| "delta_mean_abs": 2.6296034775441512e-05 |
| }, |
| "model.layers.11.mlp.down_proj": { |
| "base_key": "model.layers.11.mlp.down_proj.weight", |
| "module": "model.layers.11.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004301145381759852, |
| "delta_mean_abs": 1.8955686755361967e-05 |
| }, |
| "model.layers.11.mlp.gate_proj": { |
| "base_key": "model.layers.11.mlp.gate_proj.weight", |
| "module": "model.layers.11.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006238978239707649, |
| "delta_mean_abs": 3.518013545544818e-05 |
| }, |
| "model.layers.11.mlp.up_proj": { |
| "base_key": "model.layers.11.mlp.up_proj.weight", |
| "module": "model.layers.11.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006969037349335849, |
| "delta_mean_abs": 3.2615807867841795e-05 |
| }, |
| "model.layers.11.self_attn.k_proj": { |
| "base_key": "model.layers.11.self_attn.k_proj.weight", |
| "module": "model.layers.11.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0003391368663869798, |
| "delta_mean_abs": 2.689389293664135e-05 |
| }, |
| "model.layers.11.self_attn.o_proj": { |
| "base_key": "model.layers.11.self_attn.o_proj.weight", |
| "module": "model.layers.11.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.000488903431687504, |
| "delta_mean_abs": 2.680330908333417e-05 |
| }, |
| "model.layers.11.self_attn.q_proj": { |
| "base_key": "model.layers.11.self_attn.q_proj.weight", |
| "module": "model.layers.11.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0003848126798402518, |
| "delta_mean_abs": 2.6009003704530187e-05 |
| }, |
| "model.layers.11.self_attn.v_proj": { |
| "base_key": "model.layers.11.self_attn.v_proj.weight", |
| "module": "model.layers.11.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0003552336129359901, |
| "delta_mean_abs": 2.5538458430673927e-05 |
| }, |
| "model.layers.12.mlp.down_proj": { |
| "base_key": "model.layers.12.mlp.down_proj.weight", |
| "module": "model.layers.12.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00048704337677918375, |
| "delta_mean_abs": 2.0789399059140123e-05 |
| }, |
| "model.layers.12.mlp.gate_proj": { |
| "base_key": "model.layers.12.mlp.gate_proj.weight", |
| "module": "model.layers.12.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0008524228469468653, |
| "delta_mean_abs": 3.566720988601446e-05 |
| }, |
| "model.layers.12.mlp.up_proj": { |
| "base_key": "model.layers.12.mlp.up_proj.weight", |
| "module": "model.layers.12.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.000682688842061907, |
| "delta_mean_abs": 3.426895273150876e-05 |
| }, |
| "model.layers.12.self_attn.k_proj": { |
| "base_key": "model.layers.12.self_attn.k_proj.weight", |
| "module": "model.layers.12.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00037905716453678906, |
| "delta_mean_abs": 3.047188147320412e-05 |
| }, |
| "model.layers.12.self_attn.o_proj": { |
| "base_key": "model.layers.12.self_attn.o_proj.weight", |
| "module": "model.layers.12.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005136913387104869, |
| "delta_mean_abs": 2.8447948352550156e-05 |
| }, |
| "model.layers.12.self_attn.q_proj": { |
| "base_key": "model.layers.12.self_attn.q_proj.weight", |
| "module": "model.layers.12.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005524496082216501, |
| "delta_mean_abs": 2.9502025427063927e-05 |
| }, |
| "model.layers.12.self_attn.v_proj": { |
| "base_key": "model.layers.12.self_attn.v_proj.weight", |
| "module": "model.layers.12.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004195676010567695, |
| "delta_mean_abs": 2.7149908419232816e-05 |
| }, |
| "model.layers.13.mlp.down_proj": { |
| "base_key": "model.layers.13.mlp.down_proj.weight", |
| "module": "model.layers.13.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0003899582370650023, |
| "delta_mean_abs": 1.9898227037629113e-05 |
| }, |
| "model.layers.13.mlp.gate_proj": { |
| "base_key": "model.layers.13.mlp.gate_proj.weight", |
| "module": "model.layers.13.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006656635086983442, |
| "delta_mean_abs": 3.629798811743967e-05 |
| }, |
| "model.layers.13.mlp.up_proj": { |
| "base_key": "model.layers.13.mlp.up_proj.weight", |
| "module": "model.layers.13.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.000809995923191309, |
| "delta_mean_abs": 3.668562931125052e-05 |
| }, |
| "model.layers.13.self_attn.k_proj": { |
| "base_key": "model.layers.13.self_attn.k_proj.weight", |
| "module": "model.layers.13.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00035128198214806616, |
| "delta_mean_abs": 2.7485046302899718e-05 |
| }, |
| "model.layers.13.self_attn.o_proj": { |
| "base_key": "model.layers.13.self_attn.o_proj.weight", |
| "module": "model.layers.13.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005378679488785565, |
| "delta_mean_abs": 2.9396260288194753e-05 |
| }, |
| "model.layers.13.self_attn.q_proj": { |
| "base_key": "model.layers.13.self_attn.q_proj.weight", |
| "module": "model.layers.13.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004775848356075585, |
| "delta_mean_abs": 2.4935630790423602e-05 |
| }, |
| "model.layers.13.self_attn.v_proj": { |
| "base_key": "model.layers.13.self_attn.v_proj.weight", |
| "module": "model.layers.13.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004722838057205081, |
| "delta_mean_abs": 2.789047903206665e-05 |
| }, |
| "model.layers.14.mlp.down_proj": { |
| "base_key": "model.layers.14.mlp.down_proj.weight", |
| "module": "model.layers.14.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004644313594326377, |
| "delta_mean_abs": 2.0793066141777672e-05 |
| }, |
| "model.layers.14.mlp.gate_proj": { |
| "base_key": "model.layers.14.mlp.gate_proj.weight", |
| "module": "model.layers.14.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0008109782938845456, |
| "delta_mean_abs": 3.6049219488631934e-05 |
| }, |
| "model.layers.14.mlp.up_proj": { |
| "base_key": "model.layers.14.mlp.up_proj.weight", |
| "module": "model.layers.14.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0009067971259355545, |
| "delta_mean_abs": 3.781237319344655e-05 |
| }, |
| "model.layers.14.self_attn.k_proj": { |
| "base_key": "model.layers.14.self_attn.k_proj.weight", |
| "module": "model.layers.14.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0003097833541687578, |
| "delta_mean_abs": 2.7849862817674875e-05 |
| }, |
| "model.layers.14.self_attn.o_proj": { |
| "base_key": "model.layers.14.self_attn.o_proj.weight", |
| "module": "model.layers.14.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006474938709288836, |
| "delta_mean_abs": 2.9250222723931074e-05 |
| }, |
| "model.layers.14.self_attn.q_proj": { |
| "base_key": "model.layers.14.self_attn.q_proj.weight", |
| "module": "model.layers.14.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004542551760096103, |
| "delta_mean_abs": 2.5584147806512192e-05 |
| }, |
| "model.layers.14.self_attn.v_proj": { |
| "base_key": "model.layers.14.self_attn.v_proj.weight", |
| "module": "model.layers.14.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004860599583480507, |
| "delta_mean_abs": 2.7947869966737926e-05 |
| }, |
| "model.layers.15.self_attn.q_proj": { |
| "base_key": "model.layers.15.self_attn.q_proj.weight", |
| "module": "model.layers.15.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004839968460146338, |
| "delta_mean_abs": 2.6510871975915506e-05 |
| }, |
| "model.layers.15.mlp.down_proj": { |
| "base_key": "model.layers.15.mlp.down_proj.weight", |
| "module": "model.layers.15.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00039482233114540577, |
| "delta_mean_abs": 2.0277902876841836e-05 |
| }, |
| "model.layers.15.mlp.gate_proj": { |
| "base_key": "model.layers.15.mlp.gate_proj.weight", |
| "module": "model.layers.15.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0009372858912684023, |
| "delta_mean_abs": 3.905146149918437e-05 |
| }, |
| "model.layers.15.mlp.up_proj": { |
| "base_key": "model.layers.15.mlp.up_proj.weight", |
| "module": "model.layers.15.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0012898265849798918, |
| "delta_mean_abs": 3.96422874473501e-05 |
| }, |
| "model.layers.15.self_attn.k_proj": { |
| "base_key": "model.layers.15.self_attn.k_proj.weight", |
| "module": "model.layers.15.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005392345483414829, |
| "delta_mean_abs": 2.829216464306228e-05 |
| }, |
| "model.layers.15.self_attn.o_proj": { |
| "base_key": "model.layers.15.self_attn.o_proj.weight", |
| "module": "model.layers.15.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005605737678706646, |
| "delta_mean_abs": 2.846170355041977e-05 |
| }, |
| "model.layers.15.self_attn.v_proj": { |
| "base_key": "model.layers.15.self_attn.v_proj.weight", |
| "module": "model.layers.15.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005755904130637646, |
| "delta_mean_abs": 3.540561374393292e-05 |
| }, |
| "model.layers.16.mlp.down_proj": { |
| "base_key": "model.layers.16.mlp.down_proj.weight", |
| "module": "model.layers.16.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00032310711685568094, |
| "delta_mean_abs": 1.8940079826279543e-05 |
| }, |
| "model.layers.16.mlp.gate_proj": { |
| "base_key": "model.layers.16.mlp.gate_proj.weight", |
| "module": "model.layers.16.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0007166087161749601, |
| "delta_mean_abs": 3.5575350921135396e-05 |
| }, |
| "model.layers.16.mlp.up_proj": { |
| "base_key": "model.layers.16.mlp.up_proj.weight", |
| "module": "model.layers.16.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0011048492742702365, |
| "delta_mean_abs": 3.841549187200144e-05 |
| }, |
| "model.layers.16.self_attn.k_proj": { |
| "base_key": "model.layers.16.self_attn.k_proj.weight", |
| "module": "model.layers.16.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0003247972927056253, |
| "delta_mean_abs": 2.7299880457576364e-05 |
| }, |
| "model.layers.16.self_attn.o_proj": { |
| "base_key": "model.layers.16.self_attn.o_proj.weight", |
| "module": "model.layers.16.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00047759374137967825, |
| "delta_mean_abs": 2.9183374863350764e-05 |
| }, |
| "model.layers.16.self_attn.q_proj": { |
| "base_key": "model.layers.16.self_attn.q_proj.weight", |
| "module": "model.layers.16.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00048805735423229635, |
| "delta_mean_abs": 2.7698981284629554e-05 |
| }, |
| "model.layers.16.self_attn.v_proj": { |
| "base_key": "model.layers.16.self_attn.v_proj.weight", |
| "module": "model.layers.16.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004193338972982019, |
| "delta_mean_abs": 3.265432678745128e-05 |
| }, |
| "model.layers.17.mlp.down_proj": { |
| "base_key": "model.layers.17.mlp.down_proj.weight", |
| "module": "model.layers.17.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00033790242741815746, |
| "delta_mean_abs": 1.8925655240309425e-05 |
| }, |
| "model.layers.17.mlp.gate_proj": { |
| "base_key": "model.layers.17.mlp.gate_proj.weight", |
| "module": "model.layers.17.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.000707303115632385, |
| "delta_mean_abs": 3.6994260881328955e-05 |
| }, |
| "model.layers.17.mlp.up_proj": { |
| "base_key": "model.layers.17.mlp.up_proj.weight", |
| "module": "model.layers.17.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0007223768625408411, |
| "delta_mean_abs": 3.585245940485038e-05 |
| }, |
| "model.layers.17.self_attn.k_proj": { |
| "base_key": "model.layers.17.self_attn.k_proj.weight", |
| "module": "model.layers.17.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00035532109905034304, |
| "delta_mean_abs": 2.6820809580385685e-05 |
| }, |
| "model.layers.17.self_attn.o_proj": { |
| "base_key": "model.layers.17.self_attn.o_proj.weight", |
| "module": "model.layers.17.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004767296486534178, |
| "delta_mean_abs": 2.8032882255502045e-05 |
| }, |
| "model.layers.17.self_attn.q_proj": { |
| "base_key": "model.layers.17.self_attn.q_proj.weight", |
| "module": "model.layers.17.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005394977051764727, |
| "delta_mean_abs": 2.6448475182405673e-05 |
| }, |
| "model.layers.17.self_attn.v_proj": { |
| "base_key": "model.layers.17.self_attn.v_proj.weight", |
| "module": "model.layers.17.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00039558453136123717, |
| "delta_mean_abs": 2.9297125365701504e-05 |
| }, |
| "model.layers.18.mlp.down_proj": { |
| "base_key": "model.layers.18.mlp.down_proj.weight", |
| "module": "model.layers.18.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0012150118127465248, |
| "delta_mean_abs": 2.1763864424428903e-05 |
| }, |
| "model.layers.18.mlp.gate_proj": { |
| "base_key": "model.layers.18.mlp.gate_proj.weight", |
| "module": "model.layers.18.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0007604326820001006, |
| "delta_mean_abs": 3.606493191909976e-05 |
| }, |
| "model.layers.18.mlp.up_proj": { |
| "base_key": "model.layers.18.mlp.up_proj.weight", |
| "module": "model.layers.18.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0007464013760909438, |
| "delta_mean_abs": 3.3829768653959036e-05 |
| }, |
| "model.layers.18.self_attn.k_proj": { |
| "base_key": "model.layers.18.self_attn.k_proj.weight", |
| "module": "model.layers.18.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00030617782613262534, |
| "delta_mean_abs": 2.910135663114488e-05 |
| }, |
| "model.layers.18.self_attn.o_proj": { |
| "base_key": "model.layers.18.self_attn.o_proj.weight", |
| "module": "model.layers.18.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004923552041873336, |
| "delta_mean_abs": 2.75628062809119e-05 |
| }, |
| "model.layers.18.self_attn.q_proj": { |
| "base_key": "model.layers.18.self_attn.q_proj.weight", |
| "module": "model.layers.18.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005360979121178389, |
| "delta_mean_abs": 2.8786835173377767e-05 |
| }, |
| "model.layers.18.self_attn.v_proj": { |
| "base_key": "model.layers.18.self_attn.v_proj.weight", |
| "module": "model.layers.18.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0003560184268280864, |
| "delta_mean_abs": 2.963738006656058e-05 |
| }, |
| "model.layers.19.mlp.down_proj": { |
| "base_key": "model.layers.19.mlp.down_proj.weight", |
| "module": "model.layers.19.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00039569506770931184, |
| "delta_mean_abs": 2.0670006051659584e-05 |
| }, |
| "model.layers.19.mlp.gate_proj": { |
| "base_key": "model.layers.19.mlp.gate_proj.weight", |
| "module": "model.layers.19.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0007531738374382257, |
| "delta_mean_abs": 3.9080401620594785e-05 |
| }, |
| "model.layers.19.mlp.up_proj": { |
| "base_key": "model.layers.19.mlp.up_proj.weight", |
| "module": "model.layers.19.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0007639066316187382, |
| "delta_mean_abs": 3.657360139186494e-05 |
| }, |
| "model.layers.19.self_attn.k_proj": { |
| "base_key": "model.layers.19.self_attn.k_proj.weight", |
| "module": "model.layers.19.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006408033077605069, |
| "delta_mean_abs": 3.3364933187840506e-05 |
| }, |
| "model.layers.19.self_attn.o_proj": { |
| "base_key": "model.layers.19.self_attn.o_proj.weight", |
| "module": "model.layers.19.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00048011617036536336, |
| "delta_mean_abs": 3.125260991510004e-05 |
| }, |
| "model.layers.19.self_attn.q_proj": { |
| "base_key": "model.layers.19.self_attn.q_proj.weight", |
| "module": "model.layers.19.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005855232011526823, |
| "delta_mean_abs": 3.5281213058624417e-05 |
| }, |
| "model.layers.19.self_attn.v_proj": { |
| "base_key": "model.layers.19.self_attn.v_proj.weight", |
| "module": "model.layers.19.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005426667048595846, |
| "delta_mean_abs": 3.1518029572907835e-05 |
| }, |
| "model.layers.20.mlp.down_proj": { |
| "base_key": "model.layers.20.mlp.down_proj.weight", |
| "module": "model.layers.20.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0003670479927677661, |
| "delta_mean_abs": 2.044621214736253e-05 |
| }, |
| "model.layers.20.mlp.gate_proj": { |
| "base_key": "model.layers.20.mlp.gate_proj.weight", |
| "module": "model.layers.20.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006585103692486882, |
| "delta_mean_abs": 3.8028229027986526e-05 |
| }, |
| "model.layers.20.mlp.up_proj": { |
| "base_key": "model.layers.20.mlp.up_proj.weight", |
| "module": "model.layers.20.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0007734259706921875, |
| "delta_mean_abs": 3.701594687299803e-05 |
| }, |
| "model.layers.20.self_attn.k_proj": { |
| "base_key": "model.layers.20.self_attn.k_proj.weight", |
| "module": "model.layers.20.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004370283568277955, |
| "delta_mean_abs": 2.7443404178484343e-05 |
| }, |
| "model.layers.20.self_attn.o_proj": { |
| "base_key": "model.layers.20.self_attn.o_proj.weight", |
| "module": "model.layers.20.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004677482938859612, |
| "delta_mean_abs": 3.259960431023501e-05 |
| }, |
| "model.layers.20.self_attn.q_proj": { |
| "base_key": "model.layers.20.self_attn.q_proj.weight", |
| "module": "model.layers.20.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00048744806554168463, |
| "delta_mean_abs": 2.8533766453620046e-05 |
| }, |
| "model.layers.20.self_attn.v_proj": { |
| "base_key": "model.layers.20.self_attn.v_proj.weight", |
| "module": "model.layers.20.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005600369186140597, |
| "delta_mean_abs": 3.3450858609285206e-05 |
| }, |
| "model.layers.21.mlp.down_proj": { |
| "base_key": "model.layers.21.mlp.down_proj.weight", |
| "module": "model.layers.21.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00031051065889187157, |
| "delta_mean_abs": 1.8340409951633774e-05 |
| }, |
| "model.layers.21.mlp.gate_proj": { |
| "base_key": "model.layers.21.mlp.gate_proj.weight", |
| "module": "model.layers.21.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0007532697636634111, |
| "delta_mean_abs": 3.595051748561673e-05 |
| }, |
| "model.layers.21.mlp.up_proj": { |
| "base_key": "model.layers.21.mlp.up_proj.weight", |
| "module": "model.layers.21.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0007955128676258028, |
| "delta_mean_abs": 3.535371070029214e-05 |
| }, |
| "model.layers.21.self_attn.k_proj": { |
| "base_key": "model.layers.21.self_attn.k_proj.weight", |
| "module": "model.layers.21.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00040434542461298406, |
| "delta_mean_abs": 2.641564788063988e-05 |
| }, |
| "model.layers.21.self_attn.o_proj": { |
| "base_key": "model.layers.21.self_attn.o_proj.weight", |
| "module": "model.layers.21.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00045431414037011564, |
| "delta_mean_abs": 3.0004262953298166e-05 |
| }, |
| "model.layers.21.self_attn.q_proj": { |
| "base_key": "model.layers.21.self_attn.q_proj.weight", |
| "module": "model.layers.21.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00044989213347435, |
| "delta_mean_abs": 2.7764279366238043e-05 |
| }, |
| "model.layers.21.self_attn.v_proj": { |
| "base_key": "model.layers.21.self_attn.v_proj.weight", |
| "module": "model.layers.21.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005729124532081187, |
| "delta_mean_abs": 2.963965744129382e-05 |
| }, |
| "model.layers.22.mlp.down_proj": { |
| "base_key": "model.layers.22.mlp.down_proj.weight", |
| "module": "model.layers.22.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0003236736811231822, |
| "delta_mean_abs": 1.8887118130805902e-05 |
| }, |
| "model.layers.22.mlp.gate_proj": { |
| "base_key": "model.layers.22.mlp.gate_proj.weight", |
| "module": "model.layers.22.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0008769913692958653, |
| "delta_mean_abs": 3.745655340026133e-05 |
| }, |
| "model.layers.22.mlp.up_proj": { |
| "base_key": "model.layers.22.mlp.up_proj.weight", |
| "module": "model.layers.22.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0008579844725318253, |
| "delta_mean_abs": 3.330839172122069e-05 |
| }, |
| "model.layers.22.self_attn.k_proj": { |
| "base_key": "model.layers.22.self_attn.k_proj.weight", |
| "module": "model.layers.22.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005424704286269844, |
| "delta_mean_abs": 3.093106352025643e-05 |
| }, |
| "model.layers.22.self_attn.o_proj": { |
| "base_key": "model.layers.22.self_attn.o_proj.weight", |
| "module": "model.layers.22.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004562589165288955, |
| "delta_mean_abs": 2.9977498343214393e-05 |
| }, |
| "model.layers.22.self_attn.q_proj": { |
| "base_key": "model.layers.22.self_attn.q_proj.weight", |
| "module": "model.layers.22.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006297448417171836, |
| "delta_mean_abs": 3.027463026228361e-05 |
| }, |
| "model.layers.22.self_attn.v_proj": { |
| "base_key": "model.layers.22.self_attn.v_proj.weight", |
| "module": "model.layers.22.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004894466837868094, |
| "delta_mean_abs": 2.844944356183987e-05 |
| }, |
| "model.layers.23.mlp.down_proj": { |
| "base_key": "model.layers.23.mlp.down_proj.weight", |
| "module": "model.layers.23.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00028796502738259733, |
| "delta_mean_abs": 1.937580600497313e-05 |
| }, |
| "model.layers.23.mlp.gate_proj": { |
| "base_key": "model.layers.23.mlp.gate_proj.weight", |
| "module": "model.layers.23.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0007519855280406773, |
| "delta_mean_abs": 3.5696913982974365e-05 |
| }, |
| "model.layers.23.mlp.up_proj": { |
| "base_key": "model.layers.23.mlp.up_proj.weight", |
| "module": "model.layers.23.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006430592038668692, |
| "delta_mean_abs": 3.358514732099138e-05 |
| }, |
| "model.layers.23.self_attn.k_proj": { |
| "base_key": "model.layers.23.self_attn.k_proj.weight", |
| "module": "model.layers.23.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00047002124483697116, |
| "delta_mean_abs": 3.0779359804000705e-05 |
| }, |
| "model.layers.23.self_attn.o_proj": { |
| "base_key": "model.layers.23.self_attn.o_proj.weight", |
| "module": "model.layers.23.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005199507577344775, |
| "delta_mean_abs": 3.092791303060949e-05 |
| }, |
| "model.layers.23.self_attn.q_proj": { |
| "base_key": "model.layers.23.self_attn.q_proj.weight", |
| "module": "model.layers.23.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005169544601812959, |
| "delta_mean_abs": 2.976926953124348e-05 |
| }, |
| "model.layers.23.self_attn.v_proj": { |
| "base_key": "model.layers.23.self_attn.v_proj.weight", |
| "module": "model.layers.23.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00044459820492193103, |
| "delta_mean_abs": 2.9304595955181867e-05 |
| }, |
| "model.layers.24.mlp.down_proj": { |
| "base_key": "model.layers.24.mlp.down_proj.weight", |
| "module": "model.layers.24.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00031682950793765485, |
| "delta_mean_abs": 1.925398646562826e-05 |
| }, |
| "model.layers.24.mlp.gate_proj": { |
| "base_key": "model.layers.24.mlp.gate_proj.weight", |
| "module": "model.layers.24.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006070475210435688, |
| "delta_mean_abs": 3.6154619010630995e-05 |
| }, |
| "model.layers.24.mlp.up_proj": { |
| "base_key": "model.layers.24.mlp.up_proj.weight", |
| "module": "model.layers.24.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006107209483161569, |
| "delta_mean_abs": 3.5046785342274234e-05 |
| }, |
| "model.layers.24.self_attn.k_proj": { |
| "base_key": "model.layers.24.self_attn.k_proj.weight", |
| "module": "model.layers.24.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00044853624422103167, |
| "delta_mean_abs": 2.999662683578208e-05 |
| }, |
| "model.layers.24.self_attn.o_proj": { |
| "base_key": "model.layers.24.self_attn.o_proj.weight", |
| "module": "model.layers.24.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005410450976341963, |
| "delta_mean_abs": 2.946443419205025e-05 |
| }, |
| "model.layers.24.self_attn.q_proj": { |
| "base_key": "model.layers.24.self_attn.q_proj.weight", |
| "module": "model.layers.24.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005078237736597657, |
| "delta_mean_abs": 2.978332486236468e-05 |
| }, |
| "model.layers.24.self_attn.v_proj": { |
| "base_key": "model.layers.24.self_attn.v_proj.weight", |
| "module": "model.layers.24.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004762047319673002, |
| "delta_mean_abs": 3.099181049037725e-05 |
| }, |
| "model.layers.25.self_attn.q_proj": { |
| "base_key": "model.layers.25.self_attn.q_proj.weight", |
| "module": "model.layers.25.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004628753231372684, |
| "delta_mean_abs": 2.8914271752000786e-05 |
| }, |
| "model.layers.25.mlp.down_proj": { |
| "base_key": "model.layers.25.mlp.down_proj.weight", |
| "module": "model.layers.25.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0003075392742175609, |
| "delta_mean_abs": 2.0207653506076895e-05 |
| }, |
| "model.layers.25.mlp.gate_proj": { |
| "base_key": "model.layers.25.mlp.gate_proj.weight", |
| "module": "model.layers.25.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006210744031704962, |
| "delta_mean_abs": 3.5870703868567944e-05 |
| }, |
| "model.layers.25.mlp.up_proj": { |
| "base_key": "model.layers.25.mlp.up_proj.weight", |
| "module": "model.layers.25.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005775247700512409, |
| "delta_mean_abs": 3.385163290658966e-05 |
| }, |
| "model.layers.25.self_attn.k_proj": { |
| "base_key": "model.layers.25.self_attn.k_proj.weight", |
| "module": "model.layers.25.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0002738982148002833, |
| "delta_mean_abs": 2.500504706404172e-05 |
| }, |
| "model.layers.25.self_attn.o_proj": { |
| "base_key": "model.layers.25.self_attn.o_proj.weight", |
| "module": "model.layers.25.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005219061858952045, |
| "delta_mean_abs": 2.9936094506410882e-05 |
| }, |
| "model.layers.25.self_attn.v_proj": { |
| "base_key": "model.layers.25.self_attn.v_proj.weight", |
| "module": "model.layers.25.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004845963267143816, |
| "delta_mean_abs": 3.082540933974087e-05 |
| }, |
| "model.layers.26.mlp.down_proj": { |
| "base_key": "model.layers.26.mlp.down_proj.weight", |
| "module": "model.layers.26.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0003063615586142987, |
| "delta_mean_abs": 2.0365259842947125e-05 |
| }, |
| "model.layers.26.mlp.gate_proj": { |
| "base_key": "model.layers.26.mlp.gate_proj.weight", |
| "module": "model.layers.26.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0007160595851019025, |
| "delta_mean_abs": 3.268890213803388e-05 |
| }, |
| "model.layers.26.mlp.up_proj": { |
| "base_key": "model.layers.26.mlp.up_proj.weight", |
| "module": "model.layers.26.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0007485139067284763, |
| "delta_mean_abs": 3.1657491490477696e-05 |
| }, |
| "model.layers.26.self_attn.k_proj": { |
| "base_key": "model.layers.26.self_attn.k_proj.weight", |
| "module": "model.layers.26.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004628219467122108, |
| "delta_mean_abs": 2.808139834087342e-05 |
| }, |
| "model.layers.26.self_attn.o_proj": { |
| "base_key": "model.layers.26.self_attn.o_proj.weight", |
| "module": "model.layers.26.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005135575775057077, |
| "delta_mean_abs": 3.048473081435077e-05 |
| }, |
| "model.layers.26.self_attn.q_proj": { |
| "base_key": "model.layers.26.self_attn.q_proj.weight", |
| "module": "model.layers.26.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005703856586478651, |
| "delta_mean_abs": 3.2326548534911126e-05 |
| }, |
| "model.layers.26.self_attn.v_proj": { |
| "base_key": "model.layers.26.self_attn.v_proj.weight", |
| "module": "model.layers.26.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005065468139946461, |
| "delta_mean_abs": 3.0895920644979924e-05 |
| }, |
| "model.layers.27.mlp.down_proj": { |
| "base_key": "model.layers.27.mlp.down_proj.weight", |
| "module": "model.layers.27.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00041634959052316844, |
| "delta_mean_abs": 2.1655549062415957e-05 |
| }, |
| "model.layers.27.mlp.gate_proj": { |
| "base_key": "model.layers.27.mlp.gate_proj.weight", |
| "module": "model.layers.27.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0007026033708825707, |
| "delta_mean_abs": 3.719523738254793e-05 |
| }, |
| "model.layers.27.mlp.up_proj": { |
| "base_key": "model.layers.27.mlp.up_proj.weight", |
| "module": "model.layers.27.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006436676485463977, |
| "delta_mean_abs": 3.587271930882707e-05 |
| }, |
| "model.layers.27.self_attn.k_proj": { |
| "base_key": "model.layers.27.self_attn.k_proj.weight", |
| "module": "model.layers.27.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004348110524006188, |
| "delta_mean_abs": 2.8866721550002694e-05 |
| }, |
| "model.layers.27.self_attn.o_proj": { |
| "base_key": "model.layers.27.self_attn.o_proj.weight", |
| "module": "model.layers.27.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005425090785138309, |
| "delta_mean_abs": 3.136127270408906e-05 |
| }, |
| "model.layers.27.self_attn.q_proj": { |
| "base_key": "model.layers.27.self_attn.q_proj.weight", |
| "module": "model.layers.27.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005659640301018953, |
| "delta_mean_abs": 3.276155621279031e-05 |
| }, |
| "model.layers.27.self_attn.v_proj": { |
| "base_key": "model.layers.27.self_attn.v_proj.weight", |
| "module": "model.layers.27.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.000424560799729079, |
| "delta_mean_abs": 3.2723146432545036e-05 |
| }, |
| "model.layers.28.mlp.down_proj": { |
| "base_key": "model.layers.28.mlp.down_proj.weight", |
| "module": "model.layers.28.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.000376499374397099, |
| "delta_mean_abs": 2.0816356482100673e-05 |
| }, |
| "model.layers.28.mlp.gate_proj": { |
| "base_key": "model.layers.28.mlp.gate_proj.weight", |
| "module": "model.layers.28.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0007759013678878546, |
| "delta_mean_abs": 3.880199074046686e-05 |
| }, |
| "model.layers.28.mlp.up_proj": { |
| "base_key": "model.layers.28.mlp.up_proj.weight", |
| "module": "model.layers.28.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.000749202270526439, |
| "delta_mean_abs": 3.534151255735196e-05 |
| }, |
| "model.layers.28.self_attn.k_proj": { |
| "base_key": "model.layers.28.self_attn.k_proj.weight", |
| "module": "model.layers.28.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005018810625188053, |
| "delta_mean_abs": 3.025566002179403e-05 |
| }, |
| "model.layers.28.self_attn.o_proj": { |
| "base_key": "model.layers.28.self_attn.o_proj.weight", |
| "module": "model.layers.28.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004431013949215412, |
| "delta_mean_abs": 3.1759314879309386e-05 |
| }, |
| "model.layers.28.self_attn.q_proj": { |
| "base_key": "model.layers.28.self_attn.q_proj.weight", |
| "module": "model.layers.28.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004962035454809666, |
| "delta_mean_abs": 2.915566074079834e-05 |
| }, |
| "model.layers.28.self_attn.v_proj": { |
| "base_key": "model.layers.28.self_attn.v_proj.weight", |
| "module": "model.layers.28.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00047562684630975127, |
| "delta_mean_abs": 3.2382387871621177e-05 |
| }, |
| "model.layers.29.mlp.down_proj": { |
| "base_key": "model.layers.29.mlp.down_proj.weight", |
| "module": "model.layers.29.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00041410772246308625, |
| "delta_mean_abs": 2.083051367662847e-05 |
| }, |
| "model.layers.29.mlp.gate_proj": { |
| "base_key": "model.layers.29.mlp.gate_proj.weight", |
| "module": "model.layers.29.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0008486849255859852, |
| "delta_mean_abs": 3.923393160221167e-05 |
| }, |
| "model.layers.29.mlp.up_proj": { |
| "base_key": "model.layers.29.mlp.up_proj.weight", |
| "module": "model.layers.29.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0007861411431804299, |
| "delta_mean_abs": 3.4819782740669325e-05 |
| }, |
| "model.layers.29.self_attn.k_proj": { |
| "base_key": "model.layers.29.self_attn.k_proj.weight", |
| "module": "model.layers.29.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00046382981236092746, |
| "delta_mean_abs": 3.087160439463332e-05 |
| }, |
| "model.layers.29.self_attn.o_proj": { |
| "base_key": "model.layers.29.self_attn.o_proj.weight", |
| "module": "model.layers.29.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005282893544062972, |
| "delta_mean_abs": 3.1298783142119646e-05 |
| }, |
| "model.layers.29.self_attn.q_proj": { |
| "base_key": "model.layers.29.self_attn.q_proj.weight", |
| "module": "model.layers.29.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00048768805572763085, |
| "delta_mean_abs": 2.771464642137289e-05 |
| }, |
| "model.layers.29.self_attn.v_proj": { |
| "base_key": "model.layers.29.self_attn.v_proj.weight", |
| "module": "model.layers.29.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00046866273623891175, |
| "delta_mean_abs": 3.2133561035152525e-05 |
| }, |
| "model.layers.30.mlp.down_proj": { |
| "base_key": "model.layers.30.mlp.down_proj.weight", |
| "module": "model.layers.30.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0003852120426017791, |
| "delta_mean_abs": 2.0445459085749462e-05 |
| }, |
| "model.layers.30.mlp.gate_proj": { |
| "base_key": "model.layers.30.mlp.gate_proj.weight", |
| "module": "model.layers.30.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006641994696110487, |
| "delta_mean_abs": 3.699332955875434e-05 |
| }, |
| "model.layers.30.mlp.up_proj": { |
| "base_key": "model.layers.30.mlp.up_proj.weight", |
| "module": "model.layers.30.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006927794311195612, |
| "delta_mean_abs": 3.272937101428397e-05 |
| }, |
| "model.layers.30.self_attn.k_proj": { |
| "base_key": "model.layers.30.self_attn.k_proj.weight", |
| "module": "model.layers.30.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.000526847317814827, |
| "delta_mean_abs": 3.015199581568595e-05 |
| }, |
| "model.layers.30.self_attn.o_proj": { |
| "base_key": "model.layers.30.self_attn.o_proj.weight", |
| "module": "model.layers.30.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004556751227937639, |
| "delta_mean_abs": 3.15938304993324e-05 |
| }, |
| "model.layers.30.self_attn.q_proj": { |
| "base_key": "model.layers.30.self_attn.q_proj.weight", |
| "module": "model.layers.30.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006128204404376447, |
| "delta_mean_abs": 2.982766818604432e-05 |
| }, |
| "model.layers.30.self_attn.v_proj": { |
| "base_key": "model.layers.30.self_attn.v_proj.weight", |
| "module": "model.layers.30.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.000492256076540798, |
| "delta_mean_abs": 3.2863510568859056e-05 |
| }, |
| "model.layers.31.mlp.down_proj": { |
| "base_key": "model.layers.31.mlp.down_proj.weight", |
| "module": "model.layers.31.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00046290218597278, |
| "delta_mean_abs": 2.188756116083823e-05 |
| }, |
| "model.layers.31.mlp.gate_proj": { |
| "base_key": "model.layers.31.mlp.gate_proj.weight", |
| "module": "model.layers.31.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0009979824535548687, |
| "delta_mean_abs": 3.967043085140176e-05 |
| }, |
| "model.layers.31.mlp.up_proj": { |
| "base_key": "model.layers.31.mlp.up_proj.weight", |
| "module": "model.layers.31.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006500134477391839, |
| "delta_mean_abs": 3.566565646906383e-05 |
| }, |
| "model.layers.31.self_attn.k_proj": { |
| "base_key": "model.layers.31.self_attn.k_proj.weight", |
| "module": "model.layers.31.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005368662532418966, |
| "delta_mean_abs": 3.209201167919673e-05 |
| }, |
| "model.layers.31.self_attn.o_proj": { |
| "base_key": "model.layers.31.self_attn.o_proj.weight", |
| "module": "model.layers.31.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005189056391827762, |
| "delta_mean_abs": 3.260055746068247e-05 |
| }, |
| "model.layers.31.self_attn.q_proj": { |
| "base_key": "model.layers.31.self_attn.q_proj.weight", |
| "module": "model.layers.31.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.000686001731082797, |
| "delta_mean_abs": 2.9941722459625453e-05 |
| }, |
| "model.layers.31.self_attn.v_proj": { |
| "base_key": "model.layers.31.self_attn.v_proj.weight", |
| "module": "model.layers.31.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006456708651967347, |
| "delta_mean_abs": 3.5762597690336406e-05 |
| }, |
| "model.layers.32.mlp.down_proj": { |
| "base_key": "model.layers.32.mlp.down_proj.weight", |
| "module": "model.layers.32.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00047211372293531895, |
| "delta_mean_abs": 2.2208392692846246e-05 |
| }, |
| "model.layers.32.mlp.gate_proj": { |
| "base_key": "model.layers.32.mlp.gate_proj.weight", |
| "module": "model.layers.32.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006780275725759566, |
| "delta_mean_abs": 3.9647307858103886e-05 |
| }, |
| "model.layers.32.mlp.up_proj": { |
| "base_key": "model.layers.32.mlp.up_proj.weight", |
| "module": "model.layers.32.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0007835898431949317, |
| "delta_mean_abs": 3.651916267699562e-05 |
| }, |
| "model.layers.32.self_attn.k_proj": { |
| "base_key": "model.layers.32.self_attn.k_proj.weight", |
| "module": "model.layers.32.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005122994771227241, |
| "delta_mean_abs": 3.1651914468966424e-05 |
| }, |
| "model.layers.32.self_attn.o_proj": { |
| "base_key": "model.layers.32.self_attn.o_proj.weight", |
| "module": "model.layers.32.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005488019087351859, |
| "delta_mean_abs": 3.2963282137643546e-05 |
| }, |
| "model.layers.32.self_attn.q_proj": { |
| "base_key": "model.layers.32.self_attn.q_proj.weight", |
| "module": "model.layers.32.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005603755125775933, |
| "delta_mean_abs": 3.066295539611019e-05 |
| }, |
| "model.layers.32.self_attn.v_proj": { |
| "base_key": "model.layers.32.self_attn.v_proj.weight", |
| "module": "model.layers.32.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004880889900960028, |
| "delta_mean_abs": 3.6247285606805235e-05 |
| }, |
| "model.layers.33.mlp.down_proj": { |
| "base_key": "model.layers.33.mlp.down_proj.weight", |
| "module": "model.layers.33.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00046494242269545794, |
| "delta_mean_abs": 1.967673233593814e-05 |
| }, |
| "model.layers.33.mlp.gate_proj": { |
| "base_key": "model.layers.33.mlp.gate_proj.weight", |
| "module": "model.layers.33.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0010205315193161368, |
| "delta_mean_abs": 4.296499173506163e-05 |
| }, |
| "model.layers.33.mlp.up_proj": { |
| "base_key": "model.layers.33.mlp.up_proj.weight", |
| "module": "model.layers.33.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0008991762297227979, |
| "delta_mean_abs": 3.632656080299057e-05 |
| }, |
| "model.layers.33.self_attn.k_proj": { |
| "base_key": "model.layers.33.self_attn.k_proj.weight", |
| "module": "model.layers.33.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0003362589341122657, |
| "delta_mean_abs": 3.2607415050733835e-05 |
| }, |
| "model.layers.33.self_attn.o_proj": { |
| "base_key": "model.layers.33.self_attn.o_proj.weight", |
| "module": "model.layers.33.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005127033218741417, |
| "delta_mean_abs": 3.3575688576092944e-05 |
| }, |
| "model.layers.33.self_attn.q_proj": { |
| "base_key": "model.layers.33.self_attn.q_proj.weight", |
| "module": "model.layers.33.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005462816916406155, |
| "delta_mean_abs": 3.1112300348468125e-05 |
| }, |
| "model.layers.33.self_attn.v_proj": { |
| "base_key": "model.layers.33.self_attn.v_proj.weight", |
| "module": "model.layers.33.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004778730508405715, |
| "delta_mean_abs": 3.6407338484423235e-05 |
| }, |
| "model.layers.34.mlp.down_proj": { |
| "base_key": "model.layers.34.mlp.down_proj.weight", |
| "module": "model.layers.34.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00035231365473009646, |
| "delta_mean_abs": 1.7413749446859583e-05 |
| }, |
| "model.layers.34.mlp.gate_proj": { |
| "base_key": "model.layers.34.mlp.gate_proj.weight", |
| "module": "model.layers.34.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0009908298961818218, |
| "delta_mean_abs": 4.174706919002347e-05 |
| }, |
| "model.layers.34.mlp.up_proj": { |
| "base_key": "model.layers.34.mlp.up_proj.weight", |
| "module": "model.layers.34.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.000783766561653465, |
| "delta_mean_abs": 3.5336717701284215e-05 |
| }, |
| "model.layers.34.self_attn.k_proj": { |
| "base_key": "model.layers.34.self_attn.k_proj.weight", |
| "module": "model.layers.34.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004040884377900511, |
| "delta_mean_abs": 3.1170005968306214e-05 |
| }, |
| "model.layers.34.self_attn.o_proj": { |
| "base_key": "model.layers.34.self_attn.o_proj.weight", |
| "module": "model.layers.34.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00043990896665491164, |
| "delta_mean_abs": 2.9985816581756808e-05 |
| }, |
| "model.layers.34.self_attn.q_proj": { |
| "base_key": "model.layers.34.self_attn.q_proj.weight", |
| "module": "model.layers.34.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00046395423123613, |
| "delta_mean_abs": 3.0289735150290653e-05 |
| }, |
| "model.layers.34.self_attn.v_proj": { |
| "base_key": "model.layers.34.self_attn.v_proj.weight", |
| "module": "model.layers.34.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005479517276398838, |
| "delta_mean_abs": 3.35903387167491e-05 |
| }, |
| "model.layers.35.self_attn.q_proj": { |
| "base_key": "model.layers.35.self_attn.q_proj.weight", |
| "module": "model.layers.35.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005421487148851156, |
| "delta_mean_abs": 3.139608816127293e-05 |
| }, |
| "model.layers.35.mlp.down_proj": { |
| "base_key": "model.layers.35.mlp.down_proj.weight", |
| "module": "model.layers.35.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00032082918914966285, |
| "delta_mean_abs": 1.8702803572523408e-05 |
| }, |
| "model.layers.35.mlp.gate_proj": { |
| "base_key": "model.layers.35.mlp.gate_proj.weight", |
| "module": "model.layers.35.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0011147449258714914, |
| "delta_mean_abs": 3.892763197654858e-05 |
| }, |
| "model.layers.35.mlp.up_proj": { |
| "base_key": "model.layers.35.mlp.up_proj.weight", |
| "module": "model.layers.35.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0007120087975636125, |
| "delta_mean_abs": 3.5402390494709834e-05 |
| }, |
| "model.layers.35.self_attn.k_proj": { |
| "base_key": "model.layers.35.self_attn.k_proj.weight", |
| "module": "model.layers.35.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0003369122860021889, |
| "delta_mean_abs": 2.97386541205924e-05 |
| }, |
| "model.layers.35.self_attn.o_proj": { |
| "base_key": "model.layers.35.self_attn.o_proj.weight", |
| "module": "model.layers.35.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006077519501559436, |
| "delta_mean_abs": 2.8890683097415604e-05 |
| }, |
| "model.layers.35.self_attn.v_proj": { |
| "base_key": "model.layers.35.self_attn.v_proj.weight", |
| "module": "model.layers.35.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005003415863029659, |
| "delta_mean_abs": 3.181271313223988e-05 |
| }, |
| "model.layers.36.mlp.down_proj": { |
| "base_key": "model.layers.36.mlp.down_proj.weight", |
| "module": "model.layers.36.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00031107431277632713, |
| "delta_mean_abs": 1.830846849770751e-05 |
| }, |
| "model.layers.36.mlp.gate_proj": { |
| "base_key": "model.layers.36.mlp.gate_proj.weight", |
| "module": "model.layers.36.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0008636223501525819, |
| "delta_mean_abs": 4.07787665608339e-05 |
| }, |
| "model.layers.36.mlp.up_proj": { |
| "base_key": "model.layers.36.mlp.up_proj.weight", |
| "module": "model.layers.36.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.001006239908747375, |
| "delta_mean_abs": 3.894712062901817e-05 |
| }, |
| "model.layers.36.self_attn.k_proj": { |
| "base_key": "model.layers.36.self_attn.k_proj.weight", |
| "module": "model.layers.36.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0008924414869397879, |
| "delta_mean_abs": 3.314469722681679e-05 |
| }, |
| "model.layers.36.self_attn.o_proj": { |
| "base_key": "model.layers.36.self_attn.o_proj.weight", |
| "module": "model.layers.36.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00048726610839366913, |
| "delta_mean_abs": 2.8322330763330683e-05 |
| }, |
| "model.layers.36.self_attn.q_proj": { |
| "base_key": "model.layers.36.self_attn.q_proj.weight", |
| "module": "model.layers.36.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00091399229131639, |
| "delta_mean_abs": 3.5189310437999666e-05 |
| }, |
| "model.layers.36.self_attn.v_proj": { |
| "base_key": "model.layers.36.self_attn.v_proj.weight", |
| "module": "model.layers.36.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004785800410900265, |
| "delta_mean_abs": 3.151838245685212e-05 |
| }, |
| "model.layers.37.mlp.down_proj": { |
| "base_key": "model.layers.37.mlp.down_proj.weight", |
| "module": "model.layers.37.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0003155853773932904, |
| "delta_mean_abs": 1.697808511380572e-05 |
| }, |
| "model.layers.37.mlp.gate_proj": { |
| "base_key": "model.layers.37.mlp.gate_proj.weight", |
| "module": "model.layers.37.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0008523616124875844, |
| "delta_mean_abs": 4.009905023849569e-05 |
| }, |
| "model.layers.37.mlp.up_proj": { |
| "base_key": "model.layers.37.mlp.up_proj.weight", |
| "module": "model.layers.37.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.000742932315915823, |
| "delta_mean_abs": 3.849756467388943e-05 |
| }, |
| "model.layers.37.self_attn.k_proj": { |
| "base_key": "model.layers.37.self_attn.k_proj.weight", |
| "module": "model.layers.37.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.000447834754595533, |
| "delta_mean_abs": 3.186934191035107e-05 |
| }, |
| "model.layers.37.self_attn.o_proj": { |
| "base_key": "model.layers.37.self_attn.o_proj.weight", |
| "module": "model.layers.37.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004809541569557041, |
| "delta_mean_abs": 2.7444939405540936e-05 |
| }, |
| "model.layers.37.self_attn.q_proj": { |
| "base_key": "model.layers.37.self_attn.q_proj.weight", |
| "module": "model.layers.37.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005997818661853671, |
| "delta_mean_abs": 2.9726596039836295e-05 |
| }, |
| "model.layers.37.self_attn.v_proj": { |
| "base_key": "model.layers.37.self_attn.v_proj.weight", |
| "module": "model.layers.37.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004297611303627491, |
| "delta_mean_abs": 3.052594547625631e-05 |
| }, |
| "model.layers.38.mlp.down_proj": { |
| "base_key": "model.layers.38.mlp.down_proj.weight", |
| "module": "model.layers.38.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00028839349397458136, |
| "delta_mean_abs": 1.6918413166422397e-05 |
| }, |
| "model.layers.38.mlp.gate_proj": { |
| "base_key": "model.layers.38.mlp.gate_proj.weight", |
| "module": "model.layers.38.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.001065364689566195, |
| "delta_mean_abs": 4.0387018088949844e-05 |
| }, |
| "model.layers.38.mlp.up_proj": { |
| "base_key": "model.layers.38.mlp.up_proj.weight", |
| "module": "model.layers.38.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006954649579711258, |
| "delta_mean_abs": 3.480457962723449e-05 |
| }, |
| "model.layers.38.self_attn.k_proj": { |
| "base_key": "model.layers.38.self_attn.k_proj.weight", |
| "module": "model.layers.38.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00040716829244047403, |
| "delta_mean_abs": 2.9893722967244685e-05 |
| }, |
| "model.layers.38.self_attn.o_proj": { |
| "base_key": "model.layers.38.self_attn.o_proj.weight", |
| "module": "model.layers.38.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004174862406216562, |
| "delta_mean_abs": 2.7689127819030546e-05 |
| }, |
| "model.layers.38.self_attn.q_proj": { |
| "base_key": "model.layers.38.self_attn.q_proj.weight", |
| "module": "model.layers.38.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006464465404860675, |
| "delta_mean_abs": 2.9984816137584858e-05 |
| }, |
| "model.layers.38.self_attn.v_proj": { |
| "base_key": "model.layers.38.self_attn.v_proj.weight", |
| "module": "model.layers.38.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0003740675456356257, |
| "delta_mean_abs": 3.0346342100529e-05 |
| }, |
| "model.layers.39.mlp.down_proj": { |
| "base_key": "model.layers.39.mlp.down_proj.weight", |
| "module": "model.layers.39.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0002961461723316461, |
| "delta_mean_abs": 1.8194896256318316e-05 |
| }, |
| "model.layers.39.mlp.gate_proj": { |
| "base_key": "model.layers.39.mlp.gate_proj.weight", |
| "module": "model.layers.39.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0007970178266987205, |
| "delta_mean_abs": 3.92308684240561e-05 |
| }, |
| "model.layers.39.mlp.up_proj": { |
| "base_key": "model.layers.39.mlp.up_proj.weight", |
| "module": "model.layers.39.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0007019301410764456, |
| "delta_mean_abs": 3.728358569787815e-05 |
| }, |
| "model.layers.39.self_attn.k_proj": { |
| "base_key": "model.layers.39.self_attn.k_proj.weight", |
| "module": "model.layers.39.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0003473837277851999, |
| "delta_mean_abs": 2.6527020963840187e-05 |
| }, |
| "model.layers.39.self_attn.o_proj": { |
| "base_key": "model.layers.39.self_attn.o_proj.weight", |
| "module": "model.layers.39.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00042731009307317436, |
| "delta_mean_abs": 2.6828305635717697e-05 |
| }, |
| "model.layers.39.self_attn.q_proj": { |
| "base_key": "model.layers.39.self_attn.q_proj.weight", |
| "module": "model.layers.39.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006152298301458359, |
| "delta_mean_abs": 2.8421343813533895e-05 |
| }, |
| "model.layers.39.self_attn.v_proj": { |
| "base_key": "model.layers.39.self_attn.v_proj.weight", |
| "module": "model.layers.39.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00039450175245292485, |
| "delta_mean_abs": 2.8525952075142413e-05 |
| }, |
| "model.layers.40.mlp.down_proj": { |
| "base_key": "model.layers.40.mlp.down_proj.weight", |
| "module": "model.layers.40.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00029594526859000325, |
| "delta_mean_abs": 1.709214484435506e-05 |
| }, |
| "model.layers.40.mlp.gate_proj": { |
| "base_key": "model.layers.40.mlp.gate_proj.weight", |
| "module": "model.layers.40.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.000681317993439734, |
| "delta_mean_abs": 3.6720015486935154e-05 |
| }, |
| "model.layers.40.mlp.up_proj": { |
| "base_key": "model.layers.40.mlp.up_proj.weight", |
| "module": "model.layers.40.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0008296065498143435, |
| "delta_mean_abs": 3.917914000339806e-05 |
| }, |
| "model.layers.40.self_attn.k_proj": { |
| "base_key": "model.layers.40.self_attn.k_proj.weight", |
| "module": "model.layers.40.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00034495291765779257, |
| "delta_mean_abs": 2.9910184821346775e-05 |
| }, |
| "model.layers.40.self_attn.o_proj": { |
| "base_key": "model.layers.40.self_attn.o_proj.weight", |
| "module": "model.layers.40.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005356971523724496, |
| "delta_mean_abs": 2.947970642708242e-05 |
| }, |
| "model.layers.40.self_attn.q_proj": { |
| "base_key": "model.layers.40.self_attn.q_proj.weight", |
| "module": "model.layers.40.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00046502085751853883, |
| "delta_mean_abs": 2.851252065738663e-05 |
| }, |
| "model.layers.40.self_attn.v_proj": { |
| "base_key": "model.layers.40.self_attn.v_proj.weight", |
| "module": "model.layers.40.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004806818615179509, |
| "delta_mean_abs": 2.9176389944041148e-05 |
| }, |
| "model.layers.41.mlp.down_proj": { |
| "base_key": "model.layers.41.mlp.down_proj.weight", |
| "module": "model.layers.41.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00025182575336657465, |
| "delta_mean_abs": 1.7500367903267033e-05 |
| }, |
| "model.layers.41.mlp.gate_proj": { |
| "base_key": "model.layers.41.mlp.gate_proj.weight", |
| "module": "model.layers.41.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0007002067868597806, |
| "delta_mean_abs": 3.7423909816425294e-05 |
| }, |
| "model.layers.41.mlp.up_proj": { |
| "base_key": "model.layers.41.mlp.up_proj.weight", |
| "module": "model.layers.41.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0008171783993020654, |
| "delta_mean_abs": 3.675529660540633e-05 |
| }, |
| "model.layers.41.self_attn.k_proj": { |
| "base_key": "model.layers.41.self_attn.k_proj.weight", |
| "module": "model.layers.41.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004009816038887948, |
| "delta_mean_abs": 3.074146661674604e-05 |
| }, |
| "model.layers.41.self_attn.o_proj": { |
| "base_key": "model.layers.41.self_attn.o_proj.weight", |
| "module": "model.layers.41.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.000497576838824898, |
| "delta_mean_abs": 2.7241538191447034e-05 |
| }, |
| "model.layers.41.self_attn.q_proj": { |
| "base_key": "model.layers.41.self_attn.q_proj.weight", |
| "module": "model.layers.41.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004348670190665871, |
| "delta_mean_abs": 2.6945595891447738e-05 |
| }, |
| "model.layers.41.self_attn.v_proj": { |
| "base_key": "model.layers.41.self_attn.v_proj.weight", |
| "module": "model.layers.41.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00046172470320016146, |
| "delta_mean_abs": 2.7789252271759324e-05 |
| }, |
| "model.layers.42.mlp.down_proj": { |
| "base_key": "model.layers.42.mlp.down_proj.weight", |
| "module": "model.layers.42.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0002663386694621295, |
| "delta_mean_abs": 1.7591384676052257e-05 |
| }, |
| "model.layers.42.mlp.gate_proj": { |
| "base_key": "model.layers.42.mlp.gate_proj.weight", |
| "module": "model.layers.42.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0007347972132265568, |
| "delta_mean_abs": 3.646871118689887e-05 |
| }, |
| "model.layers.42.mlp.up_proj": { |
| "base_key": "model.layers.42.mlp.up_proj.weight", |
| "module": "model.layers.42.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005968305049464107, |
| "delta_mean_abs": 3.379353802301921e-05 |
| }, |
| "model.layers.42.self_attn.k_proj": { |
| "base_key": "model.layers.42.self_attn.k_proj.weight", |
| "module": "model.layers.42.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00044519209768623114, |
| "delta_mean_abs": 2.7669648261507973e-05 |
| }, |
| "model.layers.42.self_attn.o_proj": { |
| "base_key": "model.layers.42.self_attn.o_proj.weight", |
| "module": "model.layers.42.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005367975682020187, |
| "delta_mean_abs": 2.765120007097721e-05 |
| }, |
| "model.layers.42.self_attn.q_proj": { |
| "base_key": "model.layers.42.self_attn.q_proj.weight", |
| "module": "model.layers.42.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004673278599511832, |
| "delta_mean_abs": 2.6646030164556578e-05 |
| }, |
| "model.layers.42.self_attn.v_proj": { |
| "base_key": "model.layers.42.self_attn.v_proj.weight", |
| "module": "model.layers.42.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004160058160778135, |
| "delta_mean_abs": 2.740255331445951e-05 |
| }, |
| "model.layers.43.mlp.down_proj": { |
| "base_key": "model.layers.43.mlp.down_proj.weight", |
| "module": "model.layers.43.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00023772021813783795, |
| "delta_mean_abs": 1.7026000932673924e-05 |
| }, |
| "model.layers.43.mlp.gate_proj": { |
| "base_key": "model.layers.43.mlp.gate_proj.weight", |
| "module": "model.layers.43.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0007745756884105504, |
| "delta_mean_abs": 3.784118962357752e-05 |
| }, |
| "model.layers.43.mlp.up_proj": { |
| "base_key": "model.layers.43.mlp.up_proj.weight", |
| "module": "model.layers.43.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006806864985264838, |
| "delta_mean_abs": 3.7699832319049165e-05 |
| }, |
| "model.layers.43.self_attn.k_proj": { |
| "base_key": "model.layers.43.self_attn.k_proj.weight", |
| "module": "model.layers.43.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0003728914598468691, |
| "delta_mean_abs": 2.775310531433206e-05 |
| }, |
| "model.layers.43.self_attn.o_proj": { |
| "base_key": "model.layers.43.self_attn.o_proj.weight", |
| "module": "model.layers.43.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00048579019494354725, |
| "delta_mean_abs": 2.784214484563563e-05 |
| }, |
| "model.layers.43.self_attn.q_proj": { |
| "base_key": "model.layers.43.self_attn.q_proj.weight", |
| "module": "model.layers.43.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004595101345330477, |
| "delta_mean_abs": 2.9440008802339435e-05 |
| }, |
| "model.layers.43.self_attn.v_proj": { |
| "base_key": "model.layers.43.self_attn.v_proj.weight", |
| "module": "model.layers.43.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00042334754834882915, |
| "delta_mean_abs": 2.896552177844569e-05 |
| }, |
| "model.layers.44.mlp.down_proj": { |
| "base_key": "model.layers.44.mlp.down_proj.weight", |
| "module": "model.layers.44.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.000286812981357798, |
| "delta_mean_abs": 1.7443613614887e-05 |
| }, |
| "model.layers.44.mlp.gate_proj": { |
| "base_key": "model.layers.44.mlp.gate_proj.weight", |
| "module": "model.layers.44.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005417307838797569, |
| "delta_mean_abs": 3.3523647289257497e-05 |
| }, |
| "model.layers.44.mlp.up_proj": { |
| "base_key": "model.layers.44.mlp.up_proj.weight", |
| "module": "model.layers.44.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0010184006532654166, |
| "delta_mean_abs": 3.684139301185496e-05 |
| }, |
| "model.layers.44.self_attn.k_proj": { |
| "base_key": "model.layers.44.self_attn.k_proj.weight", |
| "module": "model.layers.44.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00048543131561018527, |
| "delta_mean_abs": 3.130015102215111e-05 |
| }, |
| "model.layers.44.self_attn.o_proj": { |
| "base_key": "model.layers.44.self_attn.o_proj.weight", |
| "module": "model.layers.44.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006107743247412145, |
| "delta_mean_abs": 3.0081435397733003e-05 |
| }, |
| "model.layers.44.self_attn.q_proj": { |
| "base_key": "model.layers.44.self_attn.q_proj.weight", |
| "module": "model.layers.44.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0007446313393302262, |
| "delta_mean_abs": 3.0549181246897206e-05 |
| }, |
| "model.layers.44.self_attn.v_proj": { |
| "base_key": "model.layers.44.self_attn.v_proj.weight", |
| "module": "model.layers.44.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004254269879311323, |
| "delta_mean_abs": 3.108121745754033e-05 |
| }, |
| "model.layers.45.self_attn.q_proj": { |
| "base_key": "model.layers.45.self_attn.q_proj.weight", |
| "module": "model.layers.45.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006016346160322428, |
| "delta_mean_abs": 2.9297618311829865e-05 |
| }, |
| "model.layers.45.mlp.down_proj": { |
| "base_key": "model.layers.45.mlp.down_proj.weight", |
| "module": "model.layers.45.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00027781628887169063, |
| "delta_mean_abs": 1.7283231500186957e-05 |
| }, |
| "model.layers.45.mlp.gate_proj": { |
| "base_key": "model.layers.45.mlp.gate_proj.weight", |
| "module": "model.layers.45.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0008376310579478741, |
| "delta_mean_abs": 3.581325654522516e-05 |
| }, |
| "model.layers.45.mlp.up_proj": { |
| "base_key": "model.layers.45.mlp.up_proj.weight", |
| "module": "model.layers.45.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005828288267366588, |
| "delta_mean_abs": 3.467064016149379e-05 |
| }, |
| "model.layers.45.self_attn.k_proj": { |
| "base_key": "model.layers.45.self_attn.k_proj.weight", |
| "module": "model.layers.45.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00031261175172403455, |
| "delta_mean_abs": 2.6185036404058337e-05 |
| }, |
| "model.layers.45.self_attn.o_proj": { |
| "base_key": "model.layers.45.self_attn.o_proj.weight", |
| "module": "model.layers.45.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005854540504515171, |
| "delta_mean_abs": 3.0488052289001644e-05 |
| }, |
| "model.layers.45.self_attn.v_proj": { |
| "base_key": "model.layers.45.self_attn.v_proj.weight", |
| "module": "model.layers.45.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00041751848766580224, |
| "delta_mean_abs": 3.188758637406863e-05 |
| }, |
| "model.layers.46.mlp.down_proj": { |
| "base_key": "model.layers.46.mlp.down_proj.weight", |
| "module": "model.layers.46.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00025634642224758863, |
| "delta_mean_abs": 1.715251346467994e-05 |
| }, |
| "model.layers.46.mlp.gate_proj": { |
| "base_key": "model.layers.46.mlp.gate_proj.weight", |
| "module": "model.layers.46.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0008066649897955358, |
| "delta_mean_abs": 3.869008651236072e-05 |
| }, |
| "model.layers.46.mlp.up_proj": { |
| "base_key": "model.layers.46.mlp.up_proj.weight", |
| "module": "model.layers.46.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005771343712694943, |
| "delta_mean_abs": 3.398791886866093e-05 |
| }, |
| "model.layers.46.self_attn.k_proj": { |
| "base_key": "model.layers.46.self_attn.k_proj.weight", |
| "module": "model.layers.46.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00023649884678889066, |
| "delta_mean_abs": 2.430964377708733e-05 |
| }, |
| "model.layers.46.self_attn.o_proj": { |
| "base_key": "model.layers.46.self_attn.o_proj.weight", |
| "module": "model.layers.46.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00038045173278078437, |
| "delta_mean_abs": 2.6320951292291284e-05 |
| }, |
| "model.layers.46.self_attn.q_proj": { |
| "base_key": "model.layers.46.self_attn.q_proj.weight", |
| "module": "model.layers.46.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00027432109345681965, |
| "delta_mean_abs": 2.448376835673116e-05 |
| }, |
| "model.layers.46.self_attn.v_proj": { |
| "base_key": "model.layers.46.self_attn.v_proj.weight", |
| "module": "model.layers.46.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004108718712814152, |
| "delta_mean_abs": 2.765154385997448e-05 |
| }, |
| "model.layers.47.mlp.down_proj": { |
| "base_key": "model.layers.47.mlp.down_proj.weight", |
| "module": "model.layers.47.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0002678571327123791, |
| "delta_mean_abs": 1.715244616207201e-05 |
| }, |
| "model.layers.47.mlp.gate_proj": { |
| "base_key": "model.layers.47.mlp.gate_proj.weight", |
| "module": "model.layers.47.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005976863903924823, |
| "delta_mean_abs": 3.364206349942833e-05 |
| }, |
| "model.layers.47.mlp.up_proj": { |
| "base_key": "model.layers.47.mlp.up_proj.weight", |
| "module": "model.layers.47.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0010268824407830834, |
| "delta_mean_abs": 3.6555495171342045e-05 |
| }, |
| "model.layers.47.self_attn.k_proj": { |
| "base_key": "model.layers.47.self_attn.k_proj.weight", |
| "module": "model.layers.47.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0003631594590842724, |
| "delta_mean_abs": 3.1008774385554716e-05 |
| }, |
| "model.layers.47.self_attn.o_proj": { |
| "base_key": "model.layers.47.self_attn.o_proj.weight", |
| "module": "model.layers.47.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005022079567424953, |
| "delta_mean_abs": 2.8746029784088023e-05 |
| }, |
| "model.layers.47.self_attn.q_proj": { |
| "base_key": "model.layers.47.self_attn.q_proj.weight", |
| "module": "model.layers.47.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00042845410644076765, |
| "delta_mean_abs": 2.812365346471779e-05 |
| }, |
| "model.layers.47.self_attn.v_proj": { |
| "base_key": "model.layers.47.self_attn.v_proj.weight", |
| "module": "model.layers.47.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.000507188553456217, |
| "delta_mean_abs": 3.129264950985089e-05 |
| }, |
| "model.layers.48.mlp.down_proj": { |
| "base_key": "model.layers.48.mlp.down_proj.weight", |
| "module": "model.layers.48.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0003066085046157241, |
| "delta_mean_abs": 1.7241323803318664e-05 |
| }, |
| "model.layers.48.mlp.gate_proj": { |
| "base_key": "model.layers.48.mlp.gate_proj.weight", |
| "module": "model.layers.48.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.000737904803827405, |
| "delta_mean_abs": 3.523816849337891e-05 |
| }, |
| "model.layers.48.mlp.up_proj": { |
| "base_key": "model.layers.48.mlp.up_proj.weight", |
| "module": "model.layers.48.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.000698602176271379, |
| "delta_mean_abs": 3.6929963243892416e-05 |
| }, |
| "model.layers.48.self_attn.k_proj": { |
| "base_key": "model.layers.48.self_attn.k_proj.weight", |
| "module": "model.layers.48.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00026751539553515613, |
| "delta_mean_abs": 2.9061064196866937e-05 |
| }, |
| "model.layers.48.self_attn.o_proj": { |
| "base_key": "model.layers.48.self_attn.o_proj.weight", |
| "module": "model.layers.48.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004805081116501242, |
| "delta_mean_abs": 2.7203084755456075e-05 |
| }, |
| "model.layers.48.self_attn.q_proj": { |
| "base_key": "model.layers.48.self_attn.q_proj.weight", |
| "module": "model.layers.48.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004430412664078176, |
| "delta_mean_abs": 2.6943038392346352e-05 |
| }, |
| "model.layers.48.self_attn.v_proj": { |
| "base_key": "model.layers.48.self_attn.v_proj.weight", |
| "module": "model.layers.48.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004731325025204569, |
| "delta_mean_abs": 3.0082144803600386e-05 |
| }, |
| "model.layers.49.mlp.down_proj": { |
| "base_key": "model.layers.49.mlp.down_proj.weight", |
| "module": "model.layers.49.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0002612618845887482, |
| "delta_mean_abs": 1.658838846196886e-05 |
| }, |
| "model.layers.49.mlp.gate_proj": { |
| "base_key": "model.layers.49.mlp.gate_proj.weight", |
| "module": "model.layers.49.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006817448884248734, |
| "delta_mean_abs": 3.279067459516227e-05 |
| }, |
| "model.layers.49.mlp.up_proj": { |
| "base_key": "model.layers.49.mlp.up_proj.weight", |
| "module": "model.layers.49.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.000571961048990488, |
| "delta_mean_abs": 3.372990249772556e-05 |
| }, |
| "model.layers.49.self_attn.k_proj": { |
| "base_key": "model.layers.49.self_attn.k_proj.weight", |
| "module": "model.layers.49.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004050247371196747, |
| "delta_mean_abs": 3.160976484650746e-05 |
| }, |
| "model.layers.49.self_attn.o_proj": { |
| "base_key": "model.layers.49.self_attn.o_proj.weight", |
| "module": "model.layers.49.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004134149639867246, |
| "delta_mean_abs": 2.948636029032059e-05 |
| }, |
| "model.layers.49.self_attn.q_proj": { |
| "base_key": "model.layers.49.self_attn.q_proj.weight", |
| "module": "model.layers.49.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004713641828857362, |
| "delta_mean_abs": 2.7362541004549712e-05 |
| }, |
| "model.layers.49.self_attn.v_proj": { |
| "base_key": "model.layers.49.self_attn.v_proj.weight", |
| "module": "model.layers.49.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00043007114436477423, |
| "delta_mean_abs": 2.9732573239016347e-05 |
| }, |
| "model.layers.50.mlp.down_proj": { |
| "base_key": "model.layers.50.mlp.down_proj.weight", |
| "module": "model.layers.50.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00023194488312583417, |
| "delta_mean_abs": 1.7242582543985918e-05 |
| }, |
| "model.layers.50.mlp.gate_proj": { |
| "base_key": "model.layers.50.mlp.gate_proj.weight", |
| "module": "model.layers.50.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005742249777540565, |
| "delta_mean_abs": 3.279751763329841e-05 |
| }, |
| "model.layers.50.mlp.up_proj": { |
| "base_key": "model.layers.50.mlp.up_proj.weight", |
| "module": "model.layers.50.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0007408225792460144, |
| "delta_mean_abs": 3.439130159677006e-05 |
| }, |
| "model.layers.50.self_attn.k_proj": { |
| "base_key": "model.layers.50.self_attn.k_proj.weight", |
| "module": "model.layers.50.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00019026474910788238, |
| "delta_mean_abs": 2.4938890419434756e-05 |
| }, |
| "model.layers.50.self_attn.o_proj": { |
| "base_key": "model.layers.50.self_attn.o_proj.weight", |
| "module": "model.layers.50.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004952774615958333, |
| "delta_mean_abs": 2.636972931213677e-05 |
| }, |
| "model.layers.50.self_attn.q_proj": { |
| "base_key": "model.layers.50.self_attn.q_proj.weight", |
| "module": "model.layers.50.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00034488431992940605, |
| "delta_mean_abs": 2.5767822080524638e-05 |
| }, |
| "model.layers.50.self_attn.v_proj": { |
| "base_key": "model.layers.50.self_attn.v_proj.weight", |
| "module": "model.layers.50.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004133190377615392, |
| "delta_mean_abs": 2.7670121198752895e-05 |
| }, |
| "model.layers.51.mlp.down_proj": { |
| "base_key": "model.layers.51.mlp.down_proj.weight", |
| "module": "model.layers.51.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00023741941549815238, |
| "delta_mean_abs": 1.7018424841808155e-05 |
| }, |
| "model.layers.51.mlp.gate_proj": { |
| "base_key": "model.layers.51.mlp.gate_proj.weight", |
| "module": "model.layers.51.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0007643967983312905, |
| "delta_mean_abs": 3.5127741284668446e-05 |
| }, |
| "model.layers.51.mlp.up_proj": { |
| "base_key": "model.layers.51.mlp.up_proj.weight", |
| "module": "model.layers.51.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0007651571067981422, |
| "delta_mean_abs": 3.326529622427188e-05 |
| }, |
| "model.layers.51.self_attn.k_proj": { |
| "base_key": "model.layers.51.self_attn.k_proj.weight", |
| "module": "model.layers.51.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00025259843096137047, |
| "delta_mean_abs": 2.628485526656732e-05 |
| }, |
| "model.layers.51.self_attn.o_proj": { |
| "base_key": "model.layers.51.self_attn.o_proj.weight", |
| "module": "model.layers.51.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004929337301291525, |
| "delta_mean_abs": 2.7723737730411813e-05 |
| }, |
| "model.layers.51.self_attn.q_proj": { |
| "base_key": "model.layers.51.self_attn.q_proj.weight", |
| "module": "model.layers.51.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00037992766010574996, |
| "delta_mean_abs": 2.6464798793313093e-05 |
| }, |
| "model.layers.51.self_attn.v_proj": { |
| "base_key": "model.layers.51.self_attn.v_proj.weight", |
| "module": "model.layers.51.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004395891446620226, |
| "delta_mean_abs": 2.937245517387055e-05 |
| }, |
| "model.layers.52.mlp.down_proj": { |
| "base_key": "model.layers.52.mlp.down_proj.weight", |
| "module": "model.layers.52.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00028310270863585174, |
| "delta_mean_abs": 1.8282893506693654e-05 |
| }, |
| "model.layers.52.mlp.gate_proj": { |
| "base_key": "model.layers.52.mlp.gate_proj.weight", |
| "module": "model.layers.52.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0007774893310852349, |
| "delta_mean_abs": 3.615232344600372e-05 |
| }, |
| "model.layers.52.mlp.up_proj": { |
| "base_key": "model.layers.52.mlp.up_proj.weight", |
| "module": "model.layers.52.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0008839613874442875, |
| "delta_mean_abs": 3.548489621607587e-05 |
| }, |
| "model.layers.52.self_attn.k_proj": { |
| "base_key": "model.layers.52.self_attn.k_proj.weight", |
| "module": "model.layers.52.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004241006972733885, |
| "delta_mean_abs": 3.1713181670056656e-05 |
| }, |
| "model.layers.52.self_attn.o_proj": { |
| "base_key": "model.layers.52.self_attn.o_proj.weight", |
| "module": "model.layers.52.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005529000773094594, |
| "delta_mean_abs": 3.08616399706807e-05 |
| }, |
| "model.layers.52.self_attn.q_proj": { |
| "base_key": "model.layers.52.self_attn.q_proj.weight", |
| "module": "model.layers.52.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.000667033193167299, |
| "delta_mean_abs": 3.088713128818199e-05 |
| }, |
| "model.layers.52.self_attn.v_proj": { |
| "base_key": "model.layers.52.self_attn.v_proj.weight", |
| "module": "model.layers.52.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00041208151378668845, |
| "delta_mean_abs": 3.337444650242105e-05 |
| }, |
| "model.layers.53.mlp.down_proj": { |
| "base_key": "model.layers.53.mlp.down_proj.weight", |
| "module": "model.layers.53.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00032677350100129843, |
| "delta_mean_abs": 1.849012915045023e-05 |
| }, |
| "model.layers.53.mlp.gate_proj": { |
| "base_key": "model.layers.53.mlp.gate_proj.weight", |
| "module": "model.layers.53.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0007463723886758089, |
| "delta_mean_abs": 3.689089135150425e-05 |
| }, |
| "model.layers.53.mlp.up_proj": { |
| "base_key": "model.layers.53.mlp.up_proj.weight", |
| "module": "model.layers.53.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.000800183741375804, |
| "delta_mean_abs": 3.469672446954064e-05 |
| }, |
| "model.layers.53.self_attn.k_proj": { |
| "base_key": "model.layers.53.self_attn.k_proj.weight", |
| "module": "model.layers.53.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0003538328455761075, |
| "delta_mean_abs": 2.924890759459231e-05 |
| }, |
| "model.layers.53.self_attn.o_proj": { |
| "base_key": "model.layers.53.self_attn.o_proj.weight", |
| "module": "model.layers.53.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0003903247707057744, |
| "delta_mean_abs": 2.7988488000119105e-05 |
| }, |
| "model.layers.53.self_attn.q_proj": { |
| "base_key": "model.layers.53.self_attn.q_proj.weight", |
| "module": "model.layers.53.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004061553336214274, |
| "delta_mean_abs": 2.8124779419158585e-05 |
| }, |
| "model.layers.53.self_attn.v_proj": { |
| "base_key": "model.layers.53.self_attn.v_proj.weight", |
| "module": "model.layers.53.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00041134460479952395, |
| "delta_mean_abs": 3.0076376788201742e-05 |
| }, |
| "model.layers.54.mlp.down_proj": { |
| "base_key": "model.layers.54.mlp.down_proj.weight", |
| "module": "model.layers.54.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00022944800730329007, |
| "delta_mean_abs": 1.732584132696502e-05 |
| }, |
| "model.layers.54.mlp.gate_proj": { |
| "base_key": "model.layers.54.mlp.gate_proj.weight", |
| "module": "model.layers.54.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004836066800635308, |
| "delta_mean_abs": 3.0529707146342844e-05 |
| }, |
| "model.layers.54.mlp.up_proj": { |
| "base_key": "model.layers.54.mlp.up_proj.weight", |
| "module": "model.layers.54.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006010723300278187, |
| "delta_mean_abs": 3.241946978960186e-05 |
| }, |
| "model.layers.54.self_attn.k_proj": { |
| "base_key": "model.layers.54.self_attn.k_proj.weight", |
| "module": "model.layers.54.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0003643657255452126, |
| "delta_mean_abs": 2.948681139969267e-05 |
| }, |
| "model.layers.54.self_attn.o_proj": { |
| "base_key": "model.layers.54.self_attn.o_proj.weight", |
| "module": "model.layers.54.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00042049153125844896, |
| "delta_mean_abs": 2.7296073312754743e-05 |
| }, |
| "model.layers.54.self_attn.q_proj": { |
| "base_key": "model.layers.54.self_attn.q_proj.weight", |
| "module": "model.layers.54.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005797162302769721, |
| "delta_mean_abs": 2.850515920727048e-05 |
| }, |
| "model.layers.54.self_attn.v_proj": { |
| "base_key": "model.layers.54.self_attn.v_proj.weight", |
| "module": "model.layers.54.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0003436410042922944, |
| "delta_mean_abs": 2.6404381060274318e-05 |
| }, |
| "model.layers.55.self_attn.q_proj": { |
| "base_key": "model.layers.55.self_attn.q_proj.weight", |
| "module": "model.layers.55.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0007317893323488533, |
| "delta_mean_abs": 3.1857900467002764e-05 |
| }, |
| "model.layers.55.mlp.down_proj": { |
| "base_key": "model.layers.55.mlp.down_proj.weight", |
| "module": "model.layers.55.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00024017966643441468, |
| "delta_mean_abs": 1.71609572134912e-05 |
| }, |
| "model.layers.55.mlp.gate_proj": { |
| "base_key": "model.layers.55.mlp.gate_proj.weight", |
| "module": "model.layers.55.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.000630101072601974, |
| "delta_mean_abs": 3.266374551458284e-05 |
| }, |
| "model.layers.55.mlp.up_proj": { |
| "base_key": "model.layers.55.mlp.up_proj.weight", |
| "module": "model.layers.55.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005690811085514724, |
| "delta_mean_abs": 3.304798519820906e-05 |
| }, |
| "model.layers.55.self_attn.k_proj": { |
| "base_key": "model.layers.55.self_attn.k_proj.weight", |
| "module": "model.layers.55.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00047077270573936403, |
| "delta_mean_abs": 3.327948797959834e-05 |
| }, |
| "model.layers.55.self_attn.o_proj": { |
| "base_key": "model.layers.55.self_attn.o_proj.weight", |
| "module": "model.layers.55.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006803169380873442, |
| "delta_mean_abs": 3.148540417896584e-05 |
| }, |
| "model.layers.55.self_attn.v_proj": { |
| "base_key": "model.layers.55.self_attn.v_proj.weight", |
| "module": "model.layers.55.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0007956264307722449, |
| "delta_mean_abs": 3.9914502849569544e-05 |
| }, |
| "model.layers.56.mlp.down_proj": { |
| "base_key": "model.layers.56.mlp.down_proj.weight", |
| "module": "model.layers.56.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0003252432798035443, |
| "delta_mean_abs": 1.75830955413403e-05 |
| }, |
| "model.layers.56.mlp.gate_proj": { |
| "base_key": "model.layers.56.mlp.gate_proj.weight", |
| "module": "model.layers.56.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0011477298103272915, |
| "delta_mean_abs": 3.319076495245099e-05 |
| }, |
| "model.layers.56.mlp.up_proj": { |
| "base_key": "model.layers.56.mlp.up_proj.weight", |
| "module": "model.layers.56.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006982828490436077, |
| "delta_mean_abs": 3.44103536917828e-05 |
| }, |
| "model.layers.56.self_attn.k_proj": { |
| "base_key": "model.layers.56.self_attn.k_proj.weight", |
| "module": "model.layers.56.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0002250112738693133, |
| "delta_mean_abs": 2.615237099234946e-05 |
| }, |
| "model.layers.56.self_attn.o_proj": { |
| "base_key": "model.layers.56.self_attn.o_proj.weight", |
| "module": "model.layers.56.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005071043269708753, |
| "delta_mean_abs": 2.814992876665201e-05 |
| }, |
| "model.layers.56.self_attn.q_proj": { |
| "base_key": "model.layers.56.self_attn.q_proj.weight", |
| "module": "model.layers.56.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005149850621819496, |
| "delta_mean_abs": 2.648120789672248e-05 |
| }, |
| "model.layers.56.self_attn.v_proj": { |
| "base_key": "model.layers.56.self_attn.v_proj.weight", |
| "module": "model.layers.56.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004076813056599349, |
| "delta_mean_abs": 3.120432666037232e-05 |
| }, |
| "model.layers.57.mlp.down_proj": { |
| "base_key": "model.layers.57.mlp.down_proj.weight", |
| "module": "model.layers.57.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00032328657107427716, |
| "delta_mean_abs": 1.756659366947133e-05 |
| }, |
| "model.layers.57.mlp.gate_proj": { |
| "base_key": "model.layers.57.mlp.gate_proj.weight", |
| "module": "model.layers.57.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0007672883803024888, |
| "delta_mean_abs": 3.496990757412277e-05 |
| }, |
| "model.layers.57.mlp.up_proj": { |
| "base_key": "model.layers.57.mlp.up_proj.weight", |
| "module": "model.layers.57.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005826873239129782, |
| "delta_mean_abs": 3.319397728773765e-05 |
| }, |
| "model.layers.57.self_attn.k_proj": { |
| "base_key": "model.layers.57.self_attn.k_proj.weight", |
| "module": "model.layers.57.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00041932318708859384, |
| "delta_mean_abs": 3.0390878237085417e-05 |
| }, |
| "model.layers.57.self_attn.o_proj": { |
| "base_key": "model.layers.57.self_attn.o_proj.weight", |
| "module": "model.layers.57.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004013587604276836, |
| "delta_mean_abs": 2.728542676777579e-05 |
| }, |
| "model.layers.57.self_attn.q_proj": { |
| "base_key": "model.layers.57.self_attn.q_proj.weight", |
| "module": "model.layers.57.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00039427896263077855, |
| "delta_mean_abs": 2.7683438020176254e-05 |
| }, |
| "model.layers.57.self_attn.v_proj": { |
| "base_key": "model.layers.57.self_attn.v_proj.weight", |
| "module": "model.layers.57.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004960961523465812, |
| "delta_mean_abs": 2.8850430680904537e-05 |
| }, |
| "model.layers.58.mlp.down_proj": { |
| "base_key": "model.layers.58.mlp.down_proj.weight", |
| "module": "model.layers.58.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0002375221229158342, |
| "delta_mean_abs": 1.8474675016477704e-05 |
| }, |
| "model.layers.58.mlp.gate_proj": { |
| "base_key": "model.layers.58.mlp.gate_proj.weight", |
| "module": "model.layers.58.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005813697935082018, |
| "delta_mean_abs": 3.141004344797693e-05 |
| }, |
| "model.layers.58.mlp.up_proj": { |
| "base_key": "model.layers.58.mlp.up_proj.weight", |
| "module": "model.layers.58.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005685397773049772, |
| "delta_mean_abs": 3.3447082387283444e-05 |
| }, |
| "model.layers.58.self_attn.k_proj": { |
| "base_key": "model.layers.58.self_attn.k_proj.weight", |
| "module": "model.layers.58.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00026678666472435, |
| "delta_mean_abs": 2.9029772122157738e-05 |
| }, |
| "model.layers.58.self_attn.o_proj": { |
| "base_key": "model.layers.58.self_attn.o_proj.weight", |
| "module": "model.layers.58.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004007090756203979, |
| "delta_mean_abs": 2.7240399504080415e-05 |
| }, |
| "model.layers.58.self_attn.q_proj": { |
| "base_key": "model.layers.58.self_attn.q_proj.weight", |
| "module": "model.layers.58.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00037795526441186666, |
| "delta_mean_abs": 2.9251541491248645e-05 |
| }, |
| "model.layers.58.self_attn.v_proj": { |
| "base_key": "model.layers.58.self_attn.v_proj.weight", |
| "module": "model.layers.58.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.000517230131663382, |
| "delta_mean_abs": 2.8717611712636426e-05 |
| }, |
| "model.layers.59.mlp.down_proj": { |
| "base_key": "model.layers.59.mlp.down_proj.weight", |
| "module": "model.layers.59.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0003167118411511183, |
| "delta_mean_abs": 1.801358666853048e-05 |
| }, |
| "model.layers.59.mlp.gate_proj": { |
| "base_key": "model.layers.59.mlp.gate_proj.weight", |
| "module": "model.layers.59.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006274434272199869, |
| "delta_mean_abs": 3.511364775476977e-05 |
| }, |
| "model.layers.59.mlp.up_proj": { |
| "base_key": "model.layers.59.mlp.up_proj.weight", |
| "module": "model.layers.59.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006253743777051568, |
| "delta_mean_abs": 3.493780241115019e-05 |
| }, |
| "model.layers.59.self_attn.k_proj": { |
| "base_key": "model.layers.59.self_attn.k_proj.weight", |
| "module": "model.layers.59.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004909363924525678, |
| "delta_mean_abs": 3.30900184053462e-05 |
| }, |
| "model.layers.59.self_attn.o_proj": { |
| "base_key": "model.layers.59.self_attn.o_proj.weight", |
| "module": "model.layers.59.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.000426191691076383, |
| "delta_mean_abs": 2.9037670174147934e-05 |
| }, |
| "model.layers.59.self_attn.q_proj": { |
| "base_key": "model.layers.59.self_attn.q_proj.weight", |
| "module": "model.layers.59.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006625897367484868, |
| "delta_mean_abs": 2.9437003831844777e-05 |
| }, |
| "model.layers.59.self_attn.v_proj": { |
| "base_key": "model.layers.59.self_attn.v_proj.weight", |
| "module": "model.layers.59.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004401344631332904, |
| "delta_mean_abs": 3.363710129633546e-05 |
| }, |
| "model.layers.60.mlp.down_proj": { |
| "base_key": "model.layers.60.mlp.down_proj.weight", |
| "module": "model.layers.60.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0003403117589186877, |
| "delta_mean_abs": 1.8032351363217458e-05 |
| }, |
| "model.layers.60.mlp.gate_proj": { |
| "base_key": "model.layers.60.mlp.gate_proj.weight", |
| "module": "model.layers.60.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005885711289010942, |
| "delta_mean_abs": 3.376022868906148e-05 |
| }, |
| "model.layers.60.mlp.up_proj": { |
| "base_key": "model.layers.60.mlp.up_proj.weight", |
| "module": "model.layers.60.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0008577533299103379, |
| "delta_mean_abs": 3.6479705158853903e-05 |
| }, |
| "model.layers.60.self_attn.k_proj": { |
| "base_key": "model.layers.60.self_attn.k_proj.weight", |
| "module": "model.layers.60.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00040963260107673705, |
| "delta_mean_abs": 3.062326504732482e-05 |
| }, |
| "model.layers.60.self_attn.o_proj": { |
| "base_key": "model.layers.60.self_attn.o_proj.weight", |
| "module": "model.layers.60.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005057673552073538, |
| "delta_mean_abs": 2.9038703360129148e-05 |
| }, |
| "model.layers.60.self_attn.q_proj": { |
| "base_key": "model.layers.60.self_attn.q_proj.weight", |
| "module": "model.layers.60.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005076383822597563, |
| "delta_mean_abs": 2.90478092210833e-05 |
| }, |
| "model.layers.60.self_attn.v_proj": { |
| "base_key": "model.layers.60.self_attn.v_proj.weight", |
| "module": "model.layers.60.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00045544508611783385, |
| "delta_mean_abs": 2.7974878321401775e-05 |
| }, |
| "model.layers.61.mlp.down_proj": { |
| "base_key": "model.layers.61.mlp.down_proj.weight", |
| "module": "model.layers.61.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0003474331460893154, |
| "delta_mean_abs": 1.7789465346140787e-05 |
| }, |
| "model.layers.61.mlp.gate_proj": { |
| "base_key": "model.layers.61.mlp.gate_proj.weight", |
| "module": "model.layers.61.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006403843872249126, |
| "delta_mean_abs": 3.265478881075978e-05 |
| }, |
| "model.layers.61.mlp.up_proj": { |
| "base_key": "model.layers.61.mlp.up_proj.weight", |
| "module": "model.layers.61.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0008014805498532951, |
| "delta_mean_abs": 3.3041374990716577e-05 |
| }, |
| "model.layers.61.self_attn.k_proj": { |
| "base_key": "model.layers.61.self_attn.k_proj.weight", |
| "module": "model.layers.61.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00020574846712406725, |
| "delta_mean_abs": 2.4439828848699108e-05 |
| }, |
| "model.layers.61.self_attn.o_proj": { |
| "base_key": "model.layers.61.self_attn.o_proj.weight", |
| "module": "model.layers.61.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004112191090825945, |
| "delta_mean_abs": 2.655314892763272e-05 |
| }, |
| "model.layers.61.self_attn.q_proj": { |
| "base_key": "model.layers.61.self_attn.q_proj.weight", |
| "module": "model.layers.61.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0003253582399338484, |
| "delta_mean_abs": 2.659644633240532e-05 |
| }, |
| "model.layers.61.self_attn.v_proj": { |
| "base_key": "model.layers.61.self_attn.v_proj.weight", |
| "module": "model.layers.61.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00042377019417472184, |
| "delta_mean_abs": 2.6528774469625205e-05 |
| }, |
| "model.layers.62.mlp.down_proj": { |
| "base_key": "model.layers.62.mlp.down_proj.weight", |
| "module": "model.layers.62.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00027904025046154857, |
| "delta_mean_abs": 1.8366310541750863e-05 |
| }, |
| "model.layers.62.mlp.gate_proj": { |
| "base_key": "model.layers.62.mlp.gate_proj.weight", |
| "module": "model.layers.62.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00043480738531798124, |
| "delta_mean_abs": 3.1796804250916466e-05 |
| }, |
| "model.layers.62.mlp.up_proj": { |
| "base_key": "model.layers.62.mlp.up_proj.weight", |
| "module": "model.layers.62.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005380992661230266, |
| "delta_mean_abs": 3.250975350965746e-05 |
| }, |
| "model.layers.62.self_attn.k_proj": { |
| "base_key": "model.layers.62.self_attn.k_proj.weight", |
| "module": "model.layers.62.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00039939151611179113, |
| "delta_mean_abs": 3.062015093746595e-05 |
| }, |
| "model.layers.62.self_attn.o_proj": { |
| "base_key": "model.layers.62.self_attn.o_proj.weight", |
| "module": "model.layers.62.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005230074748396873, |
| "delta_mean_abs": 2.8701606424874626e-05 |
| }, |
| "model.layers.62.self_attn.q_proj": { |
| "base_key": "model.layers.62.self_attn.q_proj.weight", |
| "module": "model.layers.62.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005595781840384007, |
| "delta_mean_abs": 2.8908872991451062e-05 |
| }, |
| "model.layers.62.self_attn.v_proj": { |
| "base_key": "model.layers.62.self_attn.v_proj.weight", |
| "module": "model.layers.62.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00044293981045484543, |
| "delta_mean_abs": 2.972842412418686e-05 |
| }, |
| "model.layers.63.mlp.down_proj": { |
| "base_key": "model.layers.63.mlp.down_proj.weight", |
| "module": "model.layers.63.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00030975008849054575, |
| "delta_mean_abs": 1.7318867321591824e-05 |
| }, |
| "model.layers.63.mlp.gate_proj": { |
| "base_key": "model.layers.63.mlp.gate_proj.weight", |
| "module": "model.layers.63.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006528462981805205, |
| "delta_mean_abs": 3.2834603189257905e-05 |
| }, |
| "model.layers.63.mlp.up_proj": { |
| "base_key": "model.layers.63.mlp.up_proj.weight", |
| "module": "model.layers.63.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006288392469286919, |
| "delta_mean_abs": 3.329902028781362e-05 |
| }, |
| "model.layers.63.self_attn.k_proj": { |
| "base_key": "model.layers.63.self_attn.k_proj.weight", |
| "module": "model.layers.63.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004438305040821433, |
| "delta_mean_abs": 3.373196523170918e-05 |
| }, |
| "model.layers.63.self_attn.o_proj": { |
| "base_key": "model.layers.63.self_attn.o_proj.weight", |
| "module": "model.layers.63.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006851164507679641, |
| "delta_mean_abs": 2.9773869755445048e-05 |
| }, |
| "model.layers.63.self_attn.q_proj": { |
| "base_key": "model.layers.63.self_attn.q_proj.weight", |
| "module": "model.layers.63.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004850758705288172, |
| "delta_mean_abs": 2.9002858354942873e-05 |
| }, |
| "model.layers.63.self_attn.v_proj": { |
| "base_key": "model.layers.63.self_attn.v_proj.weight", |
| "module": "model.layers.63.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.000509008183144033, |
| "delta_mean_abs": 3.334598659421317e-05 |
| }, |
| "model.layers.64.mlp.down_proj": { |
| "base_key": "model.layers.64.mlp.down_proj.weight", |
| "module": "model.layers.64.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.000435884139733389, |
| "delta_mean_abs": 1.7994438167079352e-05 |
| }, |
| "model.layers.64.mlp.gate_proj": { |
| "base_key": "model.layers.64.mlp.gate_proj.weight", |
| "module": "model.layers.64.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005222313920967281, |
| "delta_mean_abs": 3.355167791596614e-05 |
| }, |
| "model.layers.64.mlp.up_proj": { |
| "base_key": "model.layers.64.mlp.up_proj.weight", |
| "module": "model.layers.64.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0007353847031481564, |
| "delta_mean_abs": 3.324563294881955e-05 |
| }, |
| "model.layers.64.self_attn.k_proj": { |
| "base_key": "model.layers.64.self_attn.k_proj.weight", |
| "module": "model.layers.64.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00024940219009295106, |
| "delta_mean_abs": 2.564599344623275e-05 |
| }, |
| "model.layers.64.self_attn.o_proj": { |
| "base_key": "model.layers.64.self_attn.o_proj.weight", |
| "module": "model.layers.64.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004086136177647859, |
| "delta_mean_abs": 2.841260538843926e-05 |
| }, |
| "model.layers.64.self_attn.q_proj": { |
| "base_key": "model.layers.64.self_attn.q_proj.weight", |
| "module": "model.layers.64.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004364412452559918, |
| "delta_mean_abs": 2.7691814466379583e-05 |
| }, |
| "model.layers.64.self_attn.v_proj": { |
| "base_key": "model.layers.64.self_attn.v_proj.weight", |
| "module": "model.layers.64.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00040847124182619154, |
| "delta_mean_abs": 2.973815571749583e-05 |
| }, |
| "model.layers.65.self_attn.q_proj": { |
| "base_key": "model.layers.65.self_attn.q_proj.weight", |
| "module": "model.layers.65.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0016359227010980248, |
| "delta_mean_abs": 4.618140519596636e-05 |
| }, |
| "model.layers.65.mlp.down_proj": { |
| "base_key": "model.layers.65.mlp.down_proj.weight", |
| "module": "model.layers.65.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.000353805284248665, |
| "delta_mean_abs": 1.8734939658315852e-05 |
| }, |
| "model.layers.65.mlp.gate_proj": { |
| "base_key": "model.layers.65.mlp.gate_proj.weight", |
| "module": "model.layers.65.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006487490609288216, |
| "delta_mean_abs": 3.391228165128268e-05 |
| }, |
| "model.layers.65.mlp.up_proj": { |
| "base_key": "model.layers.65.mlp.up_proj.weight", |
| "module": "model.layers.65.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.000639317266177386, |
| "delta_mean_abs": 3.5004592064069584e-05 |
| }, |
| "model.layers.65.self_attn.k_proj": { |
| "base_key": "model.layers.65.self_attn.k_proj.weight", |
| "module": "model.layers.65.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0009715057094581425, |
| "delta_mean_abs": 3.872136221616529e-05 |
| }, |
| "model.layers.65.self_attn.o_proj": { |
| "base_key": "model.layers.65.self_attn.o_proj.weight", |
| "module": "model.layers.65.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004719960270449519, |
| "delta_mean_abs": 2.9004280804656446e-05 |
| }, |
| "model.layers.65.self_attn.v_proj": { |
| "base_key": "model.layers.65.self_attn.v_proj.weight", |
| "module": "model.layers.65.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005848380969837308, |
| "delta_mean_abs": 3.295863643870689e-05 |
| }, |
| "model.layers.66.mlp.down_proj": { |
| "base_key": "model.layers.66.mlp.down_proj.weight", |
| "module": "model.layers.66.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0003305491991341114, |
| "delta_mean_abs": 1.7633958123042248e-05 |
| }, |
| "model.layers.66.mlp.gate_proj": { |
| "base_key": "model.layers.66.mlp.gate_proj.weight", |
| "module": "model.layers.66.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0008325888775289059, |
| "delta_mean_abs": 3.1855521228862926e-05 |
| }, |
| "model.layers.66.mlp.up_proj": { |
| "base_key": "model.layers.66.mlp.up_proj.weight", |
| "module": "model.layers.66.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.000667522253934294, |
| "delta_mean_abs": 3.3472100767539814e-05 |
| }, |
| "model.layers.66.self_attn.k_proj": { |
| "base_key": "model.layers.66.self_attn.k_proj.weight", |
| "module": "model.layers.66.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0003795888915192336, |
| "delta_mean_abs": 3.177179314661771e-05 |
| }, |
| "model.layers.66.self_attn.o_proj": { |
| "base_key": "model.layers.66.self_attn.o_proj.weight", |
| "module": "model.layers.66.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006076532299630344, |
| "delta_mean_abs": 3.119217581115663e-05 |
| }, |
| "model.layers.66.self_attn.q_proj": { |
| "base_key": "model.layers.66.self_attn.q_proj.weight", |
| "module": "model.layers.66.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006125041400082409, |
| "delta_mean_abs": 3.14492717734538e-05 |
| }, |
| "model.layers.66.self_attn.v_proj": { |
| "base_key": "model.layers.66.self_attn.v_proj.weight", |
| "module": "model.layers.66.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004753570829052478, |
| "delta_mean_abs": 3.286154969828203e-05 |
| }, |
| "model.layers.67.mlp.down_proj": { |
| "base_key": "model.layers.67.mlp.down_proj.weight", |
| "module": "model.layers.67.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0003385959716979414, |
| "delta_mean_abs": 1.7417542039765976e-05 |
| }, |
| "model.layers.67.mlp.gate_proj": { |
| "base_key": "model.layers.67.mlp.gate_proj.weight", |
| "module": "model.layers.67.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006302095716819167, |
| "delta_mean_abs": 3.453836325206794e-05 |
| }, |
| "model.layers.67.mlp.up_proj": { |
| "base_key": "model.layers.67.mlp.up_proj.weight", |
| "module": "model.layers.67.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0009874554816633463, |
| "delta_mean_abs": 3.361910785315558e-05 |
| }, |
| "model.layers.67.self_attn.k_proj": { |
| "base_key": "model.layers.67.self_attn.k_proj.weight", |
| "module": "model.layers.67.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005051043699495494, |
| "delta_mean_abs": 3.528805973473936e-05 |
| }, |
| "model.layers.67.self_attn.o_proj": { |
| "base_key": "model.layers.67.self_attn.o_proj.weight", |
| "module": "model.layers.67.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006839020643383265, |
| "delta_mean_abs": 3.4324639273108914e-05 |
| }, |
| "model.layers.67.self_attn.q_proj": { |
| "base_key": "model.layers.67.self_attn.q_proj.weight", |
| "module": "model.layers.67.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004992749891243875, |
| "delta_mean_abs": 2.9469358196365647e-05 |
| }, |
| "model.layers.67.self_attn.v_proj": { |
| "base_key": "model.layers.67.self_attn.v_proj.weight", |
| "module": "model.layers.67.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00041802023770287633, |
| "delta_mean_abs": 3.2986248697852716e-05 |
| }, |
| "model.layers.68.mlp.down_proj": { |
| "base_key": "model.layers.68.mlp.down_proj.weight", |
| "module": "model.layers.68.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00043951772386208177, |
| "delta_mean_abs": 1.7884161934489384e-05 |
| }, |
| "model.layers.68.mlp.gate_proj": { |
| "base_key": "model.layers.68.mlp.gate_proj.weight", |
| "module": "model.layers.68.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006655589095316827, |
| "delta_mean_abs": 3.6978912248741835e-05 |
| }, |
| "model.layers.68.mlp.up_proj": { |
| "base_key": "model.layers.68.mlp.up_proj.weight", |
| "module": "model.layers.68.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006126568769104779, |
| "delta_mean_abs": 3.346798621350899e-05 |
| }, |
| "model.layers.68.self_attn.k_proj": { |
| "base_key": "model.layers.68.self_attn.k_proj.weight", |
| "module": "model.layers.68.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005210642702877522, |
| "delta_mean_abs": 3.042715798073914e-05 |
| }, |
| "model.layers.68.self_attn.o_proj": { |
| "base_key": "model.layers.68.self_attn.o_proj.weight", |
| "module": "model.layers.68.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006868410855531693, |
| "delta_mean_abs": 3.023002136615105e-05 |
| }, |
| "model.layers.68.self_attn.q_proj": { |
| "base_key": "model.layers.68.self_attn.q_proj.weight", |
| "module": "model.layers.68.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006250206497497857, |
| "delta_mean_abs": 3.134355210931972e-05 |
| }, |
| "model.layers.68.self_attn.v_proj": { |
| "base_key": "model.layers.68.self_attn.v_proj.weight", |
| "module": "model.layers.68.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005906482692807913, |
| "delta_mean_abs": 3.0426575904130004e-05 |
| }, |
| "model.layers.69.mlp.down_proj": { |
| "base_key": "model.layers.69.mlp.down_proj.weight", |
| "module": "model.layers.69.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004005744121968746, |
| "delta_mean_abs": 1.761115163390059e-05 |
| }, |
| "model.layers.69.mlp.gate_proj": { |
| "base_key": "model.layers.69.mlp.gate_proj.weight", |
| "module": "model.layers.69.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005679959431290627, |
| "delta_mean_abs": 3.1574734748573974e-05 |
| }, |
| "model.layers.69.mlp.up_proj": { |
| "base_key": "model.layers.69.mlp.up_proj.weight", |
| "module": "model.layers.69.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0007787889917381108, |
| "delta_mean_abs": 3.496246426948346e-05 |
| }, |
| "model.layers.69.self_attn.k_proj": { |
| "base_key": "model.layers.69.self_attn.k_proj.weight", |
| "module": "model.layers.69.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00036143948091194034, |
| "delta_mean_abs": 3.2853706215973943e-05 |
| }, |
| "model.layers.69.self_attn.o_proj": { |
| "base_key": "model.layers.69.self_attn.o_proj.weight", |
| "module": "model.layers.69.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005376170156523585, |
| "delta_mean_abs": 3.064176416955888e-05 |
| }, |
| "model.layers.69.self_attn.q_proj": { |
| "base_key": "model.layers.69.self_attn.q_proj.weight", |
| "module": "model.layers.69.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006360808620229363, |
| "delta_mean_abs": 3.128091339021921e-05 |
| }, |
| "model.layers.69.self_attn.v_proj": { |
| "base_key": "model.layers.69.self_attn.v_proj.weight", |
| "module": "model.layers.69.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.000496246328111738, |
| "delta_mean_abs": 3.219795326003805e-05 |
| }, |
| "model.layers.70.mlp.down_proj": { |
| "base_key": "model.layers.70.mlp.down_proj.weight", |
| "module": "model.layers.70.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00036174969864077866, |
| "delta_mean_abs": 1.7974818547372706e-05 |
| }, |
| "model.layers.70.mlp.gate_proj": { |
| "base_key": "model.layers.70.mlp.gate_proj.weight", |
| "module": "model.layers.70.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0009025035542435944, |
| "delta_mean_abs": 3.468829527264461e-05 |
| }, |
| "model.layers.70.mlp.up_proj": { |
| "base_key": "model.layers.70.mlp.up_proj.weight", |
| "module": "model.layers.70.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0008815202163532376, |
| "delta_mean_abs": 3.4279782994417474e-05 |
| }, |
| "model.layers.70.self_attn.k_proj": { |
| "base_key": "model.layers.70.self_attn.k_proj.weight", |
| "module": "model.layers.70.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00046798615949228406, |
| "delta_mean_abs": 2.801379559969064e-05 |
| }, |
| "model.layers.70.self_attn.o_proj": { |
| "base_key": "model.layers.70.self_attn.o_proj.weight", |
| "module": "model.layers.70.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006216843030415475, |
| "delta_mean_abs": 3.282462785136886e-05 |
| }, |
| "model.layers.70.self_attn.q_proj": { |
| "base_key": "model.layers.70.self_attn.q_proj.weight", |
| "module": "model.layers.70.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006040799780748785, |
| "delta_mean_abs": 2.991302426380571e-05 |
| }, |
| "model.layers.70.self_attn.v_proj": { |
| "base_key": "model.layers.70.self_attn.v_proj.weight", |
| "module": "model.layers.70.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00045949139166623354, |
| "delta_mean_abs": 3.4063912607962266e-05 |
| }, |
| "model.layers.71.mlp.down_proj": { |
| "base_key": "model.layers.71.mlp.down_proj.weight", |
| "module": "model.layers.71.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0003577861643861979, |
| "delta_mean_abs": 1.8323609765502624e-05 |
| }, |
| "model.layers.71.mlp.gate_proj": { |
| "base_key": "model.layers.71.mlp.gate_proj.weight", |
| "module": "model.layers.71.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006235248874872923, |
| "delta_mean_abs": 3.685526826302521e-05 |
| }, |
| "model.layers.71.mlp.up_proj": { |
| "base_key": "model.layers.71.mlp.up_proj.weight", |
| "module": "model.layers.71.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0007694624364376068, |
| "delta_mean_abs": 3.668199497042224e-05 |
| }, |
| "model.layers.71.self_attn.k_proj": { |
| "base_key": "model.layers.71.self_attn.k_proj.weight", |
| "module": "model.layers.71.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00031912949634715915, |
| "delta_mean_abs": 2.805414260365069e-05 |
| }, |
| "model.layers.71.self_attn.o_proj": { |
| "base_key": "model.layers.71.self_attn.o_proj.weight", |
| "module": "model.layers.71.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005239666788838804, |
| "delta_mean_abs": 3.219921563868411e-05 |
| }, |
| "model.layers.71.self_attn.q_proj": { |
| "base_key": "model.layers.71.self_attn.q_proj.weight", |
| "module": "model.layers.71.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005433373735286295, |
| "delta_mean_abs": 2.9417969926726073e-05 |
| }, |
| "model.layers.71.self_attn.v_proj": { |
| "base_key": "model.layers.71.self_attn.v_proj.weight", |
| "module": "model.layers.71.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004660948761738837, |
| "delta_mean_abs": 3.300354364910163e-05 |
| }, |
| "model.layers.72.mlp.down_proj": { |
| "base_key": "model.layers.72.mlp.down_proj.weight", |
| "module": "model.layers.72.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004198637616354972, |
| "delta_mean_abs": 1.976541170733981e-05 |
| }, |
| "model.layers.72.mlp.gate_proj": { |
| "base_key": "model.layers.72.mlp.gate_proj.weight", |
| "module": "model.layers.72.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0013648399617522955, |
| "delta_mean_abs": 4.1749510273803025e-05 |
| }, |
| "model.layers.72.mlp.up_proj": { |
| "base_key": "model.layers.72.mlp.up_proj.weight", |
| "module": "model.layers.72.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0009160926565527916, |
| "delta_mean_abs": 3.999894761363976e-05 |
| }, |
| "model.layers.72.self_attn.k_proj": { |
| "base_key": "model.layers.72.self_attn.k_proj.weight", |
| "module": "model.layers.72.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004971388261765242, |
| "delta_mean_abs": 3.343899152241647e-05 |
| }, |
| "model.layers.72.self_attn.o_proj": { |
| "base_key": "model.layers.72.self_attn.o_proj.weight", |
| "module": "model.layers.72.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005863019614480436, |
| "delta_mean_abs": 3.249990186304785e-05 |
| }, |
| "model.layers.72.self_attn.q_proj": { |
| "base_key": "model.layers.72.self_attn.q_proj.weight", |
| "module": "model.layers.72.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005925617297179997, |
| "delta_mean_abs": 3.2505042327102274e-05 |
| }, |
| "model.layers.72.self_attn.v_proj": { |
| "base_key": "model.layers.72.self_attn.v_proj.weight", |
| "module": "model.layers.72.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005048689199611545, |
| "delta_mean_abs": 3.781314808293246e-05 |
| }, |
| "model.layers.73.mlp.down_proj": { |
| "base_key": "model.layers.73.mlp.down_proj.weight", |
| "module": "model.layers.73.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004268938791938126, |
| "delta_mean_abs": 2.0288223822717555e-05 |
| }, |
| "model.layers.73.mlp.gate_proj": { |
| "base_key": "model.layers.73.mlp.gate_proj.weight", |
| "module": "model.layers.73.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0010354557307437062, |
| "delta_mean_abs": 4.065262692165561e-05 |
| }, |
| "model.layers.73.mlp.up_proj": { |
| "base_key": "model.layers.73.mlp.up_proj.weight", |
| "module": "model.layers.73.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.000910265720449388, |
| "delta_mean_abs": 3.885718251694925e-05 |
| }, |
| "model.layers.73.self_attn.k_proj": { |
| "base_key": "model.layers.73.self_attn.k_proj.weight", |
| "module": "model.layers.73.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006154202274046838, |
| "delta_mean_abs": 3.179792838636786e-05 |
| }, |
| "model.layers.73.self_attn.o_proj": { |
| "base_key": "model.layers.73.self_attn.o_proj.weight", |
| "module": "model.layers.73.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005339720519259572, |
| "delta_mean_abs": 3.178670885972679e-05 |
| }, |
| "model.layers.73.self_attn.q_proj": { |
| "base_key": "model.layers.73.self_attn.q_proj.weight", |
| "module": "model.layers.73.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005624737241305411, |
| "delta_mean_abs": 3.212135925423354e-05 |
| }, |
| "model.layers.73.self_attn.v_proj": { |
| "base_key": "model.layers.73.self_attn.v_proj.weight", |
| "module": "model.layers.73.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006142078200355172, |
| "delta_mean_abs": 3.961876791436225e-05 |
| }, |
| "model.layers.74.mlp.down_proj": { |
| "base_key": "model.layers.74.mlp.down_proj.weight", |
| "module": "model.layers.74.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00045918376417830586, |
| "delta_mean_abs": 2.1708243366447277e-05 |
| }, |
| "model.layers.74.mlp.gate_proj": { |
| "base_key": "model.layers.74.mlp.gate_proj.weight", |
| "module": "model.layers.74.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0012383580906316638, |
| "delta_mean_abs": 4.203012213110924e-05 |
| }, |
| "model.layers.74.mlp.up_proj": { |
| "base_key": "model.layers.74.mlp.up_proj.weight", |
| "module": "model.layers.74.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0011031344765797257, |
| "delta_mean_abs": 3.800862396019511e-05 |
| }, |
| "model.layers.74.self_attn.k_proj": { |
| "base_key": "model.layers.74.self_attn.k_proj.weight", |
| "module": "model.layers.74.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00042396780918352306, |
| "delta_mean_abs": 3.1558145565213636e-05 |
| }, |
| "model.layers.74.self_attn.o_proj": { |
| "base_key": "model.layers.74.self_attn.o_proj.weight", |
| "module": "model.layers.74.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006086716894060373, |
| "delta_mean_abs": 3.103741619270295e-05 |
| }, |
| "model.layers.74.self_attn.q_proj": { |
| "base_key": "model.layers.74.self_attn.q_proj.weight", |
| "module": "model.layers.74.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005084500880911946, |
| "delta_mean_abs": 3.1571169529343024e-05 |
| }, |
| "model.layers.74.self_attn.v_proj": { |
| "base_key": "model.layers.74.self_attn.v_proj.weight", |
| "module": "model.layers.74.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005451824399642646, |
| "delta_mean_abs": 3.6451820051297545e-05 |
| }, |
| "model.layers.75.self_attn.q_proj": { |
| "base_key": "model.layers.75.self_attn.q_proj.weight", |
| "module": "model.layers.75.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0003948055091314018, |
| "delta_mean_abs": 3.016556911461521e-05 |
| }, |
| "model.layers.75.mlp.down_proj": { |
| "base_key": "model.layers.75.mlp.down_proj.weight", |
| "module": "model.layers.75.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005875154747627676, |
| "delta_mean_abs": 2.100617166433949e-05 |
| }, |
| "model.layers.75.mlp.gate_proj": { |
| "base_key": "model.layers.75.mlp.gate_proj.weight", |
| "module": "model.layers.75.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0011767561081796885, |
| "delta_mean_abs": 4.46871345047839e-05 |
| }, |
| "model.layers.75.mlp.up_proj": { |
| "base_key": "model.layers.75.mlp.up_proj.weight", |
| "module": "model.layers.75.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.001166467322036624, |
| "delta_mean_abs": 4.197916859993711e-05 |
| }, |
| "model.layers.75.self_attn.k_proj": { |
| "base_key": "model.layers.75.self_attn.k_proj.weight", |
| "module": "model.layers.75.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005544250016100705, |
| "delta_mean_abs": 3.1387960916617885e-05 |
| }, |
| "model.layers.75.self_attn.o_proj": { |
| "base_key": "model.layers.75.self_attn.o_proj.weight", |
| "module": "model.layers.75.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005811215960420668, |
| "delta_mean_abs": 3.065441705984995e-05 |
| }, |
| "model.layers.75.self_attn.v_proj": { |
| "base_key": "model.layers.75.self_attn.v_proj.weight", |
| "module": "model.layers.75.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004726992337964475, |
| "delta_mean_abs": 3.5378296161070466e-05 |
| }, |
| "model.layers.76.mlp.down_proj": { |
| "base_key": "model.layers.76.mlp.down_proj.weight", |
| "module": "model.layers.76.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.000487450510263443, |
| "delta_mean_abs": 1.985048220376484e-05 |
| }, |
| "model.layers.76.mlp.gate_proj": { |
| "base_key": "model.layers.76.mlp.gate_proj.weight", |
| "module": "model.layers.76.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0014557093381881714, |
| "delta_mean_abs": 5.117774708196521e-05 |
| }, |
| "model.layers.76.mlp.up_proj": { |
| "base_key": "model.layers.76.mlp.up_proj.weight", |
| "module": "model.layers.76.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00132599042262882, |
| "delta_mean_abs": 4.367001020000316e-05 |
| }, |
| "model.layers.76.self_attn.k_proj": { |
| "base_key": "model.layers.76.self_attn.k_proj.weight", |
| "module": "model.layers.76.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006550145917572081, |
| "delta_mean_abs": 3.6823064874624833e-05 |
| }, |
| "model.layers.76.self_attn.o_proj": { |
| "base_key": "model.layers.76.self_attn.o_proj.weight", |
| "module": "model.layers.76.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.000780296977609396, |
| "delta_mean_abs": 3.433939127717167e-05 |
| }, |
| "model.layers.76.self_attn.q_proj": { |
| "base_key": "model.layers.76.self_attn.q_proj.weight", |
| "module": "model.layers.76.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006649205461144447, |
| "delta_mean_abs": 3.580806151148863e-05 |
| }, |
| "model.layers.76.self_attn.v_proj": { |
| "base_key": "model.layers.76.self_attn.v_proj.weight", |
| "module": "model.layers.76.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00048075662925839424, |
| "delta_mean_abs": 3.91623325413093e-05 |
| }, |
| "model.layers.77.mlp.down_proj": { |
| "base_key": "model.layers.77.mlp.down_proj.weight", |
| "module": "model.layers.77.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.000440429023001343, |
| "delta_mean_abs": 2.022631815634668e-05 |
| }, |
| "model.layers.77.mlp.gate_proj": { |
| "base_key": "model.layers.77.mlp.gate_proj.weight", |
| "module": "model.layers.77.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.001515861484222114, |
| "delta_mean_abs": 5.217967554926872e-05 |
| }, |
| "model.layers.77.mlp.up_proj": { |
| "base_key": "model.layers.77.mlp.up_proj.weight", |
| "module": "model.layers.77.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0014161646831780672, |
| "delta_mean_abs": 4.541931411949918e-05 |
| }, |
| "model.layers.77.self_attn.k_proj": { |
| "base_key": "model.layers.77.self_attn.k_proj.weight", |
| "module": "model.layers.77.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00040980754420161247, |
| "delta_mean_abs": 3.379135887371376e-05 |
| }, |
| "model.layers.77.self_attn.o_proj": { |
| "base_key": "model.layers.77.self_attn.o_proj.weight", |
| "module": "model.layers.77.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006230915314517915, |
| "delta_mean_abs": 2.964258055726532e-05 |
| }, |
| "model.layers.77.self_attn.q_proj": { |
| "base_key": "model.layers.77.self_attn.q_proj.weight", |
| "module": "model.layers.77.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005980563000775874, |
| "delta_mean_abs": 3.2359061151510105e-05 |
| }, |
| "model.layers.77.self_attn.v_proj": { |
| "base_key": "model.layers.77.self_attn.v_proj.weight", |
| "module": "model.layers.77.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00047072680899873376, |
| "delta_mean_abs": 3.360545088071376e-05 |
| }, |
| "model.layers.78.mlp.down_proj": { |
| "base_key": "model.layers.78.mlp.down_proj.weight", |
| "module": "model.layers.78.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004062634543515742, |
| "delta_mean_abs": 2.1243575247353874e-05 |
| }, |
| "model.layers.78.mlp.gate_proj": { |
| "base_key": "model.layers.78.mlp.gate_proj.weight", |
| "module": "model.layers.78.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0010761362500488758, |
| "delta_mean_abs": 4.8448320740135387e-05 |
| }, |
| "model.layers.78.mlp.up_proj": { |
| "base_key": "model.layers.78.mlp.up_proj.weight", |
| "module": "model.layers.78.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.001022767974063754, |
| "delta_mean_abs": 4.201831325190142e-05 |
| }, |
| "model.layers.78.self_attn.k_proj": { |
| "base_key": "model.layers.78.self_attn.k_proj.weight", |
| "module": "model.layers.78.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00039437448140233755, |
| "delta_mean_abs": 3.590131382225081e-05 |
| }, |
| "model.layers.78.self_attn.o_proj": { |
| "base_key": "model.layers.78.self_attn.o_proj.weight", |
| "module": "model.layers.78.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00047684129094704986, |
| "delta_mean_abs": 2.85476962744724e-05 |
| }, |
| "model.layers.78.self_attn.q_proj": { |
| "base_key": "model.layers.78.self_attn.q_proj.weight", |
| "module": "model.layers.78.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006162011413834989, |
| "delta_mean_abs": 3.284952254034579e-05 |
| }, |
| "model.layers.78.self_attn.v_proj": { |
| "base_key": "model.layers.78.self_attn.v_proj.weight", |
| "module": "model.layers.78.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004823432245757431, |
| "delta_mean_abs": 3.197722980985418e-05 |
| }, |
| "model.layers.79.mlp.down_proj": { |
| "base_key": "model.layers.79.mlp.down_proj.weight", |
| "module": "model.layers.79.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0003193787415511906, |
| "delta_mean_abs": 1.8103039110428654e-05 |
| }, |
| "model.layers.79.mlp.gate_proj": { |
| "base_key": "model.layers.79.mlp.gate_proj.weight", |
| "module": "model.layers.79.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.001132657751441002, |
| "delta_mean_abs": 4.3130454287165776e-05 |
| }, |
| "model.layers.79.mlp.up_proj": { |
| "base_key": "model.layers.79.mlp.up_proj.weight", |
| "module": "model.layers.79.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0009169339318759739, |
| "delta_mean_abs": 3.392613507458009e-05 |
| }, |
| "model.layers.79.self_attn.k_proj": { |
| "base_key": "model.layers.79.self_attn.k_proj.weight", |
| "module": "model.layers.79.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005257595330476761, |
| "delta_mean_abs": 3.793235373450443e-05 |
| }, |
| "model.layers.79.self_attn.o_proj": { |
| "base_key": "model.layers.79.self_attn.o_proj.weight", |
| "module": "model.layers.79.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004904827801510692, |
| "delta_mean_abs": 3.2804480724735186e-05 |
| }, |
| "model.layers.79.self_attn.q_proj": { |
| "base_key": "model.layers.79.self_attn.q_proj.weight", |
| "module": "model.layers.79.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006097261211834848, |
| "delta_mean_abs": 3.5851575376000255e-05 |
| }, |
| "model.layers.79.self_attn.v_proj": { |
| "base_key": "model.layers.79.self_attn.v_proj.weight", |
| "module": "model.layers.79.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005342763615772128, |
| "delta_mean_abs": 3.266278508817777e-05 |
| } |
| }, |
| "shards": [ |
| { |
| "status": "PASS", |
| "shard": "model-00001-of-000017.safetensors", |
| "sha256": "14d330e25424e368e8a4694505e96c8684a84b25e438f942b08527521b509694", |
| "size": 8946552816, |
| "applied": { |
| "model.layers.0.mlp.down_proj": { |
| "base_key": "model.layers.0.mlp.down_proj.weight", |
| "module": "model.layers.0.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0036352903116494417, |
| "delta_mean_abs": 3.136198574793525e-05 |
| }, |
| "model.layers.0.mlp.gate_proj": { |
| "base_key": "model.layers.0.mlp.gate_proj.weight", |
| "module": "model.layers.0.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0010508858831599355, |
| "delta_mean_abs": 5.487564703798853e-05 |
| }, |
| "model.layers.0.mlp.up_proj": { |
| "base_key": "model.layers.0.mlp.up_proj.weight", |
| "module": "model.layers.0.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0011367766419425607, |
| "delta_mean_abs": 5.960034468444064e-05 |
| }, |
| "model.layers.0.self_attn.k_proj": { |
| "base_key": "model.layers.0.self_attn.k_proj.weight", |
| "module": "model.layers.0.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.002157435519620776, |
| "delta_mean_abs": 9.706999844638631e-05 |
| }, |
| "model.layers.0.self_attn.o_proj": { |
| "base_key": "model.layers.0.self_attn.o_proj.weight", |
| "module": "model.layers.0.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.001382035086862743, |
| "delta_mean_abs": 5.239765960141085e-05 |
| }, |
| "model.layers.0.self_attn.q_proj": { |
| "base_key": "model.layers.0.self_attn.q_proj.weight", |
| "module": "model.layers.0.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.003898917231708765, |
| "delta_mean_abs": 0.0001100506997318007 |
| }, |
| "model.layers.0.self_attn.v_proj": { |
| "base_key": "model.layers.0.self_attn.v_proj.weight", |
| "module": "model.layers.0.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.009479931555688381, |
| "delta_mean_abs": 9.415674867341295e-05 |
| }, |
| "model.layers.1.mlp.down_proj": { |
| "base_key": "model.layers.1.mlp.down_proj.weight", |
| "module": "model.layers.1.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0007907061954028904, |
| "delta_mean_abs": 1.6571046216995455e-05 |
| }, |
| "model.layers.1.mlp.gate_proj": { |
| "base_key": "model.layers.1.mlp.gate_proj.weight", |
| "module": "model.layers.1.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006334871868602931, |
| "delta_mean_abs": 3.3563090255483985e-05 |
| }, |
| "model.layers.1.mlp.up_proj": { |
| "base_key": "model.layers.1.mlp.up_proj.weight", |
| "module": "model.layers.1.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.000692803761921823, |
| "delta_mean_abs": 3.019714858965017e-05 |
| }, |
| "model.layers.1.self_attn.k_proj": { |
| "base_key": "model.layers.1.self_attn.k_proj.weight", |
| "module": "model.layers.1.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0019693966023623943, |
| "delta_mean_abs": 0.00010614078200887889 |
| }, |
| "model.layers.1.self_attn.o_proj": { |
| "base_key": "model.layers.1.self_attn.o_proj.weight", |
| "module": "model.layers.1.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004677238466683775, |
| "delta_mean_abs": 2.7173442504135892e-05 |
| }, |
| "model.layers.1.self_attn.q_proj": { |
| "base_key": "model.layers.1.self_attn.q_proj.weight", |
| "module": "model.layers.1.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0030918612610548735, |
| "delta_mean_abs": 8.843185059959069e-05 |
| }, |
| "model.layers.1.self_attn.v_proj": { |
| "base_key": "model.layers.1.self_attn.v_proj.weight", |
| "module": "model.layers.1.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.002262373920530081, |
| "delta_mean_abs": 4.33517707278952e-05 |
| }, |
| "model.layers.2.mlp.down_proj": { |
| "base_key": "model.layers.2.mlp.down_proj.weight", |
| "module": "model.layers.2.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004056089965160936, |
| "delta_mean_abs": 1.764399348758161e-05 |
| }, |
| "model.layers.2.mlp.gate_proj": { |
| "base_key": "model.layers.2.mlp.gate_proj.weight", |
| "module": "model.layers.2.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.000763745978474617, |
| "delta_mean_abs": 3.423832094995305e-05 |
| }, |
| "model.layers.2.mlp.up_proj": { |
| "base_key": "model.layers.2.mlp.up_proj.weight", |
| "module": "model.layers.2.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006704325787723064, |
| "delta_mean_abs": 3.197841579094529e-05 |
| }, |
| "model.layers.2.self_attn.k_proj": { |
| "base_key": "model.layers.2.self_attn.k_proj.weight", |
| "module": "model.layers.2.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005617460119538009, |
| "delta_mean_abs": 3.201706203981303e-05 |
| }, |
| "model.layers.2.self_attn.o_proj": { |
| "base_key": "model.layers.2.self_attn.o_proj.weight", |
| "module": "model.layers.2.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005339219933375716, |
| "delta_mean_abs": 2.975089955725707e-05 |
| }, |
| "model.layers.2.self_attn.q_proj": { |
| "base_key": "model.layers.2.self_attn.q_proj.weight", |
| "module": "model.layers.2.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.000573021185118705, |
| "delta_mean_abs": 3.4839671570807695e-05 |
| }, |
| "model.layers.2.self_attn.v_proj": { |
| "base_key": "model.layers.2.self_attn.v_proj.weight", |
| "module": "model.layers.2.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0007892426801845431, |
| "delta_mean_abs": 3.2222313166130334e-05 |
| }, |
| "model.layers.3.mlp.down_proj": { |
| "base_key": "model.layers.3.mlp.down_proj.weight", |
| "module": "model.layers.3.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.003743372857570648, |
| "delta_mean_abs": 2.3273358237929642e-05 |
| }, |
| "model.layers.3.mlp.gate_proj": { |
| "base_key": "model.layers.3.mlp.gate_proj.weight", |
| "module": "model.layers.3.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0007301653386093676, |
| "delta_mean_abs": 3.1983588996808976e-05 |
| }, |
| "model.layers.3.mlp.up_proj": { |
| "base_key": "model.layers.3.mlp.up_proj.weight", |
| "module": "model.layers.3.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006075630662962794, |
| "delta_mean_abs": 3.1319137633545324e-05 |
| }, |
| "model.layers.3.self_attn.k_proj": { |
| "base_key": "model.layers.3.self_attn.k_proj.weight", |
| "module": "model.layers.3.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00039019607356749475, |
| "delta_mean_abs": 2.669027162482962e-05 |
| }, |
| "model.layers.3.self_attn.o_proj": { |
| "base_key": "model.layers.3.self_attn.o_proj.weight", |
| "module": "model.layers.3.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00043108317186124623, |
| "delta_mean_abs": 2.5481347620370798e-05 |
| }, |
| "model.layers.3.self_attn.q_proj": { |
| "base_key": "model.layers.3.self_attn.q_proj.weight", |
| "module": "model.layers.3.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005321042845025659, |
| "delta_mean_abs": 2.9094042474753223e-05 |
| }, |
| "model.layers.3.self_attn.v_proj": { |
| "base_key": "model.layers.3.self_attn.v_proj.weight", |
| "module": "model.layers.3.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004479637718759477, |
| "delta_mean_abs": 2.969752313219942e-05 |
| } |
| } |
| }, |
| { |
| "status": "PASS", |
| "shard": "model-00002-of-000017.safetensors", |
| "sha256": "0105b592a360291dd2b971ca926536dea9ac9c651783cbf404a5ab0bd2b053a8", |
| "size": 8690800056, |
| "applied": { |
| "model.layers.4.mlp.down_proj": { |
| "base_key": "model.layers.4.mlp.down_proj.weight", |
| "module": "model.layers.4.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004106768174096942, |
| "delta_mean_abs": 1.8390141121926717e-05 |
| }, |
| "model.layers.4.mlp.gate_proj": { |
| "base_key": "model.layers.4.mlp.gate_proj.weight", |
| "module": "model.layers.4.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0007407861994579434, |
| "delta_mean_abs": 3.1805113394511864e-05 |
| }, |
| "model.layers.4.mlp.up_proj": { |
| "base_key": "model.layers.4.mlp.up_proj.weight", |
| "module": "model.layers.4.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0007738450076431036, |
| "delta_mean_abs": 3.074496635235846e-05 |
| }, |
| "model.layers.4.self_attn.k_proj": { |
| "base_key": "model.layers.4.self_attn.k_proj.weight", |
| "module": "model.layers.4.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00035501853562891483, |
| "delta_mean_abs": 2.444405254209414e-05 |
| }, |
| "model.layers.4.self_attn.o_proj": { |
| "base_key": "model.layers.4.self_attn.o_proj.weight", |
| "module": "model.layers.4.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005852633621543646, |
| "delta_mean_abs": 2.7335896447766572e-05 |
| }, |
| "model.layers.4.self_attn.q_proj": { |
| "base_key": "model.layers.4.self_attn.q_proj.weight", |
| "module": "model.layers.4.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.000560253334697336, |
| "delta_mean_abs": 2.587987000879366e-05 |
| }, |
| "model.layers.4.self_attn.v_proj": { |
| "base_key": "model.layers.4.self_attn.v_proj.weight", |
| "module": "model.layers.4.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0003859245916828513, |
| "delta_mean_abs": 2.7912094083148986e-05 |
| }, |
| "model.layers.5.mlp.down_proj": { |
| "base_key": "model.layers.5.mlp.down_proj.weight", |
| "module": "model.layers.5.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.000375310773961246, |
| "delta_mean_abs": 1.9491764760459773e-05 |
| }, |
| "model.layers.5.mlp.gate_proj": { |
| "base_key": "model.layers.5.mlp.gate_proj.weight", |
| "module": "model.layers.5.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0007695615058764815, |
| "delta_mean_abs": 3.2462496164953336e-05 |
| }, |
| "model.layers.5.mlp.up_proj": { |
| "base_key": "model.layers.5.mlp.up_proj.weight", |
| "module": "model.layers.5.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.000634767347946763, |
| "delta_mean_abs": 3.161425775033422e-05 |
| }, |
| "model.layers.5.self_attn.k_proj": { |
| "base_key": "model.layers.5.self_attn.k_proj.weight", |
| "module": "model.layers.5.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004232973442412913, |
| "delta_mean_abs": 2.9286964490893297e-05 |
| }, |
| "model.layers.5.self_attn.o_proj": { |
| "base_key": "model.layers.5.self_attn.o_proj.weight", |
| "module": "model.layers.5.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006159860058687627, |
| "delta_mean_abs": 2.822165515681263e-05 |
| }, |
| "model.layers.5.self_attn.q_proj": { |
| "base_key": "model.layers.5.self_attn.q_proj.weight", |
| "module": "model.layers.5.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006719998782500625, |
| "delta_mean_abs": 2.7760039301938377e-05 |
| }, |
| "model.layers.5.self_attn.v_proj": { |
| "base_key": "model.layers.5.self_attn.v_proj.weight", |
| "module": "model.layers.5.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00048304349184036255, |
| "delta_mean_abs": 2.9610157071147114e-05 |
| }, |
| "model.layers.6.mlp.down_proj": { |
| "base_key": "model.layers.6.mlp.down_proj.weight", |
| "module": "model.layers.6.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00045023843995295465, |
| "delta_mean_abs": 1.9406774299568497e-05 |
| }, |
| "model.layers.6.mlp.gate_proj": { |
| "base_key": "model.layers.6.mlp.gate_proj.weight", |
| "module": "model.layers.6.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0007773020770400763, |
| "delta_mean_abs": 3.3434360375395045e-05 |
| }, |
| "model.layers.6.mlp.up_proj": { |
| "base_key": "model.layers.6.mlp.up_proj.weight", |
| "module": "model.layers.6.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006371574709191918, |
| "delta_mean_abs": 3.128765820292756e-05 |
| }, |
| "model.layers.6.self_attn.k_proj": { |
| "base_key": "model.layers.6.self_attn.k_proj.weight", |
| "module": "model.layers.6.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00036013053613714874, |
| "delta_mean_abs": 2.4566277716076e-05 |
| }, |
| "model.layers.6.self_attn.o_proj": { |
| "base_key": "model.layers.6.self_attn.o_proj.weight", |
| "module": "model.layers.6.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005764392553828657, |
| "delta_mean_abs": 2.6879704819293693e-05 |
| }, |
| "model.layers.6.self_attn.q_proj": { |
| "base_key": "model.layers.6.self_attn.q_proj.weight", |
| "module": "model.layers.6.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004684393643401563, |
| "delta_mean_abs": 2.5988611014327034e-05 |
| }, |
| "model.layers.6.self_attn.v_proj": { |
| "base_key": "model.layers.6.self_attn.v_proj.weight", |
| "module": "model.layers.6.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004568297299556434, |
| "delta_mean_abs": 2.989911263284739e-05 |
| }, |
| "model.layers.7.mlp.down_proj": { |
| "base_key": "model.layers.7.mlp.down_proj.weight", |
| "module": "model.layers.7.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0002940115809906274, |
| "delta_mean_abs": 1.8584800272947177e-05 |
| }, |
| "model.layers.7.mlp.gate_proj": { |
| "base_key": "model.layers.7.mlp.gate_proj.weight", |
| "module": "model.layers.7.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0007729062926955521, |
| "delta_mean_abs": 3.5601809940999374e-05 |
| }, |
| "model.layers.7.mlp.up_proj": { |
| "base_key": "model.layers.7.mlp.up_proj.weight", |
| "module": "model.layers.7.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006676525226794183, |
| "delta_mean_abs": 3.286090577603318e-05 |
| }, |
| "model.layers.7.self_attn.k_proj": { |
| "base_key": "model.layers.7.self_attn.k_proj.weight", |
| "module": "model.layers.7.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004221448034513742, |
| "delta_mean_abs": 2.9217031624284573e-05 |
| }, |
| "model.layers.7.self_attn.o_proj": { |
| "base_key": "model.layers.7.self_attn.o_proj.weight", |
| "module": "model.layers.7.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006508776568807662, |
| "delta_mean_abs": 2.695505645533558e-05 |
| }, |
| "model.layers.7.self_attn.q_proj": { |
| "base_key": "model.layers.7.self_attn.q_proj.weight", |
| "module": "model.layers.7.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006043009925633669, |
| "delta_mean_abs": 2.6296636860934086e-05 |
| }, |
| "model.layers.7.self_attn.v_proj": { |
| "base_key": "model.layers.7.self_attn.v_proj.weight", |
| "module": "model.layers.7.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00044372607953846455, |
| "delta_mean_abs": 2.892112752306275e-05 |
| }, |
| "model.layers.8.mlp.down_proj": { |
| "base_key": "model.layers.8.mlp.down_proj.weight", |
| "module": "model.layers.8.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.000441599142504856, |
| "delta_mean_abs": 1.895054265332874e-05 |
| }, |
| "model.layers.8.mlp.gate_proj": { |
| "base_key": "model.layers.8.mlp.gate_proj.weight", |
| "module": "model.layers.8.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006955206044949591, |
| "delta_mean_abs": 3.6487523175310344e-05 |
| }, |
| "model.layers.8.mlp.up_proj": { |
| "base_key": "model.layers.8.mlp.up_proj.weight", |
| "module": "model.layers.8.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.000758365960791707, |
| "delta_mean_abs": 3.4612745366757736e-05 |
| }, |
| "model.layers.8.self_attn.k_proj": { |
| "base_key": "model.layers.8.self_attn.k_proj.weight", |
| "module": "model.layers.8.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004652384377550334, |
| "delta_mean_abs": 2.9981387342559174e-05 |
| }, |
| "model.layers.8.self_attn.o_proj": { |
| "base_key": "model.layers.8.self_attn.o_proj.weight", |
| "module": "model.layers.8.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00045202657929621637, |
| "delta_mean_abs": 2.738096191023942e-05 |
| }, |
| "model.layers.8.self_attn.q_proj": { |
| "base_key": "model.layers.8.self_attn.q_proj.weight", |
| "module": "model.layers.8.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006173077272251248, |
| "delta_mean_abs": 3.0444767617154866e-05 |
| }, |
| "model.layers.8.self_attn.v_proj": { |
| "base_key": "model.layers.8.self_attn.v_proj.weight", |
| "module": "model.layers.8.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.000500081165228039, |
| "delta_mean_abs": 2.864183989004232e-05 |
| }, |
| "model.layers.9.self_attn.q_proj": { |
| "base_key": "model.layers.9.self_attn.q_proj.weight", |
| "module": "model.layers.9.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005254327552393079, |
| "delta_mean_abs": 2.826799391186796e-05 |
| } |
| } |
| }, |
| { |
| "status": "PASS", |
| "shard": "model-00003-of-000017.safetensors", |
| "sha256": "88865a2aa3d18be91c24a6d390c60f712c25535f640530c7ca0669763f7a9563", |
| "size": 1577092016, |
| "applied": { |
| "model.layers.9.mlp.down_proj": { |
| "base_key": "model.layers.9.mlp.down_proj.weight", |
| "module": "model.layers.9.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0003998197498731315, |
| "delta_mean_abs": 2.020290048676543e-05 |
| }, |
| "model.layers.9.mlp.gate_proj": { |
| "base_key": "model.layers.9.mlp.gate_proj.weight", |
| "module": "model.layers.9.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006650558789260685, |
| "delta_mean_abs": 3.5049022699240595e-05 |
| }, |
| "model.layers.9.mlp.up_proj": { |
| "base_key": "model.layers.9.mlp.up_proj.weight", |
| "module": "model.layers.9.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006629395065829158, |
| "delta_mean_abs": 3.3303804229944944e-05 |
| }, |
| "model.layers.9.self_attn.k_proj": { |
| "base_key": "model.layers.9.self_attn.k_proj.weight", |
| "module": "model.layers.9.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005030132015235722, |
| "delta_mean_abs": 2.930740447482094e-05 |
| }, |
| "model.layers.9.self_attn.o_proj": { |
| "base_key": "model.layers.9.self_attn.o_proj.weight", |
| "module": "model.layers.9.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006178875919431448, |
| "delta_mean_abs": 2.754388151515741e-05 |
| }, |
| "model.layers.9.self_attn.v_proj": { |
| "base_key": "model.layers.9.self_attn.v_proj.weight", |
| "module": "model.layers.9.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005247238441370428, |
| "delta_mean_abs": 2.8585352993104607e-05 |
| } |
| } |
| }, |
| { |
| "status": "PASS", |
| "shard": "model-00004-of-000017.safetensors", |
| "sha256": "2aa356f02c51234cb86eaae4c6c6bfa0e65f177160361f35b790c80655b33500", |
| "size": 8690767120, |
| "applied": { |
| "model.layers.10.mlp.down_proj": { |
| "base_key": "model.layers.10.mlp.down_proj.weight", |
| "module": "model.layers.10.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0003578914038371295, |
| "delta_mean_abs": 1.8644699594005942e-05 |
| }, |
| "model.layers.10.mlp.gate_proj": { |
| "base_key": "model.layers.10.mlp.gate_proj.weight", |
| "module": "model.layers.10.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0007022873614914715, |
| "delta_mean_abs": 3.4550026612123474e-05 |
| }, |
| "model.layers.10.mlp.up_proj": { |
| "base_key": "model.layers.10.mlp.up_proj.weight", |
| "module": "model.layers.10.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005343113443814218, |
| "delta_mean_abs": 3.248727080062963e-05 |
| }, |
| "model.layers.10.self_attn.k_proj": { |
| "base_key": "model.layers.10.self_attn.k_proj.weight", |
| "module": "model.layers.10.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00036571527016349137, |
| "delta_mean_abs": 2.6312834961572662e-05 |
| }, |
| "model.layers.10.self_attn.o_proj": { |
| "base_key": "model.layers.10.self_attn.o_proj.weight", |
| "module": "model.layers.10.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005304256919771433, |
| "delta_mean_abs": 2.6340872864238918e-05 |
| }, |
| "model.layers.10.self_attn.q_proj": { |
| "base_key": "model.layers.10.self_attn.q_proj.weight", |
| "module": "model.layers.10.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00047226346214301884, |
| "delta_mean_abs": 2.642719482537359e-05 |
| }, |
| "model.layers.10.self_attn.v_proj": { |
| "base_key": "model.layers.10.self_attn.v_proj.weight", |
| "module": "model.layers.10.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004641205014195293, |
| "delta_mean_abs": 2.6296034775441512e-05 |
| }, |
| "model.layers.11.mlp.down_proj": { |
| "base_key": "model.layers.11.mlp.down_proj.weight", |
| "module": "model.layers.11.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004301145381759852, |
| "delta_mean_abs": 1.8955686755361967e-05 |
| }, |
| "model.layers.11.mlp.gate_proj": { |
| "base_key": "model.layers.11.mlp.gate_proj.weight", |
| "module": "model.layers.11.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006238978239707649, |
| "delta_mean_abs": 3.518013545544818e-05 |
| }, |
| "model.layers.11.mlp.up_proj": { |
| "base_key": "model.layers.11.mlp.up_proj.weight", |
| "module": "model.layers.11.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006969037349335849, |
| "delta_mean_abs": 3.2615807867841795e-05 |
| }, |
| "model.layers.11.self_attn.k_proj": { |
| "base_key": "model.layers.11.self_attn.k_proj.weight", |
| "module": "model.layers.11.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0003391368663869798, |
| "delta_mean_abs": 2.689389293664135e-05 |
| }, |
| "model.layers.11.self_attn.o_proj": { |
| "base_key": "model.layers.11.self_attn.o_proj.weight", |
| "module": "model.layers.11.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.000488903431687504, |
| "delta_mean_abs": 2.680330908333417e-05 |
| }, |
| "model.layers.11.self_attn.q_proj": { |
| "base_key": "model.layers.11.self_attn.q_proj.weight", |
| "module": "model.layers.11.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0003848126798402518, |
| "delta_mean_abs": 2.6009003704530187e-05 |
| }, |
| "model.layers.11.self_attn.v_proj": { |
| "base_key": "model.layers.11.self_attn.v_proj.weight", |
| "module": "model.layers.11.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0003552336129359901, |
| "delta_mean_abs": 2.5538458430673927e-05 |
| }, |
| "model.layers.12.mlp.down_proj": { |
| "base_key": "model.layers.12.mlp.down_proj.weight", |
| "module": "model.layers.12.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00048704337677918375, |
| "delta_mean_abs": 2.0789399059140123e-05 |
| }, |
| "model.layers.12.mlp.gate_proj": { |
| "base_key": "model.layers.12.mlp.gate_proj.weight", |
| "module": "model.layers.12.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0008524228469468653, |
| "delta_mean_abs": 3.566720988601446e-05 |
| }, |
| "model.layers.12.mlp.up_proj": { |
| "base_key": "model.layers.12.mlp.up_proj.weight", |
| "module": "model.layers.12.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.000682688842061907, |
| "delta_mean_abs": 3.426895273150876e-05 |
| }, |
| "model.layers.12.self_attn.k_proj": { |
| "base_key": "model.layers.12.self_attn.k_proj.weight", |
| "module": "model.layers.12.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00037905716453678906, |
| "delta_mean_abs": 3.047188147320412e-05 |
| }, |
| "model.layers.12.self_attn.o_proj": { |
| "base_key": "model.layers.12.self_attn.o_proj.weight", |
| "module": "model.layers.12.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005136913387104869, |
| "delta_mean_abs": 2.8447948352550156e-05 |
| }, |
| "model.layers.12.self_attn.q_proj": { |
| "base_key": "model.layers.12.self_attn.q_proj.weight", |
| "module": "model.layers.12.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005524496082216501, |
| "delta_mean_abs": 2.9502025427063927e-05 |
| }, |
| "model.layers.12.self_attn.v_proj": { |
| "base_key": "model.layers.12.self_attn.v_proj.weight", |
| "module": "model.layers.12.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004195676010567695, |
| "delta_mean_abs": 2.7149908419232816e-05 |
| }, |
| "model.layers.13.mlp.down_proj": { |
| "base_key": "model.layers.13.mlp.down_proj.weight", |
| "module": "model.layers.13.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0003899582370650023, |
| "delta_mean_abs": 1.9898227037629113e-05 |
| }, |
| "model.layers.13.mlp.gate_proj": { |
| "base_key": "model.layers.13.mlp.gate_proj.weight", |
| "module": "model.layers.13.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006656635086983442, |
| "delta_mean_abs": 3.629798811743967e-05 |
| }, |
| "model.layers.13.mlp.up_proj": { |
| "base_key": "model.layers.13.mlp.up_proj.weight", |
| "module": "model.layers.13.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.000809995923191309, |
| "delta_mean_abs": 3.668562931125052e-05 |
| }, |
| "model.layers.13.self_attn.k_proj": { |
| "base_key": "model.layers.13.self_attn.k_proj.weight", |
| "module": "model.layers.13.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00035128198214806616, |
| "delta_mean_abs": 2.7485046302899718e-05 |
| }, |
| "model.layers.13.self_attn.o_proj": { |
| "base_key": "model.layers.13.self_attn.o_proj.weight", |
| "module": "model.layers.13.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005378679488785565, |
| "delta_mean_abs": 2.9396260288194753e-05 |
| }, |
| "model.layers.13.self_attn.q_proj": { |
| "base_key": "model.layers.13.self_attn.q_proj.weight", |
| "module": "model.layers.13.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004775848356075585, |
| "delta_mean_abs": 2.4935630790423602e-05 |
| }, |
| "model.layers.13.self_attn.v_proj": { |
| "base_key": "model.layers.13.self_attn.v_proj.weight", |
| "module": "model.layers.13.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004722838057205081, |
| "delta_mean_abs": 2.789047903206665e-05 |
| }, |
| "model.layers.14.mlp.down_proj": { |
| "base_key": "model.layers.14.mlp.down_proj.weight", |
| "module": "model.layers.14.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004644313594326377, |
| "delta_mean_abs": 2.0793066141777672e-05 |
| }, |
| "model.layers.14.mlp.gate_proj": { |
| "base_key": "model.layers.14.mlp.gate_proj.weight", |
| "module": "model.layers.14.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0008109782938845456, |
| "delta_mean_abs": 3.6049219488631934e-05 |
| }, |
| "model.layers.14.mlp.up_proj": { |
| "base_key": "model.layers.14.mlp.up_proj.weight", |
| "module": "model.layers.14.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0009067971259355545, |
| "delta_mean_abs": 3.781237319344655e-05 |
| }, |
| "model.layers.14.self_attn.k_proj": { |
| "base_key": "model.layers.14.self_attn.k_proj.weight", |
| "module": "model.layers.14.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0003097833541687578, |
| "delta_mean_abs": 2.7849862817674875e-05 |
| }, |
| "model.layers.14.self_attn.o_proj": { |
| "base_key": "model.layers.14.self_attn.o_proj.weight", |
| "module": "model.layers.14.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006474938709288836, |
| "delta_mean_abs": 2.9250222723931074e-05 |
| }, |
| "model.layers.14.self_attn.q_proj": { |
| "base_key": "model.layers.14.self_attn.q_proj.weight", |
| "module": "model.layers.14.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004542551760096103, |
| "delta_mean_abs": 2.5584147806512192e-05 |
| }, |
| "model.layers.14.self_attn.v_proj": { |
| "base_key": "model.layers.14.self_attn.v_proj.weight", |
| "module": "model.layers.14.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004860599583480507, |
| "delta_mean_abs": 2.7947869966737926e-05 |
| }, |
| "model.layers.15.self_attn.q_proj": { |
| "base_key": "model.layers.15.self_attn.q_proj.weight", |
| "module": "model.layers.15.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004839968460146338, |
| "delta_mean_abs": 2.6510871975915506e-05 |
| } |
| } |
| }, |
| { |
| "status": "PASS", |
| "shard": "model-00005-of-000017.safetensors", |
| "sha256": "92e8a39085e8890f03bc0a9d8b5be324b61ddffa54bf3cb108a094b1d8eb7f29", |
| "size": 8422331432, |
| "applied": { |
| "model.layers.15.mlp.down_proj": { |
| "base_key": "model.layers.15.mlp.down_proj.weight", |
| "module": "model.layers.15.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00039482233114540577, |
| "delta_mean_abs": 2.0277902876841836e-05 |
| }, |
| "model.layers.15.mlp.gate_proj": { |
| "base_key": "model.layers.15.mlp.gate_proj.weight", |
| "module": "model.layers.15.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0009372858912684023, |
| "delta_mean_abs": 3.905146149918437e-05 |
| }, |
| "model.layers.15.mlp.up_proj": { |
| "base_key": "model.layers.15.mlp.up_proj.weight", |
| "module": "model.layers.15.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0012898265849798918, |
| "delta_mean_abs": 3.96422874473501e-05 |
| }, |
| "model.layers.15.self_attn.k_proj": { |
| "base_key": "model.layers.15.self_attn.k_proj.weight", |
| "module": "model.layers.15.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005392345483414829, |
| "delta_mean_abs": 2.829216464306228e-05 |
| }, |
| "model.layers.15.self_attn.o_proj": { |
| "base_key": "model.layers.15.self_attn.o_proj.weight", |
| "module": "model.layers.15.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005605737678706646, |
| "delta_mean_abs": 2.846170355041977e-05 |
| }, |
| "model.layers.15.self_attn.v_proj": { |
| "base_key": "model.layers.15.self_attn.v_proj.weight", |
| "module": "model.layers.15.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005755904130637646, |
| "delta_mean_abs": 3.540561374393292e-05 |
| }, |
| "model.layers.16.mlp.down_proj": { |
| "base_key": "model.layers.16.mlp.down_proj.weight", |
| "module": "model.layers.16.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00032310711685568094, |
| "delta_mean_abs": 1.8940079826279543e-05 |
| }, |
| "model.layers.16.mlp.gate_proj": { |
| "base_key": "model.layers.16.mlp.gate_proj.weight", |
| "module": "model.layers.16.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0007166087161749601, |
| "delta_mean_abs": 3.5575350921135396e-05 |
| }, |
| "model.layers.16.mlp.up_proj": { |
| "base_key": "model.layers.16.mlp.up_proj.weight", |
| "module": "model.layers.16.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0011048492742702365, |
| "delta_mean_abs": 3.841549187200144e-05 |
| }, |
| "model.layers.16.self_attn.k_proj": { |
| "base_key": "model.layers.16.self_attn.k_proj.weight", |
| "module": "model.layers.16.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0003247972927056253, |
| "delta_mean_abs": 2.7299880457576364e-05 |
| }, |
| "model.layers.16.self_attn.o_proj": { |
| "base_key": "model.layers.16.self_attn.o_proj.weight", |
| "module": "model.layers.16.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00047759374137967825, |
| "delta_mean_abs": 2.9183374863350764e-05 |
| }, |
| "model.layers.16.self_attn.q_proj": { |
| "base_key": "model.layers.16.self_attn.q_proj.weight", |
| "module": "model.layers.16.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00048805735423229635, |
| "delta_mean_abs": 2.7698981284629554e-05 |
| }, |
| "model.layers.16.self_attn.v_proj": { |
| "base_key": "model.layers.16.self_attn.v_proj.weight", |
| "module": "model.layers.16.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004193338972982019, |
| "delta_mean_abs": 3.265432678745128e-05 |
| }, |
| "model.layers.17.mlp.down_proj": { |
| "base_key": "model.layers.17.mlp.down_proj.weight", |
| "module": "model.layers.17.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00033790242741815746, |
| "delta_mean_abs": 1.8925655240309425e-05 |
| }, |
| "model.layers.17.mlp.gate_proj": { |
| "base_key": "model.layers.17.mlp.gate_proj.weight", |
| "module": "model.layers.17.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.000707303115632385, |
| "delta_mean_abs": 3.6994260881328955e-05 |
| }, |
| "model.layers.17.mlp.up_proj": { |
| "base_key": "model.layers.17.mlp.up_proj.weight", |
| "module": "model.layers.17.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0007223768625408411, |
| "delta_mean_abs": 3.585245940485038e-05 |
| }, |
| "model.layers.17.self_attn.k_proj": { |
| "base_key": "model.layers.17.self_attn.k_proj.weight", |
| "module": "model.layers.17.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00035532109905034304, |
| "delta_mean_abs": 2.6820809580385685e-05 |
| }, |
| "model.layers.17.self_attn.o_proj": { |
| "base_key": "model.layers.17.self_attn.o_proj.weight", |
| "module": "model.layers.17.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004767296486534178, |
| "delta_mean_abs": 2.8032882255502045e-05 |
| }, |
| "model.layers.17.self_attn.q_proj": { |
| "base_key": "model.layers.17.self_attn.q_proj.weight", |
| "module": "model.layers.17.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005394977051764727, |
| "delta_mean_abs": 2.6448475182405673e-05 |
| }, |
| "model.layers.17.self_attn.v_proj": { |
| "base_key": "model.layers.17.self_attn.v_proj.weight", |
| "module": "model.layers.17.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00039558453136123717, |
| "delta_mean_abs": 2.9297125365701504e-05 |
| }, |
| "model.layers.18.mlp.down_proj": { |
| "base_key": "model.layers.18.mlp.down_proj.weight", |
| "module": "model.layers.18.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0012150118127465248, |
| "delta_mean_abs": 2.1763864424428903e-05 |
| }, |
| "model.layers.18.mlp.gate_proj": { |
| "base_key": "model.layers.18.mlp.gate_proj.weight", |
| "module": "model.layers.18.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0007604326820001006, |
| "delta_mean_abs": 3.606493191909976e-05 |
| }, |
| "model.layers.18.mlp.up_proj": { |
| "base_key": "model.layers.18.mlp.up_proj.weight", |
| "module": "model.layers.18.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0007464013760909438, |
| "delta_mean_abs": 3.3829768653959036e-05 |
| }, |
| "model.layers.18.self_attn.k_proj": { |
| "base_key": "model.layers.18.self_attn.k_proj.weight", |
| "module": "model.layers.18.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00030617782613262534, |
| "delta_mean_abs": 2.910135663114488e-05 |
| }, |
| "model.layers.18.self_attn.o_proj": { |
| "base_key": "model.layers.18.self_attn.o_proj.weight", |
| "module": "model.layers.18.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004923552041873336, |
| "delta_mean_abs": 2.75628062809119e-05 |
| }, |
| "model.layers.18.self_attn.q_proj": { |
| "base_key": "model.layers.18.self_attn.q_proj.weight", |
| "module": "model.layers.18.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005360979121178389, |
| "delta_mean_abs": 2.8786835173377767e-05 |
| }, |
| "model.layers.18.self_attn.v_proj": { |
| "base_key": "model.layers.18.self_attn.v_proj.weight", |
| "module": "model.layers.18.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0003560184268280864, |
| "delta_mean_abs": 2.963738006656058e-05 |
| }, |
| "model.layers.19.mlp.down_proj": { |
| "base_key": "model.layers.19.mlp.down_proj.weight", |
| "module": "model.layers.19.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00039569506770931184, |
| "delta_mean_abs": 2.0670006051659584e-05 |
| }, |
| "model.layers.19.mlp.gate_proj": { |
| "base_key": "model.layers.19.mlp.gate_proj.weight", |
| "module": "model.layers.19.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0007531738374382257, |
| "delta_mean_abs": 3.9080401620594785e-05 |
| }, |
| "model.layers.19.mlp.up_proj": { |
| "base_key": "model.layers.19.mlp.up_proj.weight", |
| "module": "model.layers.19.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0007639066316187382, |
| "delta_mean_abs": 3.657360139186494e-05 |
| }, |
| "model.layers.19.self_attn.k_proj": { |
| "base_key": "model.layers.19.self_attn.k_proj.weight", |
| "module": "model.layers.19.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006408033077605069, |
| "delta_mean_abs": 3.3364933187840506e-05 |
| }, |
| "model.layers.19.self_attn.o_proj": { |
| "base_key": "model.layers.19.self_attn.o_proj.weight", |
| "module": "model.layers.19.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00048011617036536336, |
| "delta_mean_abs": 3.125260991510004e-05 |
| }, |
| "model.layers.19.self_attn.q_proj": { |
| "base_key": "model.layers.19.self_attn.q_proj.weight", |
| "module": "model.layers.19.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005855232011526823, |
| "delta_mean_abs": 3.5281213058624417e-05 |
| }, |
| "model.layers.19.self_attn.v_proj": { |
| "base_key": "model.layers.19.self_attn.v_proj.weight", |
| "module": "model.layers.19.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005426667048595846, |
| "delta_mean_abs": 3.1518029572907835e-05 |
| } |
| } |
| }, |
| { |
| "status": "PASS", |
| "shard": "model-00006-of-000017.safetensors", |
| "sha256": "66334b4e798b6c16209292de6efa878ec2c3b9bc04dd923c150bf4fea2155650", |
| "size": 8690767120, |
| "applied": { |
| "model.layers.20.mlp.down_proj": { |
| "base_key": "model.layers.20.mlp.down_proj.weight", |
| "module": "model.layers.20.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0003670479927677661, |
| "delta_mean_abs": 2.044621214736253e-05 |
| }, |
| "model.layers.20.mlp.gate_proj": { |
| "base_key": "model.layers.20.mlp.gate_proj.weight", |
| "module": "model.layers.20.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006585103692486882, |
| "delta_mean_abs": 3.8028229027986526e-05 |
| }, |
| "model.layers.20.mlp.up_proj": { |
| "base_key": "model.layers.20.mlp.up_proj.weight", |
| "module": "model.layers.20.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0007734259706921875, |
| "delta_mean_abs": 3.701594687299803e-05 |
| }, |
| "model.layers.20.self_attn.k_proj": { |
| "base_key": "model.layers.20.self_attn.k_proj.weight", |
| "module": "model.layers.20.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004370283568277955, |
| "delta_mean_abs": 2.7443404178484343e-05 |
| }, |
| "model.layers.20.self_attn.o_proj": { |
| "base_key": "model.layers.20.self_attn.o_proj.weight", |
| "module": "model.layers.20.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004677482938859612, |
| "delta_mean_abs": 3.259960431023501e-05 |
| }, |
| "model.layers.20.self_attn.q_proj": { |
| "base_key": "model.layers.20.self_attn.q_proj.weight", |
| "module": "model.layers.20.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00048744806554168463, |
| "delta_mean_abs": 2.8533766453620046e-05 |
| }, |
| "model.layers.20.self_attn.v_proj": { |
| "base_key": "model.layers.20.self_attn.v_proj.weight", |
| "module": "model.layers.20.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005600369186140597, |
| "delta_mean_abs": 3.3450858609285206e-05 |
| }, |
| "model.layers.21.mlp.down_proj": { |
| "base_key": "model.layers.21.mlp.down_proj.weight", |
| "module": "model.layers.21.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00031051065889187157, |
| "delta_mean_abs": 1.8340409951633774e-05 |
| }, |
| "model.layers.21.mlp.gate_proj": { |
| "base_key": "model.layers.21.mlp.gate_proj.weight", |
| "module": "model.layers.21.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0007532697636634111, |
| "delta_mean_abs": 3.595051748561673e-05 |
| }, |
| "model.layers.21.mlp.up_proj": { |
| "base_key": "model.layers.21.mlp.up_proj.weight", |
| "module": "model.layers.21.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0007955128676258028, |
| "delta_mean_abs": 3.535371070029214e-05 |
| }, |
| "model.layers.21.self_attn.k_proj": { |
| "base_key": "model.layers.21.self_attn.k_proj.weight", |
| "module": "model.layers.21.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00040434542461298406, |
| "delta_mean_abs": 2.641564788063988e-05 |
| }, |
| "model.layers.21.self_attn.o_proj": { |
| "base_key": "model.layers.21.self_attn.o_proj.weight", |
| "module": "model.layers.21.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00045431414037011564, |
| "delta_mean_abs": 3.0004262953298166e-05 |
| }, |
| "model.layers.21.self_attn.q_proj": { |
| "base_key": "model.layers.21.self_attn.q_proj.weight", |
| "module": "model.layers.21.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00044989213347435, |
| "delta_mean_abs": 2.7764279366238043e-05 |
| }, |
| "model.layers.21.self_attn.v_proj": { |
| "base_key": "model.layers.21.self_attn.v_proj.weight", |
| "module": "model.layers.21.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005729124532081187, |
| "delta_mean_abs": 2.963965744129382e-05 |
| }, |
| "model.layers.22.mlp.down_proj": { |
| "base_key": "model.layers.22.mlp.down_proj.weight", |
| "module": "model.layers.22.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0003236736811231822, |
| "delta_mean_abs": 1.8887118130805902e-05 |
| }, |
| "model.layers.22.mlp.gate_proj": { |
| "base_key": "model.layers.22.mlp.gate_proj.weight", |
| "module": "model.layers.22.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0008769913692958653, |
| "delta_mean_abs": 3.745655340026133e-05 |
| }, |
| "model.layers.22.mlp.up_proj": { |
| "base_key": "model.layers.22.mlp.up_proj.weight", |
| "module": "model.layers.22.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0008579844725318253, |
| "delta_mean_abs": 3.330839172122069e-05 |
| }, |
| "model.layers.22.self_attn.k_proj": { |
| "base_key": "model.layers.22.self_attn.k_proj.weight", |
| "module": "model.layers.22.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005424704286269844, |
| "delta_mean_abs": 3.093106352025643e-05 |
| }, |
| "model.layers.22.self_attn.o_proj": { |
| "base_key": "model.layers.22.self_attn.o_proj.weight", |
| "module": "model.layers.22.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004562589165288955, |
| "delta_mean_abs": 2.9977498343214393e-05 |
| }, |
| "model.layers.22.self_attn.q_proj": { |
| "base_key": "model.layers.22.self_attn.q_proj.weight", |
| "module": "model.layers.22.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006297448417171836, |
| "delta_mean_abs": 3.027463026228361e-05 |
| }, |
| "model.layers.22.self_attn.v_proj": { |
| "base_key": "model.layers.22.self_attn.v_proj.weight", |
| "module": "model.layers.22.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004894466837868094, |
| "delta_mean_abs": 2.844944356183987e-05 |
| }, |
| "model.layers.23.mlp.down_proj": { |
| "base_key": "model.layers.23.mlp.down_proj.weight", |
| "module": "model.layers.23.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00028796502738259733, |
| "delta_mean_abs": 1.937580600497313e-05 |
| }, |
| "model.layers.23.mlp.gate_proj": { |
| "base_key": "model.layers.23.mlp.gate_proj.weight", |
| "module": "model.layers.23.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0007519855280406773, |
| "delta_mean_abs": 3.5696913982974365e-05 |
| }, |
| "model.layers.23.mlp.up_proj": { |
| "base_key": "model.layers.23.mlp.up_proj.weight", |
| "module": "model.layers.23.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006430592038668692, |
| "delta_mean_abs": 3.358514732099138e-05 |
| }, |
| "model.layers.23.self_attn.k_proj": { |
| "base_key": "model.layers.23.self_attn.k_proj.weight", |
| "module": "model.layers.23.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00047002124483697116, |
| "delta_mean_abs": 3.0779359804000705e-05 |
| }, |
| "model.layers.23.self_attn.o_proj": { |
| "base_key": "model.layers.23.self_attn.o_proj.weight", |
| "module": "model.layers.23.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005199507577344775, |
| "delta_mean_abs": 3.092791303060949e-05 |
| }, |
| "model.layers.23.self_attn.q_proj": { |
| "base_key": "model.layers.23.self_attn.q_proj.weight", |
| "module": "model.layers.23.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005169544601812959, |
| "delta_mean_abs": 2.976926953124348e-05 |
| }, |
| "model.layers.23.self_attn.v_proj": { |
| "base_key": "model.layers.23.self_attn.v_proj.weight", |
| "module": "model.layers.23.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00044459820492193103, |
| "delta_mean_abs": 2.9304595955181867e-05 |
| }, |
| "model.layers.24.mlp.down_proj": { |
| "base_key": "model.layers.24.mlp.down_proj.weight", |
| "module": "model.layers.24.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00031682950793765485, |
| "delta_mean_abs": 1.925398646562826e-05 |
| }, |
| "model.layers.24.mlp.gate_proj": { |
| "base_key": "model.layers.24.mlp.gate_proj.weight", |
| "module": "model.layers.24.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006070475210435688, |
| "delta_mean_abs": 3.6154619010630995e-05 |
| }, |
| "model.layers.24.mlp.up_proj": { |
| "base_key": "model.layers.24.mlp.up_proj.weight", |
| "module": "model.layers.24.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006107209483161569, |
| "delta_mean_abs": 3.5046785342274234e-05 |
| }, |
| "model.layers.24.self_attn.k_proj": { |
| "base_key": "model.layers.24.self_attn.k_proj.weight", |
| "module": "model.layers.24.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00044853624422103167, |
| "delta_mean_abs": 2.999662683578208e-05 |
| }, |
| "model.layers.24.self_attn.o_proj": { |
| "base_key": "model.layers.24.self_attn.o_proj.weight", |
| "module": "model.layers.24.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005410450976341963, |
| "delta_mean_abs": 2.946443419205025e-05 |
| }, |
| "model.layers.24.self_attn.q_proj": { |
| "base_key": "model.layers.24.self_attn.q_proj.weight", |
| "module": "model.layers.24.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005078237736597657, |
| "delta_mean_abs": 2.978332486236468e-05 |
| }, |
| "model.layers.24.self_attn.v_proj": { |
| "base_key": "model.layers.24.self_attn.v_proj.weight", |
| "module": "model.layers.24.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004762047319673002, |
| "delta_mean_abs": 3.099181049037725e-05 |
| }, |
| "model.layers.25.self_attn.q_proj": { |
| "base_key": "model.layers.25.self_attn.q_proj.weight", |
| "module": "model.layers.25.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004628753231372684, |
| "delta_mean_abs": 2.8914271752000786e-05 |
| } |
| } |
| }, |
| { |
| "status": "PASS", |
| "shard": "model-00007-of-000017.safetensors", |
| "sha256": "423b2122c161b9f3b65c9d9c146537afd1e0997f7bc2948d1798346acb22c830", |
| "size": 8422331432, |
| "applied": { |
| "model.layers.25.mlp.down_proj": { |
| "base_key": "model.layers.25.mlp.down_proj.weight", |
| "module": "model.layers.25.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0003075392742175609, |
| "delta_mean_abs": 2.0207653506076895e-05 |
| }, |
| "model.layers.25.mlp.gate_proj": { |
| "base_key": "model.layers.25.mlp.gate_proj.weight", |
| "module": "model.layers.25.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006210744031704962, |
| "delta_mean_abs": 3.5870703868567944e-05 |
| }, |
| "model.layers.25.mlp.up_proj": { |
| "base_key": "model.layers.25.mlp.up_proj.weight", |
| "module": "model.layers.25.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005775247700512409, |
| "delta_mean_abs": 3.385163290658966e-05 |
| }, |
| "model.layers.25.self_attn.k_proj": { |
| "base_key": "model.layers.25.self_attn.k_proj.weight", |
| "module": "model.layers.25.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0002738982148002833, |
| "delta_mean_abs": 2.500504706404172e-05 |
| }, |
| "model.layers.25.self_attn.o_proj": { |
| "base_key": "model.layers.25.self_attn.o_proj.weight", |
| "module": "model.layers.25.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005219061858952045, |
| "delta_mean_abs": 2.9936094506410882e-05 |
| }, |
| "model.layers.25.self_attn.v_proj": { |
| "base_key": "model.layers.25.self_attn.v_proj.weight", |
| "module": "model.layers.25.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004845963267143816, |
| "delta_mean_abs": 3.082540933974087e-05 |
| }, |
| "model.layers.26.mlp.down_proj": { |
| "base_key": "model.layers.26.mlp.down_proj.weight", |
| "module": "model.layers.26.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0003063615586142987, |
| "delta_mean_abs": 2.0365259842947125e-05 |
| }, |
| "model.layers.26.mlp.gate_proj": { |
| "base_key": "model.layers.26.mlp.gate_proj.weight", |
| "module": "model.layers.26.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0007160595851019025, |
| "delta_mean_abs": 3.268890213803388e-05 |
| }, |
| "model.layers.26.mlp.up_proj": { |
| "base_key": "model.layers.26.mlp.up_proj.weight", |
| "module": "model.layers.26.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0007485139067284763, |
| "delta_mean_abs": 3.1657491490477696e-05 |
| }, |
| "model.layers.26.self_attn.k_proj": { |
| "base_key": "model.layers.26.self_attn.k_proj.weight", |
| "module": "model.layers.26.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004628219467122108, |
| "delta_mean_abs": 2.808139834087342e-05 |
| }, |
| "model.layers.26.self_attn.o_proj": { |
| "base_key": "model.layers.26.self_attn.o_proj.weight", |
| "module": "model.layers.26.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005135575775057077, |
| "delta_mean_abs": 3.048473081435077e-05 |
| }, |
| "model.layers.26.self_attn.q_proj": { |
| "base_key": "model.layers.26.self_attn.q_proj.weight", |
| "module": "model.layers.26.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005703856586478651, |
| "delta_mean_abs": 3.2326548534911126e-05 |
| }, |
| "model.layers.26.self_attn.v_proj": { |
| "base_key": "model.layers.26.self_attn.v_proj.weight", |
| "module": "model.layers.26.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005065468139946461, |
| "delta_mean_abs": 3.0895920644979924e-05 |
| }, |
| "model.layers.27.mlp.down_proj": { |
| "base_key": "model.layers.27.mlp.down_proj.weight", |
| "module": "model.layers.27.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00041634959052316844, |
| "delta_mean_abs": 2.1655549062415957e-05 |
| }, |
| "model.layers.27.mlp.gate_proj": { |
| "base_key": "model.layers.27.mlp.gate_proj.weight", |
| "module": "model.layers.27.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0007026033708825707, |
| "delta_mean_abs": 3.719523738254793e-05 |
| }, |
| "model.layers.27.mlp.up_proj": { |
| "base_key": "model.layers.27.mlp.up_proj.weight", |
| "module": "model.layers.27.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006436676485463977, |
| "delta_mean_abs": 3.587271930882707e-05 |
| }, |
| "model.layers.27.self_attn.k_proj": { |
| "base_key": "model.layers.27.self_attn.k_proj.weight", |
| "module": "model.layers.27.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004348110524006188, |
| "delta_mean_abs": 2.8866721550002694e-05 |
| }, |
| "model.layers.27.self_attn.o_proj": { |
| "base_key": "model.layers.27.self_attn.o_proj.weight", |
| "module": "model.layers.27.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005425090785138309, |
| "delta_mean_abs": 3.136127270408906e-05 |
| }, |
| "model.layers.27.self_attn.q_proj": { |
| "base_key": "model.layers.27.self_attn.q_proj.weight", |
| "module": "model.layers.27.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005659640301018953, |
| "delta_mean_abs": 3.276155621279031e-05 |
| }, |
| "model.layers.27.self_attn.v_proj": { |
| "base_key": "model.layers.27.self_attn.v_proj.weight", |
| "module": "model.layers.27.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.000424560799729079, |
| "delta_mean_abs": 3.2723146432545036e-05 |
| }, |
| "model.layers.28.mlp.down_proj": { |
| "base_key": "model.layers.28.mlp.down_proj.weight", |
| "module": "model.layers.28.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.000376499374397099, |
| "delta_mean_abs": 2.0816356482100673e-05 |
| }, |
| "model.layers.28.mlp.gate_proj": { |
| "base_key": "model.layers.28.mlp.gate_proj.weight", |
| "module": "model.layers.28.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0007759013678878546, |
| "delta_mean_abs": 3.880199074046686e-05 |
| }, |
| "model.layers.28.mlp.up_proj": { |
| "base_key": "model.layers.28.mlp.up_proj.weight", |
| "module": "model.layers.28.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.000749202270526439, |
| "delta_mean_abs": 3.534151255735196e-05 |
| }, |
| "model.layers.28.self_attn.k_proj": { |
| "base_key": "model.layers.28.self_attn.k_proj.weight", |
| "module": "model.layers.28.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005018810625188053, |
| "delta_mean_abs": 3.025566002179403e-05 |
| }, |
| "model.layers.28.self_attn.o_proj": { |
| "base_key": "model.layers.28.self_attn.o_proj.weight", |
| "module": "model.layers.28.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004431013949215412, |
| "delta_mean_abs": 3.1759314879309386e-05 |
| }, |
| "model.layers.28.self_attn.q_proj": { |
| "base_key": "model.layers.28.self_attn.q_proj.weight", |
| "module": "model.layers.28.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004962035454809666, |
| "delta_mean_abs": 2.915566074079834e-05 |
| }, |
| "model.layers.28.self_attn.v_proj": { |
| "base_key": "model.layers.28.self_attn.v_proj.weight", |
| "module": "model.layers.28.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00047562684630975127, |
| "delta_mean_abs": 3.2382387871621177e-05 |
| }, |
| "model.layers.29.mlp.down_proj": { |
| "base_key": "model.layers.29.mlp.down_proj.weight", |
| "module": "model.layers.29.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00041410772246308625, |
| "delta_mean_abs": 2.083051367662847e-05 |
| }, |
| "model.layers.29.mlp.gate_proj": { |
| "base_key": "model.layers.29.mlp.gate_proj.weight", |
| "module": "model.layers.29.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0008486849255859852, |
| "delta_mean_abs": 3.923393160221167e-05 |
| }, |
| "model.layers.29.mlp.up_proj": { |
| "base_key": "model.layers.29.mlp.up_proj.weight", |
| "module": "model.layers.29.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0007861411431804299, |
| "delta_mean_abs": 3.4819782740669325e-05 |
| }, |
| "model.layers.29.self_attn.k_proj": { |
| "base_key": "model.layers.29.self_attn.k_proj.weight", |
| "module": "model.layers.29.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00046382981236092746, |
| "delta_mean_abs": 3.087160439463332e-05 |
| }, |
| "model.layers.29.self_attn.o_proj": { |
| "base_key": "model.layers.29.self_attn.o_proj.weight", |
| "module": "model.layers.29.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005282893544062972, |
| "delta_mean_abs": 3.1298783142119646e-05 |
| }, |
| "model.layers.29.self_attn.q_proj": { |
| "base_key": "model.layers.29.self_attn.q_proj.weight", |
| "module": "model.layers.29.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00048768805572763085, |
| "delta_mean_abs": 2.771464642137289e-05 |
| }, |
| "model.layers.29.self_attn.v_proj": { |
| "base_key": "model.layers.29.self_attn.v_proj.weight", |
| "module": "model.layers.29.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00046866273623891175, |
| "delta_mean_abs": 3.2133561035152525e-05 |
| } |
| } |
| }, |
| { |
| "status": "PASS", |
| "shard": "model-00008-of-000017.safetensors", |
| "sha256": "65c926e1ceb2adfffc54eccc21bbcf56a59d77da7dc0c24bcf624acf9aea6f80", |
| "size": 8690767120, |
| "applied": { |
| "model.layers.30.mlp.down_proj": { |
| "base_key": "model.layers.30.mlp.down_proj.weight", |
| "module": "model.layers.30.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0003852120426017791, |
| "delta_mean_abs": 2.0445459085749462e-05 |
| }, |
| "model.layers.30.mlp.gate_proj": { |
| "base_key": "model.layers.30.mlp.gate_proj.weight", |
| "module": "model.layers.30.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006641994696110487, |
| "delta_mean_abs": 3.699332955875434e-05 |
| }, |
| "model.layers.30.mlp.up_proj": { |
| "base_key": "model.layers.30.mlp.up_proj.weight", |
| "module": "model.layers.30.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006927794311195612, |
| "delta_mean_abs": 3.272937101428397e-05 |
| }, |
| "model.layers.30.self_attn.k_proj": { |
| "base_key": "model.layers.30.self_attn.k_proj.weight", |
| "module": "model.layers.30.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.000526847317814827, |
| "delta_mean_abs": 3.015199581568595e-05 |
| }, |
| "model.layers.30.self_attn.o_proj": { |
| "base_key": "model.layers.30.self_attn.o_proj.weight", |
| "module": "model.layers.30.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004556751227937639, |
| "delta_mean_abs": 3.15938304993324e-05 |
| }, |
| "model.layers.30.self_attn.q_proj": { |
| "base_key": "model.layers.30.self_attn.q_proj.weight", |
| "module": "model.layers.30.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006128204404376447, |
| "delta_mean_abs": 2.982766818604432e-05 |
| }, |
| "model.layers.30.self_attn.v_proj": { |
| "base_key": "model.layers.30.self_attn.v_proj.weight", |
| "module": "model.layers.30.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.000492256076540798, |
| "delta_mean_abs": 3.2863510568859056e-05 |
| }, |
| "model.layers.31.mlp.down_proj": { |
| "base_key": "model.layers.31.mlp.down_proj.weight", |
| "module": "model.layers.31.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00046290218597278, |
| "delta_mean_abs": 2.188756116083823e-05 |
| }, |
| "model.layers.31.mlp.gate_proj": { |
| "base_key": "model.layers.31.mlp.gate_proj.weight", |
| "module": "model.layers.31.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0009979824535548687, |
| "delta_mean_abs": 3.967043085140176e-05 |
| }, |
| "model.layers.31.mlp.up_proj": { |
| "base_key": "model.layers.31.mlp.up_proj.weight", |
| "module": "model.layers.31.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006500134477391839, |
| "delta_mean_abs": 3.566565646906383e-05 |
| }, |
| "model.layers.31.self_attn.k_proj": { |
| "base_key": "model.layers.31.self_attn.k_proj.weight", |
| "module": "model.layers.31.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005368662532418966, |
| "delta_mean_abs": 3.209201167919673e-05 |
| }, |
| "model.layers.31.self_attn.o_proj": { |
| "base_key": "model.layers.31.self_attn.o_proj.weight", |
| "module": "model.layers.31.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005189056391827762, |
| "delta_mean_abs": 3.260055746068247e-05 |
| }, |
| "model.layers.31.self_attn.q_proj": { |
| "base_key": "model.layers.31.self_attn.q_proj.weight", |
| "module": "model.layers.31.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.000686001731082797, |
| "delta_mean_abs": 2.9941722459625453e-05 |
| }, |
| "model.layers.31.self_attn.v_proj": { |
| "base_key": "model.layers.31.self_attn.v_proj.weight", |
| "module": "model.layers.31.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006456708651967347, |
| "delta_mean_abs": 3.5762597690336406e-05 |
| }, |
| "model.layers.32.mlp.down_proj": { |
| "base_key": "model.layers.32.mlp.down_proj.weight", |
| "module": "model.layers.32.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00047211372293531895, |
| "delta_mean_abs": 2.2208392692846246e-05 |
| }, |
| "model.layers.32.mlp.gate_proj": { |
| "base_key": "model.layers.32.mlp.gate_proj.weight", |
| "module": "model.layers.32.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006780275725759566, |
| "delta_mean_abs": 3.9647307858103886e-05 |
| }, |
| "model.layers.32.mlp.up_proj": { |
| "base_key": "model.layers.32.mlp.up_proj.weight", |
| "module": "model.layers.32.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0007835898431949317, |
| "delta_mean_abs": 3.651916267699562e-05 |
| }, |
| "model.layers.32.self_attn.k_proj": { |
| "base_key": "model.layers.32.self_attn.k_proj.weight", |
| "module": "model.layers.32.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005122994771227241, |
| "delta_mean_abs": 3.1651914468966424e-05 |
| }, |
| "model.layers.32.self_attn.o_proj": { |
| "base_key": "model.layers.32.self_attn.o_proj.weight", |
| "module": "model.layers.32.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005488019087351859, |
| "delta_mean_abs": 3.2963282137643546e-05 |
| }, |
| "model.layers.32.self_attn.q_proj": { |
| "base_key": "model.layers.32.self_attn.q_proj.weight", |
| "module": "model.layers.32.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005603755125775933, |
| "delta_mean_abs": 3.066295539611019e-05 |
| }, |
| "model.layers.32.self_attn.v_proj": { |
| "base_key": "model.layers.32.self_attn.v_proj.weight", |
| "module": "model.layers.32.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004880889900960028, |
| "delta_mean_abs": 3.6247285606805235e-05 |
| }, |
| "model.layers.33.mlp.down_proj": { |
| "base_key": "model.layers.33.mlp.down_proj.weight", |
| "module": "model.layers.33.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00046494242269545794, |
| "delta_mean_abs": 1.967673233593814e-05 |
| }, |
| "model.layers.33.mlp.gate_proj": { |
| "base_key": "model.layers.33.mlp.gate_proj.weight", |
| "module": "model.layers.33.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0010205315193161368, |
| "delta_mean_abs": 4.296499173506163e-05 |
| }, |
| "model.layers.33.mlp.up_proj": { |
| "base_key": "model.layers.33.mlp.up_proj.weight", |
| "module": "model.layers.33.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0008991762297227979, |
| "delta_mean_abs": 3.632656080299057e-05 |
| }, |
| "model.layers.33.self_attn.k_proj": { |
| "base_key": "model.layers.33.self_attn.k_proj.weight", |
| "module": "model.layers.33.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0003362589341122657, |
| "delta_mean_abs": 3.2607415050733835e-05 |
| }, |
| "model.layers.33.self_attn.o_proj": { |
| "base_key": "model.layers.33.self_attn.o_proj.weight", |
| "module": "model.layers.33.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005127033218741417, |
| "delta_mean_abs": 3.3575688576092944e-05 |
| }, |
| "model.layers.33.self_attn.q_proj": { |
| "base_key": "model.layers.33.self_attn.q_proj.weight", |
| "module": "model.layers.33.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005462816916406155, |
| "delta_mean_abs": 3.1112300348468125e-05 |
| }, |
| "model.layers.33.self_attn.v_proj": { |
| "base_key": "model.layers.33.self_attn.v_proj.weight", |
| "module": "model.layers.33.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004778730508405715, |
| "delta_mean_abs": 3.6407338484423235e-05 |
| }, |
| "model.layers.34.mlp.down_proj": { |
| "base_key": "model.layers.34.mlp.down_proj.weight", |
| "module": "model.layers.34.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00035231365473009646, |
| "delta_mean_abs": 1.7413749446859583e-05 |
| }, |
| "model.layers.34.mlp.gate_proj": { |
| "base_key": "model.layers.34.mlp.gate_proj.weight", |
| "module": "model.layers.34.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0009908298961818218, |
| "delta_mean_abs": 4.174706919002347e-05 |
| }, |
| "model.layers.34.mlp.up_proj": { |
| "base_key": "model.layers.34.mlp.up_proj.weight", |
| "module": "model.layers.34.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.000783766561653465, |
| "delta_mean_abs": 3.5336717701284215e-05 |
| }, |
| "model.layers.34.self_attn.k_proj": { |
| "base_key": "model.layers.34.self_attn.k_proj.weight", |
| "module": "model.layers.34.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004040884377900511, |
| "delta_mean_abs": 3.1170005968306214e-05 |
| }, |
| "model.layers.34.self_attn.o_proj": { |
| "base_key": "model.layers.34.self_attn.o_proj.weight", |
| "module": "model.layers.34.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00043990896665491164, |
| "delta_mean_abs": 2.9985816581756808e-05 |
| }, |
| "model.layers.34.self_attn.q_proj": { |
| "base_key": "model.layers.34.self_attn.q_proj.weight", |
| "module": "model.layers.34.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00046395423123613, |
| "delta_mean_abs": 3.0289735150290653e-05 |
| }, |
| "model.layers.34.self_attn.v_proj": { |
| "base_key": "model.layers.34.self_attn.v_proj.weight", |
| "module": "model.layers.34.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005479517276398838, |
| "delta_mean_abs": 3.35903387167491e-05 |
| }, |
| "model.layers.35.self_attn.q_proj": { |
| "base_key": "model.layers.35.self_attn.q_proj.weight", |
| "module": "model.layers.35.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005421487148851156, |
| "delta_mean_abs": 3.139608816127293e-05 |
| } |
| } |
| }, |
| { |
| "status": "PASS", |
| "shard": "model-00009-of-000017.safetensors", |
| "sha256": "d0e6e50e55b7c3e6da466e73351836b438d9e7dd4728fae10d8b30121b57ef23", |
| "size": 8422331432, |
| "applied": { |
| "model.layers.35.mlp.down_proj": { |
| "base_key": "model.layers.35.mlp.down_proj.weight", |
| "module": "model.layers.35.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00032082918914966285, |
| "delta_mean_abs": 1.8702803572523408e-05 |
| }, |
| "model.layers.35.mlp.gate_proj": { |
| "base_key": "model.layers.35.mlp.gate_proj.weight", |
| "module": "model.layers.35.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0011147449258714914, |
| "delta_mean_abs": 3.892763197654858e-05 |
| }, |
| "model.layers.35.mlp.up_proj": { |
| "base_key": "model.layers.35.mlp.up_proj.weight", |
| "module": "model.layers.35.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0007120087975636125, |
| "delta_mean_abs": 3.5402390494709834e-05 |
| }, |
| "model.layers.35.self_attn.k_proj": { |
| "base_key": "model.layers.35.self_attn.k_proj.weight", |
| "module": "model.layers.35.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0003369122860021889, |
| "delta_mean_abs": 2.97386541205924e-05 |
| }, |
| "model.layers.35.self_attn.o_proj": { |
| "base_key": "model.layers.35.self_attn.o_proj.weight", |
| "module": "model.layers.35.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006077519501559436, |
| "delta_mean_abs": 2.8890683097415604e-05 |
| }, |
| "model.layers.35.self_attn.v_proj": { |
| "base_key": "model.layers.35.self_attn.v_proj.weight", |
| "module": "model.layers.35.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005003415863029659, |
| "delta_mean_abs": 3.181271313223988e-05 |
| }, |
| "model.layers.36.mlp.down_proj": { |
| "base_key": "model.layers.36.mlp.down_proj.weight", |
| "module": "model.layers.36.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00031107431277632713, |
| "delta_mean_abs": 1.830846849770751e-05 |
| }, |
| "model.layers.36.mlp.gate_proj": { |
| "base_key": "model.layers.36.mlp.gate_proj.weight", |
| "module": "model.layers.36.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0008636223501525819, |
| "delta_mean_abs": 4.07787665608339e-05 |
| }, |
| "model.layers.36.mlp.up_proj": { |
| "base_key": "model.layers.36.mlp.up_proj.weight", |
| "module": "model.layers.36.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.001006239908747375, |
| "delta_mean_abs": 3.894712062901817e-05 |
| }, |
| "model.layers.36.self_attn.k_proj": { |
| "base_key": "model.layers.36.self_attn.k_proj.weight", |
| "module": "model.layers.36.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0008924414869397879, |
| "delta_mean_abs": 3.314469722681679e-05 |
| }, |
| "model.layers.36.self_attn.o_proj": { |
| "base_key": "model.layers.36.self_attn.o_proj.weight", |
| "module": "model.layers.36.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00048726610839366913, |
| "delta_mean_abs": 2.8322330763330683e-05 |
| }, |
| "model.layers.36.self_attn.q_proj": { |
| "base_key": "model.layers.36.self_attn.q_proj.weight", |
| "module": "model.layers.36.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00091399229131639, |
| "delta_mean_abs": 3.5189310437999666e-05 |
| }, |
| "model.layers.36.self_attn.v_proj": { |
| "base_key": "model.layers.36.self_attn.v_proj.weight", |
| "module": "model.layers.36.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004785800410900265, |
| "delta_mean_abs": 3.151838245685212e-05 |
| }, |
| "model.layers.37.mlp.down_proj": { |
| "base_key": "model.layers.37.mlp.down_proj.weight", |
| "module": "model.layers.37.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0003155853773932904, |
| "delta_mean_abs": 1.697808511380572e-05 |
| }, |
| "model.layers.37.mlp.gate_proj": { |
| "base_key": "model.layers.37.mlp.gate_proj.weight", |
| "module": "model.layers.37.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0008523616124875844, |
| "delta_mean_abs": 4.009905023849569e-05 |
| }, |
| "model.layers.37.mlp.up_proj": { |
| "base_key": "model.layers.37.mlp.up_proj.weight", |
| "module": "model.layers.37.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.000742932315915823, |
| "delta_mean_abs": 3.849756467388943e-05 |
| }, |
| "model.layers.37.self_attn.k_proj": { |
| "base_key": "model.layers.37.self_attn.k_proj.weight", |
| "module": "model.layers.37.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.000447834754595533, |
| "delta_mean_abs": 3.186934191035107e-05 |
| }, |
| "model.layers.37.self_attn.o_proj": { |
| "base_key": "model.layers.37.self_attn.o_proj.weight", |
| "module": "model.layers.37.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004809541569557041, |
| "delta_mean_abs": 2.7444939405540936e-05 |
| }, |
| "model.layers.37.self_attn.q_proj": { |
| "base_key": "model.layers.37.self_attn.q_proj.weight", |
| "module": "model.layers.37.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005997818661853671, |
| "delta_mean_abs": 2.9726596039836295e-05 |
| }, |
| "model.layers.37.self_attn.v_proj": { |
| "base_key": "model.layers.37.self_attn.v_proj.weight", |
| "module": "model.layers.37.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004297611303627491, |
| "delta_mean_abs": 3.052594547625631e-05 |
| }, |
| "model.layers.38.mlp.down_proj": { |
| "base_key": "model.layers.38.mlp.down_proj.weight", |
| "module": "model.layers.38.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00028839349397458136, |
| "delta_mean_abs": 1.6918413166422397e-05 |
| }, |
| "model.layers.38.mlp.gate_proj": { |
| "base_key": "model.layers.38.mlp.gate_proj.weight", |
| "module": "model.layers.38.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.001065364689566195, |
| "delta_mean_abs": 4.0387018088949844e-05 |
| }, |
| "model.layers.38.mlp.up_proj": { |
| "base_key": "model.layers.38.mlp.up_proj.weight", |
| "module": "model.layers.38.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006954649579711258, |
| "delta_mean_abs": 3.480457962723449e-05 |
| }, |
| "model.layers.38.self_attn.k_proj": { |
| "base_key": "model.layers.38.self_attn.k_proj.weight", |
| "module": "model.layers.38.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00040716829244047403, |
| "delta_mean_abs": 2.9893722967244685e-05 |
| }, |
| "model.layers.38.self_attn.o_proj": { |
| "base_key": "model.layers.38.self_attn.o_proj.weight", |
| "module": "model.layers.38.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004174862406216562, |
| "delta_mean_abs": 2.7689127819030546e-05 |
| }, |
| "model.layers.38.self_attn.q_proj": { |
| "base_key": "model.layers.38.self_attn.q_proj.weight", |
| "module": "model.layers.38.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006464465404860675, |
| "delta_mean_abs": 2.9984816137584858e-05 |
| }, |
| "model.layers.38.self_attn.v_proj": { |
| "base_key": "model.layers.38.self_attn.v_proj.weight", |
| "module": "model.layers.38.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0003740675456356257, |
| "delta_mean_abs": 3.0346342100529e-05 |
| }, |
| "model.layers.39.mlp.down_proj": { |
| "base_key": "model.layers.39.mlp.down_proj.weight", |
| "module": "model.layers.39.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0002961461723316461, |
| "delta_mean_abs": 1.8194896256318316e-05 |
| }, |
| "model.layers.39.mlp.gate_proj": { |
| "base_key": "model.layers.39.mlp.gate_proj.weight", |
| "module": "model.layers.39.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0007970178266987205, |
| "delta_mean_abs": 3.92308684240561e-05 |
| }, |
| "model.layers.39.mlp.up_proj": { |
| "base_key": "model.layers.39.mlp.up_proj.weight", |
| "module": "model.layers.39.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0007019301410764456, |
| "delta_mean_abs": 3.728358569787815e-05 |
| }, |
| "model.layers.39.self_attn.k_proj": { |
| "base_key": "model.layers.39.self_attn.k_proj.weight", |
| "module": "model.layers.39.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0003473837277851999, |
| "delta_mean_abs": 2.6527020963840187e-05 |
| }, |
| "model.layers.39.self_attn.o_proj": { |
| "base_key": "model.layers.39.self_attn.o_proj.weight", |
| "module": "model.layers.39.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00042731009307317436, |
| "delta_mean_abs": 2.6828305635717697e-05 |
| }, |
| "model.layers.39.self_attn.q_proj": { |
| "base_key": "model.layers.39.self_attn.q_proj.weight", |
| "module": "model.layers.39.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006152298301458359, |
| "delta_mean_abs": 2.8421343813533895e-05 |
| }, |
| "model.layers.39.self_attn.v_proj": { |
| "base_key": "model.layers.39.self_attn.v_proj.weight", |
| "module": "model.layers.39.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00039450175245292485, |
| "delta_mean_abs": 2.8525952075142413e-05 |
| } |
| } |
| }, |
| { |
| "status": "PASS", |
| "shard": "model-00010-of-000017.safetensors", |
| "sha256": "eb4e26cd8ec354a85afb5b8ebfdd1d66985b4f081a8494979a19b7d095104b12", |
| "size": 8690767120, |
| "applied": { |
| "model.layers.40.mlp.down_proj": { |
| "base_key": "model.layers.40.mlp.down_proj.weight", |
| "module": "model.layers.40.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00029594526859000325, |
| "delta_mean_abs": 1.709214484435506e-05 |
| }, |
| "model.layers.40.mlp.gate_proj": { |
| "base_key": "model.layers.40.mlp.gate_proj.weight", |
| "module": "model.layers.40.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.000681317993439734, |
| "delta_mean_abs": 3.6720015486935154e-05 |
| }, |
| "model.layers.40.mlp.up_proj": { |
| "base_key": "model.layers.40.mlp.up_proj.weight", |
| "module": "model.layers.40.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0008296065498143435, |
| "delta_mean_abs": 3.917914000339806e-05 |
| }, |
| "model.layers.40.self_attn.k_proj": { |
| "base_key": "model.layers.40.self_attn.k_proj.weight", |
| "module": "model.layers.40.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00034495291765779257, |
| "delta_mean_abs": 2.9910184821346775e-05 |
| }, |
| "model.layers.40.self_attn.o_proj": { |
| "base_key": "model.layers.40.self_attn.o_proj.weight", |
| "module": "model.layers.40.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005356971523724496, |
| "delta_mean_abs": 2.947970642708242e-05 |
| }, |
| "model.layers.40.self_attn.q_proj": { |
| "base_key": "model.layers.40.self_attn.q_proj.weight", |
| "module": "model.layers.40.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00046502085751853883, |
| "delta_mean_abs": 2.851252065738663e-05 |
| }, |
| "model.layers.40.self_attn.v_proj": { |
| "base_key": "model.layers.40.self_attn.v_proj.weight", |
| "module": "model.layers.40.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004806818615179509, |
| "delta_mean_abs": 2.9176389944041148e-05 |
| }, |
| "model.layers.41.mlp.down_proj": { |
| "base_key": "model.layers.41.mlp.down_proj.weight", |
| "module": "model.layers.41.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00025182575336657465, |
| "delta_mean_abs": 1.7500367903267033e-05 |
| }, |
| "model.layers.41.mlp.gate_proj": { |
| "base_key": "model.layers.41.mlp.gate_proj.weight", |
| "module": "model.layers.41.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0007002067868597806, |
| "delta_mean_abs": 3.7423909816425294e-05 |
| }, |
| "model.layers.41.mlp.up_proj": { |
| "base_key": "model.layers.41.mlp.up_proj.weight", |
| "module": "model.layers.41.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0008171783993020654, |
| "delta_mean_abs": 3.675529660540633e-05 |
| }, |
| "model.layers.41.self_attn.k_proj": { |
| "base_key": "model.layers.41.self_attn.k_proj.weight", |
| "module": "model.layers.41.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004009816038887948, |
| "delta_mean_abs": 3.074146661674604e-05 |
| }, |
| "model.layers.41.self_attn.o_proj": { |
| "base_key": "model.layers.41.self_attn.o_proj.weight", |
| "module": "model.layers.41.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.000497576838824898, |
| "delta_mean_abs": 2.7241538191447034e-05 |
| }, |
| "model.layers.41.self_attn.q_proj": { |
| "base_key": "model.layers.41.self_attn.q_proj.weight", |
| "module": "model.layers.41.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004348670190665871, |
| "delta_mean_abs": 2.6945595891447738e-05 |
| }, |
| "model.layers.41.self_attn.v_proj": { |
| "base_key": "model.layers.41.self_attn.v_proj.weight", |
| "module": "model.layers.41.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00046172470320016146, |
| "delta_mean_abs": 2.7789252271759324e-05 |
| }, |
| "model.layers.42.mlp.down_proj": { |
| "base_key": "model.layers.42.mlp.down_proj.weight", |
| "module": "model.layers.42.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0002663386694621295, |
| "delta_mean_abs": 1.7591384676052257e-05 |
| }, |
| "model.layers.42.mlp.gate_proj": { |
| "base_key": "model.layers.42.mlp.gate_proj.weight", |
| "module": "model.layers.42.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0007347972132265568, |
| "delta_mean_abs": 3.646871118689887e-05 |
| }, |
| "model.layers.42.mlp.up_proj": { |
| "base_key": "model.layers.42.mlp.up_proj.weight", |
| "module": "model.layers.42.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005968305049464107, |
| "delta_mean_abs": 3.379353802301921e-05 |
| }, |
| "model.layers.42.self_attn.k_proj": { |
| "base_key": "model.layers.42.self_attn.k_proj.weight", |
| "module": "model.layers.42.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00044519209768623114, |
| "delta_mean_abs": 2.7669648261507973e-05 |
| }, |
| "model.layers.42.self_attn.o_proj": { |
| "base_key": "model.layers.42.self_attn.o_proj.weight", |
| "module": "model.layers.42.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005367975682020187, |
| "delta_mean_abs": 2.765120007097721e-05 |
| }, |
| "model.layers.42.self_attn.q_proj": { |
| "base_key": "model.layers.42.self_attn.q_proj.weight", |
| "module": "model.layers.42.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004673278599511832, |
| "delta_mean_abs": 2.6646030164556578e-05 |
| }, |
| "model.layers.42.self_attn.v_proj": { |
| "base_key": "model.layers.42.self_attn.v_proj.weight", |
| "module": "model.layers.42.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004160058160778135, |
| "delta_mean_abs": 2.740255331445951e-05 |
| }, |
| "model.layers.43.mlp.down_proj": { |
| "base_key": "model.layers.43.mlp.down_proj.weight", |
| "module": "model.layers.43.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00023772021813783795, |
| "delta_mean_abs": 1.7026000932673924e-05 |
| }, |
| "model.layers.43.mlp.gate_proj": { |
| "base_key": "model.layers.43.mlp.gate_proj.weight", |
| "module": "model.layers.43.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0007745756884105504, |
| "delta_mean_abs": 3.784118962357752e-05 |
| }, |
| "model.layers.43.mlp.up_proj": { |
| "base_key": "model.layers.43.mlp.up_proj.weight", |
| "module": "model.layers.43.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006806864985264838, |
| "delta_mean_abs": 3.7699832319049165e-05 |
| }, |
| "model.layers.43.self_attn.k_proj": { |
| "base_key": "model.layers.43.self_attn.k_proj.weight", |
| "module": "model.layers.43.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0003728914598468691, |
| "delta_mean_abs": 2.775310531433206e-05 |
| }, |
| "model.layers.43.self_attn.o_proj": { |
| "base_key": "model.layers.43.self_attn.o_proj.weight", |
| "module": "model.layers.43.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00048579019494354725, |
| "delta_mean_abs": 2.784214484563563e-05 |
| }, |
| "model.layers.43.self_attn.q_proj": { |
| "base_key": "model.layers.43.self_attn.q_proj.weight", |
| "module": "model.layers.43.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004595101345330477, |
| "delta_mean_abs": 2.9440008802339435e-05 |
| }, |
| "model.layers.43.self_attn.v_proj": { |
| "base_key": "model.layers.43.self_attn.v_proj.weight", |
| "module": "model.layers.43.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00042334754834882915, |
| "delta_mean_abs": 2.896552177844569e-05 |
| }, |
| "model.layers.44.mlp.down_proj": { |
| "base_key": "model.layers.44.mlp.down_proj.weight", |
| "module": "model.layers.44.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.000286812981357798, |
| "delta_mean_abs": 1.7443613614887e-05 |
| }, |
| "model.layers.44.mlp.gate_proj": { |
| "base_key": "model.layers.44.mlp.gate_proj.weight", |
| "module": "model.layers.44.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005417307838797569, |
| "delta_mean_abs": 3.3523647289257497e-05 |
| }, |
| "model.layers.44.mlp.up_proj": { |
| "base_key": "model.layers.44.mlp.up_proj.weight", |
| "module": "model.layers.44.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0010184006532654166, |
| "delta_mean_abs": 3.684139301185496e-05 |
| }, |
| "model.layers.44.self_attn.k_proj": { |
| "base_key": "model.layers.44.self_attn.k_proj.weight", |
| "module": "model.layers.44.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00048543131561018527, |
| "delta_mean_abs": 3.130015102215111e-05 |
| }, |
| "model.layers.44.self_attn.o_proj": { |
| "base_key": "model.layers.44.self_attn.o_proj.weight", |
| "module": "model.layers.44.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006107743247412145, |
| "delta_mean_abs": 3.0081435397733003e-05 |
| }, |
| "model.layers.44.self_attn.q_proj": { |
| "base_key": "model.layers.44.self_attn.q_proj.weight", |
| "module": "model.layers.44.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0007446313393302262, |
| "delta_mean_abs": 3.0549181246897206e-05 |
| }, |
| "model.layers.44.self_attn.v_proj": { |
| "base_key": "model.layers.44.self_attn.v_proj.weight", |
| "module": "model.layers.44.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004254269879311323, |
| "delta_mean_abs": 3.108121745754033e-05 |
| }, |
| "model.layers.45.self_attn.q_proj": { |
| "base_key": "model.layers.45.self_attn.q_proj.weight", |
| "module": "model.layers.45.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006016346160322428, |
| "delta_mean_abs": 2.9297618311829865e-05 |
| } |
| } |
| }, |
| { |
| "status": "PASS", |
| "shard": "model-00011-of-000017.safetensors", |
| "sha256": "1d4964f30f8ff64070ced443d5df9d8d4ac195eb59a23b867b587cde8d6907cc", |
| "size": 8422331432, |
| "applied": { |
| "model.layers.45.mlp.down_proj": { |
| "base_key": "model.layers.45.mlp.down_proj.weight", |
| "module": "model.layers.45.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00027781628887169063, |
| "delta_mean_abs": 1.7283231500186957e-05 |
| }, |
| "model.layers.45.mlp.gate_proj": { |
| "base_key": "model.layers.45.mlp.gate_proj.weight", |
| "module": "model.layers.45.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0008376310579478741, |
| "delta_mean_abs": 3.581325654522516e-05 |
| }, |
| "model.layers.45.mlp.up_proj": { |
| "base_key": "model.layers.45.mlp.up_proj.weight", |
| "module": "model.layers.45.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005828288267366588, |
| "delta_mean_abs": 3.467064016149379e-05 |
| }, |
| "model.layers.45.self_attn.k_proj": { |
| "base_key": "model.layers.45.self_attn.k_proj.weight", |
| "module": "model.layers.45.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00031261175172403455, |
| "delta_mean_abs": 2.6185036404058337e-05 |
| }, |
| "model.layers.45.self_attn.o_proj": { |
| "base_key": "model.layers.45.self_attn.o_proj.weight", |
| "module": "model.layers.45.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005854540504515171, |
| "delta_mean_abs": 3.0488052289001644e-05 |
| }, |
| "model.layers.45.self_attn.v_proj": { |
| "base_key": "model.layers.45.self_attn.v_proj.weight", |
| "module": "model.layers.45.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00041751848766580224, |
| "delta_mean_abs": 3.188758637406863e-05 |
| }, |
| "model.layers.46.mlp.down_proj": { |
| "base_key": "model.layers.46.mlp.down_proj.weight", |
| "module": "model.layers.46.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00025634642224758863, |
| "delta_mean_abs": 1.715251346467994e-05 |
| }, |
| "model.layers.46.mlp.gate_proj": { |
| "base_key": "model.layers.46.mlp.gate_proj.weight", |
| "module": "model.layers.46.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0008066649897955358, |
| "delta_mean_abs": 3.869008651236072e-05 |
| }, |
| "model.layers.46.mlp.up_proj": { |
| "base_key": "model.layers.46.mlp.up_proj.weight", |
| "module": "model.layers.46.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005771343712694943, |
| "delta_mean_abs": 3.398791886866093e-05 |
| }, |
| "model.layers.46.self_attn.k_proj": { |
| "base_key": "model.layers.46.self_attn.k_proj.weight", |
| "module": "model.layers.46.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00023649884678889066, |
| "delta_mean_abs": 2.430964377708733e-05 |
| }, |
| "model.layers.46.self_attn.o_proj": { |
| "base_key": "model.layers.46.self_attn.o_proj.weight", |
| "module": "model.layers.46.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00038045173278078437, |
| "delta_mean_abs": 2.6320951292291284e-05 |
| }, |
| "model.layers.46.self_attn.q_proj": { |
| "base_key": "model.layers.46.self_attn.q_proj.weight", |
| "module": "model.layers.46.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00027432109345681965, |
| "delta_mean_abs": 2.448376835673116e-05 |
| }, |
| "model.layers.46.self_attn.v_proj": { |
| "base_key": "model.layers.46.self_attn.v_proj.weight", |
| "module": "model.layers.46.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004108718712814152, |
| "delta_mean_abs": 2.765154385997448e-05 |
| }, |
| "model.layers.47.mlp.down_proj": { |
| "base_key": "model.layers.47.mlp.down_proj.weight", |
| "module": "model.layers.47.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0002678571327123791, |
| "delta_mean_abs": 1.715244616207201e-05 |
| }, |
| "model.layers.47.mlp.gate_proj": { |
| "base_key": "model.layers.47.mlp.gate_proj.weight", |
| "module": "model.layers.47.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005976863903924823, |
| "delta_mean_abs": 3.364206349942833e-05 |
| }, |
| "model.layers.47.mlp.up_proj": { |
| "base_key": "model.layers.47.mlp.up_proj.weight", |
| "module": "model.layers.47.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0010268824407830834, |
| "delta_mean_abs": 3.6555495171342045e-05 |
| }, |
| "model.layers.47.self_attn.k_proj": { |
| "base_key": "model.layers.47.self_attn.k_proj.weight", |
| "module": "model.layers.47.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0003631594590842724, |
| "delta_mean_abs": 3.1008774385554716e-05 |
| }, |
| "model.layers.47.self_attn.o_proj": { |
| "base_key": "model.layers.47.self_attn.o_proj.weight", |
| "module": "model.layers.47.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005022079567424953, |
| "delta_mean_abs": 2.8746029784088023e-05 |
| }, |
| "model.layers.47.self_attn.q_proj": { |
| "base_key": "model.layers.47.self_attn.q_proj.weight", |
| "module": "model.layers.47.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00042845410644076765, |
| "delta_mean_abs": 2.812365346471779e-05 |
| }, |
| "model.layers.47.self_attn.v_proj": { |
| "base_key": "model.layers.47.self_attn.v_proj.weight", |
| "module": "model.layers.47.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.000507188553456217, |
| "delta_mean_abs": 3.129264950985089e-05 |
| }, |
| "model.layers.48.mlp.down_proj": { |
| "base_key": "model.layers.48.mlp.down_proj.weight", |
| "module": "model.layers.48.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0003066085046157241, |
| "delta_mean_abs": 1.7241323803318664e-05 |
| }, |
| "model.layers.48.mlp.gate_proj": { |
| "base_key": "model.layers.48.mlp.gate_proj.weight", |
| "module": "model.layers.48.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.000737904803827405, |
| "delta_mean_abs": 3.523816849337891e-05 |
| }, |
| "model.layers.48.mlp.up_proj": { |
| "base_key": "model.layers.48.mlp.up_proj.weight", |
| "module": "model.layers.48.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.000698602176271379, |
| "delta_mean_abs": 3.6929963243892416e-05 |
| }, |
| "model.layers.48.self_attn.k_proj": { |
| "base_key": "model.layers.48.self_attn.k_proj.weight", |
| "module": "model.layers.48.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00026751539553515613, |
| "delta_mean_abs": 2.9061064196866937e-05 |
| }, |
| "model.layers.48.self_attn.o_proj": { |
| "base_key": "model.layers.48.self_attn.o_proj.weight", |
| "module": "model.layers.48.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004805081116501242, |
| "delta_mean_abs": 2.7203084755456075e-05 |
| }, |
| "model.layers.48.self_attn.q_proj": { |
| "base_key": "model.layers.48.self_attn.q_proj.weight", |
| "module": "model.layers.48.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004430412664078176, |
| "delta_mean_abs": 2.6943038392346352e-05 |
| }, |
| "model.layers.48.self_attn.v_proj": { |
| "base_key": "model.layers.48.self_attn.v_proj.weight", |
| "module": "model.layers.48.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004731325025204569, |
| "delta_mean_abs": 3.0082144803600386e-05 |
| }, |
| "model.layers.49.mlp.down_proj": { |
| "base_key": "model.layers.49.mlp.down_proj.weight", |
| "module": "model.layers.49.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0002612618845887482, |
| "delta_mean_abs": 1.658838846196886e-05 |
| }, |
| "model.layers.49.mlp.gate_proj": { |
| "base_key": "model.layers.49.mlp.gate_proj.weight", |
| "module": "model.layers.49.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006817448884248734, |
| "delta_mean_abs": 3.279067459516227e-05 |
| }, |
| "model.layers.49.mlp.up_proj": { |
| "base_key": "model.layers.49.mlp.up_proj.weight", |
| "module": "model.layers.49.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.000571961048990488, |
| "delta_mean_abs": 3.372990249772556e-05 |
| }, |
| "model.layers.49.self_attn.k_proj": { |
| "base_key": "model.layers.49.self_attn.k_proj.weight", |
| "module": "model.layers.49.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004050247371196747, |
| "delta_mean_abs": 3.160976484650746e-05 |
| }, |
| "model.layers.49.self_attn.o_proj": { |
| "base_key": "model.layers.49.self_attn.o_proj.weight", |
| "module": "model.layers.49.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004134149639867246, |
| "delta_mean_abs": 2.948636029032059e-05 |
| }, |
| "model.layers.49.self_attn.q_proj": { |
| "base_key": "model.layers.49.self_attn.q_proj.weight", |
| "module": "model.layers.49.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004713641828857362, |
| "delta_mean_abs": 2.7362541004549712e-05 |
| }, |
| "model.layers.49.self_attn.v_proj": { |
| "base_key": "model.layers.49.self_attn.v_proj.weight", |
| "module": "model.layers.49.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00043007114436477423, |
| "delta_mean_abs": 2.9732573239016347e-05 |
| } |
| } |
| }, |
| { |
| "status": "PASS", |
| "shard": "model-00012-of-000017.safetensors", |
| "sha256": "a8f24f6f58ac4bf549a5c0f2e2cf3a5140d0b2aad7c63b0932c708a430635f59", |
| "size": 8690767120, |
| "applied": { |
| "model.layers.50.mlp.down_proj": { |
| "base_key": "model.layers.50.mlp.down_proj.weight", |
| "module": "model.layers.50.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00023194488312583417, |
| "delta_mean_abs": 1.7242582543985918e-05 |
| }, |
| "model.layers.50.mlp.gate_proj": { |
| "base_key": "model.layers.50.mlp.gate_proj.weight", |
| "module": "model.layers.50.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005742249777540565, |
| "delta_mean_abs": 3.279751763329841e-05 |
| }, |
| "model.layers.50.mlp.up_proj": { |
| "base_key": "model.layers.50.mlp.up_proj.weight", |
| "module": "model.layers.50.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0007408225792460144, |
| "delta_mean_abs": 3.439130159677006e-05 |
| }, |
| "model.layers.50.self_attn.k_proj": { |
| "base_key": "model.layers.50.self_attn.k_proj.weight", |
| "module": "model.layers.50.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00019026474910788238, |
| "delta_mean_abs": 2.4938890419434756e-05 |
| }, |
| "model.layers.50.self_attn.o_proj": { |
| "base_key": "model.layers.50.self_attn.o_proj.weight", |
| "module": "model.layers.50.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004952774615958333, |
| "delta_mean_abs": 2.636972931213677e-05 |
| }, |
| "model.layers.50.self_attn.q_proj": { |
| "base_key": "model.layers.50.self_attn.q_proj.weight", |
| "module": "model.layers.50.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00034488431992940605, |
| "delta_mean_abs": 2.5767822080524638e-05 |
| }, |
| "model.layers.50.self_attn.v_proj": { |
| "base_key": "model.layers.50.self_attn.v_proj.weight", |
| "module": "model.layers.50.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004133190377615392, |
| "delta_mean_abs": 2.7670121198752895e-05 |
| }, |
| "model.layers.51.mlp.down_proj": { |
| "base_key": "model.layers.51.mlp.down_proj.weight", |
| "module": "model.layers.51.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00023741941549815238, |
| "delta_mean_abs": 1.7018424841808155e-05 |
| }, |
| "model.layers.51.mlp.gate_proj": { |
| "base_key": "model.layers.51.mlp.gate_proj.weight", |
| "module": "model.layers.51.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0007643967983312905, |
| "delta_mean_abs": 3.5127741284668446e-05 |
| }, |
| "model.layers.51.mlp.up_proj": { |
| "base_key": "model.layers.51.mlp.up_proj.weight", |
| "module": "model.layers.51.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0007651571067981422, |
| "delta_mean_abs": 3.326529622427188e-05 |
| }, |
| "model.layers.51.self_attn.k_proj": { |
| "base_key": "model.layers.51.self_attn.k_proj.weight", |
| "module": "model.layers.51.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00025259843096137047, |
| "delta_mean_abs": 2.628485526656732e-05 |
| }, |
| "model.layers.51.self_attn.o_proj": { |
| "base_key": "model.layers.51.self_attn.o_proj.weight", |
| "module": "model.layers.51.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004929337301291525, |
| "delta_mean_abs": 2.7723737730411813e-05 |
| }, |
| "model.layers.51.self_attn.q_proj": { |
| "base_key": "model.layers.51.self_attn.q_proj.weight", |
| "module": "model.layers.51.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00037992766010574996, |
| "delta_mean_abs": 2.6464798793313093e-05 |
| }, |
| "model.layers.51.self_attn.v_proj": { |
| "base_key": "model.layers.51.self_attn.v_proj.weight", |
| "module": "model.layers.51.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004395891446620226, |
| "delta_mean_abs": 2.937245517387055e-05 |
| }, |
| "model.layers.52.mlp.down_proj": { |
| "base_key": "model.layers.52.mlp.down_proj.weight", |
| "module": "model.layers.52.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00028310270863585174, |
| "delta_mean_abs": 1.8282893506693654e-05 |
| }, |
| "model.layers.52.mlp.gate_proj": { |
| "base_key": "model.layers.52.mlp.gate_proj.weight", |
| "module": "model.layers.52.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0007774893310852349, |
| "delta_mean_abs": 3.615232344600372e-05 |
| }, |
| "model.layers.52.mlp.up_proj": { |
| "base_key": "model.layers.52.mlp.up_proj.weight", |
| "module": "model.layers.52.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0008839613874442875, |
| "delta_mean_abs": 3.548489621607587e-05 |
| }, |
| "model.layers.52.self_attn.k_proj": { |
| "base_key": "model.layers.52.self_attn.k_proj.weight", |
| "module": "model.layers.52.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004241006972733885, |
| "delta_mean_abs": 3.1713181670056656e-05 |
| }, |
| "model.layers.52.self_attn.o_proj": { |
| "base_key": "model.layers.52.self_attn.o_proj.weight", |
| "module": "model.layers.52.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005529000773094594, |
| "delta_mean_abs": 3.08616399706807e-05 |
| }, |
| "model.layers.52.self_attn.q_proj": { |
| "base_key": "model.layers.52.self_attn.q_proj.weight", |
| "module": "model.layers.52.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.000667033193167299, |
| "delta_mean_abs": 3.088713128818199e-05 |
| }, |
| "model.layers.52.self_attn.v_proj": { |
| "base_key": "model.layers.52.self_attn.v_proj.weight", |
| "module": "model.layers.52.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00041208151378668845, |
| "delta_mean_abs": 3.337444650242105e-05 |
| }, |
| "model.layers.53.mlp.down_proj": { |
| "base_key": "model.layers.53.mlp.down_proj.weight", |
| "module": "model.layers.53.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00032677350100129843, |
| "delta_mean_abs": 1.849012915045023e-05 |
| }, |
| "model.layers.53.mlp.gate_proj": { |
| "base_key": "model.layers.53.mlp.gate_proj.weight", |
| "module": "model.layers.53.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0007463723886758089, |
| "delta_mean_abs": 3.689089135150425e-05 |
| }, |
| "model.layers.53.mlp.up_proj": { |
| "base_key": "model.layers.53.mlp.up_proj.weight", |
| "module": "model.layers.53.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.000800183741375804, |
| "delta_mean_abs": 3.469672446954064e-05 |
| }, |
| "model.layers.53.self_attn.k_proj": { |
| "base_key": "model.layers.53.self_attn.k_proj.weight", |
| "module": "model.layers.53.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0003538328455761075, |
| "delta_mean_abs": 2.924890759459231e-05 |
| }, |
| "model.layers.53.self_attn.o_proj": { |
| "base_key": "model.layers.53.self_attn.o_proj.weight", |
| "module": "model.layers.53.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0003903247707057744, |
| "delta_mean_abs": 2.7988488000119105e-05 |
| }, |
| "model.layers.53.self_attn.q_proj": { |
| "base_key": "model.layers.53.self_attn.q_proj.weight", |
| "module": "model.layers.53.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004061553336214274, |
| "delta_mean_abs": 2.8124779419158585e-05 |
| }, |
| "model.layers.53.self_attn.v_proj": { |
| "base_key": "model.layers.53.self_attn.v_proj.weight", |
| "module": "model.layers.53.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00041134460479952395, |
| "delta_mean_abs": 3.0076376788201742e-05 |
| }, |
| "model.layers.54.mlp.down_proj": { |
| "base_key": "model.layers.54.mlp.down_proj.weight", |
| "module": "model.layers.54.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00022944800730329007, |
| "delta_mean_abs": 1.732584132696502e-05 |
| }, |
| "model.layers.54.mlp.gate_proj": { |
| "base_key": "model.layers.54.mlp.gate_proj.weight", |
| "module": "model.layers.54.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004836066800635308, |
| "delta_mean_abs": 3.0529707146342844e-05 |
| }, |
| "model.layers.54.mlp.up_proj": { |
| "base_key": "model.layers.54.mlp.up_proj.weight", |
| "module": "model.layers.54.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006010723300278187, |
| "delta_mean_abs": 3.241946978960186e-05 |
| }, |
| "model.layers.54.self_attn.k_proj": { |
| "base_key": "model.layers.54.self_attn.k_proj.weight", |
| "module": "model.layers.54.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0003643657255452126, |
| "delta_mean_abs": 2.948681139969267e-05 |
| }, |
| "model.layers.54.self_attn.o_proj": { |
| "base_key": "model.layers.54.self_attn.o_proj.weight", |
| "module": "model.layers.54.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00042049153125844896, |
| "delta_mean_abs": 2.7296073312754743e-05 |
| }, |
| "model.layers.54.self_attn.q_proj": { |
| "base_key": "model.layers.54.self_attn.q_proj.weight", |
| "module": "model.layers.54.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005797162302769721, |
| "delta_mean_abs": 2.850515920727048e-05 |
| }, |
| "model.layers.54.self_attn.v_proj": { |
| "base_key": "model.layers.54.self_attn.v_proj.weight", |
| "module": "model.layers.54.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0003436410042922944, |
| "delta_mean_abs": 2.6404381060274318e-05 |
| }, |
| "model.layers.55.self_attn.q_proj": { |
| "base_key": "model.layers.55.self_attn.q_proj.weight", |
| "module": "model.layers.55.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0007317893323488533, |
| "delta_mean_abs": 3.1857900467002764e-05 |
| } |
| } |
| }, |
| { |
| "status": "PASS", |
| "shard": "model-00013-of-000017.safetensors", |
| "sha256": "919d0101b9e4a512743786d1683f45e85cfd79da344241dd2fc86b9abddc4704", |
| "size": 8422331432, |
| "applied": { |
| "model.layers.55.mlp.down_proj": { |
| "base_key": "model.layers.55.mlp.down_proj.weight", |
| "module": "model.layers.55.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00024017966643441468, |
| "delta_mean_abs": 1.71609572134912e-05 |
| }, |
| "model.layers.55.mlp.gate_proj": { |
| "base_key": "model.layers.55.mlp.gate_proj.weight", |
| "module": "model.layers.55.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.000630101072601974, |
| "delta_mean_abs": 3.266374551458284e-05 |
| }, |
| "model.layers.55.mlp.up_proj": { |
| "base_key": "model.layers.55.mlp.up_proj.weight", |
| "module": "model.layers.55.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005690811085514724, |
| "delta_mean_abs": 3.304798519820906e-05 |
| }, |
| "model.layers.55.self_attn.k_proj": { |
| "base_key": "model.layers.55.self_attn.k_proj.weight", |
| "module": "model.layers.55.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00047077270573936403, |
| "delta_mean_abs": 3.327948797959834e-05 |
| }, |
| "model.layers.55.self_attn.o_proj": { |
| "base_key": "model.layers.55.self_attn.o_proj.weight", |
| "module": "model.layers.55.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006803169380873442, |
| "delta_mean_abs": 3.148540417896584e-05 |
| }, |
| "model.layers.55.self_attn.v_proj": { |
| "base_key": "model.layers.55.self_attn.v_proj.weight", |
| "module": "model.layers.55.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0007956264307722449, |
| "delta_mean_abs": 3.9914502849569544e-05 |
| }, |
| "model.layers.56.mlp.down_proj": { |
| "base_key": "model.layers.56.mlp.down_proj.weight", |
| "module": "model.layers.56.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0003252432798035443, |
| "delta_mean_abs": 1.75830955413403e-05 |
| }, |
| "model.layers.56.mlp.gate_proj": { |
| "base_key": "model.layers.56.mlp.gate_proj.weight", |
| "module": "model.layers.56.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0011477298103272915, |
| "delta_mean_abs": 3.319076495245099e-05 |
| }, |
| "model.layers.56.mlp.up_proj": { |
| "base_key": "model.layers.56.mlp.up_proj.weight", |
| "module": "model.layers.56.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006982828490436077, |
| "delta_mean_abs": 3.44103536917828e-05 |
| }, |
| "model.layers.56.self_attn.k_proj": { |
| "base_key": "model.layers.56.self_attn.k_proj.weight", |
| "module": "model.layers.56.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0002250112738693133, |
| "delta_mean_abs": 2.615237099234946e-05 |
| }, |
| "model.layers.56.self_attn.o_proj": { |
| "base_key": "model.layers.56.self_attn.o_proj.weight", |
| "module": "model.layers.56.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005071043269708753, |
| "delta_mean_abs": 2.814992876665201e-05 |
| }, |
| "model.layers.56.self_attn.q_proj": { |
| "base_key": "model.layers.56.self_attn.q_proj.weight", |
| "module": "model.layers.56.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005149850621819496, |
| "delta_mean_abs": 2.648120789672248e-05 |
| }, |
| "model.layers.56.self_attn.v_proj": { |
| "base_key": "model.layers.56.self_attn.v_proj.weight", |
| "module": "model.layers.56.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004076813056599349, |
| "delta_mean_abs": 3.120432666037232e-05 |
| }, |
| "model.layers.57.mlp.down_proj": { |
| "base_key": "model.layers.57.mlp.down_proj.weight", |
| "module": "model.layers.57.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00032328657107427716, |
| "delta_mean_abs": 1.756659366947133e-05 |
| }, |
| "model.layers.57.mlp.gate_proj": { |
| "base_key": "model.layers.57.mlp.gate_proj.weight", |
| "module": "model.layers.57.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0007672883803024888, |
| "delta_mean_abs": 3.496990757412277e-05 |
| }, |
| "model.layers.57.mlp.up_proj": { |
| "base_key": "model.layers.57.mlp.up_proj.weight", |
| "module": "model.layers.57.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005826873239129782, |
| "delta_mean_abs": 3.319397728773765e-05 |
| }, |
| "model.layers.57.self_attn.k_proj": { |
| "base_key": "model.layers.57.self_attn.k_proj.weight", |
| "module": "model.layers.57.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00041932318708859384, |
| "delta_mean_abs": 3.0390878237085417e-05 |
| }, |
| "model.layers.57.self_attn.o_proj": { |
| "base_key": "model.layers.57.self_attn.o_proj.weight", |
| "module": "model.layers.57.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004013587604276836, |
| "delta_mean_abs": 2.728542676777579e-05 |
| }, |
| "model.layers.57.self_attn.q_proj": { |
| "base_key": "model.layers.57.self_attn.q_proj.weight", |
| "module": "model.layers.57.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00039427896263077855, |
| "delta_mean_abs": 2.7683438020176254e-05 |
| }, |
| "model.layers.57.self_attn.v_proj": { |
| "base_key": "model.layers.57.self_attn.v_proj.weight", |
| "module": "model.layers.57.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004960961523465812, |
| "delta_mean_abs": 2.8850430680904537e-05 |
| }, |
| "model.layers.58.mlp.down_proj": { |
| "base_key": "model.layers.58.mlp.down_proj.weight", |
| "module": "model.layers.58.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0002375221229158342, |
| "delta_mean_abs": 1.8474675016477704e-05 |
| }, |
| "model.layers.58.mlp.gate_proj": { |
| "base_key": "model.layers.58.mlp.gate_proj.weight", |
| "module": "model.layers.58.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005813697935082018, |
| "delta_mean_abs": 3.141004344797693e-05 |
| }, |
| "model.layers.58.mlp.up_proj": { |
| "base_key": "model.layers.58.mlp.up_proj.weight", |
| "module": "model.layers.58.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005685397773049772, |
| "delta_mean_abs": 3.3447082387283444e-05 |
| }, |
| "model.layers.58.self_attn.k_proj": { |
| "base_key": "model.layers.58.self_attn.k_proj.weight", |
| "module": "model.layers.58.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00026678666472435, |
| "delta_mean_abs": 2.9029772122157738e-05 |
| }, |
| "model.layers.58.self_attn.o_proj": { |
| "base_key": "model.layers.58.self_attn.o_proj.weight", |
| "module": "model.layers.58.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004007090756203979, |
| "delta_mean_abs": 2.7240399504080415e-05 |
| }, |
| "model.layers.58.self_attn.q_proj": { |
| "base_key": "model.layers.58.self_attn.q_proj.weight", |
| "module": "model.layers.58.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00037795526441186666, |
| "delta_mean_abs": 2.9251541491248645e-05 |
| }, |
| "model.layers.58.self_attn.v_proj": { |
| "base_key": "model.layers.58.self_attn.v_proj.weight", |
| "module": "model.layers.58.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.000517230131663382, |
| "delta_mean_abs": 2.8717611712636426e-05 |
| }, |
| "model.layers.59.mlp.down_proj": { |
| "base_key": "model.layers.59.mlp.down_proj.weight", |
| "module": "model.layers.59.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0003167118411511183, |
| "delta_mean_abs": 1.801358666853048e-05 |
| }, |
| "model.layers.59.mlp.gate_proj": { |
| "base_key": "model.layers.59.mlp.gate_proj.weight", |
| "module": "model.layers.59.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006274434272199869, |
| "delta_mean_abs": 3.511364775476977e-05 |
| }, |
| "model.layers.59.mlp.up_proj": { |
| "base_key": "model.layers.59.mlp.up_proj.weight", |
| "module": "model.layers.59.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006253743777051568, |
| "delta_mean_abs": 3.493780241115019e-05 |
| }, |
| "model.layers.59.self_attn.k_proj": { |
| "base_key": "model.layers.59.self_attn.k_proj.weight", |
| "module": "model.layers.59.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004909363924525678, |
| "delta_mean_abs": 3.30900184053462e-05 |
| }, |
| "model.layers.59.self_attn.o_proj": { |
| "base_key": "model.layers.59.self_attn.o_proj.weight", |
| "module": "model.layers.59.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.000426191691076383, |
| "delta_mean_abs": 2.9037670174147934e-05 |
| }, |
| "model.layers.59.self_attn.q_proj": { |
| "base_key": "model.layers.59.self_attn.q_proj.weight", |
| "module": "model.layers.59.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006625897367484868, |
| "delta_mean_abs": 2.9437003831844777e-05 |
| }, |
| "model.layers.59.self_attn.v_proj": { |
| "base_key": "model.layers.59.self_attn.v_proj.weight", |
| "module": "model.layers.59.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004401344631332904, |
| "delta_mean_abs": 3.363710129633546e-05 |
| } |
| } |
| }, |
| { |
| "status": "PASS", |
| "shard": "model-00014-of-000017.safetensors", |
| "sha256": "8e9b7b897b6df3a1bbc225a4e481ba6273fe69e906bdfbe132e6727bd8eb9211", |
| "size": 8690767120, |
| "applied": { |
| "model.layers.60.mlp.down_proj": { |
| "base_key": "model.layers.60.mlp.down_proj.weight", |
| "module": "model.layers.60.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0003403117589186877, |
| "delta_mean_abs": 1.8032351363217458e-05 |
| }, |
| "model.layers.60.mlp.gate_proj": { |
| "base_key": "model.layers.60.mlp.gate_proj.weight", |
| "module": "model.layers.60.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005885711289010942, |
| "delta_mean_abs": 3.376022868906148e-05 |
| }, |
| "model.layers.60.mlp.up_proj": { |
| "base_key": "model.layers.60.mlp.up_proj.weight", |
| "module": "model.layers.60.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0008577533299103379, |
| "delta_mean_abs": 3.6479705158853903e-05 |
| }, |
| "model.layers.60.self_attn.k_proj": { |
| "base_key": "model.layers.60.self_attn.k_proj.weight", |
| "module": "model.layers.60.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00040963260107673705, |
| "delta_mean_abs": 3.062326504732482e-05 |
| }, |
| "model.layers.60.self_attn.o_proj": { |
| "base_key": "model.layers.60.self_attn.o_proj.weight", |
| "module": "model.layers.60.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005057673552073538, |
| "delta_mean_abs": 2.9038703360129148e-05 |
| }, |
| "model.layers.60.self_attn.q_proj": { |
| "base_key": "model.layers.60.self_attn.q_proj.weight", |
| "module": "model.layers.60.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005076383822597563, |
| "delta_mean_abs": 2.90478092210833e-05 |
| }, |
| "model.layers.60.self_attn.v_proj": { |
| "base_key": "model.layers.60.self_attn.v_proj.weight", |
| "module": "model.layers.60.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00045544508611783385, |
| "delta_mean_abs": 2.7974878321401775e-05 |
| }, |
| "model.layers.61.mlp.down_proj": { |
| "base_key": "model.layers.61.mlp.down_proj.weight", |
| "module": "model.layers.61.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0003474331460893154, |
| "delta_mean_abs": 1.7789465346140787e-05 |
| }, |
| "model.layers.61.mlp.gate_proj": { |
| "base_key": "model.layers.61.mlp.gate_proj.weight", |
| "module": "model.layers.61.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006403843872249126, |
| "delta_mean_abs": 3.265478881075978e-05 |
| }, |
| "model.layers.61.mlp.up_proj": { |
| "base_key": "model.layers.61.mlp.up_proj.weight", |
| "module": "model.layers.61.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0008014805498532951, |
| "delta_mean_abs": 3.3041374990716577e-05 |
| }, |
| "model.layers.61.self_attn.k_proj": { |
| "base_key": "model.layers.61.self_attn.k_proj.weight", |
| "module": "model.layers.61.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00020574846712406725, |
| "delta_mean_abs": 2.4439828848699108e-05 |
| }, |
| "model.layers.61.self_attn.o_proj": { |
| "base_key": "model.layers.61.self_attn.o_proj.weight", |
| "module": "model.layers.61.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004112191090825945, |
| "delta_mean_abs": 2.655314892763272e-05 |
| }, |
| "model.layers.61.self_attn.q_proj": { |
| "base_key": "model.layers.61.self_attn.q_proj.weight", |
| "module": "model.layers.61.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0003253582399338484, |
| "delta_mean_abs": 2.659644633240532e-05 |
| }, |
| "model.layers.61.self_attn.v_proj": { |
| "base_key": "model.layers.61.self_attn.v_proj.weight", |
| "module": "model.layers.61.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00042377019417472184, |
| "delta_mean_abs": 2.6528774469625205e-05 |
| }, |
| "model.layers.62.mlp.down_proj": { |
| "base_key": "model.layers.62.mlp.down_proj.weight", |
| "module": "model.layers.62.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00027904025046154857, |
| "delta_mean_abs": 1.8366310541750863e-05 |
| }, |
| "model.layers.62.mlp.gate_proj": { |
| "base_key": "model.layers.62.mlp.gate_proj.weight", |
| "module": "model.layers.62.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00043480738531798124, |
| "delta_mean_abs": 3.1796804250916466e-05 |
| }, |
| "model.layers.62.mlp.up_proj": { |
| "base_key": "model.layers.62.mlp.up_proj.weight", |
| "module": "model.layers.62.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005380992661230266, |
| "delta_mean_abs": 3.250975350965746e-05 |
| }, |
| "model.layers.62.self_attn.k_proj": { |
| "base_key": "model.layers.62.self_attn.k_proj.weight", |
| "module": "model.layers.62.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00039939151611179113, |
| "delta_mean_abs": 3.062015093746595e-05 |
| }, |
| "model.layers.62.self_attn.o_proj": { |
| "base_key": "model.layers.62.self_attn.o_proj.weight", |
| "module": "model.layers.62.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005230074748396873, |
| "delta_mean_abs": 2.8701606424874626e-05 |
| }, |
| "model.layers.62.self_attn.q_proj": { |
| "base_key": "model.layers.62.self_attn.q_proj.weight", |
| "module": "model.layers.62.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005595781840384007, |
| "delta_mean_abs": 2.8908872991451062e-05 |
| }, |
| "model.layers.62.self_attn.v_proj": { |
| "base_key": "model.layers.62.self_attn.v_proj.weight", |
| "module": "model.layers.62.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00044293981045484543, |
| "delta_mean_abs": 2.972842412418686e-05 |
| }, |
| "model.layers.63.mlp.down_proj": { |
| "base_key": "model.layers.63.mlp.down_proj.weight", |
| "module": "model.layers.63.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00030975008849054575, |
| "delta_mean_abs": 1.7318867321591824e-05 |
| }, |
| "model.layers.63.mlp.gate_proj": { |
| "base_key": "model.layers.63.mlp.gate_proj.weight", |
| "module": "model.layers.63.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006528462981805205, |
| "delta_mean_abs": 3.2834603189257905e-05 |
| }, |
| "model.layers.63.mlp.up_proj": { |
| "base_key": "model.layers.63.mlp.up_proj.weight", |
| "module": "model.layers.63.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006288392469286919, |
| "delta_mean_abs": 3.329902028781362e-05 |
| }, |
| "model.layers.63.self_attn.k_proj": { |
| "base_key": "model.layers.63.self_attn.k_proj.weight", |
| "module": "model.layers.63.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004438305040821433, |
| "delta_mean_abs": 3.373196523170918e-05 |
| }, |
| "model.layers.63.self_attn.o_proj": { |
| "base_key": "model.layers.63.self_attn.o_proj.weight", |
| "module": "model.layers.63.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006851164507679641, |
| "delta_mean_abs": 2.9773869755445048e-05 |
| }, |
| "model.layers.63.self_attn.q_proj": { |
| "base_key": "model.layers.63.self_attn.q_proj.weight", |
| "module": "model.layers.63.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004850758705288172, |
| "delta_mean_abs": 2.9002858354942873e-05 |
| }, |
| "model.layers.63.self_attn.v_proj": { |
| "base_key": "model.layers.63.self_attn.v_proj.weight", |
| "module": "model.layers.63.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.000509008183144033, |
| "delta_mean_abs": 3.334598659421317e-05 |
| }, |
| "model.layers.64.mlp.down_proj": { |
| "base_key": "model.layers.64.mlp.down_proj.weight", |
| "module": "model.layers.64.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.000435884139733389, |
| "delta_mean_abs": 1.7994438167079352e-05 |
| }, |
| "model.layers.64.mlp.gate_proj": { |
| "base_key": "model.layers.64.mlp.gate_proj.weight", |
| "module": "model.layers.64.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005222313920967281, |
| "delta_mean_abs": 3.355167791596614e-05 |
| }, |
| "model.layers.64.mlp.up_proj": { |
| "base_key": "model.layers.64.mlp.up_proj.weight", |
| "module": "model.layers.64.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0007353847031481564, |
| "delta_mean_abs": 3.324563294881955e-05 |
| }, |
| "model.layers.64.self_attn.k_proj": { |
| "base_key": "model.layers.64.self_attn.k_proj.weight", |
| "module": "model.layers.64.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00024940219009295106, |
| "delta_mean_abs": 2.564599344623275e-05 |
| }, |
| "model.layers.64.self_attn.o_proj": { |
| "base_key": "model.layers.64.self_attn.o_proj.weight", |
| "module": "model.layers.64.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004086136177647859, |
| "delta_mean_abs": 2.841260538843926e-05 |
| }, |
| "model.layers.64.self_attn.q_proj": { |
| "base_key": "model.layers.64.self_attn.q_proj.weight", |
| "module": "model.layers.64.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004364412452559918, |
| "delta_mean_abs": 2.7691814466379583e-05 |
| }, |
| "model.layers.64.self_attn.v_proj": { |
| "base_key": "model.layers.64.self_attn.v_proj.weight", |
| "module": "model.layers.64.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00040847124182619154, |
| "delta_mean_abs": 2.973815571749583e-05 |
| }, |
| "model.layers.65.self_attn.q_proj": { |
| "base_key": "model.layers.65.self_attn.q_proj.weight", |
| "module": "model.layers.65.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0016359227010980248, |
| "delta_mean_abs": 4.618140519596636e-05 |
| } |
| } |
| }, |
| { |
| "status": "PASS", |
| "shard": "model-00015-of-000017.safetensors", |
| "sha256": "9de03d46bd6f31145385ef56769f97ce2f2ec0abac7afb5e68bebc1270616fcd", |
| "size": 8422331432, |
| "applied": { |
| "model.layers.65.mlp.down_proj": { |
| "base_key": "model.layers.65.mlp.down_proj.weight", |
| "module": "model.layers.65.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.000353805284248665, |
| "delta_mean_abs": 1.8734939658315852e-05 |
| }, |
| "model.layers.65.mlp.gate_proj": { |
| "base_key": "model.layers.65.mlp.gate_proj.weight", |
| "module": "model.layers.65.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006487490609288216, |
| "delta_mean_abs": 3.391228165128268e-05 |
| }, |
| "model.layers.65.mlp.up_proj": { |
| "base_key": "model.layers.65.mlp.up_proj.weight", |
| "module": "model.layers.65.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.000639317266177386, |
| "delta_mean_abs": 3.5004592064069584e-05 |
| }, |
| "model.layers.65.self_attn.k_proj": { |
| "base_key": "model.layers.65.self_attn.k_proj.weight", |
| "module": "model.layers.65.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0009715057094581425, |
| "delta_mean_abs": 3.872136221616529e-05 |
| }, |
| "model.layers.65.self_attn.o_proj": { |
| "base_key": "model.layers.65.self_attn.o_proj.weight", |
| "module": "model.layers.65.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004719960270449519, |
| "delta_mean_abs": 2.9004280804656446e-05 |
| }, |
| "model.layers.65.self_attn.v_proj": { |
| "base_key": "model.layers.65.self_attn.v_proj.weight", |
| "module": "model.layers.65.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005848380969837308, |
| "delta_mean_abs": 3.295863643870689e-05 |
| }, |
| "model.layers.66.mlp.down_proj": { |
| "base_key": "model.layers.66.mlp.down_proj.weight", |
| "module": "model.layers.66.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0003305491991341114, |
| "delta_mean_abs": 1.7633958123042248e-05 |
| }, |
| "model.layers.66.mlp.gate_proj": { |
| "base_key": "model.layers.66.mlp.gate_proj.weight", |
| "module": "model.layers.66.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0008325888775289059, |
| "delta_mean_abs": 3.1855521228862926e-05 |
| }, |
| "model.layers.66.mlp.up_proj": { |
| "base_key": "model.layers.66.mlp.up_proj.weight", |
| "module": "model.layers.66.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.000667522253934294, |
| "delta_mean_abs": 3.3472100767539814e-05 |
| }, |
| "model.layers.66.self_attn.k_proj": { |
| "base_key": "model.layers.66.self_attn.k_proj.weight", |
| "module": "model.layers.66.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0003795888915192336, |
| "delta_mean_abs": 3.177179314661771e-05 |
| }, |
| "model.layers.66.self_attn.o_proj": { |
| "base_key": "model.layers.66.self_attn.o_proj.weight", |
| "module": "model.layers.66.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006076532299630344, |
| "delta_mean_abs": 3.119217581115663e-05 |
| }, |
| "model.layers.66.self_attn.q_proj": { |
| "base_key": "model.layers.66.self_attn.q_proj.weight", |
| "module": "model.layers.66.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006125041400082409, |
| "delta_mean_abs": 3.14492717734538e-05 |
| }, |
| "model.layers.66.self_attn.v_proj": { |
| "base_key": "model.layers.66.self_attn.v_proj.weight", |
| "module": "model.layers.66.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004753570829052478, |
| "delta_mean_abs": 3.286154969828203e-05 |
| }, |
| "model.layers.67.mlp.down_proj": { |
| "base_key": "model.layers.67.mlp.down_proj.weight", |
| "module": "model.layers.67.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0003385959716979414, |
| "delta_mean_abs": 1.7417542039765976e-05 |
| }, |
| "model.layers.67.mlp.gate_proj": { |
| "base_key": "model.layers.67.mlp.gate_proj.weight", |
| "module": "model.layers.67.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006302095716819167, |
| "delta_mean_abs": 3.453836325206794e-05 |
| }, |
| "model.layers.67.mlp.up_proj": { |
| "base_key": "model.layers.67.mlp.up_proj.weight", |
| "module": "model.layers.67.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0009874554816633463, |
| "delta_mean_abs": 3.361910785315558e-05 |
| }, |
| "model.layers.67.self_attn.k_proj": { |
| "base_key": "model.layers.67.self_attn.k_proj.weight", |
| "module": "model.layers.67.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005051043699495494, |
| "delta_mean_abs": 3.528805973473936e-05 |
| }, |
| "model.layers.67.self_attn.o_proj": { |
| "base_key": "model.layers.67.self_attn.o_proj.weight", |
| "module": "model.layers.67.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006839020643383265, |
| "delta_mean_abs": 3.4324639273108914e-05 |
| }, |
| "model.layers.67.self_attn.q_proj": { |
| "base_key": "model.layers.67.self_attn.q_proj.weight", |
| "module": "model.layers.67.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004992749891243875, |
| "delta_mean_abs": 2.9469358196365647e-05 |
| }, |
| "model.layers.67.self_attn.v_proj": { |
| "base_key": "model.layers.67.self_attn.v_proj.weight", |
| "module": "model.layers.67.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00041802023770287633, |
| "delta_mean_abs": 3.2986248697852716e-05 |
| }, |
| "model.layers.68.mlp.down_proj": { |
| "base_key": "model.layers.68.mlp.down_proj.weight", |
| "module": "model.layers.68.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00043951772386208177, |
| "delta_mean_abs": 1.7884161934489384e-05 |
| }, |
| "model.layers.68.mlp.gate_proj": { |
| "base_key": "model.layers.68.mlp.gate_proj.weight", |
| "module": "model.layers.68.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006655589095316827, |
| "delta_mean_abs": 3.6978912248741835e-05 |
| }, |
| "model.layers.68.mlp.up_proj": { |
| "base_key": "model.layers.68.mlp.up_proj.weight", |
| "module": "model.layers.68.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006126568769104779, |
| "delta_mean_abs": 3.346798621350899e-05 |
| }, |
| "model.layers.68.self_attn.k_proj": { |
| "base_key": "model.layers.68.self_attn.k_proj.weight", |
| "module": "model.layers.68.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005210642702877522, |
| "delta_mean_abs": 3.042715798073914e-05 |
| }, |
| "model.layers.68.self_attn.o_proj": { |
| "base_key": "model.layers.68.self_attn.o_proj.weight", |
| "module": "model.layers.68.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006868410855531693, |
| "delta_mean_abs": 3.023002136615105e-05 |
| }, |
| "model.layers.68.self_attn.q_proj": { |
| "base_key": "model.layers.68.self_attn.q_proj.weight", |
| "module": "model.layers.68.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006250206497497857, |
| "delta_mean_abs": 3.134355210931972e-05 |
| }, |
| "model.layers.68.self_attn.v_proj": { |
| "base_key": "model.layers.68.self_attn.v_proj.weight", |
| "module": "model.layers.68.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005906482692807913, |
| "delta_mean_abs": 3.0426575904130004e-05 |
| }, |
| "model.layers.69.mlp.down_proj": { |
| "base_key": "model.layers.69.mlp.down_proj.weight", |
| "module": "model.layers.69.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004005744121968746, |
| "delta_mean_abs": 1.761115163390059e-05 |
| }, |
| "model.layers.69.mlp.gate_proj": { |
| "base_key": "model.layers.69.mlp.gate_proj.weight", |
| "module": "model.layers.69.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005679959431290627, |
| "delta_mean_abs": 3.1574734748573974e-05 |
| }, |
| "model.layers.69.mlp.up_proj": { |
| "base_key": "model.layers.69.mlp.up_proj.weight", |
| "module": "model.layers.69.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0007787889917381108, |
| "delta_mean_abs": 3.496246426948346e-05 |
| }, |
| "model.layers.69.self_attn.k_proj": { |
| "base_key": "model.layers.69.self_attn.k_proj.weight", |
| "module": "model.layers.69.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00036143948091194034, |
| "delta_mean_abs": 3.2853706215973943e-05 |
| }, |
| "model.layers.69.self_attn.o_proj": { |
| "base_key": "model.layers.69.self_attn.o_proj.weight", |
| "module": "model.layers.69.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005376170156523585, |
| "delta_mean_abs": 3.064176416955888e-05 |
| }, |
| "model.layers.69.self_attn.q_proj": { |
| "base_key": "model.layers.69.self_attn.q_proj.weight", |
| "module": "model.layers.69.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006360808620229363, |
| "delta_mean_abs": 3.128091339021921e-05 |
| }, |
| "model.layers.69.self_attn.v_proj": { |
| "base_key": "model.layers.69.self_attn.v_proj.weight", |
| "module": "model.layers.69.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.000496246328111738, |
| "delta_mean_abs": 3.219795326003805e-05 |
| } |
| } |
| }, |
| { |
| "status": "PASS", |
| "shard": "model-00016-of-000017.safetensors", |
| "sha256": "24ce99fcede83a06afb2ad3e8d7d059711b728d98a1de16d9411b5bc3168bf6e", |
| "size": 8690767120, |
| "applied": { |
| "model.layers.70.mlp.down_proj": { |
| "base_key": "model.layers.70.mlp.down_proj.weight", |
| "module": "model.layers.70.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00036174969864077866, |
| "delta_mean_abs": 1.7974818547372706e-05 |
| }, |
| "model.layers.70.mlp.gate_proj": { |
| "base_key": "model.layers.70.mlp.gate_proj.weight", |
| "module": "model.layers.70.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0009025035542435944, |
| "delta_mean_abs": 3.468829527264461e-05 |
| }, |
| "model.layers.70.mlp.up_proj": { |
| "base_key": "model.layers.70.mlp.up_proj.weight", |
| "module": "model.layers.70.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0008815202163532376, |
| "delta_mean_abs": 3.4279782994417474e-05 |
| }, |
| "model.layers.70.self_attn.k_proj": { |
| "base_key": "model.layers.70.self_attn.k_proj.weight", |
| "module": "model.layers.70.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00046798615949228406, |
| "delta_mean_abs": 2.801379559969064e-05 |
| }, |
| "model.layers.70.self_attn.o_proj": { |
| "base_key": "model.layers.70.self_attn.o_proj.weight", |
| "module": "model.layers.70.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006216843030415475, |
| "delta_mean_abs": 3.282462785136886e-05 |
| }, |
| "model.layers.70.self_attn.q_proj": { |
| "base_key": "model.layers.70.self_attn.q_proj.weight", |
| "module": "model.layers.70.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006040799780748785, |
| "delta_mean_abs": 2.991302426380571e-05 |
| }, |
| "model.layers.70.self_attn.v_proj": { |
| "base_key": "model.layers.70.self_attn.v_proj.weight", |
| "module": "model.layers.70.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00045949139166623354, |
| "delta_mean_abs": 3.4063912607962266e-05 |
| }, |
| "model.layers.71.mlp.down_proj": { |
| "base_key": "model.layers.71.mlp.down_proj.weight", |
| "module": "model.layers.71.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0003577861643861979, |
| "delta_mean_abs": 1.8323609765502624e-05 |
| }, |
| "model.layers.71.mlp.gate_proj": { |
| "base_key": "model.layers.71.mlp.gate_proj.weight", |
| "module": "model.layers.71.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006235248874872923, |
| "delta_mean_abs": 3.685526826302521e-05 |
| }, |
| "model.layers.71.mlp.up_proj": { |
| "base_key": "model.layers.71.mlp.up_proj.weight", |
| "module": "model.layers.71.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0007694624364376068, |
| "delta_mean_abs": 3.668199497042224e-05 |
| }, |
| "model.layers.71.self_attn.k_proj": { |
| "base_key": "model.layers.71.self_attn.k_proj.weight", |
| "module": "model.layers.71.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00031912949634715915, |
| "delta_mean_abs": 2.805414260365069e-05 |
| }, |
| "model.layers.71.self_attn.o_proj": { |
| "base_key": "model.layers.71.self_attn.o_proj.weight", |
| "module": "model.layers.71.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005239666788838804, |
| "delta_mean_abs": 3.219921563868411e-05 |
| }, |
| "model.layers.71.self_attn.q_proj": { |
| "base_key": "model.layers.71.self_attn.q_proj.weight", |
| "module": "model.layers.71.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005433373735286295, |
| "delta_mean_abs": 2.9417969926726073e-05 |
| }, |
| "model.layers.71.self_attn.v_proj": { |
| "base_key": "model.layers.71.self_attn.v_proj.weight", |
| "module": "model.layers.71.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004660948761738837, |
| "delta_mean_abs": 3.300354364910163e-05 |
| }, |
| "model.layers.72.mlp.down_proj": { |
| "base_key": "model.layers.72.mlp.down_proj.weight", |
| "module": "model.layers.72.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004198637616354972, |
| "delta_mean_abs": 1.976541170733981e-05 |
| }, |
| "model.layers.72.mlp.gate_proj": { |
| "base_key": "model.layers.72.mlp.gate_proj.weight", |
| "module": "model.layers.72.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0013648399617522955, |
| "delta_mean_abs": 4.1749510273803025e-05 |
| }, |
| "model.layers.72.mlp.up_proj": { |
| "base_key": "model.layers.72.mlp.up_proj.weight", |
| "module": "model.layers.72.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0009160926565527916, |
| "delta_mean_abs": 3.999894761363976e-05 |
| }, |
| "model.layers.72.self_attn.k_proj": { |
| "base_key": "model.layers.72.self_attn.k_proj.weight", |
| "module": "model.layers.72.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004971388261765242, |
| "delta_mean_abs": 3.343899152241647e-05 |
| }, |
| "model.layers.72.self_attn.o_proj": { |
| "base_key": "model.layers.72.self_attn.o_proj.weight", |
| "module": "model.layers.72.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005863019614480436, |
| "delta_mean_abs": 3.249990186304785e-05 |
| }, |
| "model.layers.72.self_attn.q_proj": { |
| "base_key": "model.layers.72.self_attn.q_proj.weight", |
| "module": "model.layers.72.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005925617297179997, |
| "delta_mean_abs": 3.2505042327102274e-05 |
| }, |
| "model.layers.72.self_attn.v_proj": { |
| "base_key": "model.layers.72.self_attn.v_proj.weight", |
| "module": "model.layers.72.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005048689199611545, |
| "delta_mean_abs": 3.781314808293246e-05 |
| }, |
| "model.layers.73.mlp.down_proj": { |
| "base_key": "model.layers.73.mlp.down_proj.weight", |
| "module": "model.layers.73.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004268938791938126, |
| "delta_mean_abs": 2.0288223822717555e-05 |
| }, |
| "model.layers.73.mlp.gate_proj": { |
| "base_key": "model.layers.73.mlp.gate_proj.weight", |
| "module": "model.layers.73.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0010354557307437062, |
| "delta_mean_abs": 4.065262692165561e-05 |
| }, |
| "model.layers.73.mlp.up_proj": { |
| "base_key": "model.layers.73.mlp.up_proj.weight", |
| "module": "model.layers.73.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.000910265720449388, |
| "delta_mean_abs": 3.885718251694925e-05 |
| }, |
| "model.layers.73.self_attn.k_proj": { |
| "base_key": "model.layers.73.self_attn.k_proj.weight", |
| "module": "model.layers.73.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006154202274046838, |
| "delta_mean_abs": 3.179792838636786e-05 |
| }, |
| "model.layers.73.self_attn.o_proj": { |
| "base_key": "model.layers.73.self_attn.o_proj.weight", |
| "module": "model.layers.73.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005339720519259572, |
| "delta_mean_abs": 3.178670885972679e-05 |
| }, |
| "model.layers.73.self_attn.q_proj": { |
| "base_key": "model.layers.73.self_attn.q_proj.weight", |
| "module": "model.layers.73.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005624737241305411, |
| "delta_mean_abs": 3.212135925423354e-05 |
| }, |
| "model.layers.73.self_attn.v_proj": { |
| "base_key": "model.layers.73.self_attn.v_proj.weight", |
| "module": "model.layers.73.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006142078200355172, |
| "delta_mean_abs": 3.961876791436225e-05 |
| }, |
| "model.layers.74.mlp.down_proj": { |
| "base_key": "model.layers.74.mlp.down_proj.weight", |
| "module": "model.layers.74.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00045918376417830586, |
| "delta_mean_abs": 2.1708243366447277e-05 |
| }, |
| "model.layers.74.mlp.gate_proj": { |
| "base_key": "model.layers.74.mlp.gate_proj.weight", |
| "module": "model.layers.74.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0012383580906316638, |
| "delta_mean_abs": 4.203012213110924e-05 |
| }, |
| "model.layers.74.mlp.up_proj": { |
| "base_key": "model.layers.74.mlp.up_proj.weight", |
| "module": "model.layers.74.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0011031344765797257, |
| "delta_mean_abs": 3.800862396019511e-05 |
| }, |
| "model.layers.74.self_attn.k_proj": { |
| "base_key": "model.layers.74.self_attn.k_proj.weight", |
| "module": "model.layers.74.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00042396780918352306, |
| "delta_mean_abs": 3.1558145565213636e-05 |
| }, |
| "model.layers.74.self_attn.o_proj": { |
| "base_key": "model.layers.74.self_attn.o_proj.weight", |
| "module": "model.layers.74.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006086716894060373, |
| "delta_mean_abs": 3.103741619270295e-05 |
| }, |
| "model.layers.74.self_attn.q_proj": { |
| "base_key": "model.layers.74.self_attn.q_proj.weight", |
| "module": "model.layers.74.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005084500880911946, |
| "delta_mean_abs": 3.1571169529343024e-05 |
| }, |
| "model.layers.74.self_attn.v_proj": { |
| "base_key": "model.layers.74.self_attn.v_proj.weight", |
| "module": "model.layers.74.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005451824399642646, |
| "delta_mean_abs": 3.6451820051297545e-05 |
| }, |
| "model.layers.75.self_attn.q_proj": { |
| "base_key": "model.layers.75.self_attn.q_proj.weight", |
| "module": "model.layers.75.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0003948055091314018, |
| "delta_mean_abs": 3.016556911461521e-05 |
| } |
| } |
| }, |
| { |
| "status": "PASS", |
| "shard": "model-00017-of-000017.safetensors", |
| "sha256": "17793ebe1995862397b15dda61c245fda3e50899a17cd51dffd97d62752e0ba3", |
| "size": 10523694328, |
| "applied": { |
| "model.layers.75.mlp.down_proj": { |
| "base_key": "model.layers.75.mlp.down_proj.weight", |
| "module": "model.layers.75.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005875154747627676, |
| "delta_mean_abs": 2.100617166433949e-05 |
| }, |
| "model.layers.75.mlp.gate_proj": { |
| "base_key": "model.layers.75.mlp.gate_proj.weight", |
| "module": "model.layers.75.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0011767561081796885, |
| "delta_mean_abs": 4.46871345047839e-05 |
| }, |
| "model.layers.75.mlp.up_proj": { |
| "base_key": "model.layers.75.mlp.up_proj.weight", |
| "module": "model.layers.75.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.001166467322036624, |
| "delta_mean_abs": 4.197916859993711e-05 |
| }, |
| "model.layers.75.self_attn.k_proj": { |
| "base_key": "model.layers.75.self_attn.k_proj.weight", |
| "module": "model.layers.75.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005544250016100705, |
| "delta_mean_abs": 3.1387960916617885e-05 |
| }, |
| "model.layers.75.self_attn.o_proj": { |
| "base_key": "model.layers.75.self_attn.o_proj.weight", |
| "module": "model.layers.75.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005811215960420668, |
| "delta_mean_abs": 3.065441705984995e-05 |
| }, |
| "model.layers.75.self_attn.v_proj": { |
| "base_key": "model.layers.75.self_attn.v_proj.weight", |
| "module": "model.layers.75.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004726992337964475, |
| "delta_mean_abs": 3.5378296161070466e-05 |
| }, |
| "model.layers.76.mlp.down_proj": { |
| "base_key": "model.layers.76.mlp.down_proj.weight", |
| "module": "model.layers.76.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.000487450510263443, |
| "delta_mean_abs": 1.985048220376484e-05 |
| }, |
| "model.layers.76.mlp.gate_proj": { |
| "base_key": "model.layers.76.mlp.gate_proj.weight", |
| "module": "model.layers.76.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0014557093381881714, |
| "delta_mean_abs": 5.117774708196521e-05 |
| }, |
| "model.layers.76.mlp.up_proj": { |
| "base_key": "model.layers.76.mlp.up_proj.weight", |
| "module": "model.layers.76.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00132599042262882, |
| "delta_mean_abs": 4.367001020000316e-05 |
| }, |
| "model.layers.76.self_attn.k_proj": { |
| "base_key": "model.layers.76.self_attn.k_proj.weight", |
| "module": "model.layers.76.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006550145917572081, |
| "delta_mean_abs": 3.6823064874624833e-05 |
| }, |
| "model.layers.76.self_attn.o_proj": { |
| "base_key": "model.layers.76.self_attn.o_proj.weight", |
| "module": "model.layers.76.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.000780296977609396, |
| "delta_mean_abs": 3.433939127717167e-05 |
| }, |
| "model.layers.76.self_attn.q_proj": { |
| "base_key": "model.layers.76.self_attn.q_proj.weight", |
| "module": "model.layers.76.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006649205461144447, |
| "delta_mean_abs": 3.580806151148863e-05 |
| }, |
| "model.layers.76.self_attn.v_proj": { |
| "base_key": "model.layers.76.self_attn.v_proj.weight", |
| "module": "model.layers.76.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00048075662925839424, |
| "delta_mean_abs": 3.91623325413093e-05 |
| }, |
| "model.layers.77.mlp.down_proj": { |
| "base_key": "model.layers.77.mlp.down_proj.weight", |
| "module": "model.layers.77.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.000440429023001343, |
| "delta_mean_abs": 2.022631815634668e-05 |
| }, |
| "model.layers.77.mlp.gate_proj": { |
| "base_key": "model.layers.77.mlp.gate_proj.weight", |
| "module": "model.layers.77.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.001515861484222114, |
| "delta_mean_abs": 5.217967554926872e-05 |
| }, |
| "model.layers.77.mlp.up_proj": { |
| "base_key": "model.layers.77.mlp.up_proj.weight", |
| "module": "model.layers.77.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0014161646831780672, |
| "delta_mean_abs": 4.541931411949918e-05 |
| }, |
| "model.layers.77.self_attn.k_proj": { |
| "base_key": "model.layers.77.self_attn.k_proj.weight", |
| "module": "model.layers.77.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00040980754420161247, |
| "delta_mean_abs": 3.379135887371376e-05 |
| }, |
| "model.layers.77.self_attn.o_proj": { |
| "base_key": "model.layers.77.self_attn.o_proj.weight", |
| "module": "model.layers.77.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006230915314517915, |
| "delta_mean_abs": 2.964258055726532e-05 |
| }, |
| "model.layers.77.self_attn.q_proj": { |
| "base_key": "model.layers.77.self_attn.q_proj.weight", |
| "module": "model.layers.77.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005980563000775874, |
| "delta_mean_abs": 3.2359061151510105e-05 |
| }, |
| "model.layers.77.self_attn.v_proj": { |
| "base_key": "model.layers.77.self_attn.v_proj.weight", |
| "module": "model.layers.77.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00047072680899873376, |
| "delta_mean_abs": 3.360545088071376e-05 |
| }, |
| "model.layers.78.mlp.down_proj": { |
| "base_key": "model.layers.78.mlp.down_proj.weight", |
| "module": "model.layers.78.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004062634543515742, |
| "delta_mean_abs": 2.1243575247353874e-05 |
| }, |
| "model.layers.78.mlp.gate_proj": { |
| "base_key": "model.layers.78.mlp.gate_proj.weight", |
| "module": "model.layers.78.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0010761362500488758, |
| "delta_mean_abs": 4.8448320740135387e-05 |
| }, |
| "model.layers.78.mlp.up_proj": { |
| "base_key": "model.layers.78.mlp.up_proj.weight", |
| "module": "model.layers.78.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.001022767974063754, |
| "delta_mean_abs": 4.201831325190142e-05 |
| }, |
| "model.layers.78.self_attn.k_proj": { |
| "base_key": "model.layers.78.self_attn.k_proj.weight", |
| "module": "model.layers.78.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00039437448140233755, |
| "delta_mean_abs": 3.590131382225081e-05 |
| }, |
| "model.layers.78.self_attn.o_proj": { |
| "base_key": "model.layers.78.self_attn.o_proj.weight", |
| "module": "model.layers.78.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.00047684129094704986, |
| "delta_mean_abs": 2.85476962744724e-05 |
| }, |
| "model.layers.78.self_attn.q_proj": { |
| "base_key": "model.layers.78.self_attn.q_proj.weight", |
| "module": "model.layers.78.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006162011413834989, |
| "delta_mean_abs": 3.284952254034579e-05 |
| }, |
| "model.layers.78.self_attn.v_proj": { |
| "base_key": "model.layers.78.self_attn.v_proj.weight", |
| "module": "model.layers.78.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004823432245757431, |
| "delta_mean_abs": 3.197722980985418e-05 |
| }, |
| "model.layers.79.mlp.down_proj": { |
| "base_key": "model.layers.79.mlp.down_proj.weight", |
| "module": "model.layers.79.mlp.down_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 28672 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0003193787415511906, |
| "delta_mean_abs": 1.8103039110428654e-05 |
| }, |
| "model.layers.79.mlp.gate_proj": { |
| "base_key": "model.layers.79.mlp.gate_proj.weight", |
| "module": "model.layers.79.mlp.gate_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.001132657751441002, |
| "delta_mean_abs": 4.3130454287165776e-05 |
| }, |
| "model.layers.79.mlp.up_proj": { |
| "base_key": "model.layers.79.mlp.up_proj.weight", |
| "module": "model.layers.79.mlp.up_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 28672, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0009169339318759739, |
| "delta_mean_abs": 3.392613507458009e-05 |
| }, |
| "model.layers.79.self_attn.k_proj": { |
| "base_key": "model.layers.79.self_attn.k_proj.weight", |
| "module": "model.layers.79.self_attn.k_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005257595330476761, |
| "delta_mean_abs": 3.793235373450443e-05 |
| }, |
| "model.layers.79.self_attn.o_proj": { |
| "base_key": "model.layers.79.self_attn.o_proj.weight", |
| "module": "model.layers.79.self_attn.o_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0004904827801510692, |
| "delta_mean_abs": 3.2804480724735186e-05 |
| }, |
| "model.layers.79.self_attn.q_proj": { |
| "base_key": "model.layers.79.self_attn.q_proj.weight", |
| "module": "model.layers.79.self_attn.q_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 8192, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0006097261211834848, |
| "delta_mean_abs": 3.5851575376000255e-05 |
| }, |
| "model.layers.79.self_attn.v_proj": { |
| "base_key": "model.layers.79.self_attn.v_proj.weight", |
| "module": "model.layers.79.self_attn.v_proj", |
| "rank": 8, |
| "alpha": 16.0, |
| "scale": 2.0, |
| "shape": [ |
| 1024, |
| 8192 |
| ], |
| "dtype": "torch.bfloat16", |
| "delta_max_abs": 0.0005342763615772128, |
| "delta_mean_abs": 3.266278508817777e-05 |
| } |
| } |
| } |
| ] |
| } |
