Upload folder using huggingface_hub

stage1/0.8b/loss_curve.csv

@@ -0,0 +1,122 @@
+step,loss,distill_kl,agree_kl,lr,tok_per_s
+0,36.260395,36.260395,0.000000,1.000e-05,23460.3
+5,36.606621,36.606621,0.000000,6.000e-05,24472.1
+10,40.735897,40.735897,0.000000,1.100e-04,2151.6
+15,33.817451,33.817451,0.000000,1.600e-04,24337.5
+20,35.094440,35.094440,0.000000,2.100e-04,24343.7
+25,37.235092,37.235092,0.000000,2.600e-04,3642.2
+30,34.503639,34.503639,0.000000,3.100e-04,24361.1
+35,31.708145,31.708145,0.000000,3.600e-04,12538.4
+40,33.227478,33.227478,0.000000,4.100e-04,3649.9
+45,29.354061,29.354061,0.000000,4.600e-04,12527.7
+50,28.730705,28.730705,0.000000,5.100e-04,2149.0
+55,23.611305,23.611305,0.000000,5.600e-04,24321.7
+60,22.933931,22.933931,0.000000,6.100e-04,2149.1
+65,18.161081,18.161081,0.000000,6.600e-04,12515.8
+70,15.759269,15.759269,0.000000,7.100e-04,24331.2
+75,14.397238,14.397238,0.000000,7.600e-04,7700.7
+80,12.886992,12.886992,0.000000,8.100e-04,2148.9
+85,9.797360,9.797360,0.000000,8.600e-04,12540.6
+90,8.398781,8.398781,0.000000,9.100e-04,12548.2
+95,7.288176,7.288176,0.000000,9.600e-04,7692.2
+100,5.325162,5.325162,0.000000,1.000e-03,24223.8
+105,4.763614,4.763614,0.000000,9.998e-04,12553.4
+110,3.833522,3.833522,0.000000,9.990e-04,24369.9
+115,3.473099,3.473099,0.000000,9.978e-04,24373.4
+120,3.390601,3.390601,0.000000,9.961e-04,12543.7
+125,2.866962,2.866962,0.000000,9.938e-04,24368.1
+130,2.736684,2.736684,0.000000,9.911e-04,24378.9
+135,2.577870,2.577870,0.000000,9.880e-04,24342.7
+140,2.359420,2.359420,0.000000,9.843e-04,24345.0
+145,2.483137,2.483137,0.000000,9.801e-04,12539.6
+150,2.362901,2.362901,0.000000,9.755e-04,24363.5
+155,2.705151,2.705151,0.000000,9.704e-04,7698.8
+160,2.100007,2.100007,0.000000,9.649e-04,24378.6
+165,2.144134,2.144134,0.000000,9.589e-04,24335.0
+170,2.125035,2.125035,0.000000,9.524e-04,24391.0
+175,2.282474,2.282474,0.000000,9.455e-04,12539.3
+180,2.135946,2.135946,0.000000,9.382e-04,24314.2
+185,2.141824,2.141824,0.000000,9.304e-04,12532.5
+190,2.365453,2.365453,0.000000,9.222e-04,7688.2
+195,2.605087,2.605087,0.000000,9.135e-04,3644.4
+200,2.257226,2.257226,0.000000,9.045e-04,12540.8
+205,2.184510,2.184510,0.000000,8.951e-04,12560.8
+210,2.048450,2.048450,0.000000,8.853e-04,24375.1
+215,2.109840,2.109840,0.000000,8.751e-04,12545.1
+220,2.002452,2.002452,0.000000,8.645e-04,24300.3
+225,2.405103,2.405103,0.000000,8.536e-04,3641.1
+230,1.849942,1.849942,0.000000,8.423e-04,24352.9
+235,2.125109,2.125109,0.000000,8.307e-04,7701.2
+240,1.808422,1.808422,0.000000,8.187e-04,24382.1
+245,2.001122,2.001122,0.000000,8.065e-04,12535.3
+250,1.814642,1.814642,0.000000,7.939e-04,24271.3
+255,1.781193,1.781193,0.000000,7.810e-04,24385.1
+260,1.775551,1.775551,0.000000,7.679e-04,24383.9
+265,1.869654,1.869654,0.000000,7.545e-04,24181.8
+270,1.981400,1.981400,0.000000,7.409e-04,12551.5
+275,1.911845,1.911845,0.000000,7.270e-04,12521.7
+280,1.767805,1.767805,0.000000,7.129e-04,24394.0
+285,1.792970,1.792970,0.000000,6.986e-04,24331.8
+290,2.101502,2.101502,0.000000,6.841e-04,7699.6
+295,1.768139,1.768139,0.000000,6.694e-04,24383.2
+300,1.947456,1.947456,0.000000,6.545e-04,12565.5
+305,1.948907,1.948907,0.000000,6.395e-04,12423.2
+310,2.231446,2.231446,0.000000,6.243e-04,3645.7
+315,2.034647,2.034647,0.000000,6.091e-04,7696.3
+320,1.753511,1.753511,0.000000,5.937e-04,24377.5
+325,1.798271,1.798271,0.000000,5.782e-04,12558.2
+330,1.859658,1.859658,0.000000,5.627e-04,12553.2
+335,1.749497,1.749497,0.000000,5.471e-04,24349.0
+340,1.733948,1.733948,0.000000,5.314e-04,24368.1
+345,1.797558,1.797558,0.000000,5.157e-04,12553.1
+350,1.888953,1.888953,0.000000,5.000e-04,12535.1
+355,1.774325,1.774325,0.000000,4.843e-04,24172.2
+360,2.107216,2.107216,0.000000,4.686e-04,3647.2
+365,1.835690,1.835690,0.000000,4.529e-04,12548.6
+370,1.719251,1.719251,0.000000,4.373e-04,24375.9
+375,1.902813,1.902813,0.000000,4.218e-04,12541.3
+380,1.734359,1.734359,0.000000,4.063e-04,24372.7
+385,1.619053,1.619053,0.000000,3.909e-04,24349.1
+390,1.630067,1.630067,0.000000,3.757e-04,24348.0
+395,1.665148,1.665148,0.000000,3.605e-04,24254.0
+400,2.269083,2.269083,0.000000,3.455e-04,2150.5
+405,1.857728,1.857728,0.000000,3.306e-04,12526.2
+410,1.866130,1.866130,0.000000,3.159e-04,12539.3
+415,1.893616,1.893616,0.000000,3.014e-04,7697.3
+420,1.708642,1.708642,0.000000,2.871e-04,24352.8
+425,1.816054,1.816054,0.000000,2.730e-04,12538.0
+430,2.180219,2.180219,0.000000,2.591e-04,3644.2
+435,2.105157,2.105157,0.000000,2.455e-04,3640.8
+440,1.649145,1.649145,0.000000,2.321e-04,24382.1
+445,1.634491,1.634491,0.000000,2.190e-04,24344.7
+450,1.944745,1.944745,0.000000,2.061e-04,7691.6
+455,1.699265,1.699265,0.000000,1.935e-04,24340.6
+460,2.005100,2.005100,0.000000,1.813e-04,7700.4
+465,1.772515,1.772515,0.000000,1.693e-04,12554.3
+470,1.680755,1.680755,0.000000,1.577e-04,24341.3
+475,1.848207,1.848207,0.000000,1.464e-04,12514.8
+480,1.852663,1.852663,0.000000,1.355e-04,12547.7
+485,2.157216,2.157216,0.000000,1.249e-04,3642.3
+490,1.957937,1.957937,0.000000,1.147e-04,7701.1
+495,1.732060,1.732060,0.000000,1.049e-04,24316.9
+500,1.949986,1.949986,0.000000,9.549e-05,7692.4
+505,2.124868,2.124868,0.000000,8.646e-05,24356.5
+510,1.867319,1.867319,0.000000,7.784e-05,12552.1
+515,1.894930,1.894930,0.000000,6.963e-05,12570.9
+520,1.774760,1.774760,0.000000,6.185e-05,24330.9
+525,1.683455,1.683455,0.000000,5.450e-05,24250.5
+530,2.104506,2.104506,0.000000,4.759e-05,7695.7
+535,1.662714,1.662714,0.000000,4.112e-05,24373.6
+540,1.680248,1.680248,0.000000,3.511e-05,24318.7
+545,2.139166,2.139166,0.000000,2.956e-05,3648.8
+550,1.711093,1.711093,0.000000,2.447e-05,24350.3
+555,1.614073,1.614073,0.000000,1.985e-05,24384.4
+560,1.605107,1.605107,0.000000,1.571e-05,24374.7
+565,1.934373,1.934373,0.000000,1.204e-05,7696.3
+570,1.651203,1.651203,0.000000,8.856e-06,24345.4
+575,1.771540,1.771540,0.000000,6.156e-06,24365.9
+580,1.654364,1.654364,0.000000,3.943e-06,24350.9
+585,1.987633,1.987633,0.000000,2.219e-06,7702.4
+590,1.960801,1.960801,0.000000,9.866e-07,7701.1
+595,1.820015,1.820015,0.000000,2.467e-07,12528.0
+599,1.960431,1.960431,0.000000,9.870e-09,7703.4

stage1/0.8b/qwen3_5_0_8b_base_v2.json

@@ -0,0 +1,213 @@
+{
+  "model": "Qwen/Qwen3.5-0.8B-Base",
+  "calibration_size": 8,
+  "retrieval_ratio": 0.15,
+  "seq_len": 8192,
+  "metric": "paper-eq3-dual-needle",
+  "scores_per_layer": {
+    "3": [
+      0.02563737751916051,
+      0.02993820235133171,
+      0.03267090721055865,
+      0.01663368078880012,
+      0.08237696578726172,
+      0.23837296105921268,
+      0.03252467792481184,
+      0.6160814762115479
+    ],
+    "7": [
+      0.19254055991768837,
+      0.5636923052370548,
+      0.6881117597222328,
+      0.4335341826081276,
+      0.3721180781722069,
+      0.13400846999138594,
+      0.45797958225011826,
+      0.931108832359314
+    ],
+    "11": [
+      0.7446733266115189,
+      0.9405687600374222,
+      0.7798495143651962,
+      0.7953295186161995,
+      0.7983837053179741,
+      0.6812883540987968,
+      0.49565868079662323,
+      0.7492943480610847
+    ],
+    "15": [
+      0.8158092498779297,
+      0.82125024497509,
+      0.898208737373352,
+      0.6892690435051918,
+      0.78043582290411,
+      0.9417178332805634,
+      0.8911014869809151,
+      0.9743552654981613
+    ],
+    "19": [
+      0.7455862835049629,
+      0.6835766583681107,
+      0.7491354271769524,
+      0.7448243051767349,
+      0.15495550259947777,
+      0.8121444582939148,
+      0.740615226328373,
+      0.8027348443865776
+    ],
+    "23": [
+      0.6877065151929855,
+      0.57198865711689,
+      0.02371499501168728,
+      0.5292777344584465,
+      0.4410744570195675,
+      0.6382728666067123,
+      0.20930000022053719,
+      0.5222815833985806
+    ]
+  },
+  "layers": {
+    "3": {
+      "retrieval": [],
+      "local": [
+        0,
+        1,
+        2,
+        3,
+        4,
+        5,
+        6,
+        7
+      ],
+      "scores": [
+        0.025637,
+        0.029938,
+        0.032671,
+        0.016634,
+        0.082377,
+        0.238373,
+        0.032525,
+        0.616081
+      ]
+    },
+    "7": {
+      "retrieval": [
+        7
+      ],
+      "local": [
+        0,
+        1,
+        2,
+        3,
+        4,
+        5,
+        6
+      ],
+      "scores": [
+        0.192541,
+        0.563692,
+        0.688112,
+        0.433534,
+        0.372118,
+        0.134008,
+        0.45798,
+        0.931109
+      ]
+    },
+    "11": {
+      "retrieval": [
+        1
+      ],
+      "local": [
+        0,
+        2,
+        3,
+        4,
+        5,
+        6,
+        7
+      ],
+      "scores": [
+        0.744673,
+        0.940569,
+        0.77985,
+        0.79533,
+        0.798384,
+        0.681288,
+        0.495659,
+        0.749294
+      ]
+    },
+    "15": {
+      "retrieval": [
+        1,
+        2,
+        5,
+        6,
+        7
+      ],
+      "local": [
+        0,
+        3,
+        4
+      ],
+      "scores": [
+        0.815809,
+        0.82125,
+        0.898209,
+        0.689269,
+        0.780436,
+        0.941718,
+        0.891101,
+        0.974355
+      ]
+    },
+    "19": {
+      "retrieval": [],
+      "local": [
+        0,
+        1,
+        2,
+        3,
+        4,
+        5,
+        6,
+        7
+      ],
+      "scores": [
+        0.745586,
+        0.683577,
+        0.749135,
+        0.744824,
+        0.154956,
+        0.812144,
+        0.740615,
+        0.802735
+      ]
+    },
+    "23": {
+      "retrieval": [],
+      "local": [
+        0,
+        1,
+        2,
+        3,
+        4,
+        5,
+        6,
+        7
+      ],
+      "scores": [
+        0.687707,
+        0.571989,
+        0.023715,
+        0.529278,
+        0.441074,
+        0.638273,
+        0.2093,
+        0.522282
+      ]
+    }
+  },
+  "retrieval_fraction": 0.14583333333333334
+}

stage1/0.8b/stage1_summary.json

@@ -0,0 +1,37 @@
+{
+  "model": "/root/work/models/Qwen3.5-0.8B-Base",
+  "probe": "/root/work/probe_results/qwen3_5_0_8b_base_seq131k.json",
+  "steps": 600,
+  "lr": 0.001,
+  "seq_len": 131072,
+  "batch_size": 1,
+  "d_idx": 16,
+  "group_agree_loss": 0.0,
+  "init_from": null,
+  "chunked_teacher": true,
+  "cq": 2048,
+  "elapsed_min": 130.43536009731665,
+  "final_per_head_kl": {
+    "3": [
+      1.9005834124982357
+    ],
+    "7": [
+      2.2371780313551426
+    ],
+    "11": [
+      1.8176226913928986
+    ],
+    "15": [
+      1.9935339204967022,
+      2.1306826025247574
+    ],
+    "19": [
+      2.0553825963288546
+    ],
+    "23": [
+      1.6897077709436417
+    ]
+  },
+  "final_mean_kl": 1.974955860791462,
+  "final_max_kl": 2.2371780313551426
+}

stage1/0.8b/state_dict.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4c9c64368b76152a3057f702cd2f8afdb996b01eaad66bf99f545cb5b2369330
+size 120565