aLoRA retrain of cti-technique-mapping on granite-4.0-350m (gated on assistant invocation tokens)

adapter_config.json

@@ -32,23 +32,21 @@
   "qalora_group_size": 16,
   "r": 32,
   "rank_pattern": {
-    "down_proj": 32,
-    "gate_proj": 32,
+    "input_linear": 32,
     "k_proj": 32,
     "o_proj": 32,
+    "output_linear": 32,
     "q_proj": 32,
-    "up_proj": 32,
     "v_proj": 32
   },
   "revision": null,
   "target_modules": [
-    "o_proj",
-    "down_proj",
-    "v_proj",
-    "up_proj",
-    "q_proj",
     "k_proj",
-    "gate_proj"
+    "v_proj",
+    "output_linear",
+    "input_linear",
+    "o_proj",
+    "q_proj"
   ],
   "target_parameters": null,
   "task_type": "CAUSAL_LM",

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d2d8012ed118f8d945d3e8c217cd8927dbc02863a03cec9f7e95d4d288f62535
-size 56936568
+oid sha256:c53817328b96754c14be89373a8eb12fa045594a449a585e99c570db7202904a
+size 53261144

checkpoint-47/adapter_config.json

@@ -32,23 +32,21 @@
   "qalora_group_size": 16,
   "r": 32,
   "rank_pattern": {
-    "down_proj": 32,
-    "gate_proj": 32,
+    "input_linear": 32,
     "k_proj": 32,
     "o_proj": 32,
+    "output_linear": 32,
     "q_proj": 32,
-    "up_proj": 32,
     "v_proj": 32
   },
   "revision": null,
   "target_modules": [
-    "o_proj",
-    "down_proj",
-    "v_proj",
-    "up_proj",
-    "q_proj",
     "k_proj",
-    "gate_proj"
+    "v_proj",
+    "output_linear",
+    "input_linear",
+    "o_proj",
+    "q_proj"
   ],
   "target_parameters": null,
   "task_type": "CAUSAL_LM",

checkpoint-47/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:252db21c1f001cad7312cefe73167569960f2c5d41aa69862f0e9d2ed1551ec9
-size 56936568
+oid sha256:5b9929ed09931e8216e923d38f7c2c633c26f95a958a88968894bce3b67bed88
+size 53261144

checkpoint-47/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c401ae6b69708bc8302f953db9971579561ea627e2a27124ab8a9e93d0bb472b
-size 114100074
+oid sha256:b18e3326536b38c5b10b9d2f555a7a835f4b99d98acb8d93137cc9f9ee1b635e
+size 106712578

checkpoint-47/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b5cb7b6956e69dd5e09fc0bbf0f1be57422d38036f15920ac3ebd6acc1d07240
+oid sha256:275eba8d3b12cdd60d49296aac9adc57bd8616f1c8c1cd6586111be538f59f9a
 size 15024

checkpoint-47/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6aa3172c5eb3a90b2e259a8d1ef51176a5a2d5b74797dbc20cdbffe313fe228c
+oid sha256:1b5ee2a746d9c3f96ab1ed4c2fc63068afebb6190d2dbff804cfd7e0a87e5486
 size 15024

checkpoint-47/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:93f2e35e9e2c59c0b3f742a087a6b3fc15c747a4f2cd1131b7974df411ea2f3e
+oid sha256:28890307ce499e464ca161a5037b46dcea79080b7621c9deef43a568289a28ae
 size 15024

checkpoint-47/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:bd4cb1a7374f1bcc3a96c9479b16d45e286f968d793577dceeaef37ba2905f49
+oid sha256:5c11327af107ac2062249c8de836804e44a688bc5c509f9203b55f021ec246d8
 size 15024

checkpoint-47/trainer_state.json

@@ -10,472 +10,472 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "entropy": 0.8714504987001419,
+      "entropy": 0.8770920187234879,
       "epoch": 0.02127659574468085,
-      "grad_norm": 98.33157348632812,
+      "grad_norm": 101.38615417480469,
       "learning_rate": 0.0,
-      "loss": 11.492215156555176,
+      "loss": 11.496590614318848,
       "mean_token_accuracy": 0.22714237496256828,
       "num_tokens": 2486.0,
       "step": 1
     },
     {
-      "entropy": 0.9165030866861343,
+      "entropy": 0.9114853590726852,
       "epoch": 0.0425531914893617,
-      "grad_norm": 94.53471374511719,
+      "grad_norm": 96.85296630859375,
       "learning_rate": 4e-05,
-      "loss": 11.225082397460938,
+      "loss": 11.215909004211426,
       "mean_token_accuracy": 0.2649671100080013,
       "num_tokens": 4959.0,
       "step": 2
     },
     {
-      "entropy": 1.2084493041038513,
+      "entropy": 1.2038442194461823,
       "epoch": 0.06382978723404255,
-      "grad_norm": 70.36109161376953,
+      "grad_norm": 73.84027862548828,
       "learning_rate": 8e-05,
-      "loss": 9.22512435913086,
-      "mean_token_accuracy": 0.36249999701976776,
+      "loss": 9.273573875427246,
+      "mean_token_accuracy": 0.37812498956918716,
       "num_tokens": 7552.0,
       "step": 3
     },
     {
-      "entropy": 1.0519269406795502,
+      "entropy": 1.0524671226739883,
       "epoch": 0.0851063829787234,
-      "grad_norm": 31.853151321411133,
+      "grad_norm": 30.00477409362793,
       "learning_rate": 0.00012,
-      "loss": 6.967526435852051,
-      "mean_token_accuracy": 0.5811337232589722,
+      "loss": 6.997811317443848,
+      "mean_token_accuracy": 0.5748837292194366,
       "num_tokens": 10017.0,
       "step": 4
     },
     {
-      "entropy": 1.1210870146751404,
+      "entropy": 1.0943913161754608,
       "epoch": 0.10638297872340426,
-      "grad_norm": 31.122838973999023,
+      "grad_norm": 29.77185821533203,
       "learning_rate": 0.00016,
-      "loss": 5.962404251098633,
-      "mean_token_accuracy": 0.6272773295640945,
+      "loss": 5.874476432800293,
+      "mean_token_accuracy": 0.6206140369176865,
       "num_tokens": 12542.0,
       "step": 5
     },
     {
-      "entropy": 0.5732153728604317,
+      "entropy": 0.690106213092804,
       "epoch": 0.1276595744680851,
-      "grad_norm": 78.56868743896484,
+      "grad_norm": 28.456193923950195,
       "learning_rate": 0.0002,
-      "loss": 5.072619438171387,
-      "mean_token_accuracy": 0.7359775602817535,
+      "loss": 4.93742036819458,
+      "mean_token_accuracy": 0.7391025722026825,
       "num_tokens": 15108.0,
       "step": 6
     },
     {
-      "entropy": 0.4316209927201271,
+      "entropy": 0.42824630439281464,
       "epoch": 0.14893617021276595,
-      "grad_norm": 8.068974494934082,
+      "grad_norm": 8.46117877960205,
       "learning_rate": 0.00019993770622619782,
-      "loss": 4.839648246765137,
-      "mean_token_accuracy": 0.7702841460704803,
+      "loss": 4.776185512542725,
+      "mean_token_accuracy": 0.7670790255069733,
       "num_tokens": 17715.0,
       "step": 7
     },
     {
-      "entropy": 0.6428281664848328,
+      "entropy": 0.6275453418493271,
       "epoch": 0.1702127659574468,
-      "grad_norm": 7.950557708740234,
+      "grad_norm": 7.040255069732666,
       "learning_rate": 0.00019975090251507638,
-      "loss": 4.679971694946289,
-      "mean_token_accuracy": 0.7608270794153214,
+      "loss": 4.601991653442383,
+      "mean_token_accuracy": 0.7641165554523468,
       "num_tokens": 20143.0,
       "step": 8
     },
     {
-      "entropy": 0.546170249581337,
+      "entropy": 0.49182429164648056,
       "epoch": 0.19148936170212766,
-      "grad_norm": 4.716532230377197,
+      "grad_norm": 4.624846458435059,
       "learning_rate": 0.0001994398216007982,
-      "loss": 4.5612640380859375,
-      "mean_token_accuracy": 0.775667741894722,
+      "loss": 4.511226654052734,
+      "mean_token_accuracy": 0.7724626064300537,
       "num_tokens": 22656.0,
       "step": 9
     },
     {
-      "entropy": 0.6222941875457764,
+      "entropy": 0.626158207654953,
       "epoch": 0.2127659574468085,
-      "grad_norm": 4.465165615081787,
+      "grad_norm": 7.813827037811279,
       "learning_rate": 0.00019900485105144543,
-      "loss": 4.362360954284668,
+      "loss": 4.358890533447266,
       "mean_token_accuracy": 0.7753759473562241,
       "num_tokens": 25094.0,
       "step": 10
     },
     {
-      "entropy": 0.5363088622689247,
+      "entropy": 0.5334508866071701,
       "epoch": 0.23404255319148937,
-      "grad_norm": 3.769137382507324,
+      "grad_norm": 3.6953277587890625,
       "learning_rate": 0.00019844653278615833,
-      "loss": 4.292654514312744,
-      "mean_token_accuracy": 0.7902328073978424,
+      "loss": 4.273573875427246,
+      "mean_token_accuracy": 0.7838225364685059,
       "num_tokens": 27456.0,
       "step": 11
     },
     {
-      "entropy": 0.4109347388148308,
+      "entropy": 0.43513786792755127,
       "epoch": 0.2553191489361702,
-      "grad_norm": 6.13655424118042,
+      "grad_norm": 2.8964409828186035,
       "learning_rate": 0.00019776556239997146,
-      "loss": 4.145160675048828,
-      "mean_token_accuracy": 0.8742673695087433,
+      "loss": 4.085248947143555,
+      "mean_token_accuracy": 0.8868782967329025,
       "num_tokens": 29914.0,
       "step": 12
     },
     {
-      "entropy": 0.3426198214292526,
+      "entropy": 0.3346208333969116,
       "epoch": 0.2765957446808511,
-      "grad_norm": 1.4929083585739136,
+      "grad_norm": 5.369524955749512,
       "learning_rate": 0.00019696278829718883,
-      "loss": 3.946364641189575,
-      "mean_token_accuracy": 0.8999709486961365,
+      "loss": 3.968517303466797,
+      "mean_token_accuracy": 0.8908246159553528,
       "num_tokens": 32448.0,
       "step": 13
     },
     {
-      "entropy": 0.30969393253326416,
+      "entropy": 0.33365894854068756,
       "epoch": 0.2978723404255319,
-      "grad_norm": 3.301110029220581,
+      "grad_norm": 2.550203800201416,
       "learning_rate": 0.00019603921063437793,
-      "loss": 4.041735649108887,
-      "mean_token_accuracy": 0.8868667930364609,
+      "loss": 4.0511908531188965,
+      "mean_token_accuracy": 0.8805366605520248,
       "num_tokens": 34909.0,
       "step": 14
     },
     {
-      "entropy": 0.2662612944841385,
+      "entropy": 0.25230497866868973,
       "epoch": 0.3191489361702128,
-      "grad_norm": 3.146635055541992,
+      "grad_norm": 3.365182876586914,
       "learning_rate": 0.0001949959800742991,
-      "loss": 4.177207946777344,
-      "mean_token_accuracy": 0.8797287046909332,
+      "loss": 4.221365928649902,
+      "mean_token_accuracy": 0.8763548880815506,
       "num_tokens": 37476.0,
       "step": 15
     },
     {
-      "entropy": 0.2030327171087265,
+      "entropy": 0.21353815868496895,
       "epoch": 0.3404255319148936,
-      "grad_norm": 11.572010040283203,
+      "grad_norm": 1.1460461616516113,
       "learning_rate": 0.00019383439635232294,
-      "loss": 4.265796184539795,
-      "mean_token_accuracy": 0.8703853487968445,
+      "loss": 4.106394290924072,
+      "mean_token_accuracy": 0.8868418335914612,
       "num_tokens": 39881.0,
       "step": 16
     },
     {
-      "entropy": 0.21177835389971733,
+      "entropy": 0.21475103124976158,
       "epoch": 0.3617021276595745,
-      "grad_norm": 8.54330062866211,
+      "grad_norm": 3.043959617614746,
       "learning_rate": 0.00019255590665712214,
-      "loss": 4.446836471557617,
-      "mean_token_accuracy": 0.8421052694320679,
+      "loss": 4.265631675720215,
+      "mean_token_accuracy": 0.8690433949232101,
       "num_tokens": 42235.0,
       "step": 17
     },
     {
-      "entropy": 0.1886730156838894,
+      "entropy": 0.1718740463256836,
       "epoch": 0.3829787234042553,
-      "grad_norm": 2.13493013381958,
+      "grad_norm": 0.8078291416168213,
       "learning_rate": 0.0001911621038276542,
-      "loss": 4.138604164123535,
-      "mean_token_accuracy": 0.8729209899902344,
+      "loss": 4.106135368347168,
+      "mean_token_accuracy": 0.892671525478363,
       "num_tokens": 44687.0,
       "step": 18
     },
     {
-      "entropy": 0.16027140989899635,
+      "entropy": 0.16231479868292809,
       "epoch": 0.40425531914893614,
-      "grad_norm": 1.003627896308899,
+      "grad_norm": 0.9561502933502197,
       "learning_rate": 0.00018965472436868286,
-      "loss": 4.130105018615723,
-      "mean_token_accuracy": 0.8872357904911041,
+      "loss": 4.120131015777588,
+      "mean_token_accuracy": 0.8869824111461639,
       "num_tokens": 47206.0,
       "step": 19
     },
     {
-      "entropy": 0.1484772488474846,
+      "entropy": 0.14364850148558617,
       "epoch": 0.425531914893617,
-      "grad_norm": 1.5729527473449707,
+      "grad_norm": 0.9734947681427002,
       "learning_rate": 0.00018803564628730915,
-      "loss": 4.042505264282227,
-      "mean_token_accuracy": 0.8873282968997955,
+      "loss": 4.03128719329834,
+      "mean_token_accuracy": 0.8906020969152451,
       "num_tokens": 49799.0,
       "step": 20
     },
     {
-      "entropy": 0.13887983560562134,
+      "entropy": 0.13312280178070068,
       "epoch": 0.44680851063829785,
-      "grad_norm": 0.8788506388664246,
+      "grad_norm": 0.4489222764968872,
       "learning_rate": 0.00018630688675320842,
-      "loss": 4.068417072296143,
-      "mean_token_accuracy": 0.8922697156667709,
+      "loss": 4.058135032653809,
+      "mean_token_accuracy": 0.8986841887235641,
       "num_tokens": 52291.0,
       "step": 21
     },
     {
-      "entropy": 0.1481389682739973,
+      "entropy": 0.13586510717868805,
       "epoch": 0.46808510638297873,
-      "grad_norm": 1.4755803346633911,
+      "grad_norm": 0.7964894771575928,
       "learning_rate": 0.0001844705995854882,
-      "loss": 3.998652935028076,
-      "mean_token_accuracy": 0.8845166265964508,
+      "loss": 3.965346574783325,
+      "mean_token_accuracy": 0.8940518647432327,
       "num_tokens": 54812.0,
       "step": 22
     },
     {
-      "entropy": 0.15603245794773102,
+      "entropy": 0.15179748088121414,
       "epoch": 0.48936170212765956,
-      "grad_norm": 1.1161134243011475,
+      "grad_norm": 1.6409692764282227,
       "learning_rate": 0.00018252907256929775,
-      "loss": 4.17349910736084,
-      "mean_token_accuracy": 0.8755763471126556,
+      "loss": 4.181431293487549,
+      "mean_token_accuracy": 0.8783541172742844,
       "num_tokens": 57280.0,
       "step": 23
     },
     {
-      "entropy": 0.1347847804427147,
+      "entropy": 0.11831228621304035,
       "epoch": 0.5106382978723404,
-      "grad_norm": 1.2571688890457153,
+      "grad_norm": 0.7683380842208862,
       "learning_rate": 0.00018048472460553257,
-      "loss": 4.224948406219482,
-      "mean_token_accuracy": 0.8872061967849731,
+      "loss": 4.24875545501709,
+      "mean_token_accuracy": 0.8903311938047409,
       "num_tokens": 59802.0,
       "step": 24
     },
     {
-      "entropy": 0.1610047034919262,
+      "entropy": 0.15002421662211418,
       "epoch": 0.5319148936170213,
-      "grad_norm": 1.906825065612793,
+      "grad_norm": 2.176997661590576,
       "learning_rate": 0.00017834010269718526,
-      "loss": 4.16756010055542,
-      "mean_token_accuracy": 0.8820945918560028,
+      "loss": 4.15623664855957,
+      "mean_token_accuracy": 0.8820056915283203,
       "num_tokens": 62271.0,
       "step": 25
     },
     {
-      "entropy": 0.1211549025028944,
+      "entropy": 0.11383292451500893,
       "epoch": 0.5531914893617021,
-      "grad_norm": 0.33083778619766235,
+      "grad_norm": 0.5305500030517578,
       "learning_rate": 0.0001760978787760968,
-      "loss": 4.0746307373046875,
-      "mean_token_accuracy": 0.8973371386528015,
+      "loss": 4.076779365539551,
+      "mean_token_accuracy": 0.8941320031881332,
       "num_tokens": 64751.0,
       "step": 26
     },
     {
-      "entropy": 0.13322192057967186,
+      "entropy": 0.13319513201713562,
       "epoch": 0.574468085106383,
-      "grad_norm": 1.251117467880249,
+      "grad_norm": 1.0374587774276733,
       "learning_rate": 0.00017376084637406222,
-      "loss": 4.0360331535339355,
-      "mean_token_accuracy": 0.8868589699268341,
+      "loss": 4.014706611633301,
+      "mean_token_accuracy": 0.8837339729070663,
       "num_tokens": 67220.0,
       "step": 27
     },
     {
-      "entropy": 0.14910954982042313,
+      "entropy": 0.13686015084385872,
       "epoch": 0.5957446808510638,
-      "grad_norm": 2.55302095413208,
+      "grad_norm": 1.387152075767517,
       "learning_rate": 0.00017133191714243805,
-      "loss": 4.062708377838135,
-      "mean_token_accuracy": 0.8652693033218384,
+      "loss": 4.015310764312744,
+      "mean_token_accuracy": 0.8779295533895493,
       "num_tokens": 69678.0,
       "step": 28
     },
     {
-      "entropy": 0.15168574452400208,
+      "entropy": 0.13596300408244133,
       "epoch": 0.6170212765957447,
-      "grad_norm": 1.1428608894348145,
+      "grad_norm": 1.2698324918746948,
       "learning_rate": 0.00016881411722458688,
-      "loss": 4.023514270782471,
-      "mean_token_accuracy": 0.8867939561605453,
+      "loss": 3.992821455001831,
+      "mean_token_accuracy": 0.8992939442396164,
       "num_tokens": 72158.0,
       "step": 29
     },
     {
-      "entropy": 0.15590869635343552,
+      "entropy": 0.13812873139977455,
       "epoch": 0.6382978723404256,
-      "grad_norm": 1.40220308303833,
+      "grad_norm": 1.9082021713256836,
       "learning_rate": 0.00016621058348568007,
-      "loss": 4.084057807922363,
-      "mean_token_accuracy": 0.8829438984394073,
+      "loss": 4.059752464294434,
+      "mean_token_accuracy": 0.886225238442421,
       "num_tokens": 74621.0,
       "step": 30
     },
     {
-      "entropy": 0.1768706664443016,
+      "entropy": 0.1471228487789631,
       "epoch": 0.6595744680851063,
-      "grad_norm": 0.9734287858009338,
+      "grad_norm": 1.0790090560913086,
       "learning_rate": 0.00016352455960455387,
-      "loss": 3.9810099601745605,
-      "mean_token_accuracy": 0.8919207155704498,
+      "loss": 3.956774950027466,
+      "mean_token_accuracy": 0.8888719379901886,
       "num_tokens": 77122.0,
       "step": 31
     },
     {
-      "entropy": 0.1485825888812542,
+      "entropy": 0.13649508357048035,
       "epoch": 0.6808510638297872,
-      "grad_norm": 0.7609644532203674,
+      "grad_norm": 0.8638296127319336,
       "learning_rate": 0.0001607593920324899,
-      "loss": 4.148928642272949,
-      "mean_token_accuracy": 0.8895917534828186,
+      "loss": 4.1457977294921875,
+      "mean_token_accuracy": 0.8863866329193115,
       "num_tokens": 79658.0,
       "step": 32
     },
     {
-      "entropy": 0.13068590499460697,
+      "entropy": 0.13422205485403538,
       "epoch": 0.7021276595744681,
-      "grad_norm": 1.0391005277633667,
+      "grad_norm": 0.8206593990325928,
       "learning_rate": 0.00015791852582395334,
-      "loss": 4.11308479309082,
-      "mean_token_accuracy": 0.8879216760396957,
+      "loss": 4.104636192321777,
+      "mean_token_accuracy": 0.893946647644043,
       "num_tokens": 82198.0,
       "step": 33
     },
     {
-      "entropy": 0.12392068840563297,
+      "entropy": 0.13717253133654594,
       "epoch": 0.723404255319149,
-      "grad_norm": 1.5052447319030762,
+      "grad_norm": 0.43719571828842163,
       "learning_rate": 0.00015500550034448413,
-      "loss": 4.05723762512207,
-      "mean_token_accuracy": 0.8877825438976288,
+      "loss": 4.031332015991211,
+      "mean_token_accuracy": 0.894197016954422,
       "num_tokens": 84670.0,
       "step": 34
     },
     {
-      "entropy": 0.1099250353872776,
+      "entropy": 0.1137513667345047,
       "epoch": 0.7446808510638298,
-      "grad_norm": 0.7635676264762878,
+      "grad_norm": 1.8441417217254639,
       "learning_rate": 0.0001520239448610882,
-      "loss": 4.097797393798828,
-      "mean_token_accuracy": 0.8941126763820648,
+      "loss": 4.112149715423584,
+      "mean_token_accuracy": 0.8909876793622971,
       "num_tokens": 87292.0,
       "step": 35
     },
     {
-      "entropy": 0.10329670459032059,
+      "entropy": 0.11223667673766613,
       "epoch": 0.7659574468085106,
-      "grad_norm": 0.2738026976585388,
+      "grad_norm": 0.4776930809020996,
       "learning_rate": 0.00014897757402062284,
-      "loss": 4.029165267944336,
-      "mean_token_accuracy": 0.8953875005245209,
+      "loss": 4.026741027832031,
+      "mean_token_accuracy": 0.8924113065004349,
       "num_tokens": 89830.0,
       "step": 36
     },
     {
-      "entropy": 0.11790071427822113,
+      "entropy": 0.12275257520377636,
       "epoch": 0.7872340425531915,
-      "grad_norm": 0.381307452917099,
+      "grad_norm": 0.9828954935073853,
       "learning_rate": 0.00014587018322180905,
-      "loss": 4.153435707092285,
-      "mean_token_accuracy": 0.8958049416542053,
+      "loss": 4.183655738830566,
+      "mean_token_accuracy": 0.8827805668115616,
       "num_tokens": 92344.0,
       "step": 37
     },
     {
-      "entropy": 0.10945623554289341,
+      "entropy": 0.12380701676011086,
       "epoch": 0.8085106382978723,
-      "grad_norm": 1.1360689401626587,
+      "grad_norm": 1.3710100650787354,
       "learning_rate": 0.0001427056438866376,
-      "loss": 4.012265205383301,
-      "mean_token_accuracy": 0.8903843760490417,
+      "loss": 4.003378868103027,
+      "mean_token_accuracy": 0.8812988549470901,
       "num_tokens": 94826.0,
       "step": 38
     },
     {
-      "entropy": 0.10652521438896656,
+      "entropy": 0.10968485288321972,
       "epoch": 0.8297872340425532,
-      "grad_norm": 1.8644981384277344,
+      "grad_norm": 0.37869715690612793,
       "learning_rate": 0.00013948789863705912,
-      "loss": 4.101892471313477,
-      "mean_token_accuracy": 0.8909918814897537,
+      "loss": 4.050682067871094,
+      "mean_token_accuracy": 0.8941126763820648,
       "num_tokens": 97290.0,
       "step": 39
     },
     {
-      "entropy": 0.1185864806175232,
+      "entropy": 0.12006617523729801,
       "epoch": 0.851063829787234,
-      "grad_norm": 0.35084277391433716,
+      "grad_norm": 0.5866818428039551,
       "learning_rate": 0.00013622095638296826,
-      "loss": 4.115968704223633,
-      "mean_token_accuracy": 0.8960161507129669,
+      "loss": 4.113466262817383,
+      "mean_token_accuracy": 0.8894326388835907,
       "num_tokens": 99776.0,
       "step": 40
     },
     {
-      "entropy": 0.10158661007881165,
+      "entropy": 0.09959963150322437,
       "epoch": 0.8723404255319149,
-      "grad_norm": 0.14153917133808136,
+      "grad_norm": 0.4211389422416687,
       "learning_rate": 0.000132908887327601,
-      "loss": 3.8947978019714355,
-      "mean_token_accuracy": 0.9017689526081085,
+      "loss": 3.8911352157592773,
+      "mean_token_accuracy": 0.8987201750278473,
       "num_tokens": 102278.0,
       "step": 41
     },
     {
-      "entropy": 0.1076634582132101,
+      "entropy": 0.10989912413060665,
       "epoch": 0.8936170212765957,
-      "grad_norm": 1.0466699600219727,
+      "grad_norm": 1.014770746231079,
       "learning_rate": 0.00012955581789656843,
-      "loss": 4.078176021575928,
-      "mean_token_accuracy": 0.8884615302085876,
+      "loss": 4.067371845245361,
+      "mean_token_accuracy": 0.8916666656732559,
       "num_tokens": 104764.0,
       "step": 42
     },
     {
-      "entropy": 0.11152214743196964,
+      "entropy": 0.11506957560777664,
       "epoch": 0.9148936170212766,
-      "grad_norm": 0.7619017362594604,
+      "grad_norm": 0.4580586552619934,
       "learning_rate": 0.0001261659255968441,
-      "loss": 4.1144890785217285,
-      "mean_token_accuracy": 0.890312910079956,
+      "loss": 4.101069927215576,
+      "mean_token_accuracy": 0.8936023861169815,
       "num_tokens": 107201.0,
       "step": 43
     },
     {
-      "entropy": 0.10949222184717655,
+      "entropy": 0.10838168673217297,
       "epoch": 0.9361702127659575,
-      "grad_norm": 0.7719561457633972,
+      "grad_norm": 0.4166942536830902,
       "learning_rate": 0.00012274343381211066,
-      "loss": 4.053421497344971,
-      "mean_token_accuracy": 0.8914329707622528,
+      "loss": 4.025703430175781,
+      "mean_token_accuracy": 0.8947224318981171,
       "num_tokens": 109660.0,
       "step": 44
     },
     {
-      "entropy": 0.10472714714705944,
+      "entropy": 0.09614014439284801,
       "epoch": 0.9574468085106383,
-      "grad_norm": 0.6925389766693115,
+      "grad_norm": 0.9769757390022278,
       "learning_rate": 0.00011929260654094969,
-      "loss": 4.108519554138184,
-      "mean_token_accuracy": 0.893522247672081,
+      "loss": 4.121439456939697,
+      "mean_token_accuracy": 0.8903171271085739,
       "num_tokens": 112117.0,
       "step": 45
     },
     {
-      "entropy": 0.10658742487430573,
+      "entropy": 0.11138966307044029,
       "epoch": 0.9787234042553191,
-      "grad_norm": 0.8806354999542236,
+      "grad_norm": 1.5290015935897827,
       "learning_rate": 0.0001158177430844304,
-      "loss": 4.042045593261719,
-      "mean_token_accuracy": 0.8898036479949951,
+      "loss": 4.022660732269287,
+      "mean_token_accuracy": 0.8867548704147339,
       "num_tokens": 114595.0,
       "step": 46
     },
     {
-      "entropy": 0.10867075435817242,
+      "entropy": 0.10535211116075516,
       "epoch": 1.0,
-      "grad_norm": 1.0520161390304565,
+      "grad_norm": 0.6933005452156067,
       "learning_rate": 0.00011232317268971585,
-      "loss": 4.1521220207214355,
-      "mean_token_accuracy": 0.8863234221935272,
+      "loss": 4.150263786315918,
+      "mean_token_accuracy": 0.8926616609096527,
       "num_tokens": 117080.0,
       "step": 47
     },
@@ -485,9 +485,9 @@
       "eval_loss": NaN,
       "eval_mean_token_accuracy": 0.0,
       "eval_num_tokens": 117080.0,
-      "eval_runtime": 17.3755,
-      "eval_samples_per_second": 17.266,
-      "eval_steps_per_second": 4.316,
+      "eval_runtime": 14.1972,
+      "eval_samples_per_second": 21.131,
+      "eval_steps_per_second": 5.283,
       "step": 47
     }
   ],
@@ -517,7 +517,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 196924418293760.0,
+  "total_flos": 196239614279680.0,
   "train_batch_size": 2,
   "trial_name": null,
   "trial_params": null

checkpoint-47/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0a5df61c66afc72c9c2f3e6ebf29ce0723f79ac1bd71f5ce8bf3abd116d58ac6
+oid sha256:49a5fd350f76aacdefac1c174b863cb237ced8dddcf5626a48a21ab11428ac63
 size 5496

checkpoint-94/adapter_config.json

@@ -32,23 +32,21 @@
   "qalora_group_size": 16,
   "r": 32,
   "rank_pattern": {
-    "down_proj": 32,
-    "gate_proj": 32,
+    "input_linear": 32,
     "k_proj": 32,
     "o_proj": 32,
+    "output_linear": 32,
     "q_proj": 32,
-    "up_proj": 32,
     "v_proj": 32
   },
   "revision": null,
   "target_modules": [
-    "o_proj",
-    "down_proj",
-    "v_proj",
-    "up_proj",
-    "q_proj",
     "k_proj",
-    "gate_proj"
+    "v_proj",
+    "output_linear",
+    "input_linear",
+    "o_proj",
+    "q_proj"
   ],
   "target_parameters": null,
   "task_type": "CAUSAL_LM",

checkpoint-94/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d2d8012ed118f8d945d3e8c217cd8927dbc02863a03cec9f7e95d4d288f62535
-size 56936568
+oid sha256:c53817328b96754c14be89373a8eb12fa045594a449a585e99c570db7202904a
+size 53261144

checkpoint-94/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3a50d88da6e1d85dc01027ac68411b9319c447c1f83b31de9bc0f8009e4774f8
-size 114100074
+oid sha256:82b1627bdf9d94009fdfabf7f86edd4d1c37ec8cb9478ed6e8c1b72e9a68d008
+size 106712578

checkpoint-94/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:71e7ad77c3b1db5c5cf8dd111f2c994dbd58923af86a195a337322aee0ff2893
+oid sha256:abac2db5d0310c6cdc771319af673fcea21b4e0db062931c35dac4117514fef5
 size 15024

checkpoint-94/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ce314ea180a11d6bd701685800c89364e3c3c281a400a35b0128861425ba91f6
+oid sha256:cfaef057d22d3b5da65a053256b1f394dacd89cbd6cf33d4925c56938100d677
 size 15024

checkpoint-94/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:398254995abd89d91cb2a6f73e12304f1015f27726ecb9a565e785888d00a7c6
+oid sha256:74cb7993ff2c68e21555e72cc8972b98cdddbd83a84ede67fe547e5bff780110
 size 15024

checkpoint-94/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c4119f630d716b8cfa315a48fab58d644e947bd321819198099f49910f548948
+oid sha256:ba1d0fab2d7e106334859f1f2dacce0b331719900a9c0495e597387ad5d20d6c
 size 15024

checkpoint-94/trainer_state.json

@@ -10,472 +10,472 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "entropy": 0.8714504987001419,
+      "entropy": 0.8770920187234879,
       "epoch": 0.02127659574468085,
-      "grad_norm": 98.33157348632812,
+      "grad_norm": 101.38615417480469,
       "learning_rate": 0.0,
-      "loss": 11.492215156555176,
+      "loss": 11.496590614318848,
       "mean_token_accuracy": 0.22714237496256828,
       "num_tokens": 2486.0,
       "step": 1
     },
     {
-      "entropy": 0.9165030866861343,
+      "entropy": 0.9114853590726852,
       "epoch": 0.0425531914893617,
-      "grad_norm": 94.53471374511719,
+      "grad_norm": 96.85296630859375,
       "learning_rate": 4e-05,
-      "loss": 11.225082397460938,
+      "loss": 11.215909004211426,
       "mean_token_accuracy": 0.2649671100080013,
       "num_tokens": 4959.0,
       "step": 2
     },
     {
-      "entropy": 1.2084493041038513,
+      "entropy": 1.2038442194461823,
       "epoch": 0.06382978723404255,
-      "grad_norm": 70.36109161376953,
+      "grad_norm": 73.84027862548828,
       "learning_rate": 8e-05,
-      "loss": 9.22512435913086,
-      "mean_token_accuracy": 0.36249999701976776,
+      "loss": 9.273573875427246,
+      "mean_token_accuracy": 0.37812498956918716,
       "num_tokens": 7552.0,
       "step": 3
     },
     {
-      "entropy": 1.0519269406795502,
+      "entropy": 1.0524671226739883,
       "epoch": 0.0851063829787234,
-      "grad_norm": 31.853151321411133,
+      "grad_norm": 30.00477409362793,
       "learning_rate": 0.00012,
-      "loss": 6.967526435852051,
-      "mean_token_accuracy": 0.5811337232589722,
+      "loss": 6.997811317443848,
+      "mean_token_accuracy": 0.5748837292194366,
       "num_tokens": 10017.0,
       "step": 4
     },
     {
-      "entropy": 1.1210870146751404,
+      "entropy": 1.0943913161754608,
       "epoch": 0.10638297872340426,
-      "grad_norm": 31.122838973999023,
+      "grad_norm": 29.77185821533203,
       "learning_rate": 0.00016,
-      "loss": 5.962404251098633,
-      "mean_token_accuracy": 0.6272773295640945,
+      "loss": 5.874476432800293,
+      "mean_token_accuracy": 0.6206140369176865,
       "num_tokens": 12542.0,
       "step": 5
     },
     {
-      "entropy": 0.5732153728604317,
+      "entropy": 0.690106213092804,
       "epoch": 0.1276595744680851,
-      "grad_norm": 78.56868743896484,
+      "grad_norm": 28.456193923950195,
       "learning_rate": 0.0002,
-      "loss": 5.072619438171387,
-      "mean_token_accuracy": 0.7359775602817535,
+      "loss": 4.93742036819458,
+      "mean_token_accuracy": 0.7391025722026825,
       "num_tokens": 15108.0,
       "step": 6
     },
     {
-      "entropy": 0.4316209927201271,
+      "entropy": 0.42824630439281464,
       "epoch": 0.14893617021276595,
-      "grad_norm": 8.068974494934082,
+      "grad_norm": 8.46117877960205,
       "learning_rate": 0.00019993770622619782,
-      "loss": 4.839648246765137,
-      "mean_token_accuracy": 0.7702841460704803,
+      "loss": 4.776185512542725,
+      "mean_token_accuracy": 0.7670790255069733,
       "num_tokens": 17715.0,
       "step": 7
     },
     {
-      "entropy": 0.6428281664848328,
+      "entropy": 0.6275453418493271,
       "epoch": 0.1702127659574468,
-      "grad_norm": 7.950557708740234,
+      "grad_norm": 7.040255069732666,
       "learning_rate": 0.00019975090251507638,
-      "loss": 4.679971694946289,
-      "mean_token_accuracy": 0.7608270794153214,
+      "loss": 4.601991653442383,
+      "mean_token_accuracy": 0.7641165554523468,
       "num_tokens": 20143.0,
       "step": 8
     },
     {
-      "entropy": 0.546170249581337,
+      "entropy": 0.49182429164648056,
       "epoch": 0.19148936170212766,
-      "grad_norm": 4.716532230377197,
+      "grad_norm": 4.624846458435059,
       "learning_rate": 0.0001994398216007982,
-      "loss": 4.5612640380859375,
-      "mean_token_accuracy": 0.775667741894722,
+      "loss": 4.511226654052734,
+      "mean_token_accuracy": 0.7724626064300537,
       "num_tokens": 22656.0,
       "step": 9
     },
     {
-      "entropy": 0.6222941875457764,
+      "entropy": 0.626158207654953,
       "epoch": 0.2127659574468085,
-      "grad_norm": 4.465165615081787,
+      "grad_norm": 7.813827037811279,
       "learning_rate": 0.00019900485105144543,
-      "loss": 4.362360954284668,
+      "loss": 4.358890533447266,
       "mean_token_accuracy": 0.7753759473562241,
       "num_tokens": 25094.0,
       "step": 10
     },
     {
-      "entropy": 0.5363088622689247,
+      "entropy": 0.5334508866071701,
       "epoch": 0.23404255319148937,
-      "grad_norm": 3.769137382507324,
+      "grad_norm": 3.6953277587890625,
       "learning_rate": 0.00019844653278615833,
-      "loss": 4.292654514312744,
-      "mean_token_accuracy": 0.7902328073978424,
+      "loss": 4.273573875427246,
+      "mean_token_accuracy": 0.7838225364685059,
       "num_tokens": 27456.0,
       "step": 11
     },
     {
-      "entropy": 0.4109347388148308,
+      "entropy": 0.43513786792755127,
       "epoch": 0.2553191489361702,
-      "grad_norm": 6.13655424118042,
+      "grad_norm": 2.8964409828186035,
       "learning_rate": 0.00019776556239997146,
-      "loss": 4.145160675048828,
-      "mean_token_accuracy": 0.8742673695087433,
+      "loss": 4.085248947143555,
+      "mean_token_accuracy": 0.8868782967329025,
       "num_tokens": 29914.0,
       "step": 12
     },
     {
-      "entropy": 0.3426198214292526,
+      "entropy": 0.3346208333969116,
       "epoch": 0.2765957446808511,
-      "grad_norm": 1.4929083585739136,
+      "grad_norm": 5.369524955749512,
       "learning_rate": 0.00019696278829718883,
-      "loss": 3.946364641189575,
-      "mean_token_accuracy": 0.8999709486961365,
+      "loss": 3.968517303466797,
+      "mean_token_accuracy": 0.8908246159553528,
       "num_tokens": 32448.0,
       "step": 13
     },
     {
-      "entropy": 0.30969393253326416,
+      "entropy": 0.33365894854068756,
       "epoch": 0.2978723404255319,
-      "grad_norm": 3.301110029220581,
+      "grad_norm": 2.550203800201416,
       "learning_rate": 0.00019603921063437793,
-      "loss": 4.041735649108887,
-      "mean_token_accuracy": 0.8868667930364609,
+      "loss": 4.0511908531188965,
+      "mean_token_accuracy": 0.8805366605520248,
       "num_tokens": 34909.0,
       "step": 14
     },
     {
-      "entropy": 0.2662612944841385,
+      "entropy": 0.25230497866868973,
       "epoch": 0.3191489361702128,
-      "grad_norm": 3.146635055541992,
+      "grad_norm": 3.365182876586914,
       "learning_rate": 0.0001949959800742991,
-      "loss": 4.177207946777344,
-      "mean_token_accuracy": 0.8797287046909332,
+      "loss": 4.221365928649902,
+      "mean_token_accuracy": 0.8763548880815506,
       "num_tokens": 37476.0,
       "step": 15
     },
     {
-      "entropy": 0.2030327171087265,
+      "entropy": 0.21353815868496895,
       "epoch": 0.3404255319148936,
-      "grad_norm": 11.572010040283203,
+      "grad_norm": 1.1460461616516113,
       "learning_rate": 0.00019383439635232294,
-      "loss": 4.265796184539795,
-      "mean_token_accuracy": 0.8703853487968445,
+      "loss": 4.106394290924072,
+      "mean_token_accuracy": 0.8868418335914612,
       "num_tokens": 39881.0,
       "step": 16
     },
     {
-      "entropy": 0.21177835389971733,
+      "entropy": 0.21475103124976158,
       "epoch": 0.3617021276595745,
-      "grad_norm": 8.54330062866211,
+      "grad_norm": 3.043959617614746,
       "learning_rate": 0.00019255590665712214,
-      "loss": 4.446836471557617,
-      "mean_token_accuracy": 0.8421052694320679,
+      "loss": 4.265631675720215,
+      "mean_token_accuracy": 0.8690433949232101,
       "num_tokens": 42235.0,
       "step": 17
     },
     {
-      "entropy": 0.1886730156838894,
+      "entropy": 0.1718740463256836,
       "epoch": 0.3829787234042553,
-      "grad_norm": 2.13493013381958,
+      "grad_norm": 0.8078291416168213,
       "learning_rate": 0.0001911621038276542,
-      "loss": 4.138604164123535,
-      "mean_token_accuracy": 0.8729209899902344,
+      "loss": 4.106135368347168,
+      "mean_token_accuracy": 0.892671525478363,
       "num_tokens": 44687.0,
       "step": 18
     },
     {
-      "entropy": 0.16027140989899635,
+      "entropy": 0.16231479868292809,
       "epoch": 0.40425531914893614,
-      "grad_norm": 1.003627896308899,
+      "grad_norm": 0.9561502933502197,
       "learning_rate": 0.00018965472436868286,
-      "loss": 4.130105018615723,
-      "mean_token_accuracy": 0.8872357904911041,
+      "loss": 4.120131015777588,
+      "mean_token_accuracy": 0.8869824111461639,
       "num_tokens": 47206.0,
       "step": 19
     },
     {
-      "entropy": 0.1484772488474846,
+      "entropy": 0.14364850148558617,
       "epoch": 0.425531914893617,
-      "grad_norm": 1.5729527473449707,
+      "grad_norm": 0.9734947681427002,
       "learning_rate": 0.00018803564628730915,
-      "loss": 4.042505264282227,
-      "mean_token_accuracy": 0.8873282968997955,
+      "loss": 4.03128719329834,
+      "mean_token_accuracy": 0.8906020969152451,
       "num_tokens": 49799.0,
       "step": 20
     },
     {
-      "entropy": 0.13887983560562134,
+      "entropy": 0.13312280178070068,
       "epoch": 0.44680851063829785,
-      "grad_norm": 0.8788506388664246,
+      "grad_norm": 0.4489222764968872,
       "learning_rate": 0.00018630688675320842,
-      "loss": 4.068417072296143,
-      "mean_token_accuracy": 0.8922697156667709,
+      "loss": 4.058135032653809,
+      "mean_token_accuracy": 0.8986841887235641,
       "num_tokens": 52291.0,
       "step": 21
     },
     {
-      "entropy": 0.1481389682739973,
+      "entropy": 0.13586510717868805,
       "epoch": 0.46808510638297873,
-      "grad_norm": 1.4755803346633911,
+      "grad_norm": 0.7964894771575928,
       "learning_rate": 0.0001844705995854882,
-      "loss": 3.998652935028076,
-      "mean_token_accuracy": 0.8845166265964508,
+      "loss": 3.965346574783325,
+      "mean_token_accuracy": 0.8940518647432327,
       "num_tokens": 54812.0,
       "step": 22
     },
     {
-      "entropy": 0.15603245794773102,
+      "entropy": 0.15179748088121414,
       "epoch": 0.48936170212765956,
-      "grad_norm": 1.1161134243011475,
+      "grad_norm": 1.6409692764282227,
       "learning_rate": 0.00018252907256929775,
-      "loss": 4.17349910736084,
-      "mean_token_accuracy": 0.8755763471126556,
+      "loss": 4.181431293487549,
+      "mean_token_accuracy": 0.8783541172742844,
       "num_tokens": 57280.0,
       "step": 23
     },
     {
-      "entropy": 0.1347847804427147,
+      "entropy": 0.11831228621304035,
       "epoch": 0.5106382978723404,
-      "grad_norm": 1.2571688890457153,
+      "grad_norm": 0.7683380842208862,
       "learning_rate": 0.00018048472460553257,
-      "loss": 4.224948406219482,
-      "mean_token_accuracy": 0.8872061967849731,
+      "loss": 4.24875545501709,
+      "mean_token_accuracy": 0.8903311938047409,
       "num_tokens": 59802.0,
       "step": 24
     },
     {
-      "entropy": 0.1610047034919262,
+      "entropy": 0.15002421662211418,
       "epoch": 0.5319148936170213,
-      "grad_norm": 1.906825065612793,
+      "grad_norm": 2.176997661590576,
       "learning_rate": 0.00017834010269718526,
-      "loss": 4.16756010055542,
-      "mean_token_accuracy": 0.8820945918560028,
+      "loss": 4.15623664855957,
+      "mean_token_accuracy": 0.8820056915283203,
       "num_tokens": 62271.0,
       "step": 25
     },
     {
-      "entropy": 0.1211549025028944,
+      "entropy": 0.11383292451500893,
       "epoch": 0.5531914893617021,
-      "grad_norm": 0.33083778619766235,
+      "grad_norm": 0.5305500030517578,
       "learning_rate": 0.0001760978787760968,
-      "loss": 4.0746307373046875,
-      "mean_token_accuracy": 0.8973371386528015,
+      "loss": 4.076779365539551,
+      "mean_token_accuracy": 0.8941320031881332,
       "num_tokens": 64751.0,
       "step": 26
     },
     {
-      "entropy": 0.13322192057967186,
+      "entropy": 0.13319513201713562,
       "epoch": 0.574468085106383,
-      "grad_norm": 1.251117467880249,
+      "grad_norm": 1.0374587774276733,
       "learning_rate": 0.00017376084637406222,
-      "loss": 4.0360331535339355,
-      "mean_token_accuracy": 0.8868589699268341,
+      "loss": 4.014706611633301,
+      "mean_token_accuracy": 0.8837339729070663,
       "num_tokens": 67220.0,
       "step": 27
     },
     {
-      "entropy": 0.14910954982042313,
+      "entropy": 0.13686015084385872,
       "epoch": 0.5957446808510638,
-      "grad_norm": 2.55302095413208,
+      "grad_norm": 1.387152075767517,
       "learning_rate": 0.00017133191714243805,
-      "loss": 4.062708377838135,
-      "mean_token_accuracy": 0.8652693033218384,
+      "loss": 4.015310764312744,
+      "mean_token_accuracy": 0.8779295533895493,
       "num_tokens": 69678.0,
       "step": 28
     },
     {
-      "entropy": 0.15168574452400208,
+      "entropy": 0.13596300408244133,
       "epoch": 0.6170212765957447,
-      "grad_norm": 1.1428608894348145,
+      "grad_norm": 1.2698324918746948,
       "learning_rate": 0.00016881411722458688,
-      "loss": 4.023514270782471,
-      "mean_token_accuracy": 0.8867939561605453,
+      "loss": 3.992821455001831,
+      "mean_token_accuracy": 0.8992939442396164,
       "num_tokens": 72158.0,
       "step": 29
     },
     {
-      "entropy": 0.15590869635343552,
+      "entropy": 0.13812873139977455,
       "epoch": 0.6382978723404256,
-      "grad_norm": 1.40220308303833,
+      "grad_norm": 1.9082021713256836,
       "learning_rate": 0.00016621058348568007,
-      "loss": 4.084057807922363,
-      "mean_token_accuracy": 0.8829438984394073,
+      "loss": 4.059752464294434,
+      "mean_token_accuracy": 0.886225238442421,
       "num_tokens": 74621.0,
       "step": 30
     },
     {
-      "entropy": 0.1768706664443016,
+      "entropy": 0.1471228487789631,
       "epoch": 0.6595744680851063,
-      "grad_norm": 0.9734287858009338,
+      "grad_norm": 1.0790090560913086,
       "learning_rate": 0.00016352455960455387,
-      "loss": 3.9810099601745605,
-      "mean_token_accuracy": 0.8919207155704498,
+      "loss": 3.956774950027466,
+      "mean_token_accuracy": 0.8888719379901886,
       "num_tokens": 77122.0,
       "step": 31
     },
     {
-      "entropy": 0.1485825888812542,
+      "entropy": 0.13649508357048035,
       "epoch": 0.6808510638297872,
-      "grad_norm": 0.7609644532203674,
+      "grad_norm": 0.8638296127319336,
       "learning_rate": 0.0001607593920324899,
-      "loss": 4.148928642272949,
-      "mean_token_accuracy": 0.8895917534828186,
+      "loss": 4.1457977294921875,
+      "mean_token_accuracy": 0.8863866329193115,
       "num_tokens": 79658.0,
       "step": 32
     },
     {
-      "entropy": 0.13068590499460697,
+      "entropy": 0.13422205485403538,
       "epoch": 0.7021276595744681,
-      "grad_norm": 1.0391005277633667,
+      "grad_norm": 0.8206593990325928,
       "learning_rate": 0.00015791852582395334,
-      "loss": 4.11308479309082,
-      "mean_token_accuracy": 0.8879216760396957,
+      "loss": 4.104636192321777,
+      "mean_token_accuracy": 0.893946647644043,
       "num_tokens": 82198.0,
       "step": 33
     },
     {
-      "entropy": 0.12392068840563297,
+      "entropy": 0.13717253133654594,
       "epoch": 0.723404255319149,
-      "grad_norm": 1.5052447319030762,
+      "grad_norm": 0.43719571828842163,
       "learning_rate": 0.00015500550034448413,
-      "loss": 4.05723762512207,
-      "mean_token_accuracy": 0.8877825438976288,
+      "loss": 4.031332015991211,
+      "mean_token_accuracy": 0.894197016954422,
       "num_tokens": 84670.0,
       "step": 34
     },
     {
-      "entropy": 0.1099250353872776,
+      "entropy": 0.1137513667345047,
       "epoch": 0.7446808510638298,
-      "grad_norm": 0.7635676264762878,
+      "grad_norm": 1.8441417217254639,
       "learning_rate": 0.0001520239448610882,
-      "loss": 4.097797393798828,
-      "mean_token_accuracy": 0.8941126763820648,
+      "loss": 4.112149715423584,
+      "mean_token_accuracy": 0.8909876793622971,
       "num_tokens": 87292.0,
       "step": 35
     },
     {
-      "entropy": 0.10329670459032059,
+      "entropy": 0.11223667673766613,
       "epoch": 0.7659574468085106,
-      "grad_norm": 0.2738026976585388,
+      "grad_norm": 0.4776930809020996,
       "learning_rate": 0.00014897757402062284,
-      "loss": 4.029165267944336,
-      "mean_token_accuracy": 0.8953875005245209,
+      "loss": 4.026741027832031,
+      "mean_token_accuracy": 0.8924113065004349,
       "num_tokens": 89830.0,
       "step": 36
     },
     {
-      "entropy": 0.11790071427822113,
+      "entropy": 0.12275257520377636,
       "epoch": 0.7872340425531915,
-      "grad_norm": 0.381307452917099,
+      "grad_norm": 0.9828954935073853,
       "learning_rate": 0.00014587018322180905,
-      "loss": 4.153435707092285,
-      "mean_token_accuracy": 0.8958049416542053,
+      "loss": 4.183655738830566,
+      "mean_token_accuracy": 0.8827805668115616,
       "num_tokens": 92344.0,
       "step": 37
     },
     {
-      "entropy": 0.10945623554289341,
+      "entropy": 0.12380701676011086,
       "epoch": 0.8085106382978723,
-      "grad_norm": 1.1360689401626587,
+      "grad_norm": 1.3710100650787354,
       "learning_rate": 0.0001427056438866376,
-      "loss": 4.012265205383301,
-      "mean_token_accuracy": 0.8903843760490417,
+      "loss": 4.003378868103027,
+      "mean_token_accuracy": 0.8812988549470901,
       "num_tokens": 94826.0,
       "step": 38
     },
     {
-      "entropy": 0.10652521438896656,
+      "entropy": 0.10968485288321972,
       "epoch": 0.8297872340425532,
-      "grad_norm": 1.8644981384277344,
+      "grad_norm": 0.37869715690612793,
       "learning_rate": 0.00013948789863705912,
-      "loss": 4.101892471313477,
-      "mean_token_accuracy": 0.8909918814897537,
+      "loss": 4.050682067871094,
+      "mean_token_accuracy": 0.8941126763820648,
       "num_tokens": 97290.0,
       "step": 39
     },
     {
-      "entropy": 0.1185864806175232,
+      "entropy": 0.12006617523729801,
       "epoch": 0.851063829787234,
-      "grad_norm": 0.35084277391433716,
+      "grad_norm": 0.5866818428039551,
       "learning_rate": 0.00013622095638296826,
-      "loss": 4.115968704223633,
-      "mean_token_accuracy": 0.8960161507129669,
+      "loss": 4.113466262817383,
+      "mean_token_accuracy": 0.8894326388835907,
       "num_tokens": 99776.0,
       "step": 40
     },
     {
-      "entropy": 0.10158661007881165,
+      "entropy": 0.09959963150322437,
       "epoch": 0.8723404255319149,
-      "grad_norm": 0.14153917133808136,
+      "grad_norm": 0.4211389422416687,
       "learning_rate": 0.000132908887327601,
-      "loss": 3.8947978019714355,
-      "mean_token_accuracy": 0.9017689526081085,
+      "loss": 3.8911352157592773,
+      "mean_token_accuracy": 0.8987201750278473,
       "num_tokens": 102278.0,
       "step": 41
     },
     {
-      "entropy": 0.1076634582132101,
+      "entropy": 0.10989912413060665,
       "epoch": 0.8936170212765957,
-      "grad_norm": 1.0466699600219727,
+      "grad_norm": 1.014770746231079,
       "learning_rate": 0.00012955581789656843,
-      "loss": 4.078176021575928,
-      "mean_token_accuracy": 0.8884615302085876,
+      "loss": 4.067371845245361,
+      "mean_token_accuracy": 0.8916666656732559,
       "num_tokens": 104764.0,
       "step": 42
     },
     {
-      "entropy": 0.11152214743196964,
+      "entropy": 0.11506957560777664,
       "epoch": 0.9148936170212766,
-      "grad_norm": 0.7619017362594604,
+      "grad_norm": 0.4580586552619934,
       "learning_rate": 0.0001261659255968441,
-      "loss": 4.1144890785217285,
-      "mean_token_accuracy": 0.890312910079956,
+      "loss": 4.101069927215576,
+      "mean_token_accuracy": 0.8936023861169815,
       "num_tokens": 107201.0,
       "step": 43
     },
     {
-      "entropy": 0.10949222184717655,
+      "entropy": 0.10838168673217297,
       "epoch": 0.9361702127659575,
-      "grad_norm": 0.7719561457633972,
+      "grad_norm": 0.4166942536830902,
       "learning_rate": 0.00012274343381211066,
-      "loss": 4.053421497344971,
-      "mean_token_accuracy": 0.8914329707622528,
+      "loss": 4.025703430175781,
+      "mean_token_accuracy": 0.8947224318981171,
       "num_tokens": 109660.0,
       "step": 44
     },
     {
-      "entropy": 0.10472714714705944,
+      "entropy": 0.09614014439284801,
       "epoch": 0.9574468085106383,
-      "grad_norm": 0.6925389766693115,
+      "grad_norm": 0.9769757390022278,
       "learning_rate": 0.00011929260654094969,
-      "loss": 4.108519554138184,
-      "mean_token_accuracy": 0.893522247672081,
+      "loss": 4.121439456939697,
+      "mean_token_accuracy": 0.8903171271085739,
       "num_tokens": 112117.0,
       "step": 45
     },
     {
-      "entropy": 0.10658742487430573,
+      "entropy": 0.11138966307044029,
       "epoch": 0.9787234042553191,
-      "grad_norm": 0.8806354999542236,
+      "grad_norm": 1.5290015935897827,
       "learning_rate": 0.0001158177430844304,
-      "loss": 4.042045593261719,
-      "mean_token_accuracy": 0.8898036479949951,
+      "loss": 4.022660732269287,
+      "mean_token_accuracy": 0.8867548704147339,
       "num_tokens": 114595.0,
       "step": 46
     },
     {
-      "entropy": 0.10867075435817242,
+      "entropy": 0.10535211116075516,
       "epoch": 1.0,
-      "grad_norm": 1.0520161390304565,
+      "grad_norm": 0.6933005452156067,
       "learning_rate": 0.00011232317268971585,
-      "loss": 4.1521220207214355,
-      "mean_token_accuracy": 0.8863234221935272,
+      "loss": 4.150263786315918,
+      "mean_token_accuracy": 0.8926616609096527,
       "num_tokens": 117080.0,
       "step": 47
     },
@@ -485,477 +485,477 @@
       "eval_loss": NaN,
       "eval_mean_token_accuracy": 0.0,
       "eval_num_tokens": 117080.0,
-      "eval_runtime": 17.3755,
-      "eval_samples_per_second": 17.266,
-      "eval_steps_per_second": 4.316,
+      "eval_runtime": 14.1972,
+      "eval_samples_per_second": 21.131,
+      "eval_steps_per_second": 5.283,
       "step": 47
     },
     {
-      "entropy": 0.11232210509479046,
+      "entropy": 0.10801766626536846,
       "epoch": 1.0212765957446808,
-      "grad_norm": 0.7298098206520081,
+      "grad_norm": 0.7822345495223999,
       "learning_rate": 0.00010881324915636019,
-      "loss": 4.093656539916992,
-      "mean_token_accuracy": 0.8939774483442307,
+      "loss": 4.09091854095459,
+      "mean_token_accuracy": 0.8906879872083664,
       "num_tokens": 119621.0,
       "step": 48
     },
     {
-      "entropy": 0.10608281195163727,
+      "entropy": 0.09913330711424351,
       "epoch": 1.0425531914893618,
-      "grad_norm": 1.3020367622375488,
+      "grad_norm": 0.39496442675590515,
       "learning_rate": 0.00010529234541201631,
-      "loss": 4.062341690063477,
-      "mean_token_accuracy": 0.8859083205461502,
+      "loss": 4.025051593780518,
+      "mean_token_accuracy": 0.8955279290676117,
       "num_tokens": 122130.0,
       "step": 49
     },
     {
-      "entropy": 0.11243193037807941,
+      "entropy": 0.1028544194996357,
       "epoch": 1.0638297872340425,
-      "grad_norm": 1.0893304347991943,
+      "grad_norm": 0.6361362338066101,
       "learning_rate": 0.00010176484806431288,
-      "loss": 3.958235740661621,
-      "mean_token_accuracy": 0.8887957185506821,
+      "loss": 3.9412522315979004,
+      "mean_token_accuracy": 0.8919207155704498,
       "num_tokens": 124637.0,
       "step": 50
     },
     {
-      "entropy": 0.12208659574389458,
+      "entropy": 0.11368882842361927,
       "epoch": 1.0851063829787233,
-      "grad_norm": 0.7100182175636292,
+      "grad_norm": 0.31298503279685974,
       "learning_rate": 9.823515193568715e-05,
-      "loss": 4.059011459350586,
-      "mean_token_accuracy": 0.8908425271511078,
+      "loss": 4.047340393066406,
+      "mean_token_accuracy": 0.8940476626157761,
       "num_tokens": 127112.0,
       "step": 51
     },
     {
-      "entropy": 0.12010342627763748,
+      "entropy": 0.10808366723358631,
       "epoch": 1.1063829787234043,
-      "grad_norm": 1.0296722650527954,
+      "grad_norm": 0.49920883774757385,
       "learning_rate": 9.470765458798368e-05,
-      "loss": 4.030320167541504,
-      "mean_token_accuracy": 0.8866378515958786,
+      "loss": 4.001613616943359,
+      "mean_token_accuracy": 0.889927327632904,
       "num_tokens": 129587.0,
       "step": 52
     },
     {
-      "entropy": 0.12237976677715778,
+      "entropy": 0.11089801043272018,
       "epoch": 1.127659574468085,
-      "grad_norm": 1.033589482307434,
+      "grad_norm": 0.46408575773239136,
       "learning_rate": 9.118675084363986e-05,
-      "loss": 4.0632524490356445,
-      "mean_token_accuracy": 0.8913581818342209,
+      "loss": 4.0368852615356445,
+      "mean_token_accuracy": 0.8947365581989288,
       "num_tokens": 132070.0,
       "step": 53
     },
     {
-      "entropy": 0.11414493061602116,
+      "entropy": 0.10656954161822796,
       "epoch": 1.148936170212766,
-      "grad_norm": 0.4377439320087433,
+      "grad_norm": 0.3796578049659729,
       "learning_rate": 8.767682731028415e-05,
-      "loss": 4.025282859802246,
-      "mean_token_accuracy": 0.8956041485071182,
+      "loss": 4.020212173461914,
+      "mean_token_accuracy": 0.8955279290676117,
       "num_tokens": 134579.0,
       "step": 54
     },
     {
-      "entropy": 0.12252005375921726,
+      "entropy": 0.11924762278795242,
       "epoch": 1.1702127659574468,
-      "grad_norm": 0.2526348829269409,
+      "grad_norm": 0.21246585249900818,
       "learning_rate": 8.418225691556962e-05,
-      "loss": 4.062685966491699,
+      "loss": 4.0544514656066895,
       "mean_token_accuracy": 0.8960863649845123,
       "num_tokens": 136943.0,
       "step": 55
     },
     {
-      "entropy": 0.12010006979107857,
+      "entropy": 0.11127430945634842,
       "epoch": 1.1914893617021276,
-      "grad_norm": 0.8727321624755859,
+      "grad_norm": 0.3929970860481262,
       "learning_rate": 8.070739345905032e-05,
-      "loss": 4.114907741546631,
-      "mean_token_accuracy": 0.8910819441080093,
+      "loss": 4.095489025115967,
+      "mean_token_accuracy": 0.8973319381475449,
       "num_tokens": 139492.0,
       "step": 56
     },
     {
-      "entropy": 0.11698324605822563,
+      "entropy": 0.11025936901569366,
       "epoch": 1.2127659574468086,
-      "grad_norm": 0.44399282336235046,
+      "grad_norm": 0.5039997696876526,
       "learning_rate": 7.725656618788937e-05,
-      "loss": 4.1000213623046875,
-      "mean_token_accuracy": 0.8869433104991913,
+      "loss": 4.091797828674316,
+      "mean_token_accuracy": 0.8902327716350555,
       "num_tokens": 141944.0,
       "step": 57
     },
     {
-      "entropy": 0.11351130343973637,
+      "entropy": 0.1125901136547327,
       "epoch": 1.2340425531914894,
-      "grad_norm": 0.4143531918525696,
+      "grad_norm": 0.23206305503845215,
       "learning_rate": 7.383407440315596e-05,
-      "loss": 4.076257705688477,
-      "mean_token_accuracy": 0.8947127312421799,
+      "loss": 4.056682586669922,
+      "mean_token_accuracy": 0.8979178667068481,
       "num_tokens": 144392.0,
       "step": 58
     },
     {
-      "entropy": 0.10903612524271011,
+      "entropy": 0.10212262719869614,
       "epoch": 1.2553191489361701,
-      "grad_norm": 0.19821909070014954,
+      "grad_norm": 0.05287244915962219,
       "learning_rate": 7.04441821034316e-05,
-      "loss": 3.9583330154418945,
+      "loss": 3.9536325931549072,
       "mean_token_accuracy": 0.899132177233696,
       "num_tokens": 146822.0,
       "step": 59
     },
     {
-      "entropy": 0.10224357433617115,
+      "entropy": 0.10311507619917393,
       "epoch": 1.2765957446808511,
-      "grad_norm": 0.13308529555797577,
+      "grad_norm": 0.22314538061618805,
       "learning_rate": 6.7091112672399e-05,
-      "loss": 4.060159683227539,
-      "mean_token_accuracy": 0.8979781568050385,
+      "loss": 4.06229829788208,
+      "mean_token_accuracy": 0.894688680768013,
       "num_tokens": 149381.0,
       "step": 60
     },
     {
-      "entropy": 0.11792953498661518,
+      "entropy": 0.11741508357226849,
       "epoch": 1.297872340425532,
-      "grad_norm": 1.1922239065170288,
+      "grad_norm": 0.8379692435264587,
       "learning_rate": 6.377904361703178e-05,
-      "loss": 4.162205696105957,
-      "mean_token_accuracy": 0.8798677921295166,
+      "loss": 4.129307270050049,
+      "mean_token_accuracy": 0.8864650130271912,
       "num_tokens": 151872.0,
       "step": 61
     },
     {
-      "entropy": 0.10444029048085213,
+      "entropy": 0.10069144517183304,
       "epoch": 1.3191489361702127,
-      "grad_norm": 0.0910131111741066,
+      "grad_norm": 0.07896701246500015,
       "learning_rate": 6.051210136294089e-05,
-      "loss": 3.9983675479888916,
+      "loss": 3.9954659938812256,
       "mean_token_accuracy": 0.8986529260873795,
       "num_tokens": 154366.0,
       "step": 62
     },
     {
-      "entropy": 0.10578916594386101,
+      "entropy": 0.10588889196515083,
       "epoch": 1.3404255319148937,
-      "grad_norm": 0.4810399115085602,
+      "grad_norm": 0.37503722310066223,
       "learning_rate": 5.729435611336239e-05,
-      "loss": 4.0733442306518555,
+      "loss": 4.065610885620117,
       "mean_token_accuracy": 0.8914495259523392,
       "num_tokens": 156920.0,
       "step": 63
     },
     {
-      "entropy": 0.11232149787247181,
+      "entropy": 0.10409239679574966,
       "epoch": 1.3617021276595744,
-      "grad_norm": 0.6053599715232849,
+      "grad_norm": 0.33657321333885193,
       "learning_rate": 5.4129816778190936e-05,
-      "loss": 3.9595446586608887,
-      "mean_token_accuracy": 0.8930288255214691,
+      "loss": 3.9452602863311768,
+      "mean_token_accuracy": 0.8961538225412369,
       "num_tokens": 159260.0,
       "step": 64
     },
     {
-      "entropy": 0.1037730760872364,
+      "entropy": 0.10675412602722645,
       "epoch": 1.3829787234042552,
-      "grad_norm": 0.025716669857501984,
+      "grad_norm": 0.23680685460567474,
       "learning_rate": 5.102242597937717e-05,
-      "loss": 4.07431697845459,
+      "loss": 4.065974235534668,
       "mean_token_accuracy": 0.8973371386528015,
       "num_tokens": 161698.0,
       "step": 65
     },
     {
-      "entropy": 0.11597166769206524,
+      "entropy": 0.11763200350105762,
       "epoch": 1.4042553191489362,
-      "grad_norm": 0.5439313054084778,
+      "grad_norm": 0.48635369539260864,
       "learning_rate": 4.797605513891179e-05,
-      "loss": 4.174406051635742,
-      "mean_token_accuracy": 0.8888326436281204,
+      "loss": 4.164796829223633,
+      "mean_token_accuracy": 0.8921221196651459,
       "num_tokens": 164193.0,
       "step": 66
     },
     {
-      "entropy": 0.10735544376075268,
+      "entropy": 0.09798314236104488,
       "epoch": 1.425531914893617,
-      "grad_norm": 0.4331194758415222,
+      "grad_norm": 0.14462092518806458,
       "learning_rate": 4.4994499655515865e-05,
-      "loss": 4.058930397033691,
-      "mean_token_accuracy": 0.8946283906698227,
+      "loss": 4.042727947235107,
+      "mean_token_accuracy": 0.8979178667068481,
       "num_tokens": 166683.0,
       "step": 67
     },
     {
-      "entropy": 0.11135114170610905,
+      "entropy": 0.10395899415016174,
       "epoch": 1.4468085106382977,
-      "grad_norm": 0.16449232399463654,
+      "grad_norm": 0.06169198080897331,
       "learning_rate": 4.2081474176046646e-05,
-      "loss": 4.138002872467041,
+      "loss": 4.123526573181152,
       "mean_token_accuracy": 0.8957811146974564,
       "num_tokens": 169151.0,
       "step": 68
     },
     {
-      "entropy": 0.10969364643096924,
+      "entropy": 0.10421585291624069,
       "epoch": 1.4680851063829787,
-      "grad_norm": 0.15414418280124664,
+      "grad_norm": 1.1794296503067017,
       "learning_rate": 3.924060796751012e-05,
-      "loss": 4.1435699462890625,
-      "mean_token_accuracy": 0.8953413814306259,
+      "loss": 4.157026767730713,
+      "mean_token_accuracy": 0.888762429356575,
       "num_tokens": 171681.0,
       "step": 69
     },
     {
-      "entropy": 0.10336218029260635,
+      "entropy": 0.09568466432392597,
       "epoch": 1.4893617021276595,
-      "grad_norm": 0.1289367526769638,
+      "grad_norm": 0.02028118632733822,
       "learning_rate": 3.647544039544615e-05,
-      "loss": 4.0548601150512695,
+      "loss": 4.053896427154541,
       "mean_token_accuracy": 0.8979416936635971,
       "num_tokens": 174203.0,
       "step": 70
     },
     {
-      "entropy": 0.10815661773085594,
+      "entropy": 0.10582420416176319,
       "epoch": 1.5106382978723403,
-      "grad_norm": 0.15281076729297638,
+      "grad_norm": 0.1698542982339859,
       "learning_rate": 3.378941651431996e-05,
-      "loss": 4.0012993812561035,
+      "loss": 3.990485191345215,
       "mean_token_accuracy": 0.898457407951355,
       "num_tokens": 176621.0,
       "step": 71
     },
     {
-      "entropy": 0.11221517436206341,
+      "entropy": 0.1072782352566719,
       "epoch": 1.5319148936170213,
-      "grad_norm": 0.475345641374588,
+      "grad_norm": 0.6851707100868225,
       "learning_rate": 3.118588277541312e-05,
-      "loss": 4.170467376708984,
-      "mean_token_accuracy": 0.891996756196022,
+      "loss": 4.162285327911377,
+      "mean_token_accuracy": 0.8921699970960617,
       "num_tokens": 179126.0,
       "step": 72
     },
     {
-      "entropy": 0.10897163301706314,
+      "entropy": 0.10387748293578625,
       "epoch": 1.5531914893617023,
-      "grad_norm": 0.2501252293586731,
+      "grad_norm": 0.37178531289100647,
       "learning_rate": 2.8668082857562005e-05,
-      "loss": 4.078862190246582,
-      "mean_token_accuracy": 0.8966623693704605,
+      "loss": 4.084682464599609,
+      "mean_token_accuracy": 0.8933728933334351,
       "num_tokens": 181650.0,
       "step": 73
     },
     {
-      "entropy": 0.10291661694645882,
+      "entropy": 0.10093368589878082,
       "epoch": 1.574468085106383,
-      "grad_norm": 0.03701994568109512,
+      "grad_norm": 0.03769727796316147,
       "learning_rate": 2.6239153625937784e-05,
-      "loss": 4.060463905334473,
+      "loss": 4.045993804931641,
       "mean_token_accuracy": 0.8973006755113602,
       "num_tokens": 184077.0,
       "step": 74
     },
     {
-      "entropy": 0.10492184944450855,
+      "entropy": 0.09933018498122692,
       "epoch": 1.5957446808510638,
-      "grad_norm": 0.1324034184217453,
+      "grad_norm": 0.06748200207948685,
       "learning_rate": 2.390212122390323e-05,
-      "loss": 4.004302024841309,
+      "loss": 3.9992191791534424,
       "mean_token_accuracy": 0.8992939442396164,
       "num_tokens": 186549.0,
       "step": 75
     },
     {
-      "entropy": 0.10788590088486671,
+      "entropy": 0.10636681690812111,
       "epoch": 1.6170212765957448,
-      "grad_norm": 0.06939677894115448,
+      "grad_norm": 0.21794308722019196,
       "learning_rate": 2.1659897302814747e-05,
-      "loss": 4.080732345581055,
+      "loss": 4.072402000427246,
       "mean_token_accuracy": 0.8967273831367493,
       "num_tokens": 189001.0,
       "step": 76
     },
     {
-      "entropy": 0.09719057753682137,
+      "entropy": 0.09489450231194496,
       "epoch": 1.6382978723404256,
-      "grad_norm": 0.25152677297592163,
+      "grad_norm": 0.20672070980072021,
       "learning_rate": 1.9515275394467446e-05,
-      "loss": 3.8840041160583496,
+      "loss": 3.8761157989501953,
       "mean_token_accuracy": 0.9024099707603455,
       "num_tokens": 191502.0,
       "step": 77
     },
     {
-      "entropy": 0.09232526086270809,
+      "entropy": 0.09246066771447659,
       "epoch": 1.6595744680851063,
-      "grad_norm": 0.18408416211605072,
+      "grad_norm": 0.2982085943222046,
       "learning_rate": 1.7470927430702277e-05,
-      "loss": 4.008336067199707,
+      "loss": 4.011519432067871,
       "mean_token_accuracy": 0.8992336541414261,
       "num_tokens": 194063.0,
       "step": 78
     },
     {
-      "entropy": 0.09508595243096352,
+      "entropy": 0.09505276940762997,
       "epoch": 1.6808510638297873,
-      "grad_norm": 0.13421863317489624,
+      "grad_norm": 0.41892215609550476,
       "learning_rate": 1.5529400414511806e-05,
-      "loss": 3.9353599548339844,
-      "mean_token_accuracy": 0.9011882245540619,
+      "loss": 3.942739725112915,
+      "mean_token_accuracy": 0.8950906693935394,
       "num_tokens": 196634.0,
       "step": 79
     },
     {
-      "entropy": 0.10699102841317654,
+      "entropy": 0.10412091203033924,
       "epoch": 1.702127659574468,
-      "grad_norm": 0.6064008474349976,
+      "grad_norm": 0.6482428312301636,
       "learning_rate": 1.3693113246791589e-05,
-      "loss": 4.157015323638916,
+      "loss": 4.158843517303467,
       "mean_token_accuracy": 0.8888326436281204,
       "num_tokens": 199099.0,
       "step": 80
     },
     {
-      "entropy": 0.09753298759460449,
+      "entropy": 0.09425530955195427,
       "epoch": 1.7234042553191489,
-      "grad_norm": 0.2060132771730423,
+      "grad_norm": 0.07691552489995956,
       "learning_rate": 1.196435371269089e-05,
-      "loss": 4.060751438140869,
+      "loss": 4.058327674865723,
       "mean_token_accuracy": 0.8973006755113602,
       "num_tokens": 201577.0,
       "step": 81
     },
     {
-      "entropy": 0.10865146666765213,
+      "entropy": 0.10262792184948921,
       "epoch": 1.7446808510638299,
-      "grad_norm": 0.8439809083938599,
+      "grad_norm": 0.8844386339187622,
       "learning_rate": 1.0345275631317163e-05,
-      "loss": 4.154847145080566,
-      "mean_token_accuracy": 0.8925727605819702,
+      "loss": 4.1612653732299805,
+      "mean_token_accuracy": 0.8891943842172623,
       "num_tokens": 204137.0,
       "step": 82
     },
     {
-      "entropy": 0.10184266604483128,
+      "entropy": 0.09942430071532726,
       "epoch": 1.7659574468085106,
-      "grad_norm": 0.6061291694641113,
+      "grad_norm": 0.3644232749938965,
       "learning_rate": 8.837896172345827e-06,
-      "loss": 4.028115272521973,
+      "loss": 4.026100158691406,
       "mean_token_accuracy": 0.8954477906227112,
       "num_tokens": 206717.0,
       "step": 83
     },
     {
-      "entropy": 0.10267733037471771,
+      "entropy": 0.0988146997988224,
       "epoch": 1.7872340425531914,
-      "grad_norm": 0.33293449878692627,
+      "grad_norm": 0.2477438896894455,
       "learning_rate": 7.4440933428779e-06,
-      "loss": 3.871464490890503,
-      "mean_token_accuracy": 0.8987042456865311,
+      "loss": 3.8571834564208984,
+      "mean_token_accuracy": 0.9018292427062988,
       "num_tokens": 209169.0,
       "step": 84
     },
     {
-      "entropy": 0.1105765737593174,
+      "entropy": 0.10586683638393879,
       "epoch": 1.8085106382978724,
-      "grad_norm": 0.4586741626262665,
+      "grad_norm": 0.26259544491767883,
       "learning_rate": 6.165603647677054e-06,
-      "loss": 3.979715347290039,
+      "loss": 3.961254358291626,
       "mean_token_accuracy": 0.8967323154211044,
       "num_tokens": 211565.0,
       "step": 85
     },
     {
-      "entropy": 0.10341055132448673,
+      "entropy": 0.10182979516685009,
       "epoch": 1.8297872340425532,
-      "grad_norm": 0.05060029402375221,
+      "grad_norm": 0.1101575642824173,
       "learning_rate": 5.00401992570092e-06,
-      "loss": 4.136866569519043,
+      "loss": 4.137490272521973,
       "mean_token_accuracy": 0.89537513256073,
       "num_tokens": 214053.0,
       "step": 86
     },
     {
-      "entropy": 0.11312008649110794,
+      "entropy": 0.10845262557268143,
       "epoch": 1.851063829787234,
-      "grad_norm": 0.10508228838443756,
+      "grad_norm": 0.048552438616752625,
       "learning_rate": 3.960789365622075e-06,
-      "loss": 4.2039642333984375,
+      "loss": 4.191901206970215,
       "mean_token_accuracy": 0.8933143615722656,
       "num_tokens": 216498.0,
       "step": 87
     },
     {
-      "entropy": 0.10082121007144451,
+      "entropy": 0.10030228458344936,
       "epoch": 1.872340425531915,
-      "grad_norm": 0.837336540222168,
+      "grad_norm": 0.3250134289264679,
       "learning_rate": 3.0372117028111825e-06,
-      "loss": 4.1200056076049805,
-      "mean_token_accuracy": 0.8876234292984009,
+      "loss": 4.097182750701904,
+      "mean_token_accuracy": 0.8939535617828369,
       "num_tokens": 218982.0,
       "step": 88
     },
     {
-      "entropy": 0.1132755447179079,
+      "entropy": 0.11286174692213535,
       "epoch": 1.8936170212765957,
-      "grad_norm": 0.7579889893531799,
+      "grad_norm": 0.7210190892219543,
       "learning_rate": 2.2344376000285604e-06,
-      "loss": 4.0363874435424805,
-      "mean_token_accuracy": 0.891628697514534,
+      "loss": 4.022359848022461,
+      "mean_token_accuracy": 0.8914642333984375,
       "num_tokens": 221444.0,
       "step": 89
     },
     {
-      "entropy": 0.10484139621257782,
+      "entropy": 0.10275478102266788,
       "epoch": 1.9148936170212765,
-      "grad_norm": 0.2707573473453522,
+      "grad_norm": 0.3212966322898865,
       "learning_rate": 1.553467213841664e-06,
-      "loss": 4.049415111541748,
-      "mean_token_accuracy": 0.8979416936635971,
+      "loss": 4.045522689819336,
+      "mean_token_accuracy": 0.8947365581989288,
       "num_tokens": 223969.0,
       "step": 90
     },
     {
-      "entropy": 0.1005912534892559,
+      "entropy": 0.10150141455233097,
       "epoch": 1.9361702127659575,
-      "grad_norm": 0.14874200522899628,
+      "grad_norm": 0.23617830872535706,
       "learning_rate": 9.951489485545695e-07,
-      "loss": 4.040111541748047,
+      "loss": 4.035382270812988,
       "mean_token_accuracy": 0.8986529260873795,
       "num_tokens": 226520.0,
       "step": 91
     },
     {
-      "entropy": 0.10434216633439064,
+      "entropy": 0.10262680239975452,
       "epoch": 1.9574468085106385,
-      "grad_norm": 0.5901132225990295,
+      "grad_norm": 0.4545777440071106,
       "learning_rate": 5.60178399201805e-07,
-      "loss": 4.092916965484619,
-      "mean_token_accuracy": 0.8905990719795227,
+      "loss": 4.09911584854126,
+      "mean_token_accuracy": 0.8873096108436584,
       "num_tokens": 229047.0,
       "step": 92
     },
     {
-      "entropy": 0.1012870017439127,
+      "entropy": 0.09875928983092308,
       "epoch": 1.978723404255319,
-      "grad_norm": 0.0976901575922966,
+      "grad_norm": 0.07097365707159042,
       "learning_rate": 2.490974849236216e-07,
-      "loss": 4.024156093597412,
+      "loss": 4.016970157623291,
       "mean_token_accuracy": 0.8986529260873795,
       "num_tokens": 231567.0,
       "step": 93
     },
     {
-      "entropy": 0.09745646640658379,
+      "entropy": 0.09762230329215527,
       "epoch": 2.0,
-      "grad_norm": 0.4953719973564148,
+      "grad_norm": 0.3958527445793152,
       "learning_rate": 6.229377380218005e-08,
-      "loss": 4.083987236022949,
+      "loss": 4.0623064041137695,
       "mean_token_accuracy": 0.894871786236763,
       "num_tokens": 234138.0,
       "step": 94
@@ -966,9 +966,9 @@
       "eval_loss": NaN,
       "eval_mean_token_accuracy": 0.0,
       "eval_num_tokens": 234138.0,
-      "eval_runtime": 16.1248,
-      "eval_samples_per_second": 18.605,
-      "eval_steps_per_second": 4.651,
+      "eval_runtime": 13.8638,
+      "eval_samples_per_second": 21.639,
+      "eval_steps_per_second": 5.41,
       "step": 94
     }
   ],
@@ -998,7 +998,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 393994475667456.0,
+  "total_flos": 392624361701376.0,
   "train_batch_size": 2,
   "trial_name": null,
   "trial_params": null

checkpoint-94/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0a5df61c66afc72c9c2f3e6ebf29ce0723f79ac1bd71f5ce8bf3abd116d58ac6
+oid sha256:49a5fd350f76aacdefac1c174b863cb237ced8dddcf5626a48a21ab11428ac63
 size 5496

predictions.jsonl

@@ -60,7 +60,7 @@
 {"messages": [{"role": "user", "content": "What ATT&CK technique does the following CTI procedure sentence describe?\n\n<cti>\nFELIXROOT adds a shortcut file to the startup folder for persistence.\n</cti>"}], "ground_truth": "T1547.001", "generated_content": "T1547.001"}
 {"messages": [{"role": "user", "content": "What ATT&CK technique does the following CTI procedure sentence describe?\n\n<cti>\nEmotet has used a self-extracting RAR file to deliver modules to victims. Emotet has also extracted embedded executables from files using hard-coded buffer offsets.\n</cti>"}], "ground_truth": "T1140", "generated_content": "T1140"}
 {"messages": [{"role": "user", "content": "What ATT&CK technique does the following CTI procedure sentence describe?\n\n<cti>\nAPT37's Freenki malware lists running processes using the Microsoft Windows API.\n</cti>"}], "ground_truth": "T1057", "generated_content": "T1057"}
-{"messages": [{"role": "user", "content": "What ATT&CK technique does the following CTI procedure sentence describe?\n\n<cti>\nINC Ransomware can use the Microsoft Win32 Restart Manager to kill processes with a specific handle or that are accessing resources it wants to encrypt.\n</cti>"}], "ground_truth": "T1057", "generated_content": "T1070.004"}
+{"messages": [{"role": "user", "content": "What ATT&CK technique does the following CTI procedure sentence describe?\n\n<cti>\nINC Ransomware can use the Microsoft Win32 Restart Manager to kill processes with a specific handle or that are accessing resources it wants to encrypt.\n</cti>"}], "ground_truth": "T1057", "generated_content": "T1059.003"}
 {"messages": [{"role": "user", "content": "What ATT&CK technique does the following CTI procedure sentence describe?\n\n<cti>\nHAFNIUM has used `tasklist` to enumerate processes.\n</cti>"}], "ground_truth": "T1057", "generated_content": "T1057"}
 {"messages": [{"role": "user", "content": "What ATT&CK technique does the following CTI procedure sentence describe?\n\n<cti>\nOne persistence mechanism used by CozyCar is to set itself to be executed at system startup by adding a Registry value under one of the following Registry keys: <br><code>HKLM\\Software\\Microsoft\\Windows\\CurrentVersion\\Run\\</code> <br><code>HKCU\\Software\\Microsoft\\Windows\\CurrentVersion\\Run\\</code> <br><code>HKLM\\Software\\Microsoft\\Windows\\CurrentVersion\\Policies\\Explorer\\Run</code> <br><code>HKCU\\Software\\Microsoft\\Windows\\CurrentVersion\\Policies\\Explorer\\Run</code>\n</cti>"}], "ground_truth": "T1547.001", "generated_content": "T1547.001"}
 {"messages": [{"role": "user", "content": "What ATT&CK technique does the following CTI procedure sentence describe?\n\n<cti>\nCORESHELL downloads another dropper from its C2 server.\n</cti>"}], "ground_truth": "T1105", "generated_content": "T1105"}
@@ -126,7 +126,7 @@
 {"messages": [{"role": "user", "content": "What ATT&CK technique does the following CTI procedure sentence describe?\n\n<cti>\nCosmicDuke can use HTTP or HTTPS for command and control to hard-coded C2 servers.\n</cti>"}], "ground_truth": "T1071.001", "generated_content": "T1071.001"}
 {"messages": [{"role": "user", "content": "What ATT&CK technique does the following CTI procedure sentence describe?\n\n<cti>\nCardinal RAT can download and execute additional payloads.\n</cti>"}], "ground_truth": "T1105", "generated_content": "T1105"}
 {"messages": [{"role": "user", "content": "What ATT&CK technique does the following CTI procedure sentence describe?\n\n<cti>\nDaggerfly utilizes victim machine operating system information to create custom User Agent strings for subsequent command and control communication.\n</cti>"}], "ground_truth": "T1082", "generated_content": "T1082"}
-{"messages": [{"role": "user", "content": "What ATT&CK technique does the following CTI procedure sentence describe?\n\n<cti>\nThe IceApple Server Variable Dumper module iterates over all server variables present for the current request and returns them to the adversary.\n</cti>"}], "ground_truth": "T1082", "generated_content": "T1083"}
+{"messages": [{"role": "user", "content": "What ATT&CK technique does the following CTI procedure sentence describe?\n\n<cti>\nThe IceApple Server Variable Dumper module iterates over all server variables present for the current request and returns them to the adversary.\n</cti>"}], "ground_truth": "T1082", "generated_content": "T1082"}
 {"messages": [{"role": "user", "content": "What ATT&CK technique does the following CTI procedure sentence describe?\n\n<cti>\nChChes collects the victim hostname, window resolution, and Microsoft Windows version.\n</cti>"}], "ground_truth": "T1082", "generated_content": "T1082"}
 {"messages": [{"role": "user", "content": "What ATT&CK technique does the following CTI procedure sentence describe?\n\n<cti>\nIPsec Helper connects to command and control servers via HTTP POST requests based on parameters hard-coded into the malware.\n</cti>"}], "ground_truth": "T1071.001", "generated_content": "T1071.001"}
 {"messages": [{"role": "user", "content": "What ATT&CK technique does the following CTI procedure sentence describe?\n\n<cti>\nDaserf uses HTTP for C2.\n</cti>"}], "ground_truth": "T1071.001", "generated_content": "T1071.001"}
@@ -141,7 +141,7 @@
 {"messages": [{"role": "user", "content": "What ATT&CK technique does the following CTI procedure sentence describe?\n\n<cti>\nAppleJeus has collected the victim host information after infection.\n</cti>"}], "ground_truth": "T1082", "generated_content": "T1082"}
 {"messages": [{"role": "user", "content": "What ATT&CK technique does the following CTI procedure sentence describe?\n\n<cti>\nFIN8 has used HTTPS for command and control.\n</cti>"}], "ground_truth": "T1071.001", "generated_content": "T1071.001"}
 {"messages": [{"role": "user", "content": "What ATT&CK technique does the following CTI procedure sentence describe?\n\n<cti>\nFIN7 has downloaded additional malware to execute on the victim's machine, including by using a PowerShell script to launch shellcode that retrieves an additional payload.\n</cti>"}], "ground_truth": "T1105", "generated_content": "T1059.003"}
-{"messages": [{"role": "user", "content": "What ATT&CK technique does the following CTI procedure sentence describe?\n\n<cti>\nDEADEYE can run `cmd /c copy /y /b C:\\Users\\public\\syslog_6-*.dat C:\\Users\\public\\syslog.dll` to combine separated sections of code into a single DLL prior to execution.\n</cti>"}], "ground_truth": "T1059.003", "generated_content": "T1105"}
+{"messages": [{"role": "user", "content": "What ATT&CK technique does the following CTI procedure sentence describe?\n\n<cti>\nDEADEYE can run `cmd /c copy /y /b C:\\Users\\public\\syslog_6-*.dat C:\\Users\\public\\syslog.dll` to combine separated sections of code into a single DLL prior to execution.\n</cti>"}], "ground_truth": "T1059.003", "generated_content": "T1059.003"}
 {"messages": [{"role": "user", "content": "What ATT&CK technique does the following CTI procedure sentence describe?\n\n<cti>\nAPT41 used <code>cmd.exe /c</code> to execute commands on remote machines. APT41 used a batch file to install persistence for the Cobalt Strike BEACON loader.\n</cti>"}], "ground_truth": "T1059.003", "generated_content": "T1059.003"}
 {"messages": [{"role": "user", "content": "What ATT&CK technique does the following CTI procedure sentence describe?\n\n<cti>\nKV Botnet Activity removes on-disk copies of tools and other artifacts after it the primary botnet payload has been loaded into memory on the victim device.\n</cti>"}], "ground_truth": "T1070.004", "generated_content": "T1070.004"}
 {"messages": [{"role": "user", "content": "What ATT&CK technique does the following CTI procedure sentence describe?\n\n<cti>\nBrute Ratel C4 can enumerate all processes and locate specific process IDs .\n</cti>"}], "ground_truth": "T1057", "generated_content": "T1057"}
@@ -176,7 +176,7 @@
 {"messages": [{"role": "user", "content": "What ATT&CK technique does the following CTI procedure sentence describe?\n\n<cti>\nExplosive has used HTTP for communication.\n</cti>"}], "ground_truth": "T1071.001", "generated_content": "T1071.001"}
 {"messages": [{"role": "user", "content": "What ATT&CK technique does the following CTI procedure sentence describe?\n\n<cti>\nTurla has used shellcode to download Meterpreter after compromising a victim.\n</cti>"}], "ground_truth": "T1105", "generated_content": "T1105"}
 {"messages": [{"role": "user", "content": "What ATT&CK technique does the following CTI procedure sentence describe?\n\n<cti>\nCardinal RAT can execute commands.\n</cti>"}], "ground_truth": "T1059.003", "generated_content": "T1059.003"}
-{"messages": [{"role": "user", "content": "What ATT&CK technique does the following CTI procedure sentence describe?\n\n<cti>\nDragonfly has used batch scripts to enumerate network information, including information about trusts, zones, and the domain.\n</cti>"}], "ground_truth": "T1016", "generated_content": "T1083"}
+{"messages": [{"role": "user", "content": "What ATT&CK technique does the following CTI procedure sentence describe?\n\n<cti>\nDragonfly has used batch scripts to enumerate network information, including information about trusts, zones, and the domain.\n</cti>"}], "ground_truth": "T1016", "generated_content": "T1082"}
 {"messages": [{"role": "user", "content": "What ATT&CK technique does the following CTI procedure sentence describe?\n\n<cti>\nWinter Vivern delivered exploit payloads via base64-encoded payloads in malicious email messages.\n</cti>"}], "ground_truth": "T1140", "generated_content": "T1140"}
 {"messages": [{"role": "user", "content": "What ATT&CK technique does the following CTI procedure sentence describe?\n\n<cti>\nBundlore has used <code>openssl</code> to decrypt AES encrypted payload data. Bundlore has also used base64 and RC4 with a hardcoded key to deobfuscate data.\n</cti>"}], "ground_truth": "T1140", "generated_content": "T1140"}
 {"messages": [{"role": "user", "content": "What ATT&CK technique does the following CTI procedure sentence describe?\n\n<cti>\nRocke has deleted files on infected machines.\n</cti>"}], "ground_truth": "T1070.004", "generated_content": "T1070.004"}
@@ -244,7 +244,7 @@
 {"messages": [{"role": "user", "content": "What ATT&CK technique does the following CTI procedure sentence describe?\n\n<cti>\nMoonstone Sleet has gathered information on victim systems.\n</cti>"}], "ground_truth": "T1082", "generated_content": "T1082"}
 {"messages": [{"role": "user", "content": "What ATT&CK technique does the following CTI procedure sentence describe?\n\n<cti>\nAPT3 places scripts in the startup folder for persistence.\n</cti>"}], "ground_truth": "T1547.001", "generated_content": "T1547.001"}
 {"messages": [{"role": "user", "content": "What ATT&CK technique does the following CTI procedure sentence describe?\n\n<cti>\nDUSTTRAP can execute commands via `cmd.exe`.\n</cti>"}], "ground_truth": "T1059.003", "generated_content": "T1059.003"}
-{"messages": [{"role": "user", "content": "What ATT&CK technique does the following CTI procedure sentence describe?\n\n<cti>\nGravityRAT collects the MAC address, computer name, and CPU information.\n</cti>"}], "ground_truth": "T1082", "generated_content": "T1082"}
+{"messages": [{"role": "user", "content": "What ATT&CK technique does the following CTI procedure sentence describe?\n\n<cti>\nGravityRAT collects the MAC address, computer name, and CPU information.\n</cti>"}], "ground_truth": "T1082", "generated_content": "T1016"}
 {"messages": [{"role": "user", "content": "What ATT&CK technique does the following CTI procedure sentence describe?\n\n<cti>\nGazer communicates with its C2 servers over HTTP.\n</cti>"}], "ground_truth": "T1071.001", "generated_content": "T1071.001"}
 {"messages": [{"role": "user", "content": "What ATT&CK technique does the following CTI procedure sentence describe?\n\n<cti>\nEarth Lusca has used certutil to decode a string into a cabinet file.\n</cti>"}], "ground_truth": "T1140", "generated_content": "T1140"}
 {"messages": [{"role": "user", "content": "What ATT&CK technique does the following CTI procedure sentence describe?\n\n<cti>\nGamaredon Group tools can delete files used during an operation.\n</cti>"}], "ground_truth": "T1070.004", "generated_content": "T1070.004"}
@@ -280,7 +280,7 @@
 {"messages": [{"role": "user", "content": "What ATT&CK technique does the following CTI procedure sentence describe?\n\n<cti>\nAria-body has used HTTP in C2 communications.\n</cti>"}], "ground_truth": "T1071.001", "generated_content": "T1071.001"}
 {"messages": [{"role": "user", "content": "What ATT&CK technique does the following CTI procedure sentence describe?\n\n<cti>\nCrimson has the ability to delete files from a compromised host.\n</cti>"}], "ground_truth": "T1070.004", "generated_content": "T1070.004"}
 {"messages": [{"role": "user", "content": "What ATT&CK technique does the following CTI procedure sentence describe?\n\n<cti>\nVarious implementations of CHOPSTICK communicate with C2 over HTTP.\n</cti>"}], "ground_truth": "T1071.001", "generated_content": "T1071.001"}
-{"messages": [{"role": "user", "content": "What ATT&CK technique does the following CTI procedure sentence describe?\n\n<cti>\nDuring its initial execution, BACKSPACE extracts operating system information from the infected host.\n</cti>"}], "ground_truth": "T1082", "generated_content": "T1082"}
+{"messages": [{"role": "user", "content": "What ATT&CK technique does the following CTI procedure sentence describe?\n\n<cti>\nDuring its initial execution, BACKSPACE extracts operating system information from the infected host.\n</cti>"}], "ground_truth": "T1082", "generated_content": "T1083"}
 {"messages": [{"role": "user", "content": "What ATT&CK technique does the following CTI procedure sentence describe?\n\n<cti>\nOne variant of CloudDuke uses HTTP and HTTPS for C2.\n</cti>"}], "ground_truth": "T1071.001", "generated_content": "T1071.001"}
 {"messages": [{"role": "user", "content": "What ATT&CK technique does the following CTI procedure sentence describe?\n\n<cti>\nDownPaper uses the command line.\n</cti>"}], "ground_truth": "T1059.003", "generated_content": "T1059.003"}
 {"messages": [{"role": "user", "content": "What ATT&CK technique does the following CTI procedure sentence describe?\n\n<cti>\nBackdoor.Oldrea collects information about running processes.\n</cti>"}], "ground_truth": "T1057", "generated_content": "T1057"}

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0a5df61c66afc72c9c2f3e6ebf29ce0723f79ac1bd71f5ce8bf3abd116d58ac6
+oid sha256:49a5fd350f76aacdefac1c174b863cb237ced8dddcf5626a48a21ab11428ac63
 size 5496