diff --git "a/trainer_state.json" "b/trainer_state.json"
deleted file mode 100644--- "a/trainer_state.json"
+++ /dev/null
@@ -1,18823 +0,0 @@
-{
-  "best_metric": null,
-  "best_model_checkpoint": null,
-  "epoch": 0.99996003996004,
-  "eval_steps": 10000,
-  "global_step": 12512,
-  "is_hyper_param_search": false,
-  "is_local_process_zero": true,
-  "is_world_process_zero": true,
-  "log_history": [
-    {
-      "epoch": 0.0007992007992007992,
-      "grad_norm": 10.772292137145996,
-      "learning_rate": 3.9936102236421726e-08,
-      "logits/chosen": -7.600287914276123,
-      "logits/rejected": -7.6064043045043945,
-      "logps/chosen": -38.066959381103516,
-      "logps/rejected": -39.09525680541992,
-      "loss": 0.9358,
-      "rewards/accuracies": 0.4375,
-      "rewards/chosen": 3.0866305828094482,
-      "rewards/margins": -0.1574566811323166,
-      "rewards/rejected": 3.2440872192382812,
-      "step": 10
-    },
-    {
-      "epoch": 0.0015984015984015984,
-      "grad_norm": 10.018013000488281,
-      "learning_rate": 7.987220447284345e-08,
-      "logits/chosen": -7.648618221282959,
-      "logits/rejected": -7.633522033691406,
-      "logps/chosen": -38.70015335083008,
-      "logps/rejected": -42.91053771972656,
-      "loss": 0.9398,
-      "rewards/accuracies": 0.4625000059604645,
-      "rewards/chosen": 3.2691116333007812,
-      "rewards/margins": -0.15530334413051605,
-      "rewards/rejected": 3.424414873123169,
-      "step": 20
-    },
-    {
-      "epoch": 0.0023976023976023976,
-      "grad_norm": 9.619175910949707,
-      "learning_rate": 1.198083067092652e-07,
-      "logits/chosen": -7.60400390625,
-      "logits/rejected": -7.530278205871582,
-      "logps/chosen": -39.2209358215332,
-      "logps/rejected": -42.06409454345703,
-      "loss": 0.8955,
-      "rewards/accuracies": 0.5,
-      "rewards/chosen": 3.0832536220550537,
-      "rewards/margins": -0.06541115790605545,
-      "rewards/rejected": 3.148664712905884,
-      "step": 30
-    },
-    {
-      "epoch": 0.003196803196803197,
-      "grad_norm": 10.953289031982422,
-      "learning_rate": 1.597444089456869e-07,
-      "logits/chosen": -7.665109157562256,
-      "logits/rejected": -7.661889553070068,
-      "logps/chosen": -42.742462158203125,
-      "logps/rejected": -42.284690856933594,
-      "loss": 0.9856,
-      "rewards/accuracies": 0.4375,
-      "rewards/chosen": 3.1971795558929443,
-      "rewards/margins": -0.10852270573377609,
-      "rewards/rejected": 3.305701971054077,
-      "step": 40
-    },
-    {
-      "epoch": 0.003996003996003996,
-      "grad_norm": 8.53232192993164,
-      "learning_rate": 1.9968051118210864e-07,
-      "logits/chosen": -7.695519924163818,
-      "logits/rejected": -7.5535736083984375,
-      "logps/chosen": -38.437198638916016,
-      "logps/rejected": -40.44890213012695,
-      "loss": 0.9483,
-      "rewards/accuracies": 0.4749999940395355,
-      "rewards/chosen": 3.396397352218628,
-      "rewards/margins": -0.09994807094335556,
-      "rewards/rejected": 3.4963455200195312,
-      "step": 50
-    },
-    {
-      "epoch": 0.004795204795204795,
-      "grad_norm": 9.508108139038086,
-      "learning_rate": 2.396166134185304e-07,
-      "logits/chosen": -7.6459527015686035,
-      "logits/rejected": -7.631233215332031,
-      "logps/chosen": -36.9679069519043,
-      "logps/rejected": -36.722755432128906,
-      "loss": 0.9866,
-      "rewards/accuracies": 0.375,
-      "rewards/chosen": 3.241772174835205,
-      "rewards/margins": -0.17087379097938538,
-      "rewards/rejected": 3.4126458168029785,
-      "step": 60
-    },
-    {
-      "epoch": 0.005594405594405594,
-      "grad_norm": 11.01330280303955,
-      "learning_rate": 2.7955271565495207e-07,
-      "logits/chosen": -7.722896099090576,
-      "logits/rejected": -7.6275458335876465,
-      "logps/chosen": -39.52242660522461,
-      "logps/rejected": -39.90603256225586,
-      "loss": 1.1576,
-      "rewards/accuracies": 0.2750000059604645,
-      "rewards/chosen": 3.0969159603118896,
-      "rewards/margins": -0.5339263677597046,
-      "rewards/rejected": 3.630842685699463,
-      "step": 70
-    },
-    {
-      "epoch": 0.006393606393606394,
-      "grad_norm": 9.976761817932129,
-      "learning_rate": 3.194888178913738e-07,
-      "logits/chosen": -7.662831783294678,
-      "logits/rejected": -7.574650764465332,
-      "logps/chosen": -40.98800277709961,
-      "logps/rejected": -39.64659881591797,
-      "loss": 1.0388,
-      "rewards/accuracies": 0.4625000059604645,
-      "rewards/chosen": 3.276463270187378,
-      "rewards/margins": -0.26923805475234985,
-      "rewards/rejected": 3.545701265335083,
-      "step": 80
-    },
-    {
-      "epoch": 0.007192807192807193,
-      "grad_norm": 9.109944343566895,
-      "learning_rate": 3.594249201277956e-07,
-      "logits/chosen": -7.598524570465088,
-      "logits/rejected": -7.610738277435303,
-      "logps/chosen": -40.70936965942383,
-      "logps/rejected": -41.253238677978516,
-      "loss": 1.0047,
-      "rewards/accuracies": 0.4000000059604645,
-      "rewards/chosen": 2.980196714401245,
-      "rewards/margins": -0.22133313119411469,
-      "rewards/rejected": 3.2015297412872314,
-      "step": 90
-    },
-    {
-      "epoch": 0.007992007992007992,
-      "grad_norm": 11.5081205368042,
-      "learning_rate": 3.993610223642173e-07,
-      "logits/chosen": -7.4948930740356445,
-      "logits/rejected": -7.547598361968994,
-      "logps/chosen": -38.9138069152832,
-      "logps/rejected": -46.054534912109375,
-      "loss": 0.9985,
-      "rewards/accuracies": 0.4124999940395355,
-      "rewards/chosen": 3.134085178375244,
-      "rewards/margins": -0.17030642926692963,
-      "rewards/rejected": 3.304391622543335,
-      "step": 100
-    },
-    {
-      "epoch": 0.008791208791208791,
-      "grad_norm": 10.060518264770508,
-      "learning_rate": 4.39297124600639e-07,
-      "logits/chosen": -7.656929016113281,
-      "logits/rejected": -7.5938286781311035,
-      "logps/chosen": -40.55401611328125,
-      "logps/rejected": -36.981075286865234,
-      "loss": 1.0486,
-      "rewards/accuracies": 0.48750001192092896,
-      "rewards/chosen": 3.303391218185425,
-      "rewards/margins": -0.2725646197795868,
-      "rewards/rejected": 3.575956106185913,
-      "step": 110
-    },
-    {
-      "epoch": 0.00959040959040959,
-      "grad_norm": 12.666457176208496,
-      "learning_rate": 4.792332268370608e-07,
-      "logits/chosen": -7.658522129058838,
-      "logits/rejected": -7.637611389160156,
-      "logps/chosen": -39.06342697143555,
-      "logps/rejected": -41.50210189819336,
-      "loss": 1.0753,
-      "rewards/accuracies": 0.4375,
-      "rewards/chosen": 3.188400983810425,
-      "rewards/margins": -0.33130407333374023,
-      "rewards/rejected": 3.519705295562744,
-      "step": 120
-    },
-    {
-      "epoch": 0.01038961038961039,
-      "grad_norm": 8.32593822479248,
-      "learning_rate": 5.191693290734825e-07,
-      "logits/chosen": -7.654026985168457,
-      "logits/rejected": -7.636133670806885,
-      "logps/chosen": -39.409217834472656,
-      "logps/rejected": -40.180171966552734,
-      "loss": 0.9314,
-      "rewards/accuracies": 0.5,
-      "rewards/chosen": 3.164048671722412,
-      "rewards/margins": -0.15078404545783997,
-      "rewards/rejected": 3.3148326873779297,
-      "step": 130
-    },
-    {
-      "epoch": 0.011188811188811189,
-      "grad_norm": 11.011466979980469,
-      "learning_rate": 5.591054313099041e-07,
-      "logits/chosen": -7.688101291656494,
-      "logits/rejected": -7.679931640625,
-      "logps/chosen": -41.563720703125,
-      "logps/rejected": -40.20808792114258,
-      "loss": 1.0015,
-      "rewards/accuracies": 0.5,
-      "rewards/chosen": 3.3001694679260254,
-      "rewards/margins": -0.08632836490869522,
-      "rewards/rejected": 3.386497974395752,
-      "step": 140
-    },
-    {
-      "epoch": 0.011988011988011988,
-      "grad_norm": 9.578335762023926,
-      "learning_rate": 5.99041533546326e-07,
-      "logits/chosen": -7.632326602935791,
-      "logits/rejected": -7.601800441741943,
-      "logps/chosen": -38.280860900878906,
-      "logps/rejected": -41.48507308959961,
-      "loss": 0.8923,
-      "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": 3.3376197814941406,
-      "rewards/margins": 0.02991967462003231,
-      "rewards/rejected": 3.3077003955841064,
-      "step": 150
-    },
-    {
-      "epoch": 0.012787212787212787,
-      "grad_norm": 8.890835762023926,
-      "learning_rate": 6.389776357827476e-07,
-      "logits/chosen": -7.667271614074707,
-      "logits/rejected": -7.637221813201904,
-      "logps/chosen": -41.122371673583984,
-      "logps/rejected": -39.70975875854492,
-      "loss": 0.9828,
-      "rewards/accuracies": 0.4749999940395355,
-      "rewards/chosen": 3.362185001373291,
-      "rewards/margins": -0.14797566831111908,
-      "rewards/rejected": 3.5101609230041504,
-      "step": 160
-    },
-    {
-      "epoch": 0.013586413586413586,
-      "grad_norm": 10.098383903503418,
-      "learning_rate": 6.789137380191694e-07,
-      "logits/chosen": -7.438290596008301,
-      "logits/rejected": -7.504277229309082,
-      "logps/chosen": -40.70044708251953,
-      "logps/rejected": -42.78274154663086,
-      "loss": 0.9694,
-      "rewards/accuracies": 0.4749999940395355,
-      "rewards/chosen": 3.1208603382110596,
-      "rewards/margins": -0.16865074634552002,
-      "rewards/rejected": 3.289510726928711,
-      "step": 170
-    },
-    {
-      "epoch": 0.014385614385614386,
-      "grad_norm": 8.56761360168457,
-      "learning_rate": 7.188498402555912e-07,
-      "logits/chosen": -7.584410190582275,
-      "logits/rejected": -7.511959075927734,
-      "logps/chosen": -41.755550384521484,
-      "logps/rejected": -43.4820671081543,
-      "loss": 1.0403,
-      "rewards/accuracies": 0.5,
-      "rewards/chosen": 3.1240603923797607,
-      "rewards/margins": -0.2801520526409149,
-      "rewards/rejected": 3.404212713241577,
-      "step": 180
-    },
-    {
-      "epoch": 0.015184815184815185,
-      "grad_norm": 12.132024765014648,
-      "learning_rate": 7.587859424920128e-07,
-      "logits/chosen": -7.410823822021484,
-      "logits/rejected": -7.629261016845703,
-      "logps/chosen": -37.89729690551758,
-      "logps/rejected": -52.01679229736328,
-      "loss": 1.0236,
-      "rewards/accuracies": 0.4749999940395355,
-      "rewards/chosen": 3.1932618618011475,
-      "rewards/margins": -0.2851470410823822,
-      "rewards/rejected": 3.4784088134765625,
-      "step": 190
-    },
-    {
-      "epoch": 0.015984015984015984,
-      "grad_norm": 12.89160442352295,
-      "learning_rate": 7.987220447284346e-07,
-      "logits/chosen": -7.473165988922119,
-      "logits/rejected": -7.5601325035095215,
-      "logps/chosen": -41.2888298034668,
-      "logps/rejected": -49.0833854675293,
-      "loss": 1.0153,
-      "rewards/accuracies": 0.48750001192092896,
-      "rewards/chosen": 3.0613045692443848,
-      "rewards/margins": -0.23332662880420685,
-      "rewards/rejected": 3.294631242752075,
-      "step": 200
-    },
-    {
-      "epoch": 0.016783216783216783,
-      "grad_norm": 8.658669471740723,
-      "learning_rate": 8.386581469648564e-07,
-      "logits/chosen": -7.6641998291015625,
-      "logits/rejected": -7.578223705291748,
-      "logps/chosen": -37.058265686035156,
-      "logps/rejected": -36.27176284790039,
-      "loss": 1.0076,
-      "rewards/accuracies": 0.4124999940395355,
-      "rewards/chosen": 3.1987969875335693,
-      "rewards/margins": -0.24879248440265656,
-      "rewards/rejected": 3.447589159011841,
-      "step": 210
-    },
-    {
-      "epoch": 0.017582417582417582,
-      "grad_norm": 7.522388935089111,
-      "learning_rate": 8.78594249201278e-07,
-      "logits/chosen": -7.6894354820251465,
-      "logits/rejected": -7.607281684875488,
-      "logps/chosen": -38.99390411376953,
-      "logps/rejected": -39.1852912902832,
-      "loss": 0.956,
-      "rewards/accuracies": 0.48750001192092896,
-      "rewards/chosen": 3.2833690643310547,
-      "rewards/margins": -0.11126412451267242,
-      "rewards/rejected": 3.3946335315704346,
-      "step": 220
-    },
-    {
-      "epoch": 0.01838161838161838,
-      "grad_norm": 12.318349838256836,
-      "learning_rate": 9.185303514376998e-07,
-      "logits/chosen": -7.671786785125732,
-      "logits/rejected": -7.59027624130249,
-      "logps/chosen": -39.07917022705078,
-      "logps/rejected": -36.472999572753906,
-      "loss": 0.8457,
-      "rewards/accuracies": 0.48750001192092896,
-      "rewards/chosen": 3.3632943630218506,
-      "rewards/margins": 0.03813569247722626,
-      "rewards/rejected": 3.3251588344573975,
-      "step": 230
-    },
-    {
-      "epoch": 0.01918081918081918,
-      "grad_norm": 9.865697860717773,
-      "learning_rate": 9.584664536741215e-07,
-      "logits/chosen": -7.679706573486328,
-      "logits/rejected": -7.613624095916748,
-      "logps/chosen": -40.86072540283203,
-      "logps/rejected": -39.576168060302734,
-      "loss": 1.0037,
-      "rewards/accuracies": 0.4375,
-      "rewards/chosen": 3.191575765609741,
-      "rewards/margins": -0.21723675727844238,
-      "rewards/rejected": 3.4088127613067627,
-      "step": 240
-    },
-    {
-      "epoch": 0.01998001998001998,
-      "grad_norm": 6.2519307136535645,
-      "learning_rate": 9.984025559105434e-07,
-      "logits/chosen": -7.667039394378662,
-      "logits/rejected": -7.589197635650635,
-      "logps/chosen": -41.00186538696289,
-      "logps/rejected": -39.89643096923828,
-      "loss": 0.9396,
-      "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": 3.2628173828125,
-      "rewards/margins": -0.02534637413918972,
-      "rewards/rejected": 3.288163423538208,
-      "step": 250
-    },
-    {
-      "epoch": 0.02077922077922078,
-      "grad_norm": 12.450461387634277,
-      "learning_rate": 1.038338658146965e-06,
-      "logits/chosen": -7.692509651184082,
-      "logits/rejected": -7.7002997398376465,
-      "logps/chosen": -41.452972412109375,
-      "logps/rejected": -45.48783493041992,
-      "loss": 1.1365,
-      "rewards/accuracies": 0.4000000059604645,
-      "rewards/chosen": 3.079045057296753,
-      "rewards/margins": -0.41189107298851013,
-      "rewards/rejected": 3.490936279296875,
-      "step": 260
-    },
-    {
-      "epoch": 0.02157842157842158,
-      "grad_norm": 11.251632690429688,
-      "learning_rate": 1.0782747603833866e-06,
-      "logits/chosen": -7.693075656890869,
-      "logits/rejected": -7.673638343811035,
-      "logps/chosen": -38.9761848449707,
-      "logps/rejected": -36.27731704711914,
-      "loss": 1.0847,
-      "rewards/accuracies": 0.36250001192092896,
-      "rewards/chosen": 3.2061784267425537,
-      "rewards/margins": -0.3687199652194977,
-      "rewards/rejected": 3.5748977661132812,
-      "step": 270
-    },
-    {
-      "epoch": 0.022377622377622378,
-      "grad_norm": 9.708857536315918,
-      "learning_rate": 1.1182108626198083e-06,
-      "logits/chosen": -7.557948589324951,
-      "logits/rejected": -7.627660274505615,
-      "logps/chosen": -41.709564208984375,
-      "logps/rejected": -44.22492599487305,
-      "loss": 0.8507,
-      "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": 3.2132492065429688,
-      "rewards/margins": 0.07925846427679062,
-      "rewards/rejected": 3.133991003036499,
-      "step": 280
-    },
-    {
-      "epoch": 0.023176823176823177,
-      "grad_norm": 10.500368118286133,
-      "learning_rate": 1.1581469648562301e-06,
-      "logits/chosen": -7.655808925628662,
-      "logits/rejected": -7.5748610496521,
-      "logps/chosen": -40.64021301269531,
-      "logps/rejected": -42.83000564575195,
-      "loss": 0.9573,
-      "rewards/accuracies": 0.5250000357627869,
-      "rewards/chosen": 3.180380344390869,
-      "rewards/margins": -0.09243179857730865,
-      "rewards/rejected": 3.2728123664855957,
-      "step": 290
-    },
-    {
-      "epoch": 0.023976023976023976,
-      "grad_norm": 9.524947166442871,
-      "learning_rate": 1.198083067092652e-06,
-      "logits/chosen": -7.639985084533691,
-      "logits/rejected": -7.601684093475342,
-      "logps/chosen": -42.4692497253418,
-      "logps/rejected": -41.79701232910156,
-      "loss": 0.8997,
-      "rewards/accuracies": 0.4749999940395355,
-      "rewards/chosen": 3.2179863452911377,
-      "rewards/margins": -0.005262363236397505,
-      "rewards/rejected": 3.2232491970062256,
-      "step": 300
-    },
-    {
-      "epoch": 0.024775224775224775,
-      "grad_norm": 8.98021125793457,
-      "learning_rate": 1.2380191693290736e-06,
-      "logits/chosen": -7.446446895599365,
-      "logits/rejected": -7.513009548187256,
-      "logps/chosen": -39.54001998901367,
-      "logps/rejected": -44.0929069519043,
-      "loss": 0.9187,
-      "rewards/accuracies": 0.45000001788139343,
-      "rewards/chosen": 3.1456916332244873,
-      "rewards/margins": -0.03485720604658127,
-      "rewards/rejected": 3.180548906326294,
-      "step": 310
-    },
-    {
-      "epoch": 0.025574425574425574,
-      "grad_norm": 10.143122673034668,
-      "learning_rate": 1.2779552715654952e-06,
-      "logits/chosen": -7.6690216064453125,
-      "logits/rejected": -7.587955474853516,
-      "logps/chosen": -39.9936637878418,
-      "logps/rejected": -43.00313186645508,
-      "loss": 0.9242,
-      "rewards/accuracies": 0.4625000059604645,
-      "rewards/chosen": 3.0766303539276123,
-      "rewards/margins": -0.13719169795513153,
-      "rewards/rejected": 3.21382212638855,
-      "step": 320
-    },
-    {
-      "epoch": 0.026373626373626374,
-      "grad_norm": 9.851761817932129,
-      "learning_rate": 1.3178913738019169e-06,
-      "logits/chosen": -7.582467555999756,
-      "logits/rejected": -7.588290691375732,
-      "logps/chosen": -40.25715637207031,
-      "logps/rejected": -48.05508041381836,
-      "loss": 0.7566,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": 3.3140320777893066,
-      "rewards/margins": 0.17424726486206055,
-      "rewards/rejected": 3.139784574508667,
-      "step": 330
-    },
-    {
-      "epoch": 0.027172827172827173,
-      "grad_norm": 8.66518783569336,
-      "learning_rate": 1.3578274760383387e-06,
-      "logits/chosen": -7.712961673736572,
-      "logits/rejected": -7.637712001800537,
-      "logps/chosen": -41.09788513183594,
-      "logps/rejected": -42.51398849487305,
-      "loss": 0.7392,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": 3.4395034313201904,
-      "rewards/margins": 0.26389989256858826,
-      "rewards/rejected": 3.1756036281585693,
-      "step": 340
-    },
-    {
-      "epoch": 0.027972027972027972,
-      "grad_norm": 9.417163848876953,
-      "learning_rate": 1.3977635782747605e-06,
-      "logits/chosen": -7.6864333152771,
-      "logits/rejected": -7.5792388916015625,
-      "logps/chosen": -42.39601516723633,
-      "logps/rejected": -43.80206298828125,
-      "loss": 0.6909,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": 3.5058350563049316,
-      "rewards/margins": 0.4015202224254608,
-      "rewards/rejected": 3.1043152809143066,
-      "step": 350
-    },
-    {
-      "epoch": 0.02877122877122877,
-      "grad_norm": 11.99042797088623,
-      "learning_rate": 1.4376996805111824e-06,
-      "logits/chosen": -7.6724748611450195,
-      "logits/rejected": -7.705264568328857,
-      "logps/chosen": -40.00273513793945,
-      "logps/rejected": -46.300907135009766,
-      "loss": 0.9456,
-      "rewards/accuracies": 0.5375000238418579,
-      "rewards/chosen": 3.2930126190185547,
-      "rewards/margins": -0.08574222028255463,
-      "rewards/rejected": 3.3787548542022705,
-      "step": 360
-    },
-    {
-      "epoch": 0.02957042957042957,
-      "grad_norm": 8.556150436401367,
-      "learning_rate": 1.477635782747604e-06,
-      "logits/chosen": -7.66719388961792,
-      "logits/rejected": -7.694938659667969,
-      "logps/chosen": -37.99689865112305,
-      "logps/rejected": -40.288970947265625,
-      "loss": 1.1324,
-      "rewards/accuracies": 0.38750001788139343,
-      "rewards/chosen": 3.0645101070404053,
-      "rewards/margins": -0.44527116417884827,
-      "rewards/rejected": 3.5097808837890625,
-      "step": 370
-    },
-    {
-      "epoch": 0.03036963036963037,
-      "grad_norm": 10.442875862121582,
-      "learning_rate": 1.5175718849840257e-06,
-      "logits/chosen": -7.717287540435791,
-      "logits/rejected": -7.640286445617676,
-      "logps/chosen": -39.48725128173828,
-      "logps/rejected": -42.20526123046875,
-      "loss": 0.9239,
-      "rewards/accuracies": 0.4625000059604645,
-      "rewards/chosen": 2.930732488632202,
-      "rewards/margins": -0.12516584992408752,
-      "rewards/rejected": 3.0558977127075195,
-      "step": 380
-    },
-    {
-      "epoch": 0.03116883116883117,
-      "grad_norm": 10.448299407958984,
-      "learning_rate": 1.5575079872204473e-06,
-      "logits/chosen": -7.649921417236328,
-      "logits/rejected": -7.619688510894775,
-      "logps/chosen": -38.955135345458984,
-      "logps/rejected": -44.30516815185547,
-      "loss": 0.9215,
-      "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": 3.087367057800293,
-      "rewards/margins": -0.0514339879155159,
-      "rewards/rejected": 3.138801097869873,
-      "step": 390
-    },
-    {
-      "epoch": 0.03196803196803197,
-      "grad_norm": 8.60340404510498,
-      "learning_rate": 1.5974440894568691e-06,
-      "logits/chosen": -7.690820217132568,
-      "logits/rejected": -7.665621280670166,
-      "logps/chosen": -38.584144592285156,
-      "logps/rejected": -43.70823287963867,
-      "loss": 0.8526,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": 3.116621732711792,
-      "rewards/margins": 0.001969108125194907,
-      "rewards/rejected": 3.114652633666992,
-      "step": 400
-    },
-    {
-      "epoch": 0.03276723276723277,
-      "grad_norm": 7.8150410652160645,
-      "learning_rate": 1.6373801916932908e-06,
-      "logits/chosen": -7.8178391456604,
-      "logits/rejected": -7.7787017822265625,
-      "logps/chosen": -39.36774826049805,
-      "logps/rejected": -45.232887268066406,
-      "loss": 0.7432,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": 3.563555955886841,
-      "rewards/margins": 0.3336162269115448,
-      "rewards/rejected": 3.2299396991729736,
-      "step": 410
-    },
-    {
-      "epoch": 0.033566433566433566,
-      "grad_norm": 11.778448104858398,
-      "learning_rate": 1.6773162939297128e-06,
-      "logits/chosen": -7.745136260986328,
-      "logits/rejected": -7.732022285461426,
-      "logps/chosen": -39.71867752075195,
-      "logps/rejected": -41.45455551147461,
-      "loss": 0.9086,
-      "rewards/accuracies": 0.4749999940395355,
-      "rewards/chosen": 3.305072069168091,
-      "rewards/margins": 0.015759039670228958,
-      "rewards/rejected": 3.2893128395080566,
-      "step": 420
-    },
-    {
-      "epoch": 0.03436563436563437,
-      "grad_norm": 10.234705924987793,
-      "learning_rate": 1.7172523961661345e-06,
-      "logits/chosen": -7.8083815574646,
-      "logits/rejected": -7.757164001464844,
-      "logps/chosen": -43.37578582763672,
-      "logps/rejected": -43.48426055908203,
-      "loss": 0.9767,
-      "rewards/accuracies": 0.45000001788139343,
-      "rewards/chosen": 3.050279378890991,
-      "rewards/margins": -0.12324059009552002,
-      "rewards/rejected": 3.1735198497772217,
-      "step": 430
-    },
-    {
-      "epoch": 0.035164835164835165,
-      "grad_norm": 12.886808395385742,
-      "learning_rate": 1.757188498402556e-06,
-      "logits/chosen": -7.7803192138671875,
-      "logits/rejected": -7.83193826675415,
-      "logps/chosen": -39.28300094604492,
-      "logps/rejected": -43.702186584472656,
-      "loss": 0.9135,
-      "rewards/accuracies": 0.4749999940395355,
-      "rewards/chosen": 2.929250955581665,
-      "rewards/margins": -0.044751811772584915,
-      "rewards/rejected": 2.9740028381347656,
-      "step": 440
-    },
-    {
-      "epoch": 0.03596403596403597,
-      "grad_norm": 9.622274398803711,
-      "learning_rate": 1.7971246006389777e-06,
-      "logits/chosen": -7.708813667297363,
-      "logits/rejected": -7.727663516998291,
-      "logps/chosen": -42.62513732910156,
-      "logps/rejected": -47.713111877441406,
-      "loss": 0.7701,
-      "rewards/accuracies": 0.5250000357627869,
-      "rewards/chosen": 3.1425492763519287,
-      "rewards/margins": 0.11799657344818115,
-      "rewards/rejected": 3.024552583694458,
-      "step": 450
-    },
-    {
-      "epoch": 0.03676323676323676,
-      "grad_norm": 9.056583404541016,
-      "learning_rate": 1.8370607028753996e-06,
-      "logits/chosen": -7.841561794281006,
-      "logits/rejected": -7.817006587982178,
-      "logps/chosen": -38.9039421081543,
-      "logps/rejected": -42.429588317871094,
-      "loss": 0.9744,
-      "rewards/accuracies": 0.4625000059604645,
-      "rewards/chosen": 3.0360360145568848,
-      "rewards/margins": -0.15860283374786377,
-      "rewards/rejected": 3.194638967514038,
-      "step": 460
-    },
-    {
-      "epoch": 0.037562437562437566,
-      "grad_norm": 9.531208038330078,
-      "learning_rate": 1.8769968051118212e-06,
-      "logits/chosen": -7.782096862792969,
-      "logits/rejected": -7.689637184143066,
-      "logps/chosen": -41.65514373779297,
-      "logps/rejected": -44.37314987182617,
-      "loss": 0.8143,
-      "rewards/accuracies": 0.4749999940395355,
-      "rewards/chosen": 3.1150214672088623,
-      "rewards/margins": 0.043950047343969345,
-      "rewards/rejected": 3.0710716247558594,
-      "step": 470
-    },
-    {
-      "epoch": 0.03836163836163836,
-      "grad_norm": 9.843318939208984,
-      "learning_rate": 1.916932907348243e-06,
-      "logits/chosen": -7.828926086425781,
-      "logits/rejected": -7.818706035614014,
-      "logps/chosen": -41.4467658996582,
-      "logps/rejected": -44.70206069946289,
-      "loss": 0.8807,
-      "rewards/accuracies": 0.4625000059604645,
-      "rewards/chosen": 2.9161736965179443,
-      "rewards/margins": -0.06261247396469116,
-      "rewards/rejected": 2.978785991668701,
-      "step": 480
-    },
-    {
-      "epoch": 0.039160839160839164,
-      "grad_norm": 11.220504760742188,
-      "learning_rate": 1.9568690095846645e-06,
-      "logits/chosen": -7.802455425262451,
-      "logits/rejected": -7.806884765625,
-      "logps/chosen": -42.45248031616211,
-      "logps/rejected": -44.262325286865234,
-      "loss": 1.0429,
-      "rewards/accuracies": 0.375,
-      "rewards/chosen": 2.669381856918335,
-      "rewards/margins": -0.4036366939544678,
-      "rewards/rejected": 3.073018789291382,
-      "step": 490
-    },
-    {
-      "epoch": 0.03996003996003996,
-      "grad_norm": 9.236295700073242,
-      "learning_rate": 1.9968051118210867e-06,
-      "logits/chosen": -7.791174411773682,
-      "logits/rejected": -7.81088399887085,
-      "logps/chosen": -41.7435302734375,
-      "logps/rejected": -43.280250549316406,
-      "loss": 0.809,
-      "rewards/accuracies": 0.5250000357627869,
-      "rewards/chosen": 3.272351026535034,
-      "rewards/margins": 0.14965872466564178,
-      "rewards/rejected": 3.122692346572876,
-      "step": 500
-    },
-    {
-      "epoch": 0.04075924075924076,
-      "grad_norm": 11.817867279052734,
-      "learning_rate": 2.036741214057508e-06,
-      "logits/chosen": -7.777113437652588,
-      "logits/rejected": -7.799007415771484,
-      "logps/chosen": -43.14939498901367,
-      "logps/rejected": -50.444236755371094,
-      "loss": 0.8643,
-      "rewards/accuracies": 0.5375000238418579,
-      "rewards/chosen": 3.0027694702148438,
-      "rewards/margins": 0.0006221175426617265,
-      "rewards/rejected": 3.0021474361419678,
-      "step": 510
-    },
-    {
-      "epoch": 0.04155844155844156,
-      "grad_norm": 11.305262565612793,
-      "learning_rate": 2.07667731629393e-06,
-      "logits/chosen": -7.802015781402588,
-      "logits/rejected": -7.864040374755859,
-      "logps/chosen": -41.20371627807617,
-      "logps/rejected": -51.2141227722168,
-      "loss": 0.8347,
-      "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": 3.081130266189575,
-      "rewards/margins": 0.06094294786453247,
-      "rewards/rejected": 3.0201876163482666,
-      "step": 520
-    },
-    {
-      "epoch": 0.04235764235764236,
-      "grad_norm": 7.650259017944336,
-      "learning_rate": 2.1166134185303514e-06,
-      "logits/chosen": -7.872048854827881,
-      "logits/rejected": -7.837674140930176,
-      "logps/chosen": -44.558345794677734,
-      "logps/rejected": -42.968238830566406,
-      "loss": 0.6939,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": 3.2913405895233154,
-      "rewards/margins": 0.28411319851875305,
-      "rewards/rejected": 3.007227659225464,
-      "step": 530
-    },
-    {
-      "epoch": 0.04315684315684316,
-      "grad_norm": 13.491942405700684,
-      "learning_rate": 2.1565495207667733e-06,
-      "logits/chosen": -7.903354644775391,
-      "logits/rejected": -7.841526031494141,
-      "logps/chosen": -44.515926361083984,
-      "logps/rejected": -43.5880241394043,
-      "loss": 1.0084,
-      "rewards/accuracies": 0.38750001788139343,
-      "rewards/chosen": 2.9244306087493896,
-      "rewards/margins": -0.3297668397426605,
-      "rewards/rejected": 3.254197835922241,
-      "step": 540
-    },
-    {
-      "epoch": 0.04395604395604396,
-      "grad_norm": 9.516120910644531,
-      "learning_rate": 2.196485623003195e-06,
-      "logits/chosen": -7.896944522857666,
-      "logits/rejected": -7.855966091156006,
-      "logps/chosen": -41.538597106933594,
-      "logps/rejected": -48.686981201171875,
-      "loss": 0.9448,
-      "rewards/accuracies": 0.4749999940395355,
-      "rewards/chosen": 2.777926206588745,
-      "rewards/margins": -0.12051226943731308,
-      "rewards/rejected": 2.8984386920928955,
-      "step": 550
-    },
-    {
-      "epoch": 0.044755244755244755,
-      "grad_norm": 9.481618881225586,
-      "learning_rate": 2.2364217252396165e-06,
-      "logits/chosen": -7.87398099899292,
-      "logits/rejected": -7.824643135070801,
-      "logps/chosen": -43.4268684387207,
-      "logps/rejected": -50.8296012878418,
-      "loss": 0.8161,
-      "rewards/accuracies": 0.5250000357627869,
-      "rewards/chosen": 2.9566104412078857,
-      "rewards/margins": 0.13083016872406006,
-      "rewards/rejected": 2.825780153274536,
-      "step": 560
-    },
-    {
-      "epoch": 0.04555444555444556,
-      "grad_norm": 11.93233871459961,
-      "learning_rate": 2.2763578274760384e-06,
-      "logits/chosen": -7.741659641265869,
-      "logits/rejected": -7.810840129852295,
-      "logps/chosen": -46.115962982177734,
-      "logps/rejected": -48.21393966674805,
-      "loss": 0.7856,
-      "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": 2.884819507598877,
-      "rewards/margins": 0.10625582933425903,
-      "rewards/rejected": 2.7785637378692627,
-      "step": 570
-    },
-    {
-      "epoch": 0.046353646353646354,
-      "grad_norm": 10.899465560913086,
-      "learning_rate": 2.3162939297124602e-06,
-      "logits/chosen": -7.948541164398193,
-      "logits/rejected": -7.917975902557373,
-      "logps/chosen": -43.36311721801758,
-      "logps/rejected": -45.88932418823242,
-      "loss": 0.8594,
-      "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": 2.766517162322998,
-      "rewards/margins": -0.047191914170980453,
-      "rewards/rejected": 2.813709020614624,
-      "step": 580
-    },
-    {
-      "epoch": 0.047152847152847156,
-      "grad_norm": 9.698094367980957,
-      "learning_rate": 2.356230031948882e-06,
-      "logits/chosen": -7.8796539306640625,
-      "logits/rejected": -7.924831390380859,
-      "logps/chosen": -43.27035903930664,
-      "logps/rejected": -50.13059616088867,
-      "loss": 0.7279,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": 2.9470417499542236,
-      "rewards/margins": 0.2508575916290283,
-      "rewards/rejected": 2.6961841583251953,
-      "step": 590
-    },
-    {
-      "epoch": 0.04795204795204795,
-      "grad_norm": 13.034021377563477,
-      "learning_rate": 2.396166134185304e-06,
-      "logits/chosen": -7.984201908111572,
-      "logits/rejected": -7.917106628417969,
-      "logps/chosen": -43.913814544677734,
-      "logps/rejected": -46.33149337768555,
-      "loss": 0.8096,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": 2.9455642700195312,
-      "rewards/margins": 0.11783243715763092,
-      "rewards/rejected": 2.8277318477630615,
-      "step": 600
-    },
-    {
-      "epoch": 0.048751248751248755,
-      "grad_norm": 11.605043411254883,
-      "learning_rate": 2.4361022364217253e-06,
-      "logits/chosen": -7.932804107666016,
-      "logits/rejected": -7.966372966766357,
-      "logps/chosen": -47.445552825927734,
-      "logps/rejected": -53.64095687866211,
-      "loss": 0.8804,
-      "rewards/accuracies": 0.45000001788139343,
-      "rewards/chosen": 2.635793924331665,
-      "rewards/margins": -0.10752470791339874,
-      "rewards/rejected": 2.743318557739258,
-      "step": 610
-    },
-    {
-      "epoch": 0.04955044955044955,
-      "grad_norm": 9.33888053894043,
-      "learning_rate": 2.476038338658147e-06,
-      "logits/chosen": -7.795604228973389,
-      "logits/rejected": -7.935007572174072,
-      "logps/chosen": -46.48645782470703,
-      "logps/rejected": -58.73756790161133,
-      "loss": 0.6737,
-      "rewards/accuracies": 0.5875000357627869,
-      "rewards/chosen": 3.017953634262085,
-      "rewards/margins": 0.38412901759147644,
-      "rewards/rejected": 2.633824348449707,
-      "step": 620
-    },
-    {
-      "epoch": 0.05034965034965035,
-      "grad_norm": 12.623750686645508,
-      "learning_rate": 2.515974440894569e-06,
-      "logits/chosen": -7.986891269683838,
-      "logits/rejected": -8.00345516204834,
-      "logps/chosen": -48.04877853393555,
-      "logps/rejected": -54.673492431640625,
-      "loss": 0.7497,
-      "rewards/accuracies": 0.5375000238418579,
-      "rewards/chosen": 2.724353551864624,
-      "rewards/margins": 0.18675349652767181,
-      "rewards/rejected": 2.537600040435791,
-      "step": 630
-    },
-    {
-      "epoch": 0.05114885114885115,
-      "grad_norm": 9.63351058959961,
-      "learning_rate": 2.5559105431309904e-06,
-      "logits/chosen": -8.031598091125488,
-      "logits/rejected": -7.999667644500732,
-      "logps/chosen": -47.81797409057617,
-      "logps/rejected": -55.66168975830078,
-      "loss": 0.8449,
-      "rewards/accuracies": 0.5375000238418579,
-      "rewards/chosen": 2.450789213180542,
-      "rewards/margins": -0.023081457242369652,
-      "rewards/rejected": 2.4738709926605225,
-      "step": 640
-    },
-    {
-      "epoch": 0.05194805194805195,
-      "grad_norm": 12.155508995056152,
-      "learning_rate": 2.5958466453674123e-06,
-      "logits/chosen": -8.06650161743164,
-      "logits/rejected": -8.056830406188965,
-      "logps/chosen": -46.27900695800781,
-      "logps/rejected": -52.38481521606445,
-      "loss": 0.8038,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": 2.556885242462158,
-      "rewards/margins": 0.18576882779598236,
-      "rewards/rejected": 2.3711163997650146,
-      "step": 650
-    },
-    {
-      "epoch": 0.05274725274725275,
-      "grad_norm": 11.306816101074219,
-      "learning_rate": 2.6357827476038337e-06,
-      "logits/chosen": -8.018638610839844,
-      "logits/rejected": -8.113482475280762,
-      "logps/chosen": -46.61680221557617,
-      "logps/rejected": -56.37105941772461,
-      "loss": 0.8834,
-      "rewards/accuracies": 0.5375000238418579,
-      "rewards/chosen": 2.37644362449646,
-      "rewards/margins": 0.015388269908726215,
-      "rewards/rejected": 2.3610551357269287,
-      "step": 660
-    },
-    {
-      "epoch": 0.05354645354645355,
-      "grad_norm": 11.837052345275879,
-      "learning_rate": 2.675718849840256e-06,
-      "logits/chosen": -8.242222785949707,
-      "logits/rejected": -8.185019493103027,
-      "logps/chosen": -47.2098274230957,
-      "logps/rejected": -48.34408187866211,
-      "loss": 0.7931,
-      "rewards/accuracies": 0.5,
-      "rewards/chosen": 2.488115072250366,
-      "rewards/margins": 0.08373232930898666,
-      "rewards/rejected": 2.4043824672698975,
-      "step": 670
-    },
-    {
-      "epoch": 0.054345654345654346,
-      "grad_norm": 8.400740623474121,
-      "learning_rate": 2.7156549520766774e-06,
-      "logits/chosen": -8.075019836425781,
-      "logits/rejected": -8.1636323928833,
-      "logps/chosen": -48.5149040222168,
-      "logps/rejected": -55.56972885131836,
-      "loss": 0.7801,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": 2.3642938137054443,
-      "rewards/margins": 0.22013552486896515,
-      "rewards/rejected": 2.144158363342285,
-      "step": 680
-    },
-    {
-      "epoch": 0.05514485514485515,
-      "grad_norm": 10.385449409484863,
-      "learning_rate": 2.7555910543130992e-06,
-      "logits/chosen": -8.109994888305664,
-      "logits/rejected": -8.19114875793457,
-      "logps/chosen": -49.08882522583008,
-      "logps/rejected": -58.7880744934082,
-      "loss": 0.8518,
-      "rewards/accuracies": 0.45000001788139343,
-      "rewards/chosen": 2.2802441120147705,
-      "rewards/margins": 0.08611059188842773,
-      "rewards/rejected": 2.1941335201263428,
-      "step": 690
-    },
-    {
-      "epoch": 0.055944055944055944,
-      "grad_norm": 8.919368743896484,
-      "learning_rate": 2.795527156549521e-06,
-      "logits/chosen": -8.19044017791748,
-      "logits/rejected": -8.116991996765137,
-      "logps/chosen": -48.398433685302734,
-      "logps/rejected": -54.6253547668457,
-      "loss": 0.7628,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": 2.2787158489227295,
-      "rewards/margins": 0.1386861652135849,
-      "rewards/rejected": 2.1400296688079834,
-      "step": 700
-    },
-    {
-      "epoch": 0.05674325674325675,
-      "grad_norm": 10.989197731018066,
-      "learning_rate": 2.8354632587859425e-06,
-      "logits/chosen": -8.191237449645996,
-      "logits/rejected": -8.133883476257324,
-      "logps/chosen": -51.068336486816406,
-      "logps/rejected": -48.82586669921875,
-      "loss": 0.7454,
-      "rewards/accuracies": 0.5375000238418579,
-      "rewards/chosen": 2.269320249557495,
-      "rewards/margins": 0.19282734394073486,
-      "rewards/rejected": 2.0764927864074707,
-      "step": 710
-    },
-    {
-      "epoch": 0.05754245754245754,
-      "grad_norm": 9.337309837341309,
-      "learning_rate": 2.8753993610223648e-06,
-      "logits/chosen": -8.1231107711792,
-      "logits/rejected": -8.196731567382812,
-      "logps/chosen": -47.43552017211914,
-      "logps/rejected": -62.961219787597656,
-      "loss": 0.6214,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": 2.0796561241149902,
-      "rewards/margins": 0.5399475693702698,
-      "rewards/rejected": 1.5397084951400757,
-      "step": 720
-    },
-    {
-      "epoch": 0.058341658341658345,
-      "grad_norm": 10.511343955993652,
-      "learning_rate": 2.915335463258786e-06,
-      "logits/chosen": -8.190321922302246,
-      "logits/rejected": -8.20262622833252,
-      "logps/chosen": -50.10573959350586,
-      "logps/rejected": -56.38605880737305,
-      "loss": 0.7403,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": 2.325929880142212,
-      "rewards/margins": 0.26561981439590454,
-      "rewards/rejected": 2.060310125350952,
-      "step": 730
-    },
-    {
-      "epoch": 0.05914085914085914,
-      "grad_norm": 11.828329086303711,
-      "learning_rate": 2.955271565495208e-06,
-      "logits/chosen": -8.302579879760742,
-      "logits/rejected": -8.209403038024902,
-      "logps/chosen": -51.29810333251953,
-      "logps/rejected": -53.7548942565918,
-      "loss": 0.7672,
-      "rewards/accuracies": 0.5375000238418579,
-      "rewards/chosen": 1.9456299543380737,
-      "rewards/margins": 0.11358864605426788,
-      "rewards/rejected": 1.8320411443710327,
-      "step": 740
-    },
-    {
-      "epoch": 0.059940059940059943,
-      "grad_norm": 8.120183944702148,
-      "learning_rate": 2.9952076677316295e-06,
-      "logits/chosen": -8.302461624145508,
-      "logits/rejected": -8.379423141479492,
-      "logps/chosen": -49.91122817993164,
-      "logps/rejected": -63.1845588684082,
-      "loss": 0.7319,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": 1.995306372642517,
-      "rewards/margins": 0.24632492661476135,
-      "rewards/rejected": 1.7489814758300781,
-      "step": 750
-    },
-    {
-      "epoch": 0.06073926073926074,
-      "grad_norm": 8.874757766723633,
-      "learning_rate": 3.0351437699680513e-06,
-      "logits/chosen": -8.478800773620605,
-      "logits/rejected": -8.399639129638672,
-      "logps/chosen": -51.680633544921875,
-      "logps/rejected": -57.44747543334961,
-      "loss": 0.6957,
-      "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": 2.2107791900634766,
-      "rewards/margins": 0.27144598960876465,
-      "rewards/rejected": 1.9393333196640015,
-      "step": 760
-    },
-    {
-      "epoch": 0.06153846153846154,
-      "grad_norm": 10.546675682067871,
-      "learning_rate": 3.075079872204473e-06,
-      "logits/chosen": -8.519871711730957,
-      "logits/rejected": -8.45196533203125,
-      "logps/chosen": -55.83811569213867,
-      "logps/rejected": -58.5568962097168,
-      "loss": 0.6499,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": 1.9453003406524658,
-      "rewards/margins": 0.5009440779685974,
-      "rewards/rejected": 1.4443563222885132,
-      "step": 770
-    },
-    {
-      "epoch": 0.06233766233766234,
-      "grad_norm": 10.698816299438477,
-      "learning_rate": 3.1150159744408946e-06,
-      "logits/chosen": -8.529873847961426,
-      "logits/rejected": -8.482592582702637,
-      "logps/chosen": -55.25311279296875,
-      "logps/rejected": -62.57723617553711,
-      "loss": 0.5828,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": 2.1050281524658203,
-      "rewards/margins": 0.6187090277671814,
-      "rewards/rejected": 1.4863190650939941,
-      "step": 780
-    },
-    {
-      "epoch": 0.06313686313686313,
-      "grad_norm": 13.495732307434082,
-      "learning_rate": 3.154952076677317e-06,
-      "logits/chosen": -8.532597541809082,
-      "logits/rejected": -8.494662284851074,
-      "logps/chosen": -53.01536178588867,
-      "logps/rejected": -58.662353515625,
-      "loss": 0.6178,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": 1.9007701873779297,
-      "rewards/margins": 0.653425395488739,
-      "rewards/rejected": 1.247344732284546,
-      "step": 790
-    },
-    {
-      "epoch": 0.06393606393606394,
-      "grad_norm": 10.57509708404541,
-      "learning_rate": 3.1948881789137383e-06,
-      "logits/chosen": -8.638677597045898,
-      "logits/rejected": -8.593525886535645,
-      "logps/chosen": -53.70016098022461,
-      "logps/rejected": -60.98574447631836,
-      "loss": 0.6249,
-      "rewards/accuracies": 0.6500000357627869,
-      "rewards/chosen": 2.0145130157470703,
-      "rewards/margins": 0.5168043971061707,
-      "rewards/rejected": 1.4977085590362549,
-      "step": 800
-    },
-    {
-      "epoch": 0.06473526473526474,
-      "grad_norm": 13.850438117980957,
-      "learning_rate": 3.23482428115016e-06,
-      "logits/chosen": -8.65740966796875,
-      "logits/rejected": -8.487887382507324,
-      "logps/chosen": -59.192291259765625,
-      "logps/rejected": -63.74581527709961,
-      "loss": 0.6507,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": 1.5935490131378174,
-      "rewards/margins": 0.6139525771141052,
-      "rewards/rejected": 0.9795963168144226,
-      "step": 810
-    },
-    {
-      "epoch": 0.06553446553446554,
-      "grad_norm": 12.689352989196777,
-      "learning_rate": 3.2747603833865815e-06,
-      "logits/chosen": -8.615852355957031,
-      "logits/rejected": -8.592411994934082,
-      "logps/chosen": -57.14704132080078,
-      "logps/rejected": -64.06562042236328,
-      "loss": 0.5814,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": 1.4983315467834473,
-      "rewards/margins": 0.5748814940452576,
-      "rewards/rejected": 0.9234499931335449,
-      "step": 820
-    },
-    {
-      "epoch": 0.06633366633366633,
-      "grad_norm": 14.839926719665527,
-      "learning_rate": 3.3146964856230034e-06,
-      "logits/chosen": -8.570846557617188,
-      "logits/rejected": -8.599050521850586,
-      "logps/chosen": -57.85627365112305,
-      "logps/rejected": -70.94813537597656,
-      "loss": 0.6032,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": 1.6199016571044922,
-      "rewards/margins": 0.6744619607925415,
-      "rewards/rejected": 0.9454396367073059,
-      "step": 830
-    },
-    {
-      "epoch": 0.06713286713286713,
-      "grad_norm": 10.380461692810059,
-      "learning_rate": 3.3546325878594257e-06,
-      "logits/chosen": -8.63135051727295,
-      "logits/rejected": -8.590235710144043,
-      "logps/chosen": -54.97597885131836,
-      "logps/rejected": -62.73817825317383,
-      "loss": 0.5699,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": 1.4367873668670654,
-      "rewards/margins": 0.6328876614570618,
-      "rewards/rejected": 0.8038995862007141,
-      "step": 840
-    },
-    {
-      "epoch": 0.06793206793206794,
-      "grad_norm": 13.941654205322266,
-      "learning_rate": 3.394568690095847e-06,
-      "logits/chosen": -8.744150161743164,
-      "logits/rejected": -8.62113094329834,
-      "logps/chosen": -56.95928955078125,
-      "logps/rejected": -65.73368835449219,
-      "loss": 0.6035,
-      "rewards/accuracies": 0.7125000357627869,
-      "rewards/chosen": 1.4413965940475464,
-      "rewards/margins": 0.6217647790908813,
-      "rewards/rejected": 0.8196317553520203,
-      "step": 850
-    },
-    {
-      "epoch": 0.06873126873126874,
-      "grad_norm": 14.208074569702148,
-      "learning_rate": 3.434504792332269e-06,
-      "logits/chosen": -8.818414688110352,
-      "logits/rejected": -8.71935749053955,
-      "logps/chosen": -61.444942474365234,
-      "logps/rejected": -62.967864990234375,
-      "loss": 0.7408,
-      "rewards/accuracies": 0.5875000357627869,
-      "rewards/chosen": 0.984531044960022,
-      "rewards/margins": 0.3361727297306061,
-      "rewards/rejected": 0.6483583450317383,
-      "step": 860
-    },
-    {
-      "epoch": 0.06953046953046953,
-      "grad_norm": 12.318110466003418,
-      "learning_rate": 3.4744408945686903e-06,
-      "logits/chosen": -8.844193458557129,
-      "logits/rejected": -8.807513236999512,
-      "logps/chosen": -61.32583236694336,
-      "logps/rejected": -67.97479248046875,
-      "loss": 0.6027,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": 1.021340012550354,
-      "rewards/margins": 0.5690200924873352,
-      "rewards/rejected": 0.4523199498653412,
-      "step": 870
-    },
-    {
-      "epoch": 0.07032967032967033,
-      "grad_norm": 9.995491981506348,
-      "learning_rate": 3.514376996805112e-06,
-      "logits/chosen": -8.962470054626465,
-      "logits/rejected": -8.879164695739746,
-      "logps/chosen": -62.12026596069336,
-      "logps/rejected": -71.13673400878906,
-      "loss": 0.6277,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": 1.1278456449508667,
-      "rewards/margins": 0.5825280547142029,
-      "rewards/rejected": 0.5453175902366638,
-      "step": 880
-    },
-    {
-      "epoch": 0.07112887112887113,
-      "grad_norm": 14.174229621887207,
-      "learning_rate": 3.5543130990415336e-06,
-      "logits/chosen": -8.881192207336426,
-      "logits/rejected": -8.840286254882812,
-      "logps/chosen": -59.36819076538086,
-      "logps/rejected": -66.7807388305664,
-      "loss": 0.577,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": 1.0045677423477173,
-      "rewards/margins": 0.7128462791442871,
-      "rewards/rejected": 0.29172155261039734,
-      "step": 890
-    },
-    {
-      "epoch": 0.07192807192807193,
-      "grad_norm": 19.00025177001953,
-      "learning_rate": 3.5942492012779555e-06,
-      "logits/chosen": -8.936151504516602,
-      "logits/rejected": -8.936973571777344,
-      "logps/chosen": -60.37038803100586,
-      "logps/rejected": -73.55181121826172,
-      "loss": 0.649,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": 0.622625470161438,
-      "rewards/margins": 0.5160958170890808,
-      "rewards/rejected": 0.10652966797351837,
-      "step": 900
-    },
-    {
-      "epoch": 0.07272727272727272,
-      "grad_norm": 17.046993255615234,
-      "learning_rate": 3.634185303514377e-06,
-      "logits/chosen": -9.01294994354248,
-      "logits/rejected": -8.910703659057617,
-      "logps/chosen": -65.11463928222656,
-      "logps/rejected": -72.1776123046875,
-      "loss": 0.5004,
-      "rewards/accuracies": 0.7125000357627869,
-      "rewards/chosen": 0.7025814652442932,
-      "rewards/margins": 0.923648476600647,
-      "rewards/rejected": -0.22106705605983734,
-      "step": 910
-    },
-    {
-      "epoch": 0.07352647352647353,
-      "grad_norm": 12.705639839172363,
-      "learning_rate": 3.674121405750799e-06,
-      "logits/chosen": -9.081149101257324,
-      "logits/rejected": -9.078699111938477,
-      "logps/chosen": -64.64808654785156,
-      "logps/rejected": -77.8483657836914,
-      "loss": 0.5459,
-      "rewards/accuracies": 0.7125000357627869,
-      "rewards/chosen": 0.6992087364196777,
-      "rewards/margins": 0.8265895247459412,
-      "rewards/rejected": -0.12738072872161865,
-      "step": 920
-    },
-    {
-      "epoch": 0.07432567432567433,
-      "grad_norm": 13.377921104431152,
-      "learning_rate": 3.714057507987221e-06,
-      "logits/chosen": -9.108325958251953,
-      "logits/rejected": -9.016131401062012,
-      "logps/chosen": -71.29902648925781,
-      "logps/rejected": -76.86527252197266,
-      "loss": 0.5545,
-      "rewards/accuracies": 0.7125000357627869,
-      "rewards/chosen": 0.26229915022850037,
-      "rewards/margins": 0.7861421704292297,
-      "rewards/rejected": -0.523842990398407,
-      "step": 930
-    },
-    {
-      "epoch": 0.07512487512487513,
-      "grad_norm": 14.26362133026123,
-      "learning_rate": 3.7539936102236424e-06,
-      "logits/chosen": -9.199752807617188,
-      "logits/rejected": -9.147046089172363,
-      "logps/chosen": -73.8570785522461,
-      "logps/rejected": -79.31583404541016,
-      "loss": 0.5574,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": 0.07472013682126999,
-      "rewards/margins": 0.9407575726509094,
-      "rewards/rejected": -0.8660373687744141,
-      "step": 940
-    },
-    {
-      "epoch": 0.07592407592407592,
-      "grad_norm": 12.309165954589844,
-      "learning_rate": 3.7939297124600643e-06,
-      "logits/chosen": -9.250157356262207,
-      "logits/rejected": -9.188761711120605,
-      "logps/chosen": -70.28436279296875,
-      "logps/rejected": -79.42757415771484,
-      "loss": 0.7616,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": 0.05344545841217041,
-      "rewards/margins": 0.523639976978302,
-      "rewards/rejected": -0.470194548368454,
-      "step": 950
-    },
-    {
-      "epoch": 0.07672327672327672,
-      "grad_norm": 11.375068664550781,
-      "learning_rate": 3.833865814696486e-06,
-      "logits/chosen": -9.249772071838379,
-      "logits/rejected": -9.182961463928223,
-      "logps/chosen": -69.54476165771484,
-      "logps/rejected": -82.53199768066406,
-      "loss": 0.5935,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": 0.05240759998559952,
-      "rewards/margins": 0.8829003572463989,
-      "rewards/rejected": -0.8304927945137024,
-      "step": 960
-    },
-    {
-      "epoch": 0.07752247752247753,
-      "grad_norm": 8.277685165405273,
-      "learning_rate": 3.8738019169329075e-06,
-      "logits/chosen": -9.041192054748535,
-      "logits/rejected": -9.157244682312012,
-      "logps/chosen": -76.5413818359375,
-      "logps/rejected": -90.62274932861328,
-      "loss": 0.6228,
-      "rewards/accuracies": 0.6500000357627869,
-      "rewards/chosen": -0.20422516763210297,
-      "rewards/margins": 0.9243282675743103,
-      "rewards/rejected": -1.1285533905029297,
-      "step": 970
-    },
-    {
-      "epoch": 0.07832167832167833,
-      "grad_norm": 17.049131393432617,
-      "learning_rate": 3.913738019169329e-06,
-      "logits/chosen": -9.265501976013184,
-      "logits/rejected": -9.206786155700684,
-      "logps/chosen": -74.29203033447266,
-      "logps/rejected": -83.0970458984375,
-      "loss": 0.6521,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": -0.18140174448490143,
-      "rewards/margins": 0.6961486339569092,
-      "rewards/rejected": -0.8775504231452942,
-      "step": 980
-    },
-    {
-      "epoch": 0.07912087912087912,
-      "grad_norm": 18.337081909179688,
-      "learning_rate": 3.953674121405751e-06,
-      "logits/chosen": -9.268997192382812,
-      "logits/rejected": -9.29576301574707,
-      "logps/chosen": -73.6700439453125,
-      "logps/rejected": -80.6961898803711,
-      "loss": 0.7421,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": -0.20676445960998535,
-      "rewards/margins": 0.45481792092323303,
-      "rewards/rejected": -0.6615824103355408,
-      "step": 990
-    },
-    {
-      "epoch": 0.07992007992007992,
-      "grad_norm": 13.537657737731934,
-      "learning_rate": 3.9936102236421735e-06,
-      "logits/chosen": -9.144115447998047,
-      "logits/rejected": -9.191131591796875,
-      "logps/chosen": -77.4375,
-      "logps/rejected": -92.80534362792969,
-      "loss": 0.6432,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": -0.28809496760368347,
-      "rewards/margins": 0.8597502708435059,
-      "rewards/rejected": -1.1478451490402222,
-      "step": 1000
-    },
-    {
-      "epoch": 0.08071928071928072,
-      "grad_norm": 15.525980949401855,
-      "learning_rate": 4.033546325878595e-06,
-      "logits/chosen": -9.117060661315918,
-      "logits/rejected": -9.212850570678711,
-      "logps/chosen": -75.97489166259766,
-      "logps/rejected": -94.86837005615234,
-      "loss": 0.4329,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -0.38618287444114685,
-      "rewards/margins": 1.1737312078475952,
-      "rewards/rejected": -1.559914231300354,
-      "step": 1010
-    },
-    {
-      "epoch": 0.08151848151848153,
-      "grad_norm": 12.48059368133545,
-      "learning_rate": 4.073482428115016e-06,
-      "logits/chosen": -9.42807674407959,
-      "logits/rejected": -9.370818138122559,
-      "logps/chosen": -76.704833984375,
-      "logps/rejected": -90.4799575805664,
-      "loss": 0.6013,
-      "rewards/accuracies": 0.7125000357627869,
-      "rewards/chosen": -0.45278796553611755,
-      "rewards/margins": 0.8836669325828552,
-      "rewards/rejected": -1.33645498752594,
-      "step": 1020
-    },
-    {
-      "epoch": 0.08231768231768231,
-      "grad_norm": 11.954912185668945,
-      "learning_rate": 4.113418530351438e-06,
-      "logits/chosen": -9.376872062683105,
-      "logits/rejected": -9.33382797241211,
-      "logps/chosen": -77.5802993774414,
-      "logps/rejected": -87.59728240966797,
-      "loss": 0.5556,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": -0.4362776279449463,
-      "rewards/margins": 1.0314857959747314,
-      "rewards/rejected": -1.4677635431289673,
-      "step": 1030
-    },
-    {
-      "epoch": 0.08311688311688312,
-      "grad_norm": 16.00122833251953,
-      "learning_rate": 4.15335463258786e-06,
-      "logits/chosen": -9.332417488098145,
-      "logits/rejected": -9.387582778930664,
-      "logps/chosen": -78.7520751953125,
-      "logps/rejected": -96.24468231201172,
-      "loss": 0.5661,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -0.4745631217956543,
-      "rewards/margins": 1.0611029863357544,
-      "rewards/rejected": -1.5356661081314087,
-      "step": 1040
-    },
-    {
-      "epoch": 0.08391608391608392,
-      "grad_norm": 10.953011512756348,
-      "learning_rate": 4.1932907348242814e-06,
-      "logits/chosen": -9.4227933883667,
-      "logits/rejected": -9.403426170349121,
-      "logps/chosen": -78.8366928100586,
-      "logps/rejected": -87.43746185302734,
-      "loss": 0.701,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": -0.7007558941841125,
-      "rewards/margins": 0.723336398601532,
-      "rewards/rejected": -1.424092173576355,
-      "step": 1050
-    },
-    {
-      "epoch": 0.08471528471528472,
-      "grad_norm": 19.733989715576172,
-      "learning_rate": 4.233226837060703e-06,
-      "logits/chosen": -9.42930793762207,
-      "logits/rejected": -9.404022216796875,
-      "logps/chosen": -83.59581756591797,
-      "logps/rejected": -93.55966186523438,
-      "loss": 0.7199,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": -0.8814291954040527,
-      "rewards/margins": 0.6182137727737427,
-      "rewards/rejected": -1.4996429681777954,
-      "step": 1060
-    },
-    {
-      "epoch": 0.08551448551448551,
-      "grad_norm": 10.234980583190918,
-      "learning_rate": 4.273162939297125e-06,
-      "logits/chosen": -9.564750671386719,
-      "logits/rejected": -9.467818260192871,
-      "logps/chosen": -74.3668441772461,
-      "logps/rejected": -80.5134506225586,
-      "loss": 0.5001,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": 0.0918915718793869,
-      "rewards/margins": 1.173356294631958,
-      "rewards/rejected": -1.0814647674560547,
-      "step": 1070
-    },
-    {
-      "epoch": 0.08631368631368631,
-      "grad_norm": 16.457443237304688,
-      "learning_rate": 4.3130990415335465e-06,
-      "logits/chosen": -9.450125694274902,
-      "logits/rejected": -9.436369895935059,
-      "logps/chosen": -76.90260314941406,
-      "logps/rejected": -84.70674896240234,
-      "loss": 0.5569,
-      "rewards/accuracies": 0.6500000357627869,
-      "rewards/chosen": -0.34347230195999146,
-      "rewards/margins": 0.9840227961540222,
-      "rewards/rejected": -1.3274950981140137,
-      "step": 1080
-    },
-    {
-      "epoch": 0.08711288711288712,
-      "grad_norm": 20.05044937133789,
-      "learning_rate": 4.353035143769969e-06,
-      "logits/chosen": -9.49072551727295,
-      "logits/rejected": -9.444594383239746,
-      "logps/chosen": -82.86369323730469,
-      "logps/rejected": -94.20703125,
-      "loss": 0.6601,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": -0.9662967920303345,
-      "rewards/margins": 0.6728413701057434,
-      "rewards/rejected": -1.639138102531433,
-      "step": 1090
-    },
-    {
-      "epoch": 0.08791208791208792,
-      "grad_norm": 13.484999656677246,
-      "learning_rate": 4.39297124600639e-06,
-      "logits/chosen": -9.601560592651367,
-      "logits/rejected": -9.495709419250488,
-      "logps/chosen": -79.45623779296875,
-      "logps/rejected": -91.87387084960938,
-      "loss": 0.4969,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -0.6346768736839294,
-      "rewards/margins": 1.202465295791626,
-      "rewards/rejected": -1.837141990661621,
-      "step": 1100
-    },
-    {
-      "epoch": 0.08871128871128871,
-      "grad_norm": 18.60314178466797,
-      "learning_rate": 4.432907348242812e-06,
-      "logits/chosen": -9.4898099899292,
-      "logits/rejected": -9.350008010864258,
-      "logps/chosen": -85.1569595336914,
-      "logps/rejected": -92.88752746582031,
-      "loss": 0.4891,
-      "rewards/accuracies": 0.7750000357627869,
-      "rewards/chosen": -1.1340035200119019,
-      "rewards/margins": 1.2183297872543335,
-      "rewards/rejected": -2.3523333072662354,
-      "step": 1110
-    },
-    {
-      "epoch": 0.08951048951048951,
-      "grad_norm": 9.206683158874512,
-      "learning_rate": 4.472843450479233e-06,
-      "logits/chosen": -9.5941162109375,
-      "logits/rejected": -9.480677604675293,
-      "logps/chosen": -82.32011413574219,
-      "logps/rejected": -91.23483276367188,
-      "loss": 0.6346,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": -1.1008182764053345,
-      "rewards/margins": 0.9734091758728027,
-      "rewards/rejected": -2.0742273330688477,
-      "step": 1120
-    },
-    {
-      "epoch": 0.09030969030969031,
-      "grad_norm": 14.13942813873291,
-      "learning_rate": 4.512779552715655e-06,
-      "logits/chosen": -9.605704307556152,
-      "logits/rejected": -9.566485404968262,
-      "logps/chosen": -84.40730285644531,
-      "logps/rejected": -102.96392059326172,
-      "loss": 0.4186,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -1.163941502571106,
-      "rewards/margins": 1.3259172439575195,
-      "rewards/rejected": -2.489858627319336,
-      "step": 1130
-    },
-    {
-      "epoch": 0.09110889110889112,
-      "grad_norm": 19.708494186401367,
-      "learning_rate": 4.552715654952077e-06,
-      "logits/chosen": -9.555802345275879,
-      "logits/rejected": -9.620047569274902,
-      "logps/chosen": -84.46455383300781,
-      "logps/rejected": -106.61455535888672,
-      "loss": 0.622,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": -1.5165579319000244,
-      "rewards/margins": 0.9879328012466431,
-      "rewards/rejected": -2.504490613937378,
-      "step": 1140
-    },
-    {
-      "epoch": 0.0919080919080919,
-      "grad_norm": 18.047182083129883,
-      "learning_rate": 4.592651757188499e-06,
-      "logits/chosen": -9.675976753234863,
-      "logits/rejected": -9.57038402557373,
-      "logps/chosen": -88.70226287841797,
-      "logps/rejected": -103.02335357666016,
-      "loss": 0.4802,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -1.5707515478134155,
-      "rewards/margins": 1.3256628513336182,
-      "rewards/rejected": -2.896414279937744,
-      "step": 1150
-    },
-    {
-      "epoch": 0.09270729270729271,
-      "grad_norm": 18.440229415893555,
-      "learning_rate": 4.6325878594249205e-06,
-      "logits/chosen": -9.73648738861084,
-      "logits/rejected": -9.776338577270508,
-      "logps/chosen": -91.5391845703125,
-      "logps/rejected": -109.1735610961914,
-      "loss": 0.608,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -2.060546636581421,
-      "rewards/margins": 1.3435176610946655,
-      "rewards/rejected": -3.404064178466797,
-      "step": 1160
-    },
-    {
-      "epoch": 0.09350649350649351,
-      "grad_norm": 17.93333625793457,
-      "learning_rate": 4.672523961661342e-06,
-      "logits/chosen": -9.913115501403809,
-      "logits/rejected": -9.883721351623535,
-      "logps/chosen": -90.20143127441406,
-      "logps/rejected": -109.4500503540039,
-      "loss": 0.5395,
-      "rewards/accuracies": 0.7750000357627869,
-      "rewards/chosen": -1.9290742874145508,
-      "rewards/margins": 1.6518268585205078,
-      "rewards/rejected": -3.5809013843536377,
-      "step": 1170
-    },
-    {
-      "epoch": 0.09430569430569431,
-      "grad_norm": 16.75286102294922,
-      "learning_rate": 4.712460063897764e-06,
-      "logits/chosen": -9.913932800292969,
-      "logits/rejected": -9.844474792480469,
-      "logps/chosen": -94.85572814941406,
-      "logps/rejected": -107.37994384765625,
-      "loss": 0.6062,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": -2.237203359603882,
-      "rewards/margins": 1.133827567100525,
-      "rewards/rejected": -3.371030569076538,
-      "step": 1180
-    },
-    {
-      "epoch": 0.0951048951048951,
-      "grad_norm": 8.530978202819824,
-      "learning_rate": 4.7523961661341856e-06,
-      "logits/chosen": -9.921943664550781,
-      "logits/rejected": -9.837581634521484,
-      "logps/chosen": -91.4283676147461,
-      "logps/rejected": -102.99165344238281,
-      "loss": 0.7085,
-      "rewards/accuracies": 0.6500000357627869,
-      "rewards/chosen": -1.905538558959961,
-      "rewards/margins": 1.0475950241088867,
-      "rewards/rejected": -2.9531338214874268,
-      "step": 1190
-    },
-    {
-      "epoch": 0.0959040959040959,
-      "grad_norm": 14.733861923217773,
-      "learning_rate": 4.792332268370608e-06,
-      "logits/chosen": -9.721549034118652,
-      "logits/rejected": -9.847012519836426,
-      "logps/chosen": -98.00448608398438,
-      "logps/rejected": -115.90787506103516,
-      "loss": 0.6719,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": -2.5480122566223145,
-      "rewards/margins": 1.22404944896698,
-      "rewards/rejected": -3.772061586380005,
-      "step": 1200
-    },
-    {
-      "epoch": 0.0967032967032967,
-      "grad_norm": 9.225886344909668,
-      "learning_rate": 4.832268370607029e-06,
-      "logits/chosen": -9.826550483703613,
-      "logits/rejected": -9.799708366394043,
-      "logps/chosen": -89.5009994506836,
-      "logps/rejected": -105.677734375,
-      "loss": 0.492,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -1.7875438928604126,
-      "rewards/margins": 1.4356008768081665,
-      "rewards/rejected": -3.223145008087158,
-      "step": 1210
-    },
-    {
-      "epoch": 0.09750249750249751,
-      "grad_norm": 9.776506423950195,
-      "learning_rate": 4.872204472843451e-06,
-      "logits/chosen": -9.92806339263916,
-      "logits/rejected": -9.804278373718262,
-      "logps/chosen": -90.31047821044922,
-      "logps/rejected": -104.11870574951172,
-      "loss": 0.411,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -1.8978222608566284,
-      "rewards/margins": 1.5606602430343628,
-      "rewards/rejected": -3.458482503890991,
-      "step": 1220
-    },
-    {
-      "epoch": 0.0983016983016983,
-      "grad_norm": 7.396607875823975,
-      "learning_rate": 4.912140575079873e-06,
-      "logits/chosen": -9.7081937789917,
-      "logits/rejected": -9.75550365447998,
-      "logps/chosen": -90.59484100341797,
-      "logps/rejected": -116.07256317138672,
-      "loss": 0.4841,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -1.959818720817566,
-      "rewards/margins": 1.4614990949630737,
-      "rewards/rejected": -3.4213178157806396,
-      "step": 1230
-    },
-    {
-      "epoch": 0.0991008991008991,
-      "grad_norm": 19.8353214263916,
-      "learning_rate": 4.952076677316294e-06,
-      "logits/chosen": -9.948829650878906,
-      "logits/rejected": -9.882896423339844,
-      "logps/chosen": -91.51131439208984,
-      "logps/rejected": -104.915771484375,
-      "loss": 0.4407,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -1.9766762256622314,
-      "rewards/margins": 1.4270721673965454,
-      "rewards/rejected": -3.4037482738494873,
-      "step": 1240
-    },
-    {
-      "epoch": 0.0999000999000999,
-      "grad_norm": 19.53704261779785,
-      "learning_rate": 4.992012779552717e-06,
-      "logits/chosen": -9.802943229675293,
-      "logits/rejected": -9.859160423278809,
-      "logps/chosen": -101.77998352050781,
-      "logps/rejected": -120.25943756103516,
-      "loss": 0.5989,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": -2.6252965927124023,
-      "rewards/margins": 1.1689800024032593,
-      "rewards/rejected": -3.794276475906372,
-      "step": 1250
-    },
-    {
-      "epoch": 0.1006993006993007,
-      "grad_norm": 17.491119384765625,
-      "learning_rate": 4.99999377251384e-06,
-      "logits/chosen": -9.689326286315918,
-      "logits/rejected": -9.822731018066406,
-      "logps/chosen": -96.35672760009766,
-      "logps/rejected": -121.4671859741211,
-      "loss": 0.6111,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -2.3672032356262207,
-      "rewards/margins": 1.3830018043518066,
-      "rewards/rejected": -3.7502052783966064,
-      "step": 1260
-    },
-    {
-      "epoch": 0.1014985014985015,
-      "grad_norm": 12.244000434875488,
-      "learning_rate": 4.999968473404488e-06,
-      "logits/chosen": -9.873839378356934,
-      "logits/rejected": -9.855506896972656,
-      "logps/chosen": -86.57073211669922,
-      "logps/rejected": -104.87135314941406,
-      "loss": 0.3927,
-      "rewards/accuracies": 0.8375000357627869,
-      "rewards/chosen": -1.5173803567886353,
-      "rewards/margins": 1.6421905755996704,
-      "rewards/rejected": -3.1595709323883057,
-      "step": 1270
-    },
-    {
-      "epoch": 0.1022977022977023,
-      "grad_norm": 17.089445114135742,
-      "learning_rate": 4.999923713650843e-06,
-      "logits/chosen": -9.853652954101562,
-      "logits/rejected": -9.827929496765137,
-      "logps/chosen": -88.68315887451172,
-      "logps/rejected": -108.35890197753906,
-      "loss": 0.5311,
-      "rewards/accuracies": 0.7125000357627869,
-      "rewards/chosen": -1.8326797485351562,
-      "rewards/margins": 1.4945013523101807,
-      "rewards/rejected": -3.327181339263916,
-      "step": 1280
-    },
-    {
-      "epoch": 0.1030969030969031,
-      "grad_norm": 7.547091007232666,
-      "learning_rate": 4.999859493601334e-06,
-      "logits/chosen": -9.917536735534668,
-      "logits/rejected": -9.839146614074707,
-      "logps/chosen": -90.16967010498047,
-      "logps/rejected": -111.09648895263672,
-      "loss": 0.5976,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": -1.8089485168457031,
-      "rewards/margins": 1.1288902759552002,
-      "rewards/rejected": -2.937838554382324,
-      "step": 1290
-    },
-    {
-      "epoch": 0.1038961038961039,
-      "grad_norm": 11.175216674804688,
-      "learning_rate": 4.999775813755869e-06,
-      "logits/chosen": -9.971378326416016,
-      "logits/rejected": -9.90029525756836,
-      "logps/chosen": -96.5659408569336,
-      "logps/rejected": -107.66356658935547,
-      "loss": 0.5362,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -2.084070920944214,
-      "rewards/margins": 1.1332049369812012,
-      "rewards/rejected": -3.217276096343994,
-      "step": 1300
-    },
-    {
-      "epoch": 0.10469530469530469,
-      "grad_norm": 13.143838882446289,
-      "learning_rate": 4.999672674765844e-06,
-      "logits/chosen": -9.75096607208252,
-      "logits/rejected": -9.884263038635254,
-      "logps/chosen": -96.49981689453125,
-      "logps/rejected": -115.91259002685547,
-      "loss": 0.4773,
-      "rewards/accuracies": 0.7750000357627869,
-      "rewards/chosen": -2.3865301609039307,
-      "rewards/margins": 1.4379810094833374,
-      "rewards/rejected": -3.8245110511779785,
-      "step": 1310
-    },
-    {
-      "epoch": 0.1054945054945055,
-      "grad_norm": 9.412525177001953,
-      "learning_rate": 4.99955007743413e-06,
-      "logits/chosen": -10.028037071228027,
-      "logits/rejected": -9.947481155395508,
-      "logps/chosen": -89.94660186767578,
-      "logps/rejected": -104.3980712890625,
-      "loss": 0.4876,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": -1.9574517011642456,
-      "rewards/margins": 1.4134562015533447,
-      "rewards/rejected": -3.3709075450897217,
-      "step": 1320
-    },
-    {
-      "epoch": 0.1062937062937063,
-      "grad_norm": 14.555173873901367,
-      "learning_rate": 4.9994080227150665e-06,
-      "logits/chosen": -9.958174705505371,
-      "logits/rejected": -9.973993301391602,
-      "logps/chosen": -92.73445892333984,
-      "logps/rejected": -111.23880767822266,
-      "loss": 0.5463,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": -2.252150535583496,
-      "rewards/margins": 1.247558355331421,
-      "rewards/rejected": -3.499708652496338,
-      "step": 1330
-    },
-    {
-      "epoch": 0.1070929070929071,
-      "grad_norm": 16.20941162109375,
-      "learning_rate": 4.99924651171446e-06,
-      "logits/chosen": -10.092068672180176,
-      "logits/rejected": -10.008609771728516,
-      "logps/chosen": -91.9598159790039,
-      "logps/rejected": -110.3380126953125,
-      "loss": 0.436,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -2.055654287338257,
-      "rewards/margins": 1.5372854471206665,
-      "rewards/rejected": -3.5929393768310547,
-      "step": 1340
-    },
-    {
-      "epoch": 0.10789210789210789,
-      "grad_norm": 16.660919189453125,
-      "learning_rate": 4.999065545689568e-06,
-      "logits/chosen": -9.982669830322266,
-      "logits/rejected": -10.066909790039062,
-      "logps/chosen": -96.93700408935547,
-      "logps/rejected": -112.36788177490234,
-      "loss": 0.6573,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": -2.453967809677124,
-      "rewards/margins": 1.0762113332748413,
-      "rewards/rejected": -3.530179262161255,
-      "step": 1350
-    },
-    {
-      "epoch": 0.10869130869130869,
-      "grad_norm": 12.420697212219238,
-      "learning_rate": 4.998865126049095e-06,
-      "logits/chosen": -9.949244499206543,
-      "logits/rejected": -9.97769832611084,
-      "logps/chosen": -95.5306167602539,
-      "logps/rejected": -118.81241607666016,
-      "loss": 0.5404,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -2.250213623046875,
-      "rewards/margins": 1.5049253702163696,
-      "rewards/rejected": -3.755139112472534,
-      "step": 1360
-    },
-    {
-      "epoch": 0.1094905094905095,
-      "grad_norm": 11.777141571044922,
-      "learning_rate": 4.998645254353178e-06,
-      "logits/chosen": -10.19797420501709,
-      "logits/rejected": -10.054896354675293,
-      "logps/chosen": -90.45796203613281,
-      "logps/rejected": -105.9493408203125,
-      "loss": 0.5696,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": -2.179734230041504,
-      "rewards/margins": 1.193315863609314,
-      "rewards/rejected": -3.3730499744415283,
-      "step": 1370
-    },
-    {
-      "epoch": 0.1102897102897103,
-      "grad_norm": 10.7411527633667,
-      "learning_rate": 4.998405932313378e-06,
-      "logits/chosen": -10.142987251281738,
-      "logits/rejected": -10.152636528015137,
-      "logps/chosen": -95.2154541015625,
-      "logps/rejected": -114.0091781616211,
-      "loss": 0.4335,
-      "rewards/accuracies": 0.7750000357627869,
-      "rewards/chosen": -2.1592423915863037,
-      "rewards/margins": 1.7459747791290283,
-      "rewards/rejected": -3.905216932296753,
-      "step": 1380
-    },
-    {
-      "epoch": 0.11108891108891109,
-      "grad_norm": 14.350967407226562,
-      "learning_rate": 4.998147161792663e-06,
-      "logits/chosen": -10.250876426696777,
-      "logits/rejected": -10.136800765991211,
-      "logps/chosen": -93.39093780517578,
-      "logps/rejected": -104.0428466796875,
-      "loss": 0.4735,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -1.659993052482605,
-      "rewards/margins": 1.5917600393295288,
-      "rewards/rejected": -3.2517528533935547,
-      "step": 1390
-    },
-    {
-      "epoch": 0.11188811188811189,
-      "grad_norm": 5.711775302886963,
-      "learning_rate": 4.997868944805393e-06,
-      "logits/chosen": -10.189711570739746,
-      "logits/rejected": -10.110020637512207,
-      "logps/chosen": -90.20991516113281,
-      "logps/rejected": -109.25304412841797,
-      "loss": 0.4291,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -1.949884295463562,
-      "rewards/margins": 1.5021190643310547,
-      "rewards/rejected": -3.4520034790039062,
-      "step": 1400
-    },
-    {
-      "epoch": 0.11268731268731269,
-      "grad_norm": 20.19569969177246,
-      "learning_rate": 4.99757128351731e-06,
-      "logits/chosen": -10.136848449707031,
-      "logits/rejected": -10.107802391052246,
-      "logps/chosen": -92.26567077636719,
-      "logps/rejected": -109.71477508544922,
-      "loss": 0.5538,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -2.4058151245117188,
-      "rewards/margins": 1.291394591331482,
-      "rewards/rejected": -3.6972100734710693,
-      "step": 1410
-    },
-    {
-      "epoch": 0.1134865134865135,
-      "grad_norm": 15.36054515838623,
-      "learning_rate": 4.997254180245514e-06,
-      "logits/chosen": -10.161433219909668,
-      "logits/rejected": -10.097549438476562,
-      "logps/chosen": -92.0826644897461,
-      "logps/rejected": -108.17498016357422,
-      "loss": 0.5328,
-      "rewards/accuracies": 0.7750000357627869,
-      "rewards/chosen": -2.0104286670684814,
-      "rewards/margins": 1.3813085556030273,
-      "rewards/rejected": -3.391737461090088,
-      "step": 1420
-    },
-    {
-      "epoch": 0.11428571428571428,
-      "grad_norm": 17.506912231445312,
-      "learning_rate": 4.9969176374584505e-06,
-      "logits/chosen": -10.202834129333496,
-      "logits/rejected": -10.060027122497559,
-      "logps/chosen": -97.62896728515625,
-      "logps/rejected": -110.66899871826172,
-      "loss": 0.5855,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": -2.381455421447754,
-      "rewards/margins": 1.3291677236557007,
-      "rewards/rejected": -3.710623264312744,
-      "step": 1430
-    },
-    {
-      "epoch": 0.11508491508491508,
-      "grad_norm": 19.703311920166016,
-      "learning_rate": 4.996561657775888e-06,
-      "logits/chosen": -10.093143463134766,
-      "logits/rejected": -10.130782127380371,
-      "logps/chosen": -96.21048736572266,
-      "logps/rejected": -111.49187469482422,
-      "loss": 0.5574,
-      "rewards/accuracies": 0.7750000357627869,
-      "rewards/chosen": -2.467775344848633,
-      "rewards/margins": 1.2885462045669556,
-      "rewards/rejected": -3.756321668624878,
-      "step": 1440
-    },
-    {
-      "epoch": 0.11588411588411589,
-      "grad_norm": 17.116640090942383,
-      "learning_rate": 4.996186243968899e-06,
-      "logits/chosen": -10.11058521270752,
-      "logits/rejected": -10.020354270935059,
-      "logps/chosen": -94.29996490478516,
-      "logps/rejected": -112.3018569946289,
-      "loss": 0.4899,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -2.504824161529541,
-      "rewards/margins": 1.5224872827529907,
-      "rewards/rejected": -4.027311325073242,
-      "step": 1450
-    },
-    {
-      "epoch": 0.11668331668331669,
-      "grad_norm": 14.589899063110352,
-      "learning_rate": 4.995791398959838e-06,
-      "logits/chosen": -10.134989738464355,
-      "logits/rejected": -10.075016975402832,
-      "logps/chosen": -101.14336395263672,
-      "logps/rejected": -119.43151092529297,
-      "loss": 0.5435,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -3.051831007003784,
-      "rewards/margins": 1.2346693277359009,
-      "rewards/rejected": -4.286500453948975,
-      "step": 1460
-    },
-    {
-      "epoch": 0.11748251748251748,
-      "grad_norm": 15.62790584564209,
-      "learning_rate": 4.9953771258223195e-06,
-      "logits/chosen": -10.08961009979248,
-      "logits/rejected": -10.021971702575684,
-      "logps/chosen": -99.96722412109375,
-      "logps/rejected": -118.1529541015625,
-      "loss": 0.4082,
-      "rewards/accuracies": 0.8375000357627869,
-      "rewards/chosen": -2.665688991546631,
-      "rewards/margins": 1.6962623596191406,
-      "rewards/rejected": -4.3619513511657715,
-      "step": 1470
-    },
-    {
-      "epoch": 0.11828171828171828,
-      "grad_norm": 8.319695472717285,
-      "learning_rate": 4.994943427781193e-06,
-      "logits/chosen": -10.13713264465332,
-      "logits/rejected": -10.111699104309082,
-      "logps/chosen": -102.05683898925781,
-      "logps/rejected": -123.41670989990234,
-      "loss": 0.4371,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": -3.1178786754608154,
-      "rewards/margins": 1.4688730239868164,
-      "rewards/rejected": -4.586751461029053,
-      "step": 1480
-    },
-    {
-      "epoch": 0.11908091908091908,
-      "grad_norm": 18.390336990356445,
-      "learning_rate": 4.994490308212518e-06,
-      "logits/chosen": -10.078123092651367,
-      "logits/rejected": -10.117024421691895,
-      "logps/chosen": -102.60737609863281,
-      "logps/rejected": -125.7625732421875,
-      "loss": 0.5117,
-      "rewards/accuracies": 0.7750000357627869,
-      "rewards/chosen": -2.8462154865264893,
-      "rewards/margins": 1.7813814878463745,
-      "rewards/rejected": -4.627597332000732,
-      "step": 1490
-    },
-    {
-      "epoch": 0.11988011988011989,
-      "grad_norm": 13.941241264343262,
-      "learning_rate": 4.99401777064354e-06,
-      "logits/chosen": -10.370619773864746,
-      "logits/rejected": -10.212640762329102,
-      "logps/chosen": -102.4493408203125,
-      "logps/rejected": -115.78319549560547,
-      "loss": 0.4955,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -2.7543721199035645,
-      "rewards/margins": 1.691931128501892,
-      "rewards/rejected": -4.446303367614746,
-      "step": 1500
-    },
-    {
-      "epoch": 0.12067932067932068,
-      "grad_norm": 12.472670555114746,
-      "learning_rate": 4.993525818752657e-06,
-      "logits/chosen": -10.220796585083008,
-      "logits/rejected": -10.086512565612793,
-      "logps/chosen": -100.5079574584961,
-      "logps/rejected": -119.4302978515625,
-      "loss": 0.5106,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -2.9049954414367676,
-      "rewards/margins": 1.6457710266113281,
-      "rewards/rejected": -4.550766468048096,
-      "step": 1510
-    },
-    {
-      "epoch": 0.12147852147852148,
-      "grad_norm": 13.18904972076416,
-      "learning_rate": 4.9930144563694e-06,
-      "logits/chosen": -10.146728515625,
-      "logits/rejected": -10.194172859191895,
-      "logps/chosen": -103.2806396484375,
-      "logps/rejected": -126.17928314208984,
-      "loss": 0.4131,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": -3.051010847091675,
-      "rewards/margins": 1.7306263446807861,
-      "rewards/rejected": -4.781637668609619,
-      "step": 1520
-    },
-    {
-      "epoch": 0.12227772227772228,
-      "grad_norm": 18.727994918823242,
-      "learning_rate": 4.992483687474393e-06,
-      "logits/chosen": -10.22678279876709,
-      "logits/rejected": -10.138778686523438,
-      "logps/chosen": -102.8808822631836,
-      "logps/rejected": -113.6004867553711,
-      "loss": 0.6529,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": -2.9898312091827393,
-      "rewards/margins": 1.1294559240341187,
-      "rewards/rejected": -4.119287014007568,
-      "step": 1530
-    },
-    {
-      "epoch": 0.12307692307692308,
-      "grad_norm": 13.128388404846191,
-      "learning_rate": 4.9919335161993336e-06,
-      "logits/chosen": -10.038443565368652,
-      "logits/rejected": -10.072142601013184,
-      "logps/chosen": -102.25023651123047,
-      "logps/rejected": -120.08846282958984,
-      "loss": 0.4833,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": -2.9147846698760986,
-      "rewards/margins": 1.7343504428863525,
-      "rewards/rejected": -4.649135112762451,
-      "step": 1540
-    },
-    {
-      "epoch": 0.12387612387612387,
-      "grad_norm": 14.58085823059082,
-      "learning_rate": 4.991363946826948e-06,
-      "logits/chosen": -10.023974418640137,
-      "logits/rejected": -10.000086784362793,
-      "logps/chosen": -106.97850799560547,
-      "logps/rejected": -127.46048736572266,
-      "loss": 0.3992,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -3.105583667755127,
-      "rewards/margins": 1.9775856733322144,
-      "rewards/rejected": -5.083169460296631,
-      "step": 1550
-    },
-    {
-      "epoch": 0.12467532467532468,
-      "grad_norm": 16.09984016418457,
-      "learning_rate": 4.990774983790968e-06,
-      "logits/chosen": -10.101019859313965,
-      "logits/rejected": -10.01642894744873,
-      "logps/chosen": -100.00489807128906,
-      "logps/rejected": -115.7220687866211,
-      "loss": 0.4798,
-      "rewards/accuracies": 0.7750000357627869,
-      "rewards/chosen": -2.975726366043091,
-      "rewards/margins": 1.4581574201583862,
-      "rewards/rejected": -4.4338836669921875,
-      "step": 1560
-    },
-    {
-      "epoch": 0.12547452547452548,
-      "grad_norm": 17.5771427154541,
-      "learning_rate": 4.990166631676091e-06,
-      "logits/chosen": -9.866114616394043,
-      "logits/rejected": -9.957470893859863,
-      "logps/chosen": -106.75850677490234,
-      "logps/rejected": -130.42210388183594,
-      "loss": 0.4659,
-      "rewards/accuracies": 0.7750000357627869,
-      "rewards/chosen": -3.528672933578491,
-      "rewards/margins": 1.6265524625778198,
-      "rewards/rejected": -5.1552252769470215,
-      "step": 1570
-    },
-    {
-      "epoch": 0.12627372627372627,
-      "grad_norm": 10.682378768920898,
-      "learning_rate": 4.989538895217947e-06,
-      "logits/chosen": -10.002883911132812,
-      "logits/rejected": -10.040613174438477,
-      "logps/chosen": -101.77174377441406,
-      "logps/rejected": -127.24999237060547,
-      "loss": 0.4363,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -3.1586811542510986,
-      "rewards/margins": 1.8682750463485718,
-      "rewards/rejected": -5.026956081390381,
-      "step": 1580
-    },
-    {
-      "epoch": 0.12707292707292708,
-      "grad_norm": 15.138803482055664,
-      "learning_rate": 4.98889177930306e-06,
-      "logits/chosen": -10.071353912353516,
-      "logits/rejected": -10.063872337341309,
-      "logps/chosen": -102.6048583984375,
-      "logps/rejected": -125.4056625366211,
-      "loss": 0.4258,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -3.0871083736419678,
-      "rewards/margins": 1.6908775568008423,
-      "rewards/rejected": -4.7779860496521,
-      "step": 1590
-    },
-    {
-      "epoch": 0.12787212787212787,
-      "grad_norm": 12.311010360717773,
-      "learning_rate": 4.988225288968809e-06,
-      "logits/chosen": -10.163139343261719,
-      "logits/rejected": -10.09135627746582,
-      "logps/chosen": -102.862548828125,
-      "logps/rejected": -123.2742919921875,
-      "loss": 0.5751,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -3.2173023223876953,
-      "rewards/margins": 1.3935867547988892,
-      "rewards/rejected": -4.610889434814453,
-      "step": 1600
-    },
-    {
-      "epoch": 0.12867132867132866,
-      "grad_norm": 17.597091674804688,
-      "learning_rate": 4.987539429403394e-06,
-      "logits/chosen": -10.237194061279297,
-      "logits/rejected": -10.193056106567383,
-      "logps/chosen": -102.14751434326172,
-      "logps/rejected": -114.44940185546875,
-      "loss": 0.6832,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": -3.1131019592285156,
-      "rewards/margins": 0.8959885835647583,
-      "rewards/rejected": -4.009090900421143,
-      "step": 1610
-    },
-    {
-      "epoch": 0.12947052947052948,
-      "grad_norm": 21.82961082458496,
-      "learning_rate": 4.986834205945789e-06,
-      "logits/chosen": -10.047111511230469,
-      "logits/rejected": -10.230392456054688,
-      "logps/chosen": -98.47086334228516,
-      "logps/rejected": -129.47569274902344,
-      "loss": 0.4445,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -2.724752902984619,
-      "rewards/margins": 1.9712406396865845,
-      "rewards/rejected": -4.695993900299072,
-      "step": 1620
-    },
-    {
-      "epoch": 0.13026973026973027,
-      "grad_norm": 13.836714744567871,
-      "learning_rate": 4.986109624085704e-06,
-      "logits/chosen": -10.236055374145508,
-      "logits/rejected": -10.19781494140625,
-      "logps/chosen": -96.30499267578125,
-      "logps/rejected": -113.21846771240234,
-      "loss": 0.4483,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -2.4735352993011475,
-      "rewards/margins": 1.5415750741958618,
-      "rewards/rejected": -4.015110015869141,
-      "step": 1630
-    },
-    {
-      "epoch": 0.13106893106893108,
-      "grad_norm": 9.461065292358398,
-      "learning_rate": 4.985365689463544e-06,
-      "logits/chosen": -10.24090576171875,
-      "logits/rejected": -10.232792854309082,
-      "logps/chosen": -105.45094299316406,
-      "logps/rejected": -119.8552474975586,
-      "loss": 0.4938,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -2.9603073596954346,
-      "rewards/margins": 1.7001574039459229,
-      "rewards/rejected": -4.660464763641357,
-      "step": 1640
-    },
-    {
-      "epoch": 0.13186813186813187,
-      "grad_norm": 16.63050651550293,
-      "learning_rate": 4.984602407870358e-06,
-      "logits/chosen": -9.972990036010742,
-      "logits/rejected": -10.09619140625,
-      "logps/chosen": -102.4815902709961,
-      "logps/rejected": -130.9027557373047,
-      "loss": 0.4214,
-      "rewards/accuracies": 0.8375000357627869,
-      "rewards/chosen": -2.9232234954833984,
-      "rewards/margins": 2.0470833778381348,
-      "rewards/rejected": -4.970306873321533,
-      "step": 1650
-    },
-    {
-      "epoch": 0.13266733266733266,
-      "grad_norm": 17.360261917114258,
-      "learning_rate": 4.9838197852478034e-06,
-      "logits/chosen": -10.22936725616455,
-      "logits/rejected": -10.120444297790527,
-      "logps/chosen": -102.35737609863281,
-      "logps/rejected": -122.07575988769531,
-      "loss": 0.5205,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -3.3799378871917725,
-      "rewards/margins": 1.5762410163879395,
-      "rewards/rejected": -4.956179141998291,
-      "step": 1660
-    },
-    {
-      "epoch": 0.13346653346653348,
-      "grad_norm": 19.732345581054688,
-      "learning_rate": 4.983017827688092e-06,
-      "logits/chosen": -9.989781379699707,
-      "logits/rejected": -10.211400985717773,
-      "logps/chosen": -105.05391693115234,
-      "logps/rejected": -139.42767333984375,
-      "loss": 0.483,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -3.3143703937530518,
-      "rewards/margins": 1.8999630212783813,
-      "rewards/rejected": -5.2143330574035645,
-      "step": 1670
-    },
-    {
-      "epoch": 0.13426573426573427,
-      "grad_norm": 16.516754150390625,
-      "learning_rate": 4.982196541433949e-06,
-      "logits/chosen": -10.27057933807373,
-      "logits/rejected": -10.220873832702637,
-      "logps/chosen": -99.73851776123047,
-      "logps/rejected": -120.62139129638672,
-      "loss": 0.5121,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": -2.6650800704956055,
-      "rewards/margins": 1.6771820783615112,
-      "rewards/rejected": -4.342262268066406,
-      "step": 1680
-    },
-    {
-      "epoch": 0.13506493506493505,
-      "grad_norm": 10.58267879486084,
-      "learning_rate": 4.981355932878556e-06,
-      "logits/chosen": -10.342429161071777,
-      "logits/rejected": -10.341120719909668,
-      "logps/chosen": -101.88104248046875,
-      "logps/rejected": -118.0716323852539,
-      "loss": 0.476,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -2.8363938331604004,
-      "rewards/margins": 1.6401910781860352,
-      "rewards/rejected": -4.4765849113464355,
-      "step": 1690
-    },
-    {
-      "epoch": 0.13586413586413587,
-      "grad_norm": 10.375319480895996,
-      "learning_rate": 4.980496008565512e-06,
-      "logits/chosen": -10.2080659866333,
-      "logits/rejected": -10.151313781738281,
-      "logps/chosen": -106.1724624633789,
-      "logps/rejected": -120.12035369873047,
-      "loss": 0.4919,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -3.2313430309295654,
-      "rewards/margins": 1.656258463859558,
-      "rewards/rejected": -4.887601375579834,
-      "step": 1700
-    },
-    {
-      "epoch": 0.13666333666333666,
-      "grad_norm": 6.511852741241455,
-      "learning_rate": 4.979616775188772e-06,
-      "logits/chosen": -10.20080852508545,
-      "logits/rejected": -10.14319896697998,
-      "logps/chosen": -99.20613861083984,
-      "logps/rejected": -125.47991180419922,
-      "loss": 0.2357,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": -2.5830414295196533,
-      "rewards/margins": 2.4369289875030518,
-      "rewards/rejected": -5.019970417022705,
-      "step": 1710
-    },
-    {
-      "epoch": 0.13746253746253748,
-      "grad_norm": 13.485071182250977,
-      "learning_rate": 4.978718239592603e-06,
-      "logits/chosen": -10.236807823181152,
-      "logits/rejected": -10.138768196105957,
-      "logps/chosen": -105.42073059082031,
-      "logps/rejected": -120.71671295166016,
-      "loss": 0.4516,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -3.141649007797241,
-      "rewards/margins": 1.7833950519561768,
-      "rewards/rejected": -4.925044059753418,
-      "step": 1720
-    },
-    {
-      "epoch": 0.13826173826173827,
-      "grad_norm": 8.877517700195312,
-      "learning_rate": 4.977800408771524e-06,
-      "logits/chosen": -10.232836723327637,
-      "logits/rejected": -10.129507064819336,
-      "logps/chosen": -106.29924774169922,
-      "logps/rejected": -123.05046844482422,
-      "loss": 0.51,
-      "rewards/accuracies": 0.7750000357627869,
-      "rewards/chosen": -3.2934608459472656,
-      "rewards/margins": 1.8296937942504883,
-      "rewards/rejected": -5.123155117034912,
-      "step": 1730
-    },
-    {
-      "epoch": 0.13906093906093905,
-      "grad_norm": 19.016088485717773,
-      "learning_rate": 4.976863289870258e-06,
-      "logits/chosen": -10.207541465759277,
-      "logits/rejected": -10.11108684539795,
-      "logps/chosen": -104.5240707397461,
-      "logps/rejected": -121.85936737060547,
-      "loss": 0.5537,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -3.3873188495635986,
-      "rewards/margins": 1.3276243209838867,
-      "rewards/rejected": -4.714942932128906,
-      "step": 1740
-    },
-    {
-      "epoch": 0.13986013986013987,
-      "grad_norm": 9.27366828918457,
-      "learning_rate": 4.97590689018367e-06,
-      "logits/chosen": -10.181601524353027,
-      "logits/rejected": -10.147019386291504,
-      "logps/chosen": -106.0588607788086,
-      "logps/rejected": -121.5968017578125,
-      "loss": 0.5364,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -3.300557851791382,
-      "rewards/margins": 1.6216862201690674,
-      "rewards/rejected": -4.922244071960449,
-      "step": 1750
-    },
-    {
-      "epoch": 0.14065934065934066,
-      "grad_norm": 9.187593460083008,
-      "learning_rate": 4.9749312171567185e-06,
-      "logits/chosen": -10.088096618652344,
-      "logits/rejected": -10.140934944152832,
-      "logps/chosen": -109.296875,
-      "logps/rejected": -128.57553100585938,
-      "loss": 0.6324,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -3.605172872543335,
-      "rewards/margins": 1.4040310382843018,
-      "rewards/rejected": -5.009204387664795,
-      "step": 1760
-    },
-    {
-      "epoch": 0.14145854145854145,
-      "grad_norm": 13.309720993041992,
-      "learning_rate": 4.973936278384388e-06,
-      "logits/chosen": -10.099014282226562,
-      "logits/rejected": -10.047331809997559,
-      "logps/chosen": -99.32417297363281,
-      "logps/rejected": -116.6791763305664,
-      "loss": 0.5165,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -2.890038251876831,
-      "rewards/margins": 1.5505856275558472,
-      "rewards/rejected": -4.440623760223389,
-      "step": 1770
-    },
-    {
-      "epoch": 0.14225774225774226,
-      "grad_norm": 18.847681045532227,
-      "learning_rate": 4.972922081611638e-06,
-      "logits/chosen": -10.143196105957031,
-      "logits/rejected": -10.073676109313965,
-      "logps/chosen": -101.39151763916016,
-      "logps/rejected": -109.65909576416016,
-      "loss": 0.7021,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": -3.0585334300994873,
-      "rewards/margins": 0.9025046229362488,
-      "rewards/rejected": -3.96103835105896,
-      "step": 1780
-    },
-    {
-      "epoch": 0.14305694305694305,
-      "grad_norm": 14.81293773651123,
-      "learning_rate": 4.971888634733333e-06,
-      "logits/chosen": -10.155919075012207,
-      "logits/rejected": -10.141654968261719,
-      "logps/chosen": -95.3860092163086,
-      "logps/rejected": -120.10465240478516,
-      "loss": 0.4041,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -2.396016836166382,
-      "rewards/margins": 2.0678250789642334,
-      "rewards/rejected": -4.463841438293457,
-      "step": 1790
-    },
-    {
-      "epoch": 0.14385614385614387,
-      "grad_norm": 10.353547096252441,
-      "learning_rate": 4.970835945794194e-06,
-      "logits/chosen": -10.021552085876465,
-      "logits/rejected": -10.11983585357666,
-      "logps/chosen": -93.78417205810547,
-      "logps/rejected": -117.18193817138672,
-      "loss": 0.4459,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -2.2192976474761963,
-      "rewards/margins": 1.7249668836593628,
-      "rewards/rejected": -3.9442641735076904,
-      "step": 1800
-    },
-    {
-      "epoch": 0.14465534465534466,
-      "grad_norm": 13.554123878479004,
-      "learning_rate": 4.9697640229887276e-06,
-      "logits/chosen": -10.092551231384277,
-      "logits/rejected": -10.0658597946167,
-      "logps/chosen": -93.59169006347656,
-      "logps/rejected": -114.05828857421875,
-      "loss": 0.4739,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -2.422353982925415,
-      "rewards/margins": 1.5377603769302368,
-      "rewards/rejected": -3.9601142406463623,
-      "step": 1810
-    },
-    {
-      "epoch": 0.14545454545454545,
-      "grad_norm": 6.956535339355469,
-      "learning_rate": 4.968672874661159e-06,
-      "logits/chosen": -10.121858596801758,
-      "logits/rejected": -10.103334426879883,
-      "logps/chosen": -90.10269927978516,
-      "logps/rejected": -107.42964172363281,
-      "loss": 0.5056,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -2.092161178588867,
-      "rewards/margins": 1.5522947311401367,
-      "rewards/rejected": -3.644456624984741,
-      "step": 1820
-    },
-    {
-      "epoch": 0.14625374625374626,
-      "grad_norm": 13.564081192016602,
-      "learning_rate": 4.967562509305378e-06,
-      "logits/chosen": -10.01568603515625,
-      "logits/rejected": -10.167561531066895,
-      "logps/chosen": -93.3025131225586,
-      "logps/rejected": -119.9324951171875,
-      "loss": 0.4104,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -2.195702314376831,
-      "rewards/margins": 1.805596947669983,
-      "rewards/rejected": -4.0012993812561035,
-      "step": 1830
-    },
-    {
-      "epoch": 0.14705294705294705,
-      "grad_norm": 19.11313819885254,
-      "learning_rate": 4.966432935564862e-06,
-      "logits/chosen": -10.194045066833496,
-      "logits/rejected": -10.11887264251709,
-      "logps/chosen": -98.67809295654297,
-      "logps/rejected": -112.08492279052734,
-      "loss": 0.5314,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -2.6485633850097656,
-      "rewards/margins": 1.577104926109314,
-      "rewards/rejected": -4.225668430328369,
-      "step": 1840
-    },
-    {
-      "epoch": 0.14785214785214784,
-      "grad_norm": 22.837602615356445,
-      "learning_rate": 4.965284162232616e-06,
-      "logits/chosen": -9.978657722473145,
-      "logits/rejected": -10.219416618347168,
-      "logps/chosen": -104.10310363769531,
-      "logps/rejected": -134.7755584716797,
-      "loss": 0.6344,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": -3.2193329334259033,
-      "rewards/margins": 1.4594929218292236,
-      "rewards/rejected": -4.678826332092285,
-      "step": 1850
-    },
-    {
-      "epoch": 0.14865134865134866,
-      "grad_norm": 18.858062744140625,
-      "learning_rate": 4.9641161982511e-06,
-      "logits/chosen": -10.115376472473145,
-      "logits/rejected": -10.083575248718262,
-      "logps/chosen": -98.5278091430664,
-      "logps/rejected": -111.3607177734375,
-      "loss": 0.5917,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": -2.508042335510254,
-      "rewards/margins": 1.1077853441238403,
-      "rewards/rejected": -3.6158275604248047,
-      "step": 1860
-    },
-    {
-      "epoch": 0.14945054945054945,
-      "grad_norm": 18.111146926879883,
-      "learning_rate": 4.962929052712161e-06,
-      "logits/chosen": -10.154253959655762,
-      "logits/rejected": -10.181533813476562,
-      "logps/chosen": -97.85489654541016,
-      "logps/rejected": -113.0101318359375,
-      "loss": 0.629,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": -2.643599510192871,
-      "rewards/margins": 1.1570767164230347,
-      "rewards/rejected": -3.800676107406616,
-      "step": 1870
-    },
-    {
-      "epoch": 0.15024975024975026,
-      "grad_norm": 16.29718780517578,
-      "learning_rate": 4.961722734856963e-06,
-      "logits/chosen": -10.185555458068848,
-      "logits/rejected": -10.110054969787598,
-      "logps/chosen": -95.54338836669922,
-      "logps/rejected": -108.25318908691406,
-      "loss": 0.5059,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -2.4079654216766357,
-      "rewards/margins": 1.308971643447876,
-      "rewards/rejected": -3.716937303543091,
-      "step": 1880
-    },
-    {
-      "epoch": 0.15104895104895105,
-      "grad_norm": 15.559223175048828,
-      "learning_rate": 4.9604972540759125e-06,
-      "logits/chosen": -10.053236961364746,
-      "logits/rejected": -10.091010093688965,
-      "logps/chosen": -91.77141571044922,
-      "logps/rejected": -121.4176025390625,
-      "loss": 0.3699,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -2.2185380458831787,
-      "rewards/margins": 2.1277170181274414,
-      "rewards/rejected": -4.346255302429199,
-      "step": 1890
-    },
-    {
-      "epoch": 0.15184815184815184,
-      "grad_norm": 16.811777114868164,
-      "learning_rate": 4.95925261990859e-06,
-      "logits/chosen": -10.211094856262207,
-      "logits/rejected": -10.102526664733887,
-      "logps/chosen": -97.80076599121094,
-      "logps/rejected": -110.70259094238281,
-      "loss": 0.4692,
-      "rewards/accuracies": 0.7750000357627869,
-      "rewards/chosen": -2.77150559425354,
-      "rewards/margins": 1.117484450340271,
-      "rewards/rejected": -3.8889901638031006,
-      "step": 1900
-    },
-    {
-      "epoch": 0.15264735264735266,
-      "grad_norm": 14.096748352050781,
-      "learning_rate": 4.9579888420436685e-06,
-      "logits/chosen": -10.266684532165527,
-      "logits/rejected": -10.1755952835083,
-      "logps/chosen": -99.5211410522461,
-      "logps/rejected": -114.09720611572266,
-      "loss": 0.5596,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -2.6963984966278076,
-      "rewards/margins": 1.3041267395019531,
-      "rewards/rejected": -4.00052547454834,
-      "step": 1910
-    },
-    {
-      "epoch": 0.15344655344655345,
-      "grad_norm": 12.035185813903809,
-      "learning_rate": 4.956705930318846e-06,
-      "logits/chosen": -10.360005378723145,
-      "logits/rejected": -10.299495697021484,
-      "logps/chosen": -95.43568420410156,
-      "logps/rejected": -114.6996841430664,
-      "loss": 0.392,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -2.4189343452453613,
-      "rewards/margins": 1.5851348638534546,
-      "rewards/rejected": -4.004069805145264,
-      "step": 1920
-    },
-    {
-      "epoch": 0.15424575424575424,
-      "grad_norm": 21.861103057861328,
-      "learning_rate": 4.955403894720763e-06,
-      "logits/chosen": -10.309077262878418,
-      "logits/rejected": -10.233592987060547,
-      "logps/chosen": -99.86886596679688,
-      "logps/rejected": -116.4983901977539,
-      "loss": 0.4537,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -2.6419732570648193,
-      "rewards/margins": 1.5703423023223877,
-      "rewards/rejected": -4.212315559387207,
-      "step": 1930
-    },
-    {
-      "epoch": 0.15504495504495505,
-      "grad_norm": 10.469332695007324,
-      "learning_rate": 4.954082745384929e-06,
-      "logits/chosen": -10.324387550354004,
-      "logits/rejected": -10.300577163696289,
-      "logps/chosen": -97.72261047363281,
-      "logps/rejected": -118.513916015625,
-      "loss": 0.4258,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -2.6373844146728516,
-      "rewards/margins": 1.7164171934127808,
-      "rewards/rejected": -4.353801727294922,
-      "step": 1940
-    },
-    {
-      "epoch": 0.15584415584415584,
-      "grad_norm": 14.715834617614746,
-      "learning_rate": 4.95274249259564e-06,
-      "logits/chosen": -10.156550407409668,
-      "logits/rejected": -10.29610824584961,
-      "logps/chosen": -104.190673828125,
-      "logps/rejected": -128.06883239746094,
-      "loss": 0.469,
-      "rewards/accuracies": 0.7750000357627869,
-      "rewards/chosen": -3.1413841247558594,
-      "rewards/margins": 1.6889790296554565,
-      "rewards/rejected": -4.830362796783447,
-      "step": 1950
-    },
-    {
-      "epoch": 0.15664335664335666,
-      "grad_norm": 14.701240539550781,
-      "learning_rate": 4.951383146785901e-06,
-      "logits/chosen": -10.242507934570312,
-      "logits/rejected": -10.248311042785645,
-      "logps/chosen": -96.90023803710938,
-      "logps/rejected": -120.33621978759766,
-      "loss": 0.3808,
-      "rewards/accuracies": 0.8375000357627869,
-      "rewards/chosen": -2.7027809619903564,
-      "rewards/margins": 1.877740502357483,
-      "rewards/rejected": -4.580521583557129,
-      "step": 1960
-    },
-    {
-      "epoch": 0.15744255744255745,
-      "grad_norm": 6.226134300231934,
-      "learning_rate": 4.950004718537343e-06,
-      "logits/chosen": -10.3671293258667,
-      "logits/rejected": -10.343233108520508,
-      "logps/chosen": -98.64249420166016,
-      "logps/rejected": -118.4775161743164,
-      "loss": 0.5053,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -2.516207456588745,
-      "rewards/margins": 1.798583984375,
-      "rewards/rejected": -4.314791679382324,
-      "step": 1970
-    },
-    {
-      "epoch": 0.15824175824175823,
-      "grad_norm": 12.28996467590332,
-      "learning_rate": 4.948607218580142e-06,
-      "logits/chosen": -10.079442977905273,
-      "logits/rejected": -10.25589656829834,
-      "logps/chosen": -100.0615463256836,
-      "logps/rejected": -125.9248046875,
-      "loss": 0.4506,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -2.649385452270508,
-      "rewards/margins": 2.0427322387695312,
-      "rewards/rejected": -4.692117691040039,
-      "step": 1980
-    },
-    {
-      "epoch": 0.15904095904095905,
-      "grad_norm": 12.924591064453125,
-      "learning_rate": 4.9471906577929365e-06,
-      "logits/chosen": -10.13953971862793,
-      "logits/rejected": -10.229791641235352,
-      "logps/chosen": -104.78520965576172,
-      "logps/rejected": -132.2235107421875,
-      "loss": 0.4453,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -2.940340757369995,
-      "rewards/margins": 1.6776905059814453,
-      "rewards/rejected": -4.6180315017700195,
-      "step": 1990
-    },
-    {
-      "epoch": 0.15984015984015984,
-      "grad_norm": 6.821367263793945,
-      "learning_rate": 4.9457550472027384e-06,
-      "logits/chosen": -10.422721862792969,
-      "logits/rejected": -10.324386596679688,
-      "logps/chosen": -102.27294158935547,
-      "logps/rejected": -124.86006164550781,
-      "loss": 0.4928,
-      "rewards/accuracies": 0.7125000357627869,
-      "rewards/chosen": -3.0608298778533936,
-      "rewards/margins": 1.8035176992416382,
-      "rewards/rejected": -4.864346981048584,
-      "step": 2000
-    },
-    {
-      "epoch": 0.16063936063936063,
-      "grad_norm": 11.987317085266113,
-      "learning_rate": 4.944300397984853e-06,
-      "logits/chosen": -10.271040916442871,
-      "logits/rejected": -10.377135276794434,
-      "logps/chosen": -93.9191665649414,
-      "logps/rejected": -122.4543685913086,
-      "loss": 0.4329,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -2.351457357406616,
-      "rewards/margins": 2.0309689044952393,
-      "rewards/rejected": -4.382425785064697,
-      "step": 2010
-    },
-    {
-      "epoch": 0.16143856143856145,
-      "grad_norm": 15.639244079589844,
-      "learning_rate": 4.942826721462784e-06,
-      "logits/chosen": -10.323519706726074,
-      "logits/rejected": -10.255013465881348,
-      "logps/chosen": -101.62516021728516,
-      "logps/rejected": -125.8342514038086,
-      "loss": 0.5153,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": -2.931608200073242,
-      "rewards/margins": 2.0116000175476074,
-      "rewards/rejected": -4.94320821762085,
-      "step": 2020
-    },
-    {
-      "epoch": 0.16223776223776223,
-      "grad_norm": 11.495155334472656,
-      "learning_rate": 4.941334029108158e-06,
-      "logits/chosen": -10.454184532165527,
-      "logits/rejected": -10.396833419799805,
-      "logps/chosen": -101.6688003540039,
-      "logps/rejected": -115.8319320678711,
-      "loss": 0.4669,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -2.6351420879364014,
-      "rewards/margins": 1.608012080192566,
-      "rewards/rejected": -4.243154048919678,
-      "step": 2030
-    },
-    {
-      "epoch": 0.16303696303696305,
-      "grad_norm": 13.162040710449219,
-      "learning_rate": 4.93982233254062e-06,
-      "logits/chosen": -10.46829891204834,
-      "logits/rejected": -10.413200378417969,
-      "logps/chosen": -93.69514465332031,
-      "logps/rejected": -110.6511001586914,
-      "loss": 0.51,
-      "rewards/accuracies": 0.7125000357627869,
-      "rewards/chosen": -2.364210605621338,
-      "rewards/margins": 1.6308265924453735,
-      "rewards/rejected": -3.995037078857422,
-      "step": 2040
-    },
-    {
-      "epoch": 0.16383616383616384,
-      "grad_norm": 10.807441711425781,
-      "learning_rate": 4.9382916435277575e-06,
-      "logits/chosen": -10.394776344299316,
-      "logits/rejected": -10.468161582946777,
-      "logps/chosen": -98.54164123535156,
-      "logps/rejected": -124.04756927490234,
-      "loss": 0.3813,
-      "rewards/accuracies": 0.8375000357627869,
-      "rewards/chosen": -2.47340989112854,
-      "rewards/margins": 1.9082088470458984,
-      "rewards/rejected": -4.381618499755859,
-      "step": 2050
-    },
-    {
-      "epoch": 0.16463536463536463,
-      "grad_norm": 7.141294479370117,
-      "learning_rate": 4.936741973984995e-06,
-      "logits/chosen": -10.537532806396484,
-      "logits/rejected": -10.439157485961914,
-      "logps/chosen": -94.81633758544922,
-      "logps/rejected": -110.32746887207031,
-      "loss": 0.4845,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -2.2102863788604736,
-      "rewards/margins": 1.4339321851730347,
-      "rewards/rejected": -3.6442184448242188,
-      "step": 2060
-    },
-    {
-      "epoch": 0.16543456543456544,
-      "grad_norm": 17.476516723632812,
-      "learning_rate": 4.935173335975515e-06,
-      "logits/chosen": -10.514183044433594,
-      "logits/rejected": -10.434171676635742,
-      "logps/chosen": -96.48020935058594,
-      "logps/rejected": -116.2853775024414,
-      "loss": 0.4628,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -2.4519524574279785,
-      "rewards/margins": 1.7710533142089844,
-      "rewards/rejected": -4.223005771636963,
-      "step": 2070
-    },
-    {
-      "epoch": 0.16623376623376623,
-      "grad_norm": 17.681516647338867,
-      "learning_rate": 4.933585741710152e-06,
-      "logits/chosen": -10.626288414001465,
-      "logits/rejected": -10.596988677978516,
-      "logps/chosen": -99.23323822021484,
-      "logps/rejected": -108.72478485107422,
-      "loss": 0.613,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": -2.611835241317749,
-      "rewards/margins": 1.120810627937317,
-      "rewards/rejected": -3.7326462268829346,
-      "step": 2080
-    },
-    {
-      "epoch": 0.16703296703296702,
-      "grad_norm": 12.512372016906738,
-      "learning_rate": 4.931979203547305e-06,
-      "logits/chosen": -10.581969261169434,
-      "logits/rejected": -10.576395034790039,
-      "logps/chosen": -95.50096893310547,
-      "logps/rejected": -116.55360412597656,
-      "loss": 0.3764,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -2.216334342956543,
-      "rewards/margins": 1.8770655393600464,
-      "rewards/rejected": -4.093400001525879,
-      "step": 2090
-    },
-    {
-      "epoch": 0.16783216783216784,
-      "grad_norm": 6.0488691329956055,
-      "learning_rate": 4.9303537339928395e-06,
-      "logits/chosen": -10.5402193069458,
-      "logits/rejected": -10.47226333618164,
-      "logps/chosen": -101.87493896484375,
-      "logps/rejected": -121.18462371826172,
-      "loss": 0.461,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -2.8145182132720947,
-      "rewards/margins": 1.6133416891098022,
-      "rewards/rejected": -4.427859783172607,
-      "step": 2100
-    },
-    {
-      "epoch": 0.16863136863136863,
-      "grad_norm": 11.144381523132324,
-      "learning_rate": 4.928709345699988e-06,
-      "logits/chosen": -10.50192928314209,
-      "logits/rejected": -10.488444328308105,
-      "logps/chosen": -95.65481567382812,
-      "logps/rejected": -110.44354248046875,
-      "loss": 0.5163,
-      "rewards/accuracies": 0.7750000357627869,
-      "rewards/chosen": -2.476945638656616,
-      "rewards/margins": 1.4942677021026611,
-      "rewards/rejected": -3.971212863922119,
-      "step": 2110
-    },
-    {
-      "epoch": 0.16943056943056944,
-      "grad_norm": 8.291966438293457,
-      "learning_rate": 4.927046051469255e-06,
-      "logits/chosen": -10.362055778503418,
-      "logits/rejected": -10.535036087036133,
-      "logps/chosen": -96.0195541381836,
-      "logps/rejected": -127.7907943725586,
-      "loss": 0.4381,
-      "rewards/accuracies": 0.8375000357627869,
-      "rewards/chosen": -2.3037381172180176,
-      "rewards/margins": 2.326535701751709,
-      "rewards/rejected": -4.630273342132568,
-      "step": 2120
-    },
-    {
-      "epoch": 0.17022977022977023,
-      "grad_norm": 16.796388626098633,
-      "learning_rate": 4.925363864248315e-06,
-      "logits/chosen": -10.466431617736816,
-      "logits/rejected": -10.53226375579834,
-      "logps/chosen": -98.57852935791016,
-      "logps/rejected": -122.16102600097656,
-      "loss": 0.5277,
-      "rewards/accuracies": 0.7125000357627869,
-      "rewards/chosen": -2.7202389240264893,
-      "rewards/margins": 1.4860879182815552,
-      "rewards/rejected": -4.206326961517334,
-      "step": 2130
-    },
-    {
-      "epoch": 0.17102897102897102,
-      "grad_norm": 15.538496971130371,
-      "learning_rate": 4.923662797131911e-06,
-      "logits/chosen": -10.62234115600586,
-      "logits/rejected": -10.555063247680664,
-      "logps/chosen": -101.50341796875,
-      "logps/rejected": -118.88701629638672,
-      "loss": 0.4875,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -2.9368131160736084,
-      "rewards/margins": 1.6836929321289062,
-      "rewards/rejected": -4.6205058097839355,
-      "step": 2140
-    },
-    {
-      "epoch": 0.17182817182817184,
-      "grad_norm": 5.008636951446533,
-      "learning_rate": 4.921942863361756e-06,
-      "logits/chosen": -10.370814323425293,
-      "logits/rejected": -10.40635871887207,
-      "logps/chosen": -100.81613159179688,
-      "logps/rejected": -131.5894012451172,
-      "loss": 0.4278,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -2.7908005714416504,
-      "rewards/margins": 2.3979249000549316,
-      "rewards/rejected": -5.188725471496582,
-      "step": 2150
-    },
-    {
-      "epoch": 0.17262737262737263,
-      "grad_norm": 4.927359104156494,
-      "learning_rate": 4.920204076326424e-06,
-      "logits/chosen": -10.37940788269043,
-      "logits/rejected": -10.381207466125488,
-      "logps/chosen": -104.05641174316406,
-      "logps/rejected": -136.6097869873047,
-      "loss": 0.4629,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -2.833836793899536,
-      "rewards/margins": 2.2862555980682373,
-      "rewards/rejected": -5.120091915130615,
-      "step": 2160
-    },
-    {
-      "epoch": 0.17342657342657342,
-      "grad_norm": 6.2211995124816895,
-      "learning_rate": 4.918446449561256e-06,
-      "logits/chosen": -10.465067863464355,
-      "logits/rejected": -10.413912773132324,
-      "logps/chosen": -104.07014465332031,
-      "logps/rejected": -121.4678726196289,
-      "loss": 0.3821,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -2.9140536785125732,
-      "rewards/margins": 1.8531982898712158,
-      "rewards/rejected": -4.767251968383789,
-      "step": 2170
-    },
-    {
-      "epoch": 0.17422577422577423,
-      "grad_norm": 15.96093463897705,
-      "learning_rate": 4.916669996748241e-06,
-      "logits/chosen": -10.132461547851562,
-      "logits/rejected": -10.340156555175781,
-      "logps/chosen": -103.00862121582031,
-      "logps/rejected": -127.97505950927734,
-      "loss": 0.5092,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -2.9160382747650146,
-      "rewards/margins": 1.9175217151641846,
-      "rewards/rejected": -4.833560466766357,
-      "step": 2180
-    },
-    {
-      "epoch": 0.17502497502497502,
-      "grad_norm": 9.189192771911621,
-      "learning_rate": 4.914874731715922e-06,
-      "logits/chosen": -10.33960247039795,
-      "logits/rejected": -10.356241226196289,
-      "logps/chosen": -102.74974822998047,
-      "logps/rejected": -130.01589965820312,
-      "loss": 0.4957,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -2.8845901489257812,
-      "rewards/margins": 1.936846375465393,
-      "rewards/rejected": -4.821436882019043,
-      "step": 2190
-    },
-    {
-      "epoch": 0.17582417582417584,
-      "grad_norm": 19.693923950195312,
-      "learning_rate": 4.913060668439279e-06,
-      "logits/chosen": -10.48015022277832,
-      "logits/rejected": -10.415841102600098,
-      "logps/chosen": -98.791748046875,
-      "logps/rejected": -121.809814453125,
-      "loss": 0.4732,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": -2.8315017223358154,
-      "rewards/margins": 1.8437576293945312,
-      "rewards/rejected": -4.675259590148926,
-      "step": 2200
-    },
-    {
-      "epoch": 0.17662337662337663,
-      "grad_norm": 15.863086700439453,
-      "learning_rate": 4.911227821039627e-06,
-      "logits/chosen": -10.393231391906738,
-      "logits/rejected": -10.411093711853027,
-      "logps/chosen": -102.43208312988281,
-      "logps/rejected": -120.86140441894531,
-      "loss": 0.4454,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -2.911741018295288,
-      "rewards/margins": 1.8270330429077148,
-      "rewards/rejected": -4.738774299621582,
-      "step": 2210
-    },
-    {
-      "epoch": 0.17742257742257742,
-      "grad_norm": 14.732115745544434,
-      "learning_rate": 4.909376203784504e-06,
-      "logits/chosen": -10.491077423095703,
-      "logits/rejected": -10.472362518310547,
-      "logps/chosen": -102.68416595458984,
-      "logps/rejected": -120.45503234863281,
-      "loss": 0.4677,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": -3.0989975929260254,
-      "rewards/margins": 1.5316466093063354,
-      "rewards/rejected": -4.63064432144165,
-      "step": 2220
-    },
-    {
-      "epoch": 0.17822177822177823,
-      "grad_norm": 20.54006004333496,
-      "learning_rate": 4.907505831087555e-06,
-      "logits/chosen": -10.434184074401855,
-      "logits/rejected": -10.36945629119873,
-      "logps/chosen": -106.27290344238281,
-      "logps/rejected": -118.57366180419922,
-      "loss": 0.5635,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": -3.1145856380462646,
-      "rewards/margins": 1.3009637594223022,
-      "rewards/rejected": -4.4155497550964355,
-      "step": 2230
-    },
-    {
-      "epoch": 0.17902097902097902,
-      "grad_norm": 11.54550552368164,
-      "learning_rate": 4.905616717508428e-06,
-      "logits/chosen": -10.485742568969727,
-      "logits/rejected": -10.417367935180664,
-      "logps/chosen": -99.06552124023438,
-      "logps/rejected": -121.42340087890625,
-      "loss": 0.4737,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -2.7074830532073975,
-      "rewards/margins": 1.6691547632217407,
-      "rewards/rejected": -4.376637935638428,
-      "step": 2240
-    },
-    {
-      "epoch": 0.1798201798201798,
-      "grad_norm": 9.432591438293457,
-      "learning_rate": 4.903708877752656e-06,
-      "logits/chosen": -10.386659622192383,
-      "logits/rejected": -10.39673900604248,
-      "logps/chosen": -100.4712905883789,
-      "logps/rejected": -125.22148895263672,
-      "loss": 0.3651,
-      "rewards/accuracies": 0.8375000357627869,
-      "rewards/chosen": -2.791694402694702,
-      "rewards/margins": 1.851816177368164,
-      "rewards/rejected": -4.643510341644287,
-      "step": 2250
-    },
-    {
-      "epoch": 0.18061938061938063,
-      "grad_norm": 17.72370147705078,
-      "learning_rate": 4.901782326671541e-06,
-      "logits/chosen": -10.326799392700195,
-      "logits/rejected": -10.435379028320312,
-      "logps/chosen": -96.844970703125,
-      "logps/rejected": -132.67674255371094,
-      "loss": 0.4368,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -2.9108102321624756,
-      "rewards/margins": 2.0217196941375732,
-      "rewards/rejected": -4.932530403137207,
-      "step": 2260
-    },
-    {
-      "epoch": 0.18141858141858141,
-      "grad_norm": 12.109367370605469,
-      "learning_rate": 4.8998370792620445e-06,
-      "logits/chosen": -10.46472454071045,
-      "logits/rejected": -10.46088695526123,
-      "logps/chosen": -102.29825592041016,
-      "logps/rejected": -127.4029541015625,
-      "loss": 0.4331,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -2.90110445022583,
-      "rewards/margins": 1.9227485656738281,
-      "rewards/rejected": -4.823853492736816,
-      "step": 2270
-    },
-    {
-      "epoch": 0.18221778221778223,
-      "grad_norm": 16.875375747680664,
-      "learning_rate": 4.897873150666662e-06,
-      "logits/chosen": -10.621057510375977,
-      "logits/rejected": -10.593133926391602,
-      "logps/chosen": -102.56365203857422,
-      "logps/rejected": -123.83507537841797,
-      "loss": 0.4743,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -3.0794618129730225,
-      "rewards/margins": 1.761688470840454,
-      "rewards/rejected": -4.841150760650635,
-      "step": 2280
-    },
-    {
-      "epoch": 0.18301698301698302,
-      "grad_norm": 12.519655227661133,
-      "learning_rate": 4.895890556173315e-06,
-      "logits/chosen": -10.66734790802002,
-      "logits/rejected": -10.661360740661621,
-      "logps/chosen": -104.6373519897461,
-      "logps/rejected": -117.6056137084961,
-      "loss": 0.4248,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -3.164829730987549,
-      "rewards/margins": 1.5885868072509766,
-      "rewards/rejected": -4.753416538238525,
-      "step": 2290
-    },
-    {
-      "epoch": 0.1838161838161838,
-      "grad_norm": 12.09952449798584,
-      "learning_rate": 4.893889311215222e-06,
-      "logits/chosen": -10.625605583190918,
-      "logits/rejected": -10.566299438476562,
-      "logps/chosen": -104.99718475341797,
-      "logps/rejected": -129.29478454589844,
-      "loss": 0.2968,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": -3.2635974884033203,
-      "rewards/margins": 2.182102918624878,
-      "rewards/rejected": -5.445700168609619,
-      "step": 2300
-    },
-    {
-      "epoch": 0.18461538461538463,
-      "grad_norm": 10.998613357543945,
-      "learning_rate": 4.891869431370789e-06,
-      "logits/chosen": -10.69331169128418,
-      "logits/rejected": -10.592437744140625,
-      "logps/chosen": -112.79772186279297,
-      "logps/rejected": -133.6348114013672,
-      "loss": 0.4494,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -3.744283676147461,
-      "rewards/margins": 1.9825012683868408,
-      "rewards/rejected": -5.726785182952881,
-      "step": 2310
-    },
-    {
-      "epoch": 0.18541458541458541,
-      "grad_norm": 14.158156394958496,
-      "learning_rate": 4.889830932363477e-06,
-      "logits/chosen": -10.354763984680176,
-      "logits/rejected": -10.595683097839355,
-      "logps/chosen": -106.4779052734375,
-      "logps/rejected": -145.45179748535156,
-      "loss": 0.5388,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -3.5498712062835693,
-      "rewards/margins": 2.2635765075683594,
-      "rewards/rejected": -5.81344747543335,
-      "step": 2320
-    },
-    {
-      "epoch": 0.1862137862137862,
-      "grad_norm": 13.188819885253906,
-      "learning_rate": 4.8877738300616904e-06,
-      "logits/chosen": -10.630270004272461,
-      "logits/rejected": -10.619091987609863,
-      "logps/chosen": -108.4816665649414,
-      "logps/rejected": -127.9704360961914,
-      "loss": 0.5562,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -3.6334152221679688,
-      "rewards/margins": 1.7234859466552734,
-      "rewards/rejected": -5.356901168823242,
-      "step": 2330
-    },
-    {
-      "epoch": 0.18701298701298702,
-      "grad_norm": 19.97298812866211,
-      "learning_rate": 4.885698140478643e-06,
-      "logits/chosen": -10.58581256866455,
-      "logits/rejected": -10.598006248474121,
-      "logps/chosen": -107.3741455078125,
-      "logps/rejected": -135.73526000976562,
-      "loss": 0.3938,
-      "rewards/accuracies": 0.7750000357627869,
-      "rewards/chosen": -3.553593397140503,
-      "rewards/margins": 2.0507962703704834,
-      "rewards/rejected": -5.6043901443481445,
-      "step": 2340
-    },
-    {
-      "epoch": 0.1878121878121878,
-      "grad_norm": 15.139594078063965,
-      "learning_rate": 4.883603879772242e-06,
-      "logits/chosen": -10.55914306640625,
-      "logits/rejected": -10.687617301940918,
-      "logps/chosen": -108.88002014160156,
-      "logps/rejected": -129.59170532226562,
-      "loss": 0.5406,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -3.5649666786193848,
-      "rewards/margins": 1.8942726850509644,
-      "rewards/rejected": -5.459239482879639,
-      "step": 2350
-    },
-    {
-      "epoch": 0.18861138861138863,
-      "grad_norm": 2.0041840076446533,
-      "learning_rate": 4.881491064244959e-06,
-      "logits/chosen": -10.527430534362793,
-      "logits/rejected": -10.55819320678711,
-      "logps/chosen": -103.3442611694336,
-      "logps/rejected": -139.25169372558594,
-      "loss": 0.3036,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": -3.5036957263946533,
-      "rewards/margins": 2.547741651535034,
-      "rewards/rejected": -6.0514373779296875,
-      "step": 2360
-    },
-    {
-      "epoch": 0.18941058941058941,
-      "grad_norm": 13.3259916305542,
-      "learning_rate": 4.8793597103437006e-06,
-      "logits/chosen": -10.604536056518555,
-      "logits/rejected": -10.610457420349121,
-      "logps/chosen": -106.43437957763672,
-      "logps/rejected": -128.89793395996094,
-      "loss": 0.4697,
-      "rewards/accuracies": 0.7750000357627869,
-      "rewards/chosen": -3.6763055324554443,
-      "rewards/margins": 1.6551653146743774,
-      "rewards/rejected": -5.331470489501953,
-      "step": 2370
-    },
-    {
-      "epoch": 0.1902097902097902,
-      "grad_norm": 7.772974967956543,
-      "learning_rate": 4.877209834659684e-06,
-      "logits/chosen": -10.664505004882812,
-      "logits/rejected": -10.662419319152832,
-      "logps/chosen": -111.8147201538086,
-      "logps/rejected": -136.22276306152344,
-      "loss": 0.3856,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -3.7113003730773926,
-      "rewards/margins": 2.276648998260498,
-      "rewards/rejected": -5.987949371337891,
-      "step": 2380
-    },
-    {
-      "epoch": 0.19100899100899102,
-      "grad_norm": 5.815654754638672,
-      "learning_rate": 4.8750414539283086e-06,
-      "logits/chosen": -10.54272174835205,
-      "logits/rejected": -10.583903312683105,
-      "logps/chosen": -108.4759750366211,
-      "logps/rejected": -130.7399139404297,
-      "loss": 0.4655,
-      "rewards/accuracies": 0.7750000357627869,
-      "rewards/chosen": -3.7454071044921875,
-      "rewards/margins": 1.8315483331680298,
-      "rewards/rejected": -5.5769548416137695,
-      "step": 2390
-    },
-    {
-      "epoch": 0.1918081918081918,
-      "grad_norm": 9.021738052368164,
-      "learning_rate": 4.87285458502902e-06,
-      "logits/chosen": -10.70053768157959,
-      "logits/rejected": -10.579264640808105,
-      "logps/chosen": -106.84130096435547,
-      "logps/rejected": -124.8370361328125,
-      "loss": 0.3983,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -3.41552734375,
-      "rewards/margins": 1.81061589717865,
-      "rewards/rejected": -5.2261433601379395,
-      "step": 2400
-    },
-    {
-      "epoch": 0.1926073926073926,
-      "grad_norm": 15.372767448425293,
-      "learning_rate": 4.870649244985185e-06,
-      "logits/chosen": -10.577818870544434,
-      "logits/rejected": -10.540336608886719,
-      "logps/chosen": -104.9249038696289,
-      "logps/rejected": -124.8220443725586,
-      "loss": 0.5324,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -3.282487630844116,
-      "rewards/margins": 1.7564973831176758,
-      "rewards/rejected": -5.038985252380371,
-      "step": 2410
-    },
-    {
-      "epoch": 0.1934065934065934,
-      "grad_norm": 16.5902099609375,
-      "learning_rate": 4.868425450963954e-06,
-      "logits/chosen": -10.691306114196777,
-      "logits/rejected": -10.607848167419434,
-      "logps/chosen": -106.44769287109375,
-      "logps/rejected": -123.17040252685547,
-      "loss": 0.4018,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -3.202204465866089,
-      "rewards/margins": 1.971347451210022,
-      "rewards/rejected": -5.1735520362854,
-      "step": 2420
-    },
-    {
-      "epoch": 0.1942057942057942,
-      "grad_norm": 18.179473876953125,
-      "learning_rate": 4.8661832202761325e-06,
-      "logits/chosen": -10.425712585449219,
-      "logits/rejected": -10.640589714050293,
-      "logps/chosen": -100.29244232177734,
-      "logps/rejected": -133.94224548339844,
-      "loss": 0.5147,
-      "rewards/accuracies": 0.7750000357627869,
-      "rewards/chosen": -2.6288230419158936,
-      "rewards/margins": 2.320405960083008,
-      "rewards/rejected": -4.949229717254639,
-      "step": 2430
-    },
-    {
-      "epoch": 0.19500499500499502,
-      "grad_norm": 9.252065658569336,
-      "learning_rate": 4.863922570376041e-06,
-      "logits/chosen": -10.618536949157715,
-      "logits/rejected": -10.590038299560547,
-      "logps/chosen": -96.54149627685547,
-      "logps/rejected": -122.0170669555664,
-      "loss": 0.4227,
-      "rewards/accuracies": 0.8375000357627869,
-      "rewards/chosen": -2.345371723175049,
-      "rewards/margins": 1.9312533140182495,
-      "rewards/rejected": -4.276625156402588,
-      "step": 2440
-    },
-    {
-      "epoch": 0.1958041958041958,
-      "grad_norm": 19.35995864868164,
-      "learning_rate": 4.861643518861383e-06,
-      "logits/chosen": -10.65555191040039,
-      "logits/rejected": -10.66125202178955,
-      "logps/chosen": -96.7305679321289,
-      "logps/rejected": -120.8079833984375,
-      "loss": 0.4892,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -2.612701892852783,
-      "rewards/margins": 1.7611732482910156,
-      "rewards/rejected": -4.373875141143799,
-      "step": 2450
-    },
-    {
-      "epoch": 0.1966033966033966,
-      "grad_norm": 10.779239654541016,
-      "learning_rate": 4.8593460834731075e-06,
-      "logits/chosen": -10.535378456115723,
-      "logits/rejected": -10.533195495605469,
-      "logps/chosen": -97.75586700439453,
-      "logps/rejected": -121.8031005859375,
-      "loss": 0.3735,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -2.9553823471069336,
-      "rewards/margins": 2.15702748298645,
-      "rewards/rejected": -5.112409591674805,
-      "step": 2460
-    },
-    {
-      "epoch": 0.1974025974025974,
-      "grad_norm": 16.20267105102539,
-      "learning_rate": 4.857030282095269e-06,
-      "logits/chosen": -10.685494422912598,
-      "logits/rejected": -10.652603149414062,
-      "logps/chosen": -101.35970306396484,
-      "logps/rejected": -129.28610229492188,
-      "loss": 0.3043,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -2.915410041809082,
-      "rewards/margins": 2.570354461669922,
-      "rewards/rejected": -5.485764980316162,
-      "step": 2470
-    },
-    {
-      "epoch": 0.1982017982017982,
-      "grad_norm": 13.131753921508789,
-      "learning_rate": 4.854696132754888e-06,
-      "logits/chosen": -10.578619956970215,
-      "logits/rejected": -10.617962837219238,
-      "logps/chosen": -111.79339599609375,
-      "logps/rejected": -134.59812927246094,
-      "loss": 0.498,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -3.85662841796875,
-      "rewards/margins": 1.7778087854385376,
-      "rewards/rejected": -5.634437084197998,
-      "step": 2480
-    },
-    {
-      "epoch": 0.199000999000999,
-      "grad_norm": 19.224109649658203,
-      "learning_rate": 4.852343653621815e-06,
-      "logits/chosen": -10.741569519042969,
-      "logits/rejected": -10.686893463134766,
-      "logps/chosen": -104.23567962646484,
-      "logps/rejected": -130.77008056640625,
-      "loss": 0.4078,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -3.494797945022583,
-      "rewards/margins": 2.1074116230010986,
-      "rewards/rejected": -5.60221004486084,
-      "step": 2490
-    },
-    {
-      "epoch": 0.1998001998001998,
-      "grad_norm": 17.750337600708008,
-      "learning_rate": 4.849972863008585e-06,
-      "logits/chosen": -10.586366653442383,
-      "logits/rejected": -10.663454055786133,
-      "logps/chosen": -114.85462951660156,
-      "logps/rejected": -140.34112548828125,
-      "loss": 0.5046,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -3.9352054595947266,
-      "rewards/margins": 2.1099071502685547,
-      "rewards/rejected": -6.045112609863281,
-      "step": 2500
-    },
-    {
-      "epoch": 0.2005994005994006,
-      "grad_norm": 20.27694320678711,
-      "learning_rate": 4.847583779370276e-06,
-      "logits/chosen": -10.634108543395996,
-      "logits/rejected": -10.5838041305542,
-      "logps/chosen": -116.15126037597656,
-      "logps/rejected": -132.94644165039062,
-      "loss": 0.6189,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -4.2932939529418945,
-      "rewards/margins": 1.8430267572402954,
-      "rewards/rejected": -6.136319637298584,
-      "step": 2510
-    },
-    {
-      "epoch": 0.2013986013986014,
-      "grad_norm": 11.6603364944458,
-      "learning_rate": 4.845176421304364e-06,
-      "logits/chosen": -10.655728340148926,
-      "logits/rejected": -10.594734191894531,
-      "logps/chosen": -107.4415283203125,
-      "logps/rejected": -130.86790466308594,
-      "loss": 0.3706,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -3.7540199756622314,
-      "rewards/margins": 2.3693530559539795,
-      "rewards/rejected": -6.123373508453369,
-      "step": 2520
-    },
-    {
-      "epoch": 0.2021978021978022,
-      "grad_norm": 7.581463813781738,
-      "learning_rate": 4.842750807550581e-06,
-      "logits/chosen": -10.673118591308594,
-      "logits/rejected": -10.564537048339844,
-      "logps/chosen": -108.4826889038086,
-      "logps/rejected": -132.9703826904297,
-      "loss": 0.3703,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -3.7118842601776123,
-      "rewards/margins": 2.1689469814300537,
-      "rewards/rejected": -5.880831718444824,
-      "step": 2530
-    },
-    {
-      "epoch": 0.202997002997003,
-      "grad_norm": 8.599440574645996,
-      "learning_rate": 4.84030695699077e-06,
-      "logits/chosen": -10.422922134399414,
-      "logits/rejected": -10.471639633178711,
-      "logps/chosen": -109.0437240600586,
-      "logps/rejected": -144.8496856689453,
-      "loss": 0.3634,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": -3.8359177112579346,
-      "rewards/margins": 2.711230993270874,
-      "rewards/rejected": -6.54714822769165,
-      "step": 2540
-    },
-    {
-      "epoch": 0.2037962037962038,
-      "grad_norm": 9.109241485595703,
-      "learning_rate": 4.8378448886487305e-06,
-      "logits/chosen": -10.60950756072998,
-      "logits/rejected": -10.587714195251465,
-      "logps/chosen": -106.5068588256836,
-      "logps/rejected": -129.07369995117188,
-      "loss": 0.3851,
-      "rewards/accuracies": 0.8375000357627869,
-      "rewards/chosen": -3.4131009578704834,
-      "rewards/margins": 2.165586233139038,
-      "rewards/rejected": -5.5786871910095215,
-      "step": 2550
-    },
-    {
-      "epoch": 0.2045954045954046,
-      "grad_norm": 12.172859191894531,
-      "learning_rate": 4.83536462169008e-06,
-      "logits/chosen": -10.614439964294434,
-      "logits/rejected": -10.699813842773438,
-      "logps/chosen": -107.03336334228516,
-      "logps/rejected": -145.46800231933594,
-      "loss": 0.3339,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -3.572758436203003,
-      "rewards/margins": 2.8419549465179443,
-      "rewards/rejected": -6.414712429046631,
-      "step": 2560
-    },
-    {
-      "epoch": 0.20539460539460538,
-      "grad_norm": 10.977602005004883,
-      "learning_rate": 4.8328661754221e-06,
-      "logits/chosen": -10.645990371704102,
-      "logits/rejected": -10.709532737731934,
-      "logps/chosen": -100.6408920288086,
-      "logps/rejected": -129.27320861816406,
-      "loss": 0.3918,
-      "rewards/accuracies": 0.8375000357627869,
-      "rewards/chosen": -3.156209707260132,
-      "rewards/margins": 2.329537868499756,
-      "rewards/rejected": -5.485747337341309,
-      "step": 2570
-    },
-    {
-      "epoch": 0.2061938061938062,
-      "grad_norm": 12.253616333007812,
-      "learning_rate": 4.830349569293584e-06,
-      "logits/chosen": -10.813051223754883,
-      "logits/rejected": -10.803766250610352,
-      "logps/chosen": -106.31208801269531,
-      "logps/rejected": -128.00302124023438,
-      "loss": 0.4899,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -3.913928985595703,
-      "rewards/margins": 1.778978943824768,
-      "rewards/rejected": -5.692908763885498,
-      "step": 2580
-    },
-    {
-      "epoch": 0.206993006993007,
-      "grad_norm": 13.55671215057373,
-      "learning_rate": 4.82781482289469e-06,
-      "logits/chosen": -10.833529472351074,
-      "logits/rejected": -10.762760162353516,
-      "logps/chosen": -105.3536148071289,
-      "logps/rejected": -128.6766357421875,
-      "loss": 0.4626,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -3.788266897201538,
-      "rewards/margins": 1.975807785987854,
-      "rewards/rejected": -5.764074325561523,
-      "step": 2590
-    },
-    {
-      "epoch": 0.2077922077922078,
-      "grad_norm": 12.367059707641602,
-      "learning_rate": 4.825261955956787e-06,
-      "logits/chosen": -10.88508129119873,
-      "logits/rejected": -10.806241035461426,
-      "logps/chosen": -109.46379852294922,
-      "logps/rejected": -133.81765747070312,
-      "loss": 0.4478,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -3.542149305343628,
-      "rewards/margins": 2.122302532196045,
-      "rewards/rejected": -5.664452075958252,
-      "step": 2600
-    },
-    {
-      "epoch": 0.2085914085914086,
-      "grad_norm": 16.826372146606445,
-      "learning_rate": 4.822690988352299e-06,
-      "logits/chosen": -10.91463851928711,
-      "logits/rejected": -10.809722900390625,
-      "logps/chosen": -105.45574188232422,
-      "logps/rejected": -129.732421875,
-      "loss": 0.3489,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -3.4098472595214844,
-      "rewards/margins": 2.3195455074310303,
-      "rewards/rejected": -5.7293925285339355,
-      "step": 2610
-    },
-    {
-      "epoch": 0.20939060939060938,
-      "grad_norm": 12.427678108215332,
-      "learning_rate": 4.820101940094552e-06,
-      "logits/chosen": -10.920578002929688,
-      "logits/rejected": -10.790773391723633,
-      "logps/chosen": -113.67445373535156,
-      "logps/rejected": -136.81118774414062,
-      "loss": 0.4416,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -4.328470706939697,
-      "rewards/margins": 2.0502097606658936,
-      "rewards/rejected": -6.378680229187012,
-      "step": 2620
-    },
-    {
-      "epoch": 0.2101898101898102,
-      "grad_norm": 17.296348571777344,
-      "learning_rate": 4.817494831337621e-06,
-      "logits/chosen": -10.958685874938965,
-      "logits/rejected": -10.895133018493652,
-      "logps/chosen": -112.11856842041016,
-      "logps/rejected": -131.7047119140625,
-      "loss": 0.6173,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": -3.861100912094116,
-      "rewards/margins": 1.7293856143951416,
-      "rewards/rejected": -5.590486526489258,
-      "step": 2630
-    },
-    {
-      "epoch": 0.210989010989011,
-      "grad_norm": 8.671651840209961,
-      "learning_rate": 4.8148696823761675e-06,
-      "logits/chosen": -10.896438598632812,
-      "logits/rejected": -10.810698509216309,
-      "logps/chosen": -107.3486328125,
-      "logps/rejected": -135.53353881835938,
-      "loss": 0.4052,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -3.7070469856262207,
-      "rewards/margins": 2.3581295013427734,
-      "rewards/rejected": -6.065176486968994,
-      "step": 2640
-    },
-    {
-      "epoch": 0.21178821178821178,
-      "grad_norm": 16.894672393798828,
-      "learning_rate": 4.812226513645284e-06,
-      "logits/chosen": -10.88752269744873,
-      "logits/rejected": -10.866678237915039,
-      "logps/chosen": -112.63353729248047,
-      "logps/rejected": -133.41357421875,
-      "loss": 0.3777,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -4.013194561004639,
-      "rewards/margins": 2.000812292098999,
-      "rewards/rejected": -6.0140061378479,
-      "step": 2650
-    },
-    {
-      "epoch": 0.2125874125874126,
-      "grad_norm": 23.433218002319336,
-      "learning_rate": 4.809565345720339e-06,
-      "logits/chosen": -11.05213451385498,
-      "logits/rejected": -11.008994102478027,
-      "logps/chosen": -114.2861557006836,
-      "logps/rejected": -136.83627319335938,
-      "loss": 0.3715,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -4.040170192718506,
-      "rewards/margins": 2.264209270477295,
-      "rewards/rejected": -6.304379463195801,
-      "step": 2660
-    },
-    {
-      "epoch": 0.21338661338661338,
-      "grad_norm": 19.15390968322754,
-      "learning_rate": 4.806886199316808e-06,
-      "logits/chosen": -10.964703559875488,
-      "logits/rejected": -10.933959007263184,
-      "logps/chosen": -112.89806365966797,
-      "logps/rejected": -138.8131103515625,
-      "loss": 0.4118,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": -4.257689952850342,
-      "rewards/margins": 2.3521790504455566,
-      "rewards/rejected": -6.60986852645874,
-      "step": 2670
-    },
-    {
-      "epoch": 0.2141858141858142,
-      "grad_norm": 16.9531307220459,
-      "learning_rate": 4.804189095290123e-06,
-      "logits/chosen": -10.937814712524414,
-      "logits/rejected": -10.888550758361816,
-      "logps/chosen": -117.18865203857422,
-      "logps/rejected": -139.04933166503906,
-      "loss": 0.4798,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -4.850219249725342,
-      "rewards/margins": 1.782487154006958,
-      "rewards/rejected": -6.6327056884765625,
-      "step": 2680
-    },
-    {
-      "epoch": 0.214985014985015,
-      "grad_norm": 10.327271461486816,
-      "learning_rate": 4.801474054635499e-06,
-      "logits/chosen": -10.928077697753906,
-      "logits/rejected": -10.872354507446289,
-      "logps/chosen": -114.87007904052734,
-      "logps/rejected": -142.19383239746094,
-      "loss": 0.3245,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -3.8112590312957764,
-      "rewards/margins": 2.6379764080047607,
-      "rewards/rejected": -6.449235439300537,
-      "step": 2690
-    },
-    {
-      "epoch": 0.21578421578421578,
-      "grad_norm": 20.131399154663086,
-      "learning_rate": 4.798741098487779e-06,
-      "logits/chosen": -10.838517189025879,
-      "logits/rejected": -10.823667526245117,
-      "logps/chosen": -114.2899398803711,
-      "logps/rejected": -137.4205322265625,
-      "loss": 0.3645,
-      "rewards/accuracies": 0.9000000357627869,
-      "rewards/chosen": -4.253279685974121,
-      "rewards/margins": 2.2989766597747803,
-      "rewards/rejected": -6.552256107330322,
-      "step": 2700
-    },
-    {
-      "epoch": 0.2165834165834166,
-      "grad_norm": 4.147696495056152,
-      "learning_rate": 4.795990248121267e-06,
-      "logits/chosen": -10.781776428222656,
-      "logits/rejected": -10.81888484954834,
-      "logps/chosen": -112.9885025024414,
-      "logps/rejected": -146.49099731445312,
-      "loss": 0.2864,
-      "rewards/accuracies": 0.9000000357627869,
-      "rewards/chosen": -3.9949281215667725,
-      "rewards/margins": 2.8914103507995605,
-      "rewards/rejected": -6.886338233947754,
-      "step": 2710
-    },
-    {
-      "epoch": 0.21738261738261738,
-      "grad_norm": 14.387149810791016,
-      "learning_rate": 4.793221524949563e-06,
-      "logits/chosen": -11.010252952575684,
-      "logits/rejected": -10.976485252380371,
-      "logps/chosen": -106.9677963256836,
-      "logps/rejected": -132.44029235839844,
-      "loss": 0.4534,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -3.9360272884368896,
-      "rewards/margins": 2.0807347297668457,
-      "rewards/rejected": -6.016761779785156,
-      "step": 2720
-    },
-    {
-      "epoch": 0.21818181818181817,
-      "grad_norm": 17.18922996520996,
-      "learning_rate": 4.79043495052539e-06,
-      "logits/chosen": -10.88171672821045,
-      "logits/rejected": -10.869988441467285,
-      "logps/chosen": -122.45232391357422,
-      "logps/rejected": -142.23602294921875,
-      "loss": 0.6824,
-      "rewards/accuracies": 0.7125000357627869,
-      "rewards/chosen": -4.949435710906982,
-      "rewards/margins": 1.7035139799118042,
-      "rewards/rejected": -6.652949810028076,
-      "step": 2730
-    },
-    {
-      "epoch": 0.218981018981019,
-      "grad_norm": 12.906468391418457,
-      "learning_rate": 4.787630546540437e-06,
-      "logits/chosen": -10.852500915527344,
-      "logits/rejected": -10.858360290527344,
-      "logps/chosen": -116.1691665649414,
-      "logps/rejected": -138.89089965820312,
-      "loss": 0.5419,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -4.3208746910095215,
-      "rewards/margins": 1.928979516029358,
-      "rewards/rejected": -6.249853610992432,
-      "step": 2740
-    },
-    {
-      "epoch": 0.21978021978021978,
-      "grad_norm": 11.363615989685059,
-      "learning_rate": 4.784808334825183e-06,
-      "logits/chosen": -10.809206008911133,
-      "logits/rejected": -10.811616897583008,
-      "logps/chosen": -107.04508209228516,
-      "logps/rejected": -133.67333984375,
-      "loss": 0.48,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -3.578665256500244,
-      "rewards/margins": 2.0221288204193115,
-      "rewards/rejected": -5.600793838500977,
-      "step": 2750
-    },
-    {
-      "epoch": 0.2205794205794206,
-      "grad_norm": 22.233219146728516,
-      "learning_rate": 4.781968337348727e-06,
-      "logits/chosen": -10.893993377685547,
-      "logits/rejected": -10.912309646606445,
-      "logps/chosen": -110.603271484375,
-      "logps/rejected": -139.4268341064453,
-      "loss": 0.3252,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -3.9493443965911865,
-      "rewards/margins": 2.1845157146453857,
-      "rewards/rejected": -6.133860111236572,
-      "step": 2760
-    },
-    {
-      "epoch": 0.22137862137862138,
-      "grad_norm": 19.109249114990234,
-      "learning_rate": 4.779110576218621e-06,
-      "logits/chosen": -10.984148025512695,
-      "logits/rejected": -10.904632568359375,
-      "logps/chosen": -107.2940673828125,
-      "logps/rejected": -122.34378814697266,
-      "loss": 0.4466,
-      "rewards/accuracies": 0.8375000357627869,
-      "rewards/chosen": -3.5024666786193848,
-      "rewards/margins": 1.7993252277374268,
-      "rewards/rejected": -5.301791667938232,
-      "step": 2770
-    },
-    {
-      "epoch": 0.22217782217782217,
-      "grad_norm": 14.91334342956543,
-      "learning_rate": 4.776235073680695e-06,
-      "logits/chosen": -10.826943397521973,
-      "logits/rejected": -10.896329879760742,
-      "logps/chosen": -114.7727279663086,
-      "logps/rejected": -146.49737548828125,
-      "loss": 0.3535,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -4.10916805267334,
-      "rewards/margins": 2.1611945629119873,
-      "rewards/rejected": -6.270362377166748,
-      "step": 2780
-    },
-    {
-      "epoch": 0.222977022977023,
-      "grad_norm": 18.050058364868164,
-      "learning_rate": 4.773341852118882e-06,
-      "logits/chosen": -11.01706600189209,
-      "logits/rejected": -10.908744812011719,
-      "logps/chosen": -108.1183853149414,
-      "logps/rejected": -132.41404724121094,
-      "loss": 0.3527,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -3.7724900245666504,
-      "rewards/margins": 2.286005973815918,
-      "rewards/rejected": -6.058495998382568,
-      "step": 2790
-    },
-    {
-      "epoch": 0.22377622377622378,
-      "grad_norm": 24.710174560546875,
-      "learning_rate": 4.770430934055051e-06,
-      "logits/chosen": -10.988266944885254,
-      "logits/rejected": -10.912057876586914,
-      "logps/chosen": -116.1664047241211,
-      "logps/rejected": -136.4899139404297,
-      "loss": 0.4882,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -4.406286716461182,
-      "rewards/margins": 1.9268735647201538,
-      "rewards/rejected": -6.333160400390625,
-      "step": 2800
-    },
-    {
-      "epoch": 0.22457542457542456,
-      "grad_norm": 24.48606300354004,
-      "learning_rate": 4.767502342148823e-06,
-      "logits/chosen": -11.012198448181152,
-      "logits/rejected": -10.91966438293457,
-      "logps/chosen": -116.1751480102539,
-      "logps/rejected": -134.6574249267578,
-      "loss": 0.5468,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -4.4610676765441895,
-      "rewards/margins": 1.7178303003311157,
-      "rewards/rejected": -6.178897857666016,
-      "step": 2810
-    },
-    {
-      "epoch": 0.22537462537462538,
-      "grad_norm": 15.02327823638916,
-      "learning_rate": 4.7645560991973995e-06,
-      "logits/chosen": -11.02866268157959,
-      "logits/rejected": -10.976981163024902,
-      "logps/chosen": -111.14238739013672,
-      "logps/rejected": -128.3426971435547,
-      "loss": 0.4789,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -3.900484561920166,
-      "rewards/margins": 1.8336681127548218,
-      "rewards/rejected": -5.7341532707214355,
-      "step": 2820
-    },
-    {
-      "epoch": 0.22617382617382617,
-      "grad_norm": 7.818225383758545,
-      "learning_rate": 4.761592228135384e-06,
-      "logits/chosen": -10.92916488647461,
-      "logits/rejected": -10.805343627929688,
-      "logps/chosen": -111.00489807128906,
-      "logps/rejected": -126.17559051513672,
-      "loss": 0.4926,
-      "rewards/accuracies": 0.7750000357627869,
-      "rewards/chosen": -3.6459813117980957,
-      "rewards/margins": 1.6864155530929565,
-      "rewards/rejected": -5.332396984100342,
-      "step": 2830
-    },
-    {
-      "epoch": 0.226973026973027,
-      "grad_norm": 15.374308586120605,
-      "learning_rate": 4.758610752034604e-06,
-      "logits/chosen": -10.838567733764648,
-      "logits/rejected": -10.844330787658691,
-      "logps/chosen": -105.53520965576172,
-      "logps/rejected": -126.0640869140625,
-      "loss": 0.4409,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -3.469805955886841,
-      "rewards/margins": 1.8405402898788452,
-      "rewards/rejected": -5.3103461265563965,
-      "step": 2840
-    },
-    {
-      "epoch": 0.22777222777222778,
-      "grad_norm": 20.001062393188477,
-      "learning_rate": 4.755611694103931e-06,
-      "logits/chosen": -10.827131271362305,
-      "logits/rejected": -10.795989990234375,
-      "logps/chosen": -107.8458480834961,
-      "logps/rejected": -127.47689056396484,
-      "loss": 0.4679,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": -3.4508864879608154,
-      "rewards/margins": 1.7660754919052124,
-      "rewards/rejected": -5.2169623374938965,
-      "step": 2850
-    },
-    {
-      "epoch": 0.22857142857142856,
-      "grad_norm": 4.090858459472656,
-      "learning_rate": 4.7525950776890984e-06,
-      "logits/chosen": -10.600525856018066,
-      "logits/rejected": -10.715200424194336,
-      "logps/chosen": -109.4161376953125,
-      "logps/rejected": -147.1189727783203,
-      "loss": 0.3595,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -3.810591220855713,
-      "rewards/margins": 2.4729082584381104,
-      "rewards/rejected": -6.283499240875244,
-      "step": 2860
-    },
-    {
-      "epoch": 0.22937062937062938,
-      "grad_norm": 9.171509742736816,
-      "learning_rate": 4.749560926272522e-06,
-      "logits/chosen": -10.938541412353516,
-      "logits/rejected": -10.822016716003418,
-      "logps/chosen": -106.61962127685547,
-      "logps/rejected": -130.4688262939453,
-      "loss": 0.451,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": -3.624232530593872,
-      "rewards/margins": 1.9176586866378784,
-      "rewards/rejected": -5.541890621185303,
-      "step": 2870
-    },
-    {
-      "epoch": 0.23016983016983017,
-      "grad_norm": 10.207348823547363,
-      "learning_rate": 4.746509263473117e-06,
-      "logits/chosen": -10.906013488769531,
-      "logits/rejected": -10.819199562072754,
-      "logps/chosen": -112.40340423583984,
-      "logps/rejected": -135.0236358642578,
-      "loss": 0.46,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": -4.177793025970459,
-      "rewards/margins": 1.952566146850586,
-      "rewards/rejected": -6.130359649658203,
-      "step": 2880
-    },
-    {
-      "epoch": 0.23096903096903096,
-      "grad_norm": 9.469947814941406,
-      "learning_rate": 4.743440113046111e-06,
-      "logits/chosen": -11.008387565612793,
-      "logits/rejected": -10.91887092590332,
-      "logps/chosen": -112.81743621826172,
-      "logps/rejected": -136.40574645996094,
-      "loss": 0.3242,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -3.6763412952423096,
-      "rewards/margins": 2.512251853942871,
-      "rewards/rejected": -6.188593864440918,
-      "step": 2890
-    },
-    {
-      "epoch": 0.23176823176823177,
-      "grad_norm": 26.905282974243164,
-      "learning_rate": 4.740353498882865e-06,
-      "logits/chosen": -10.986650466918945,
-      "logits/rejected": -10.911989212036133,
-      "logps/chosen": -112.4732666015625,
-      "logps/rejected": -134.392822265625,
-      "loss": 0.3732,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -3.83911395072937,
-      "rewards/margins": 2.401121139526367,
-      "rewards/rejected": -6.240235328674316,
-      "step": 2900
-    },
-    {
-      "epoch": 0.23256743256743256,
-      "grad_norm": 20.84300422668457,
-      "learning_rate": 4.737249445010681e-06,
-      "logits/chosen": -10.661272048950195,
-      "logits/rejected": -10.823920249938965,
-      "logps/chosen": -116.4116439819336,
-      "logps/rejected": -152.677734375,
-      "loss": 0.3781,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": -4.430285930633545,
-      "rewards/margins": 2.453695058822632,
-      "rewards/rejected": -6.883979797363281,
-      "step": 2910
-    },
-    {
-      "epoch": 0.23336663336663338,
-      "grad_norm": 16.04476547241211,
-      "learning_rate": 4.7341279755926195e-06,
-      "logits/chosen": -10.930587768554688,
-      "logits/rejected": -10.923018455505371,
-      "logps/chosen": -111.60106658935547,
-      "logps/rejected": -134.17442321777344,
-      "loss": 0.3989,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -4.043647289276123,
-      "rewards/margins": 2.177844762802124,
-      "rewards/rejected": -6.221491813659668,
-      "step": 2920
-    },
-    {
-      "epoch": 0.23416583416583417,
-      "grad_norm": 8.613511085510254,
-      "learning_rate": 4.73098911492731e-06,
-      "logits/chosen": -10.959551811218262,
-      "logits/rejected": -10.990097045898438,
-      "logps/chosen": -118.7111587524414,
-      "logps/rejected": -147.91688537597656,
-      "loss": 0.4673,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": -4.459880352020264,
-      "rewards/margins": 2.645782232284546,
-      "rewards/rejected": -7.105662822723389,
-      "step": 2930
-    },
-    {
-      "epoch": 0.23496503496503496,
-      "grad_norm": 10.504155158996582,
-      "learning_rate": 4.727832887448759e-06,
-      "logits/chosen": -11.000977516174316,
-      "logits/rejected": -10.987174987792969,
-      "logps/chosen": -125.21553039550781,
-      "logps/rejected": -139.5205535888672,
-      "loss": 0.5279,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": -4.987060546875,
-      "rewards/margins": 1.8689903020858765,
-      "rewards/rejected": -6.856050968170166,
-      "step": 2940
-    },
-    {
-      "epoch": 0.23576423576423577,
-      "grad_norm": 17.831087112426758,
-      "learning_rate": 4.724659317726167e-06,
-      "logits/chosen": -10.737998962402344,
-      "logits/rejected": -10.906888961791992,
-      "logps/chosen": -128.28501892089844,
-      "logps/rejected": -154.0959930419922,
-      "loss": 0.6918,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": -5.276055812835693,
-      "rewards/margins": 1.6980583667755127,
-      "rewards/rejected": -6.974113464355469,
-      "step": 2950
-    },
-    {
-      "epoch": 0.23656343656343656,
-      "grad_norm": 20.923837661743164,
-      "learning_rate": 4.721468430463727e-06,
-      "logits/chosen": -10.87534236907959,
-      "logits/rejected": -10.894587516784668,
-      "logps/chosen": -113.88321685791016,
-      "logps/rejected": -135.3246307373047,
-      "loss": 0.4989,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": -4.257094383239746,
-      "rewards/margins": 2.009908437728882,
-      "rewards/rejected": -6.267002105712891,
-      "step": 2960
-    },
-    {
-      "epoch": 0.23736263736263735,
-      "grad_norm": 18.776901245117188,
-      "learning_rate": 4.718260250500445e-06,
-      "logits/chosen": -10.793301582336426,
-      "logits/rejected": -10.877842903137207,
-      "logps/chosen": -108.6339340209961,
-      "logps/rejected": -144.16409301757812,
-      "loss": 0.3415,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -3.86503529548645,
-      "rewards/margins": 2.799866199493408,
-      "rewards/rejected": -6.6649017333984375,
-      "step": 2970
-    },
-    {
-      "epoch": 0.23816183816183817,
-      "grad_norm": 14.148445129394531,
-      "learning_rate": 4.715034802809932e-06,
-      "logits/chosen": -10.75792407989502,
-      "logits/rejected": -10.781256675720215,
-      "logps/chosen": -118.99928283691406,
-      "logps/rejected": -147.9957275390625,
-      "loss": 0.3347,
-      "rewards/accuracies": 0.9000000357627869,
-      "rewards/chosen": -4.36841344833374,
-      "rewards/margins": 2.5025014877319336,
-      "rewards/rejected": -6.870915412902832,
-      "step": 2980
-    },
-    {
-      "epoch": 0.23896103896103896,
-      "grad_norm": 21.52681541442871,
-      "learning_rate": 4.711792112500222e-06,
-      "logits/chosen": -10.904582023620605,
-      "logits/rejected": -10.906264305114746,
-      "logps/chosen": -111.39432525634766,
-      "logps/rejected": -147.8531494140625,
-      "loss": 0.3855,
-      "rewards/accuracies": 0.8375000357627869,
-      "rewards/chosen": -4.316974639892578,
-      "rewards/margins": 2.5597281455993652,
-      "rewards/rejected": -6.876702785491943,
-      "step": 2990
-    },
-    {
-      "epoch": 0.23976023976023977,
-      "grad_norm": 24.595726013183594,
-      "learning_rate": 4.70853220481357e-06,
-      "logits/chosen": -11.009933471679688,
-      "logits/rejected": -10.928666114807129,
-      "logps/chosen": -117.8331069946289,
-      "logps/rejected": -140.36131286621094,
-      "loss": 0.4611,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -4.525640964508057,
-      "rewards/margins": 2.1797804832458496,
-      "rewards/rejected": -6.705421447753906,
-      "step": 3000
-    },
-    {
-      "epoch": 0.24055944055944056,
-      "grad_norm": 13.71435546875,
-      "learning_rate": 4.705255105126259e-06,
-      "logits/chosen": -11.03426742553711,
-      "logits/rejected": -11.006673812866211,
-      "logps/chosen": -114.33635711669922,
-      "logps/rejected": -139.910400390625,
-      "loss": 0.3597,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -4.489405632019043,
-      "rewards/margins": 2.1930463314056396,
-      "rewards/rejected": -6.6824517250061035,
-      "step": 3010
-    },
-    {
-      "epoch": 0.24135864135864135,
-      "grad_norm": 8.316465377807617,
-      "learning_rate": 4.7019608389483996e-06,
-      "logits/chosen": -11.047881126403809,
-      "logits/rejected": -10.970107078552246,
-      "logps/chosen": -116.10176086425781,
-      "logps/rejected": -143.08499145507812,
-      "loss": 0.2917,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": -4.425871849060059,
-      "rewards/margins": 2.565258026123047,
-      "rewards/rejected": -6.991130352020264,
-      "step": 3020
-    },
-    {
-      "epoch": 0.24215784215784217,
-      "grad_norm": 9.213641166687012,
-      "learning_rate": 4.698649431923729e-06,
-      "logits/chosen": -11.08574104309082,
-      "logits/rejected": -11.03657054901123,
-      "logps/chosen": -127.3587417602539,
-      "logps/rejected": -150.6202850341797,
-      "loss": 0.4176,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -5.380565166473389,
-      "rewards/margins": 2.178239345550537,
-      "rewards/rejected": -7.558804512023926,
-      "step": 3030
-    },
-    {
-      "epoch": 0.24295704295704296,
-      "grad_norm": 17.757253646850586,
-      "learning_rate": 4.69532090982942e-06,
-      "logits/chosen": -10.92107105255127,
-      "logits/rejected": -10.934576034545898,
-      "logps/chosen": -125.8876953125,
-      "logps/rejected": -154.0380401611328,
-      "loss": 0.5816,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -5.499910354614258,
-      "rewards/margins": 2.0258729457855225,
-      "rewards/rejected": -7.525783538818359,
-      "step": 3040
-    },
-    {
-      "epoch": 0.24375624375624375,
-      "grad_norm": 15.936699867248535,
-      "learning_rate": 4.691975298575873e-06,
-      "logits/chosen": -10.75210952758789,
-      "logits/rejected": -11.003128051757812,
-      "logps/chosen": -120.03985595703125,
-      "logps/rejected": -159.59144592285156,
-      "loss": 0.4068,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": -4.880809783935547,
-      "rewards/margins": 2.8549282550811768,
-      "rewards/rejected": -7.7357378005981445,
-      "step": 3050
-    },
-    {
-      "epoch": 0.24455544455544456,
-      "grad_norm": 19.01104164123535,
-      "learning_rate": 4.688612624206516e-06,
-      "logits/chosen": -10.98422908782959,
-      "logits/rejected": -10.912976264953613,
-      "logps/chosen": -114.7066650390625,
-      "logps/rejected": -135.5327606201172,
-      "loss": 0.4624,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": -4.130919933319092,
-      "rewards/margins": 1.9711884260177612,
-      "rewards/rejected": -6.102107524871826,
-      "step": 3060
-    },
-    {
-      "epoch": 0.24535464535464535,
-      "grad_norm": 14.351190567016602,
-      "learning_rate": 4.685232912897605e-06,
-      "logits/chosen": -11.046852111816406,
-      "logits/rejected": -10.949774742126465,
-      "logps/chosen": -115.5030288696289,
-      "logps/rejected": -135.6511993408203,
-      "loss": 0.3135,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -4.391961574554443,
-      "rewards/margins": 2.0687472820281982,
-      "rewards/rejected": -6.4607086181640625,
-      "step": 3070
-    },
-    {
-      "epoch": 0.24615384615384617,
-      "grad_norm": 7.435042381286621,
-      "learning_rate": 4.681836190958014e-06,
-      "logits/chosen": -10.845429420471191,
-      "logits/rejected": -10.949892044067383,
-      "logps/chosen": -111.56546783447266,
-      "logps/rejected": -149.9969940185547,
-      "loss": 0.3135,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -4.222198009490967,
-      "rewards/margins": 3.064072370529175,
-      "rewards/rejected": -7.286271095275879,
-      "step": 3080
-    },
-    {
-      "epoch": 0.24695304695304696,
-      "grad_norm": 18.930419921875,
-      "learning_rate": 4.678422484829034e-06,
-      "logits/chosen": -10.939020156860352,
-      "logits/rejected": -10.933897972106934,
-      "logps/chosen": -113.1585464477539,
-      "logps/rejected": -140.92359924316406,
-      "loss": 0.4269,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -4.241700649261475,
-      "rewards/margins": 2.306051254272461,
-      "rewards/rejected": -6.547752380371094,
-      "step": 3090
-    },
-    {
-      "epoch": 0.24775224775224775,
-      "grad_norm": 18.24955177307129,
-      "learning_rate": 4.6749918210841705e-06,
-      "logits/chosen": -11.065057754516602,
-      "logits/rejected": -10.977603912353516,
-      "logps/chosen": -114.54528045654297,
-      "logps/rejected": -135.9423370361328,
-      "loss": 0.4327,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -4.298453330993652,
-      "rewards/margins": 1.8995094299316406,
-      "rewards/rejected": -6.197961807250977,
-      "step": 3100
-    },
-    {
-      "epoch": 0.24855144855144856,
-      "grad_norm": 12.965761184692383,
-      "learning_rate": 4.671544226428931e-06,
-      "logits/chosen": -10.992847442626953,
-      "logits/rejected": -10.877277374267578,
-      "logps/chosen": -112.9588623046875,
-      "logps/rejected": -145.53555297851562,
-      "loss": 0.2472,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": -3.932997226715088,
-      "rewards/margins": 2.9484670162200928,
-      "rewards/rejected": -6.88146448135376,
-      "step": 3110
-    },
-    {
-      "epoch": 0.24935064935064935,
-      "grad_norm": 25.675722122192383,
-      "learning_rate": 4.668079727700618e-06,
-      "logits/chosen": -10.96890640258789,
-      "logits/rejected": -10.932058334350586,
-      "logps/chosen": -109.23735809326172,
-      "logps/rejected": -132.6926727294922,
-      "loss": 0.4695,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -3.6113107204437256,
-      "rewards/margins": 2.003997564315796,
-      "rewards/rejected": -5.6153082847595215,
-      "step": 3120
-    },
-    {
-      "epoch": 0.25014985014985014,
-      "grad_norm": 16.3002986907959,
-      "learning_rate": 4.664598351868124e-06,
-      "logits/chosen": -10.840982437133789,
-      "logits/rejected": -10.931330680847168,
-      "logps/chosen": -106.29841613769531,
-      "logps/rejected": -141.3500213623047,
-      "loss": 0.4872,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -3.5310723781585693,
-      "rewards/margins": 2.3773586750030518,
-      "rewards/rejected": -5.908430576324463,
-      "step": 3130
-    },
-    {
-      "epoch": 0.25094905094905096,
-      "grad_norm": 18.274850845336914,
-      "learning_rate": 4.661100126031716e-06,
-      "logits/chosen": -11.053993225097656,
-      "logits/rejected": -10.965598106384277,
-      "logps/chosen": -107.51366424560547,
-      "logps/rejected": -131.13658142089844,
-      "loss": 0.3713,
-      "rewards/accuracies": 0.8375000357627869,
-      "rewards/chosen": -3.606605291366577,
-      "rewards/margins": 2.290649175643921,
-      "rewards/rejected": -5.897254467010498,
-      "step": 3140
-    },
-    {
-      "epoch": 0.2517482517482518,
-      "grad_norm": 12.054896354675293,
-      "learning_rate": 4.6575850774228304e-06,
-      "logits/chosen": -10.889887809753418,
-      "logits/rejected": -10.887163162231445,
-      "logps/chosen": -116.788818359375,
-      "logps/rejected": -143.6160125732422,
-      "loss": 0.4779,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -4.483249664306641,
-      "rewards/margins": 2.3694489002227783,
-      "rewards/rejected": -6.852698802947998,
-      "step": 3150
-    },
-    {
-      "epoch": 0.25254745254745253,
-      "grad_norm": 17.99600601196289,
-      "learning_rate": 4.654053233403854e-06,
-      "logits/chosen": -11.04506778717041,
-      "logits/rejected": -11.045677185058594,
-      "logps/chosen": -117.18670654296875,
-      "logps/rejected": -136.2570343017578,
-      "loss": 0.5422,
-      "rewards/accuracies": 0.7750000357627869,
-      "rewards/chosen": -4.177443027496338,
-      "rewards/margins": 2.1145169734954834,
-      "rewards/rejected": -6.291959762573242,
-      "step": 3160
-    },
-    {
-      "epoch": 0.25334665334665335,
-      "grad_norm": 15.124343872070312,
-      "learning_rate": 4.650504621467921e-06,
-      "logits/chosen": -10.953264236450195,
-      "logits/rejected": -10.851902961730957,
-      "logps/chosen": -113.7905502319336,
-      "logps/rejected": -140.27517700195312,
-      "loss": 0.4052,
-      "rewards/accuracies": 0.8375000357627869,
-      "rewards/chosen": -3.943930149078369,
-      "rewards/margins": 2.4566948413848877,
-      "rewards/rejected": -6.400625705718994,
-      "step": 3170
-    },
-    {
-      "epoch": 0.25414585414585417,
-      "grad_norm": 14.673730850219727,
-      "learning_rate": 4.6469392692386875e-06,
-      "logits/chosen": -10.853971481323242,
-      "logits/rejected": -10.810283660888672,
-      "logps/chosen": -106.9908676147461,
-      "logps/rejected": -133.13125610351562,
-      "loss": 0.3452,
-      "rewards/accuracies": 0.8375000357627869,
-      "rewards/chosen": -3.6714730262756348,
-      "rewards/margins": 2.0255701541900635,
-      "rewards/rejected": -5.697042942047119,
-      "step": 3180
-    },
-    {
-      "epoch": 0.2549450549450549,
-      "grad_norm": 6.663290977478027,
-      "learning_rate": 4.643357204470123e-06,
-      "logits/chosen": -10.784911155700684,
-      "logits/rejected": -10.909873008728027,
-      "logps/chosen": -113.12859344482422,
-      "logps/rejected": -145.81546020507812,
-      "loss": 0.4933,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -4.116060733795166,
-      "rewards/margins": 2.4813458919525146,
-      "rewards/rejected": -6.597405910491943,
-      "step": 3190
-    },
-    {
-      "epoch": 0.25574425574425574,
-      "grad_norm": 22.831501007080078,
-      "learning_rate": 4.639758455046298e-06,
-      "logits/chosen": -10.685839653015137,
-      "logits/rejected": -10.831932067871094,
-      "logps/chosen": -111.26412200927734,
-      "logps/rejected": -151.29969787597656,
-      "loss": 0.3531,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -4.053619384765625,
-      "rewards/margins": 2.889514684677124,
-      "rewards/rejected": -6.94313383102417,
-      "step": 3200
-    },
-    {
-      "epoch": 0.25654345654345656,
-      "grad_norm": 7.073662281036377,
-      "learning_rate": 4.636143048981155e-06,
-      "logits/chosen": -10.808653831481934,
-      "logits/rejected": -10.872891426086426,
-      "logps/chosen": -109.5636978149414,
-      "logps/rejected": -148.64907836914062,
-      "loss": 0.3256,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -3.6955623626708984,
-      "rewards/margins": 3.1040992736816406,
-      "rewards/rejected": -6.799661159515381,
-      "step": 3210
-    },
-    {
-      "epoch": 0.2573426573426573,
-      "grad_norm": 11.552987098693848,
-      "learning_rate": 4.6325110144183035e-06,
-      "logits/chosen": -10.823410034179688,
-      "logits/rejected": -10.90381145477295,
-      "logps/chosen": -116.59454345703125,
-      "logps/rejected": -145.6106414794922,
-      "loss": 0.4316,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -4.083954334259033,
-      "rewards/margins": 2.6350924968719482,
-      "rewards/rejected": -6.719046115875244,
-      "step": 3220
-    },
-    {
-      "epoch": 0.25814185814185814,
-      "grad_norm": 15.225719451904297,
-      "learning_rate": 4.628862379630793e-06,
-      "logits/chosen": -10.787749290466309,
-      "logits/rejected": -10.879558563232422,
-      "logps/chosen": -118.67707824707031,
-      "logps/rejected": -156.08311462402344,
-      "loss": 0.4681,
-      "rewards/accuracies": 0.7750000357627869,
-      "rewards/chosen": -4.722916603088379,
-      "rewards/margins": 2.6335315704345703,
-      "rewards/rejected": -7.356448650360107,
-      "step": 3230
-    },
-    {
-      "epoch": 0.25894105894105895,
-      "grad_norm": 16.662721633911133,
-      "learning_rate": 4.625197173020895e-06,
-      "logits/chosen": -10.93076229095459,
-      "logits/rejected": -10.796025276184082,
-      "logps/chosen": -124.9680404663086,
-      "logps/rejected": -142.0721435546875,
-      "loss": 0.5062,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -5.050028324127197,
-      "rewards/margins": 1.9267736673355103,
-      "rewards/rejected": -6.976802349090576,
-      "step": 3240
-    },
-    {
-      "epoch": 0.2597402597402597,
-      "grad_norm": 4.569664478302002,
-      "learning_rate": 4.621515423119882e-06,
-      "logits/chosen": -10.8897123336792,
-      "logits/rejected": -10.785490036010742,
-      "logps/chosen": -120.3673095703125,
-      "logps/rejected": -151.05860900878906,
-      "loss": 0.2977,
-      "rewards/accuracies": 0.8375000357627869,
-      "rewards/chosen": -4.9029316902160645,
-      "rewards/margins": 2.9220292568206787,
-      "rewards/rejected": -7.824961185455322,
-      "step": 3250
-    },
-    {
-      "epoch": 0.26053946053946053,
-      "grad_norm": 17.284406661987305,
-      "learning_rate": 4.617817158587805e-06,
-      "logits/chosen": -10.65257453918457,
-      "logits/rejected": -10.850028038024902,
-      "logps/chosen": -124.6682357788086,
-      "logps/rejected": -164.3677978515625,
-      "loss": 0.3649,
-      "rewards/accuracies": 0.8375000357627869,
-      "rewards/chosen": -5.075488567352295,
-      "rewards/margins": 2.894611120223999,
-      "rewards/rejected": -7.970099925994873,
-      "step": 3260
-    },
-    {
-      "epoch": 0.26133866133866135,
-      "grad_norm": 18.04502296447754,
-      "learning_rate": 4.614102408213272e-06,
-      "logits/chosen": -10.839510917663574,
-      "logits/rejected": -10.972907066345215,
-      "logps/chosen": -123.61695861816406,
-      "logps/rejected": -157.1401824951172,
-      "loss": 0.4266,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -5.112114429473877,
-      "rewards/margins": 2.5516433715820312,
-      "rewards/rejected": -7.66375732421875,
-      "step": 3270
-    },
-    {
-      "epoch": 0.26213786213786217,
-      "grad_norm": 11.22952651977539,
-      "learning_rate": 4.6103712009132214e-06,
-      "logits/chosen": -10.743083000183105,
-      "logits/rejected": -10.842226028442383,
-      "logps/chosen": -123.49077606201172,
-      "logps/rejected": -151.58590698242188,
-      "loss": 0.4307,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -5.392497539520264,
-      "rewards/margins": 2.574392557144165,
-      "rewards/rejected": -7.96688985824585,
-      "step": 3280
-    },
-    {
-      "epoch": 0.2629370629370629,
-      "grad_norm": 15.173971176147461,
-      "learning_rate": 4.6066235657326995e-06,
-      "logits/chosen": -10.695324897766113,
-      "logits/rejected": -10.919292449951172,
-      "logps/chosen": -133.7132568359375,
-      "logps/rejected": -172.5398406982422,
-      "loss": 0.5565,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": -5.995713233947754,
-      "rewards/margins": 2.6895766258239746,
-      "rewards/rejected": -8.68528938293457,
-      "step": 3290
-    },
-    {
-      "epoch": 0.26373626373626374,
-      "grad_norm": 16.572195053100586,
-      "learning_rate": 4.602859531844631e-06,
-      "logits/chosen": -10.90363597869873,
-      "logits/rejected": -10.831019401550293,
-      "logps/chosen": -121.5705337524414,
-      "logps/rejected": -144.7947235107422,
-      "loss": 0.4127,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -5.356640815734863,
-      "rewards/margins": 2.136237382888794,
-      "rewards/rejected": -7.492877960205078,
-      "step": 3300
-    },
-    {
-      "epoch": 0.26453546453546456,
-      "grad_norm": 8.897160530090332,
-      "learning_rate": 4.5990791285495976e-06,
-      "logits/chosen": -10.898272514343262,
-      "logits/rejected": -10.83046817779541,
-      "logps/chosen": -120.9656982421875,
-      "logps/rejected": -143.2664794921875,
-      "loss": 0.4477,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -4.9760966300964355,
-      "rewards/margins": 2.110779285430908,
-      "rewards/rejected": -7.086875915527344,
-      "step": 3310
-    },
-    {
-      "epoch": 0.2653346653346653,
-      "grad_norm": 20.15085792541504,
-      "learning_rate": 4.595282385275601e-06,
-      "logits/chosen": -10.82510757446289,
-      "logits/rejected": -10.807748794555664,
-      "logps/chosen": -119.2079086303711,
-      "logps/rejected": -143.563232421875,
-      "loss": 0.5241,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -4.875324249267578,
-      "rewards/margins": 2.1741631031036377,
-      "rewards/rejected": -7.049487590789795,
-      "step": 3320
-    },
-    {
-      "epoch": 0.26613386613386614,
-      "grad_norm": 2.261742115020752,
-      "learning_rate": 4.591469331577844e-06,
-      "logits/chosen": -10.748279571533203,
-      "logits/rejected": -10.887080192565918,
-      "logps/chosen": -118.4509506225586,
-      "logps/rejected": -150.45164489746094,
-      "loss": 0.3616,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -4.261892318725586,
-      "rewards/margins": 2.5474672317504883,
-      "rewards/rejected": -6.809360027313232,
-      "step": 3330
-    },
-    {
-      "epoch": 0.26693306693306695,
-      "grad_norm": 15.398425102233887,
-      "learning_rate": 4.5876399971384925e-06,
-      "logits/chosen": -10.853693962097168,
-      "logits/rejected": -10.784746170043945,
-      "logps/chosen": -113.27699279785156,
-      "logps/rejected": -143.63693237304688,
-      "loss": 0.334,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -4.377139091491699,
-      "rewards/margins": 2.923900604248047,
-      "rewards/rejected": -7.301040172576904,
-      "step": 3340
-    },
-    {
-      "epoch": 0.2677322677322677,
-      "grad_norm": 15.278942108154297,
-      "learning_rate": 4.583794411766448e-06,
-      "logits/chosen": -10.78717041015625,
-      "logits/rejected": -10.804595947265625,
-      "logps/chosen": -124.28983306884766,
-      "logps/rejected": -149.99472045898438,
-      "loss": 0.4935,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": -5.22383451461792,
-      "rewards/margins": 2.369459629058838,
-      "rewards/rejected": -7.593294620513916,
-      "step": 3350
-    },
-    {
-      "epoch": 0.26853146853146853,
-      "grad_norm": 11.385106086730957,
-      "learning_rate": 4.579932605397118e-06,
-      "logits/chosen": -10.901527404785156,
-      "logits/rejected": -10.887252807617188,
-      "logps/chosen": -124.60796356201172,
-      "logps/rejected": -149.7532501220703,
-      "loss": 0.3849,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": -5.103854656219482,
-      "rewards/margins": 2.386571168899536,
-      "rewards/rejected": -7.490426063537598,
-      "step": 3360
-    },
-    {
-      "epoch": 0.26933066933066935,
-      "grad_norm": 11.875481605529785,
-      "learning_rate": 4.5760546080921764e-06,
-      "logits/chosen": -10.9485502243042,
-      "logits/rejected": -10.858540534973145,
-      "logps/chosen": -116.72521209716797,
-      "logps/rejected": -148.48818969726562,
-      "loss": 0.3303,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -4.704689025878906,
-      "rewards/margins": 2.613240957260132,
-      "rewards/rejected": -7.317930698394775,
-      "step": 3370
-    },
-    {
-      "epoch": 0.2701298701298701,
-      "grad_norm": 17.806415557861328,
-      "learning_rate": 4.572160450039333e-06,
-      "logits/chosen": -11.021417617797852,
-      "logits/rejected": -10.940775871276855,
-      "logps/chosen": -118.53228759765625,
-      "logps/rejected": -141.04815673828125,
-      "loss": 0.4862,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -4.406890869140625,
-      "rewards/margins": 2.348252296447754,
-      "rewards/rejected": -6.755143165588379,
-      "step": 3380
-    },
-    {
-      "epoch": 0.2709290709290709,
-      "grad_norm": 21.97511863708496,
-      "learning_rate": 4.568250161552103e-06,
-      "logits/chosen": -10.92384147644043,
-      "logits/rejected": -10.922481536865234,
-      "logps/chosen": -116.38777160644531,
-      "logps/rejected": -138.46385192871094,
-      "loss": 0.5374,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": -4.345650672912598,
-      "rewards/margins": 1.9374865293502808,
-      "rewards/rejected": -6.283137321472168,
-      "step": 3390
-    },
-    {
-      "epoch": 0.27172827172827174,
-      "grad_norm": 9.408753395080566,
-      "learning_rate": 4.56432377306956e-06,
-      "logits/chosen": -10.90976619720459,
-      "logits/rejected": -10.960277557373047,
-      "logps/chosen": -117.75931549072266,
-      "logps/rejected": -142.7761688232422,
-      "loss": 0.5744,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -4.724529266357422,
-      "rewards/margins": 2.054431676864624,
-      "rewards/rejected": -6.778961181640625,
-      "step": 3400
-    },
-    {
-      "epoch": 0.2725274725274725,
-      "grad_norm": 17.630531311035156,
-      "learning_rate": 4.560381315156111e-06,
-      "logits/chosen": -10.831389427185059,
-      "logits/rejected": -10.900781631469727,
-      "logps/chosen": -116.48125457763672,
-      "logps/rejected": -149.5487823486328,
-      "loss": 0.4628,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": -4.624643325805664,
-      "rewards/margins": 2.355069160461426,
-      "rewards/rejected": -6.979712963104248,
-      "step": 3410
-    },
-    {
-      "epoch": 0.2733266733266733,
-      "grad_norm": 15.25622844696045,
-      "learning_rate": 4.556422818501253e-06,
-      "logits/chosen": -11.002695083618164,
-      "logits/rejected": -10.951990127563477,
-      "logps/chosen": -116.41981506347656,
-      "logps/rejected": -140.50015258789062,
-      "loss": 0.3325,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -4.248597621917725,
-      "rewards/margins": 2.526536226272583,
-      "rewards/rejected": -6.775134563446045,
-      "step": 3420
-    },
-    {
-      "epoch": 0.27412587412587414,
-      "grad_norm": 4.23134183883667,
-      "learning_rate": 4.55244831391933e-06,
-      "logits/chosen": -10.926270484924316,
-      "logits/rejected": -10.906249046325684,
-      "logps/chosen": -122.09199523925781,
-      "logps/rejected": -156.90701293945312,
-      "loss": 0.3542,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -4.747585296630859,
-      "rewards/margins": 2.7377564907073975,
-      "rewards/rejected": -7.485341548919678,
-      "step": 3430
-    },
-    {
-      "epoch": 0.27492507492507495,
-      "grad_norm": 15.184752464294434,
-      "learning_rate": 4.548457832349301e-06,
-      "logits/chosen": -11.011893272399902,
-      "logits/rejected": -10.915630340576172,
-      "logps/chosen": -115.81707763671875,
-      "logps/rejected": -138.1062774658203,
-      "loss": 0.4958,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -4.5808491706848145,
-      "rewards/margins": 2.0731751918792725,
-      "rewards/rejected": -6.654024600982666,
-      "step": 3440
-    },
-    {
-      "epoch": 0.2757242757242757,
-      "grad_norm": 13.697755813598633,
-      "learning_rate": 4.544451404854496e-06,
-      "logits/chosen": -10.938133239746094,
-      "logits/rejected": -10.87554931640625,
-      "logps/chosen": -122.83260345458984,
-      "logps/rejected": -144.5916290283203,
-      "loss": 0.4393,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -4.852165222167969,
-      "rewards/margins": 2.116230010986328,
-      "rewards/rejected": -6.968395233154297,
-      "step": 3450
-    },
-    {
-      "epoch": 0.27652347652347653,
-      "grad_norm": 16.020299911499023,
-      "learning_rate": 4.5404290626223705e-06,
-      "logits/chosen": -10.814127922058105,
-      "logits/rejected": -10.865537643432617,
-      "logps/chosen": -118.8655776977539,
-      "logps/rejected": -152.44393920898438,
-      "loss": 0.3268,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -4.662789344787598,
-      "rewards/margins": 3.0085370540618896,
-      "rewards/rejected": -7.67132568359375,
-      "step": 3460
-    },
-    {
-      "epoch": 0.27732267732267735,
-      "grad_norm": 10.427433013916016,
-      "learning_rate": 4.53639083696427e-06,
-      "logits/chosen": -10.999835968017578,
-      "logits/rejected": -10.97978687286377,
-      "logps/chosen": -119.28857421875,
-      "logps/rejected": -141.68748474121094,
-      "loss": 0.5876,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -4.972087383270264,
-      "rewards/margins": 1.9791492223739624,
-      "rewards/rejected": -6.951236248016357,
-      "step": 3470
-    },
-    {
-      "epoch": 0.2781218781218781,
-      "grad_norm": 20.623043060302734,
-      "learning_rate": 4.532336759315178e-06,
-      "logits/chosen": -11.031302452087402,
-      "logits/rejected": -10.975234031677246,
-      "logps/chosen": -113.26399230957031,
-      "logps/rejected": -136.5120086669922,
-      "loss": 0.3566,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -3.9642703533172607,
-      "rewards/margins": 2.586198091506958,
-      "rewards/rejected": -6.550468444824219,
-      "step": 3480
-    },
-    {
-      "epoch": 0.2789210789210789,
-      "grad_norm": 15.846445083618164,
-      "learning_rate": 4.528266861233478e-06,
-      "logits/chosen": -10.786155700683594,
-      "logits/rejected": -10.927268028259277,
-      "logps/chosen": -114.18462371826172,
-      "logps/rejected": -154.27053833007812,
-      "loss": 0.2746,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -4.222942352294922,
-      "rewards/margins": 2.964430570602417,
-      "rewards/rejected": -7.187373638153076,
-      "step": 3490
-    },
-    {
-      "epoch": 0.27972027972027974,
-      "grad_norm": 4.732353210449219,
-      "learning_rate": 4.524181174400707e-06,
-      "logits/chosen": -11.077165603637695,
-      "logits/rejected": -10.906579971313477,
-      "logps/chosen": -125.21479797363281,
-      "logps/rejected": -147.45465087890625,
-      "loss": 0.5574,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -5.154223442077637,
-      "rewards/margins": 2.2535786628723145,
-      "rewards/rejected": -7.407802104949951,
-      "step": 3500
-    },
-    {
-      "epoch": 0.2805194805194805,
-      "grad_norm": 12.709331512451172,
-      "learning_rate": 4.520079730621304e-06,
-      "logits/chosen": -10.902528762817383,
-      "logits/rejected": -10.909880638122559,
-      "logps/chosen": -111.88557434082031,
-      "logps/rejected": -147.40403747558594,
-      "loss": 0.2404,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": -4.171848297119141,
-      "rewards/margins": 2.676706552505493,
-      "rewards/rejected": -6.8485541343688965,
-      "step": 3510
-    },
-    {
-      "epoch": 0.2813186813186813,
-      "grad_norm": 11.356675148010254,
-      "learning_rate": 4.515962561822367e-06,
-      "logits/chosen": -10.958823204040527,
-      "logits/rejected": -10.86766242980957,
-      "logps/chosen": -115.6883544921875,
-      "logps/rejected": -143.24594116210938,
-      "loss": 0.3657,
-      "rewards/accuracies": 0.7750000357627869,
-      "rewards/chosen": -4.45851993560791,
-      "rewards/margins": 2.694356918334961,
-      "rewards/rejected": -7.152876377105713,
-      "step": 3520
-    },
-    {
-      "epoch": 0.28211788211788213,
-      "grad_norm": 10.391762733459473,
-      "learning_rate": 4.511829700053403e-06,
-      "logits/chosen": -10.959734916687012,
-      "logits/rejected": -10.86664867401123,
-      "logps/chosen": -113.31788635253906,
-      "logps/rejected": -136.7180633544922,
-      "loss": 0.5187,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -4.360722064971924,
-      "rewards/margins": 2.008721113204956,
-      "rewards/rejected": -6.369443416595459,
-      "step": 3530
-    },
-    {
-      "epoch": 0.2829170829170829,
-      "grad_norm": 11.747859954833984,
-      "learning_rate": 4.507681177486078e-06,
-      "logits/chosen": -10.809256553649902,
-      "logits/rejected": -10.838770866394043,
-      "logps/chosen": -116.03597259521484,
-      "logps/rejected": -148.0928497314453,
-      "loss": 0.6382,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -4.620072364807129,
-      "rewards/margins": 2.385552167892456,
-      "rewards/rejected": -7.005624294281006,
-      "step": 3540
-    },
-    {
-      "epoch": 0.2837162837162837,
-      "grad_norm": 8.82387638092041,
-      "learning_rate": 4.503517026413971e-06,
-      "logits/chosen": -10.930337905883789,
-      "logits/rejected": -10.84708309173584,
-      "logps/chosen": -110.17150115966797,
-      "logps/rejected": -133.496826171875,
-      "loss": 0.4273,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": -3.5340778827667236,
-      "rewards/margins": 2.184480667114258,
-      "rewards/rejected": -5.7185587882995605,
-      "step": 3550
-    },
-    {
-      "epoch": 0.28451548451548453,
-      "grad_norm": 15.758824348449707,
-      "learning_rate": 4.499337279252313e-06,
-      "logits/chosen": -10.917475700378418,
-      "logits/rejected": -10.893115043640137,
-      "logps/chosen": -111.2662124633789,
-      "logps/rejected": -134.28746032714844,
-      "loss": 0.4039,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -3.850860595703125,
-      "rewards/margins": 2.0992181301116943,
-      "rewards/rejected": -5.95007848739624,
-      "step": 3560
-    },
-    {
-      "epoch": 0.2853146853146853,
-      "grad_norm": 16.4167537689209,
-      "learning_rate": 4.495141968537745e-06,
-      "logits/chosen": -10.809391021728516,
-      "logits/rejected": -10.816768646240234,
-      "logps/chosen": -112.6346664428711,
-      "logps/rejected": -139.00547790527344,
-      "loss": 0.3927,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": -3.874281644821167,
-      "rewards/margins": 2.422574996948242,
-      "rewards/rejected": -6.2968573570251465,
-      "step": 3570
-    },
-    {
-      "epoch": 0.2861138861138861,
-      "grad_norm": 9.44420337677002,
-      "learning_rate": 4.490931126928058e-06,
-      "logits/chosen": -10.913290023803711,
-      "logits/rejected": -10.904637336730957,
-      "logps/chosen": -106.0438232421875,
-      "logps/rejected": -131.5537567138672,
-      "loss": 0.4227,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -3.6522140502929688,
-      "rewards/margins": 2.0631871223449707,
-      "rewards/rejected": -5.715401649475098,
-      "step": 3580
-    },
-    {
-      "epoch": 0.2869130869130869,
-      "grad_norm": 19.68636131286621,
-      "learning_rate": 4.4867047872019406e-06,
-      "logits/chosen": -10.88603687286377,
-      "logits/rejected": -10.892722129821777,
-      "logps/chosen": -115.8367919921875,
-      "logps/rejected": -138.97329711914062,
-      "loss": 0.5619,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": -4.328847408294678,
-      "rewards/margins": 2.025700330734253,
-      "rewards/rejected": -6.35454797744751,
-      "step": 3590
-    },
-    {
-      "epoch": 0.28771228771228774,
-      "grad_norm": 20.584211349487305,
-      "learning_rate": 4.4824629822587275e-06,
-      "logits/chosen": -11.01209831237793,
-      "logits/rejected": -11.003188133239746,
-      "logps/chosen": -109.67144012451172,
-      "logps/rejected": -137.8123779296875,
-      "loss": 0.3604,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -3.916273593902588,
-      "rewards/margins": 2.370434522628784,
-      "rewards/rejected": -6.286708354949951,
-      "step": 3600
-    },
-    {
-      "epoch": 0.2885114885114885,
-      "grad_norm": 16.75397491455078,
-      "learning_rate": 4.478205745118137e-06,
-      "logits/chosen": -10.814614295959473,
-      "logits/rejected": -10.955798149108887,
-      "logps/chosen": -115.36698150634766,
-      "logps/rejected": -157.5464324951172,
-      "loss": 0.3912,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -4.406135082244873,
-      "rewards/margins": 2.957414388656616,
-      "rewards/rejected": -7.36354923248291,
-      "step": 3610
-    },
-    {
-      "epoch": 0.2893106893106893,
-      "grad_norm": 18.23948860168457,
-      "learning_rate": 4.473933108920017e-06,
-      "logits/chosen": -10.993581771850586,
-      "logits/rejected": -10.95775032043457,
-      "logps/chosen": -113.6041259765625,
-      "logps/rejected": -141.8372802734375,
-      "loss": 0.3985,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -4.387929916381836,
-      "rewards/margins": 2.226003408432007,
-      "rewards/rejected": -6.613933086395264,
-      "step": 3620
-    },
-    {
-      "epoch": 0.29010989010989013,
-      "grad_norm": 16.74540901184082,
-      "learning_rate": 4.4696451069240895e-06,
-      "logits/chosen": -10.87891674041748,
-      "logits/rejected": -10.896753311157227,
-      "logps/chosen": -117.20283508300781,
-      "logps/rejected": -150.3008575439453,
-      "loss": 0.3649,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -4.631337642669678,
-      "rewards/margins": 2.589557409286499,
-      "rewards/rejected": -7.220895290374756,
-      "step": 3630
-    },
-    {
-      "epoch": 0.2909090909090909,
-      "grad_norm": 18.246065139770508,
-      "learning_rate": 4.465341772509689e-06,
-      "logits/chosen": -10.994613647460938,
-      "logits/rejected": -10.969186782836914,
-      "logps/chosen": -120.67894744873047,
-      "logps/rejected": -148.84190368652344,
-      "loss": 0.4055,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": -4.777548313140869,
-      "rewards/margins": 2.605206251144409,
-      "rewards/rejected": -7.382753849029541,
-      "step": 3640
-    },
-    {
-      "epoch": 0.2917082917082917,
-      "grad_norm": 13.251383781433105,
-      "learning_rate": 4.4610231391755e-06,
-      "logits/chosen": -11.102692604064941,
-      "logits/rejected": -11.068018913269043,
-      "logps/chosen": -116.79963684082031,
-      "logps/rejected": -145.76553344726562,
-      "loss": 0.3258,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -4.532977104187012,
-      "rewards/margins": 2.9616806507110596,
-      "rewards/rejected": -7.49465799331665,
-      "step": 3650
-    },
-    {
-      "epoch": 0.29250749250749253,
-      "grad_norm": 22.82951545715332,
-      "learning_rate": 4.456689240539302e-06,
-      "logits/chosen": -11.059904098510742,
-      "logits/rejected": -11.050941467285156,
-      "logps/chosen": -115.25200653076172,
-      "logps/rejected": -140.34536743164062,
-      "loss": 0.3619,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -4.361152648925781,
-      "rewards/margins": 2.372493267059326,
-      "rewards/rejected": -6.733645915985107,
-      "step": 3660
-    },
-    {
-      "epoch": 0.2933066933066933,
-      "grad_norm": 17.800384521484375,
-      "learning_rate": 4.452340110337704e-06,
-      "logits/chosen": -10.86427116394043,
-      "logits/rejected": -10.831857681274414,
-      "logps/chosen": -126.0827407836914,
-      "logps/rejected": -151.97552490234375,
-      "loss": 0.5101,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -5.130390644073486,
-      "rewards/margins": 2.086904525756836,
-      "rewards/rejected": -7.217294216156006,
-      "step": 3670
-    },
-    {
-      "epoch": 0.2941058941058941,
-      "grad_norm": 7.113104820251465,
-      "learning_rate": 4.447975782425884e-06,
-      "logits/chosen": -11.019858360290527,
-      "logits/rejected": -10.908228874206543,
-      "logps/chosen": -121.4881591796875,
-      "logps/rejected": -150.17971801757812,
-      "loss": 0.4571,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -4.89058256149292,
-      "rewards/margins": 2.5660440921783447,
-      "rewards/rejected": -7.456626892089844,
-      "step": 3680
-    },
-    {
-      "epoch": 0.2949050949050949,
-      "grad_norm": 20.166004180908203,
-      "learning_rate": 4.443596290777323e-06,
-      "logits/chosen": -10.978499412536621,
-      "logits/rejected": -10.94189453125,
-      "logps/chosen": -116.7060546875,
-      "logps/rejected": -147.12405395507812,
-      "loss": 0.4422,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -4.608033657073975,
-      "rewards/margins": 2.3946614265441895,
-      "rewards/rejected": -7.002694606781006,
-      "step": 3690
-    },
-    {
-      "epoch": 0.2957042957042957,
-      "grad_norm": 16.717538833618164,
-      "learning_rate": 4.439201669483544e-06,
-      "logits/chosen": -10.992826461791992,
-      "logits/rejected": -10.917204856872559,
-      "logps/chosen": -121.80244445800781,
-      "logps/rejected": -144.7339324951172,
-      "loss": 0.4072,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": -5.149653434753418,
-      "rewards/margins": 2.110015392303467,
-      "rewards/rejected": -7.259667873382568,
-      "step": 3700
-    },
-    {
-      "epoch": 0.2965034965034965,
-      "grad_norm": 19.31379508972168,
-      "learning_rate": 4.434791952753841e-06,
-      "logits/chosen": -11.015786170959473,
-      "logits/rejected": -11.005552291870117,
-      "logps/chosen": -128.9058074951172,
-      "logps/rejected": -152.67820739746094,
-      "loss": 0.4056,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -5.596781253814697,
-      "rewards/margins": 2.0842368602752686,
-      "rewards/rejected": -7.681018352508545,
-      "step": 3710
-    },
-    {
-      "epoch": 0.2973026973026973,
-      "grad_norm": 14.3607177734375,
-      "learning_rate": 4.43036717491502e-06,
-      "logits/chosen": -11.058650016784668,
-      "logits/rejected": -11.030865669250488,
-      "logps/chosen": -131.34048461914062,
-      "logps/rejected": -157.27622985839844,
-      "loss": 0.4058,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -5.617240905761719,
-      "rewards/margins": 2.2534146308898926,
-      "rewards/rejected": -7.870655059814453,
-      "step": 3720
-    },
-    {
-      "epoch": 0.2981018981018981,
-      "grad_norm": 23.008424758911133,
-      "learning_rate": 4.4259273704111255e-06,
-      "logits/chosen": -11.078301429748535,
-      "logits/rejected": -10.9492769241333,
-      "logps/chosen": -127.0863037109375,
-      "logps/rejected": -152.31321716308594,
-      "loss": 0.4948,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": -5.707759380340576,
-      "rewards/margins": 2.1356799602508545,
-      "rewards/rejected": -7.84343957901001,
-      "step": 3730
-    },
-    {
-      "epoch": 0.2989010989010989,
-      "grad_norm": 20.089584350585938,
-      "learning_rate": 4.421472573803176e-06,
-      "logits/chosen": -11.080973625183105,
-      "logits/rejected": -11.131916999816895,
-      "logps/chosen": -120.87931060791016,
-      "logps/rejected": -150.39453125,
-      "loss": 0.4976,
-      "rewards/accuracies": 0.7750000357627869,
-      "rewards/chosen": -4.832889080047607,
-      "rewards/margins": 2.2184104919433594,
-      "rewards/rejected": -7.051299571990967,
-      "step": 3740
-    },
-    {
-      "epoch": 0.2997002997002997,
-      "grad_norm": 18.062475204467773,
-      "learning_rate": 4.4170028197688946e-06,
-      "logits/chosen": -11.005610466003418,
-      "logits/rejected": -11.02857780456543,
-      "logps/chosen": -117.4045639038086,
-      "logps/rejected": -152.4534149169922,
-      "loss": 0.3542,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": -4.8840250968933105,
-      "rewards/margins": 2.6734018325805664,
-      "rewards/rejected": -7.557426452636719,
-      "step": 3750
-    },
-    {
-      "epoch": 0.3004995004995005,
-      "grad_norm": 17.00836944580078,
-      "learning_rate": 4.412518143102436e-06,
-      "logits/chosen": -11.059727668762207,
-      "logits/rejected": -11.159838676452637,
-      "logps/chosen": -116.20283508300781,
-      "logps/rejected": -146.44580078125,
-      "loss": 0.3394,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -4.518385410308838,
-      "rewards/margins": 2.554980993270874,
-      "rewards/rejected": -7.073366641998291,
-      "step": 3760
-    },
-    {
-      "epoch": 0.3012987012987013,
-      "grad_norm": 9.443163871765137,
-      "learning_rate": 4.40801857871412e-06,
-      "logits/chosen": -11.146327018737793,
-      "logits/rejected": -11.082951545715332,
-      "logps/chosen": -119.54044342041016,
-      "logps/rejected": -150.54246520996094,
-      "loss": 0.3342,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -5.059217929840088,
-      "rewards/margins": 2.752553939819336,
-      "rewards/rejected": -7.811771392822266,
-      "step": 3770
-    },
-    {
-      "epoch": 0.3020979020979021,
-      "grad_norm": 20.633556365966797,
-      "learning_rate": 4.403504161630157e-06,
-      "logits/chosen": -10.924055099487305,
-      "logits/rejected": -11.134722709655762,
-      "logps/chosen": -126.9520034790039,
-      "logps/rejected": -158.3199920654297,
-      "loss": 0.5197,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -5.593148231506348,
-      "rewards/margins": 2.4025161266326904,
-      "rewards/rejected": -7.995664119720459,
-      "step": 3780
-    },
-    {
-      "epoch": 0.3028971028971029,
-      "grad_norm": 14.347692489624023,
-      "learning_rate": 4.398974926992377e-06,
-      "logits/chosen": -11.173280715942383,
-      "logits/rejected": -11.117813110351562,
-      "logps/chosen": -126.2441635131836,
-      "logps/rejected": -144.35015869140625,
-      "loss": 0.5302,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -5.328347682952881,
-      "rewards/margins": 2.1139538288116455,
-      "rewards/rejected": -7.442301273345947,
-      "step": 3790
-    },
-    {
-      "epoch": 0.3036963036963037,
-      "grad_norm": 17.70793342590332,
-      "learning_rate": 4.394430910057953e-06,
-      "logits/chosen": -11.115402221679688,
-      "logits/rejected": -11.108474731445312,
-      "logps/chosen": -124.29450988769531,
-      "logps/rejected": -152.91156005859375,
-      "loss": 0.4852,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -5.28679895401001,
-      "rewards/margins": 2.495290517807007,
-      "rewards/rejected": -7.7820892333984375,
-      "step": 3800
-    },
-    {
-      "epoch": 0.3044955044955045,
-      "grad_norm": 19.73763084411621,
-      "learning_rate": 4.389872146199131e-06,
-      "logits/chosen": -11.063430786132812,
-      "logits/rejected": -11.00093936920166,
-      "logps/chosen": -115.728759765625,
-      "logps/rejected": -140.1971435546875,
-      "loss": 0.4596,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": -4.454066276550293,
-      "rewards/margins": 2.1964149475097656,
-      "rewards/rejected": -6.650481700897217,
-      "step": 3810
-    },
-    {
-      "epoch": 0.3052947052947053,
-      "grad_norm": 20.13469696044922,
-      "learning_rate": 4.3852986709029505e-06,
-      "logits/chosen": -11.031046867370605,
-      "logits/rejected": -10.978442192077637,
-      "logps/chosen": -123.1930160522461,
-      "logps/rejected": -143.85939025878906,
-      "loss": 0.3605,
-      "rewards/accuracies": 0.8375000357627869,
-      "rewards/chosen": -4.909556865692139,
-      "rewards/margins": 2.124016523361206,
-      "rewards/rejected": -7.033573150634766,
-      "step": 3820
-    },
-    {
-      "epoch": 0.3060939060939061,
-      "grad_norm": 19.117149353027344,
-      "learning_rate": 4.380710519770971e-06,
-      "logits/chosen": -10.842525482177734,
-      "logits/rejected": -10.88807201385498,
-      "logps/chosen": -117.79216766357422,
-      "logps/rejected": -146.6833953857422,
-      "loss": 0.5215,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": -4.661837100982666,
-      "rewards/margins": 2.1786434650421143,
-      "rewards/rejected": -6.840480327606201,
-      "step": 3830
-    },
-    {
-      "epoch": 0.3068931068931069,
-      "grad_norm": 9.562210083007812,
-      "learning_rate": 4.3761077285189944e-06,
-      "logits/chosen": -10.787657737731934,
-      "logits/rejected": -10.900628089904785,
-      "logps/chosen": -117.65953826904297,
-      "logps/rejected": -149.9417266845703,
-      "loss": 0.4388,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -4.780348777770996,
-      "rewards/margins": 2.5565185546875,
-      "rewards/rejected": -7.336867809295654,
-      "step": 3840
-    },
-    {
-      "epoch": 0.3076923076923077,
-      "grad_norm": 7.741754055023193,
-      "learning_rate": 4.371490332976785e-06,
-      "logits/chosen": -10.886703491210938,
-      "logits/rejected": -10.845261573791504,
-      "logps/chosen": -120.8351058959961,
-      "logps/rejected": -148.33517456054688,
-      "loss": 0.2512,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": -4.3019328117370605,
-      "rewards/margins": 2.726824998855591,
-      "rewards/rejected": -7.028757572174072,
-      "step": 3850
-    },
-    {
-      "epoch": 0.30849150849150847,
-      "grad_norm": 19.575355529785156,
-      "learning_rate": 4.366858369087793e-06,
-      "logits/chosen": -10.896971702575684,
-      "logits/rejected": -10.994900703430176,
-      "logps/chosen": -116.78076171875,
-      "logps/rejected": -154.10101318359375,
-      "loss": 0.5178,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -4.506631374359131,
-      "rewards/margins": 2.368802547454834,
-      "rewards/rejected": -6.875433444976807,
-      "step": 3860
-    },
-    {
-      "epoch": 0.3092907092907093,
-      "grad_norm": 19.912109375,
-      "learning_rate": 4.362211872908875e-06,
-      "logits/chosen": -11.043672561645508,
-      "logits/rejected": -10.9789457321167,
-      "logps/chosen": -116.03849029541016,
-      "logps/rejected": -145.1275634765625,
-      "loss": 0.4105,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -4.266086101531982,
-      "rewards/margins": 2.2475318908691406,
-      "rewards/rejected": -6.513618469238281,
-      "step": 3870
-    },
-    {
-      "epoch": 0.3100899100899101,
-      "grad_norm": 16.946151733398438,
-      "learning_rate": 4.3575508806100095e-06,
-      "logits/chosen": -10.786105155944824,
-      "logits/rejected": -11.042498588562012,
-      "logps/chosen": -116.54414367675781,
-      "logps/rejected": -156.69126892089844,
-      "loss": 0.4573,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -4.4445390701293945,
-      "rewards/margins": 2.808750629425049,
-      "rewards/rejected": -7.253289222717285,
-      "step": 3880
-    },
-    {
-      "epoch": 0.31088911088911086,
-      "grad_norm": 5.492505073547363,
-      "learning_rate": 4.35287542847402e-06,
-      "logits/chosen": -11.115829467773438,
-      "logits/rejected": -11.024444580078125,
-      "logps/chosen": -108.21112060546875,
-      "logps/rejected": -136.28868103027344,
-      "loss": 0.3118,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -3.6235527992248535,
-      "rewards/margins": 2.592973232269287,
-      "rewards/rejected": -6.216526031494141,
-      "step": 3890
-    },
-    {
-      "epoch": 0.3116883116883117,
-      "grad_norm": 19.17418670654297,
-      "learning_rate": 4.348185552896288e-06,
-      "logits/chosen": -10.888279914855957,
-      "logits/rejected": -11.070181846618652,
-      "logps/chosen": -112.90650939941406,
-      "logps/rejected": -150.2111053466797,
-      "loss": 0.327,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -4.273529529571533,
-      "rewards/margins": 2.744952917098999,
-      "rewards/rejected": -7.018482208251953,
-      "step": 3900
-    },
-    {
-      "epoch": 0.3124875124875125,
-      "grad_norm": 12.910696983337402,
-      "learning_rate": 4.343481290384477e-06,
-      "logits/chosen": -11.108476638793945,
-      "logits/rejected": -11.030253410339355,
-      "logps/chosen": -116.43798828125,
-      "logps/rejected": -139.5264434814453,
-      "loss": 0.5206,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -4.633407115936279,
-      "rewards/margins": 1.9944370985031128,
-      "rewards/rejected": -6.627844333648682,
-      "step": 3910
-    },
-    {
-      "epoch": 0.3132867132867133,
-      "grad_norm": 11.539154052734375,
-      "learning_rate": 4.338762677558239e-06,
-      "logits/chosen": -10.992209434509277,
-      "logits/rejected": -10.999341011047363,
-      "logps/chosen": -112.24641418457031,
-      "logps/rejected": -146.12533569335938,
-      "loss": 0.3476,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -4.139171600341797,
-      "rewards/margins": 2.6567089557647705,
-      "rewards/rejected": -6.7958807945251465,
-      "step": 3920
-    },
-    {
-      "epoch": 0.3140859140859141,
-      "grad_norm": 22.58559226989746,
-      "learning_rate": 4.334029751148935e-06,
-      "logits/chosen": -11.136322021484375,
-      "logits/rejected": -11.054377555847168,
-      "logps/chosen": -123.18364715576172,
-      "logps/rejected": -142.98477172851562,
-      "loss": 0.5163,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": -4.624068737030029,
-      "rewards/margins": 1.8977127075195312,
-      "rewards/rejected": -6.5217814445495605,
-      "step": 3930
-    },
-    {
-      "epoch": 0.3148851148851149,
-      "grad_norm": 19.0967960357666,
-      "learning_rate": 4.32928254799935e-06,
-      "logits/chosen": -10.945610046386719,
-      "logits/rejected": -11.0343599319458,
-      "logps/chosen": -117.14969635009766,
-      "logps/rejected": -146.8812255859375,
-      "loss": 0.507,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": -4.535704135894775,
-      "rewards/margins": 2.2106072902679443,
-      "rewards/rejected": -6.746312141418457,
-      "step": 3940
-    },
-    {
-      "epoch": 0.3156843156843157,
-      "grad_norm": 11.345633506774902,
-      "learning_rate": 4.324521105063402e-06,
-      "logits/chosen": -10.919096946716309,
-      "logits/rejected": -10.984095573425293,
-      "logps/chosen": -119.08155822753906,
-      "logps/rejected": -156.5112762451172,
-      "loss": 0.3024,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -4.524669647216797,
-      "rewards/margins": 3.321077346801758,
-      "rewards/rejected": -7.845747470855713,
-      "step": 3950
-    },
-    {
-      "epoch": 0.31648351648351647,
-      "grad_norm": 7.852194786071777,
-      "learning_rate": 4.319745459405856e-06,
-      "logits/chosen": -10.871066093444824,
-      "logits/rejected": -10.972114562988281,
-      "logps/chosen": -118.8388442993164,
-      "logps/rejected": -163.1884307861328,
-      "loss": 0.245,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": -4.97475528717041,
-      "rewards/margins": 3.3601295948028564,
-      "rewards/rejected": -8.334884643554688,
-      "step": 3960
-    },
-    {
-      "epoch": 0.3172827172827173,
-      "grad_norm": 9.91881275177002,
-      "learning_rate": 4.3149556482020405e-06,
-      "logits/chosen": -11.144624710083008,
-      "logits/rejected": -10.999795913696289,
-      "logps/chosen": -118.6851806640625,
-      "logps/rejected": -139.0449981689453,
-      "loss": 0.4195,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -5.0811944007873535,
-      "rewards/margins": 1.9882850646972656,
-      "rewards/rejected": -7.069479465484619,
-      "step": 3970
-    },
-    {
-      "epoch": 0.3180819180819181,
-      "grad_norm": 16.901947021484375,
-      "learning_rate": 4.3101517087375476e-06,
-      "logits/chosen": -11.125807762145996,
-      "logits/rejected": -10.992350578308105,
-      "logps/chosen": -134.23509216308594,
-      "logps/rejected": -162.49835205078125,
-      "loss": 0.4757,
-      "rewards/accuracies": 0.7750000357627869,
-      "rewards/chosen": -6.138756275177002,
-      "rewards/margins": 2.7983481884002686,
-      "rewards/rejected": -8.937105178833008,
-      "step": 3980
-    },
-    {
-      "epoch": 0.31888111888111886,
-      "grad_norm": 4.893545627593994,
-      "learning_rate": 4.305333678407954e-06,
-      "logits/chosen": -10.927640914916992,
-      "logits/rejected": -11.005168914794922,
-      "logps/chosen": -127.2559814453125,
-      "logps/rejected": -162.3601531982422,
-      "loss": 0.3398,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -5.480685234069824,
-      "rewards/margins": 2.8020551204681396,
-      "rewards/rejected": -8.282740592956543,
-      "step": 3990
-    },
-    {
-      "epoch": 0.3196803196803197,
-      "grad_norm": 25.388872146606445,
-      "learning_rate": 4.30050159471852e-06,
-      "logits/chosen": -11.14094066619873,
-      "logits/rejected": -11.068666458129883,
-      "logps/chosen": -124.82854461669922,
-      "logps/rejected": -143.60812377929688,
-      "loss": 0.5363,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -5.373419284820557,
-      "rewards/margins": 2.2448341846466064,
-      "rewards/rejected": -7.618253231048584,
-      "step": 4000
-    },
-    {
-      "epoch": 0.3204795204795205,
-      "grad_norm": 23.49039077758789,
-      "learning_rate": 4.295655495283906e-06,
-      "logits/chosen": -10.962862014770508,
-      "logits/rejected": -11.026691436767578,
-      "logps/chosen": -126.8401107788086,
-      "logps/rejected": -152.8392791748047,
-      "loss": 0.4894,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -5.357210636138916,
-      "rewards/margins": 2.148259401321411,
-      "rewards/rejected": -7.505469799041748,
-      "step": 4010
-    },
-    {
-      "epoch": 0.32127872127872126,
-      "grad_norm": 15.883101463317871,
-      "learning_rate": 4.290795417827875e-06,
-      "logits/chosen": -11.019301414489746,
-      "logits/rejected": -10.915206909179688,
-      "logps/chosen": -125.77542877197266,
-      "logps/rejected": -152.4874267578125,
-      "loss": 0.3691,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -4.951974868774414,
-      "rewards/margins": 2.749574899673462,
-      "rewards/rejected": -7.701549530029297,
-      "step": 4020
-    },
-    {
-      "epoch": 0.3220779220779221,
-      "grad_norm": 18.857650756835938,
-      "learning_rate": 4.2859214001829985e-06,
-      "logits/chosen": -10.747640609741211,
-      "logits/rejected": -10.962597846984863,
-      "logps/chosen": -117.14226531982422,
-      "logps/rejected": -167.71926879882812,
-      "loss": 0.3841,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -4.515815258026123,
-      "rewards/margins": 3.4797050952911377,
-      "rewards/rejected": -7.995520114898682,
-      "step": 4030
-    },
-    {
-      "epoch": 0.3228771228771229,
-      "grad_norm": 18.094587326049805,
-      "learning_rate": 4.281033480290366e-06,
-      "logits/chosen": -11.031801223754883,
-      "logits/rejected": -11.001874923706055,
-      "logps/chosen": -117.6884994506836,
-      "logps/rejected": -150.9893035888672,
-      "loss": 0.4641,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -4.466110706329346,
-      "rewards/margins": 2.8957650661468506,
-      "rewards/rejected": -7.361876010894775,
-      "step": 4040
-    },
-    {
-      "epoch": 0.32367632367632365,
-      "grad_norm": 10.894872665405273,
-      "learning_rate": 4.276131696199284e-06,
-      "logits/chosen": -11.011004447937012,
-      "logits/rejected": -10.910088539123535,
-      "logps/chosen": -115.50347900390625,
-      "logps/rejected": -150.2606964111328,
-      "loss": 0.2434,
-      "rewards/accuracies": 0.9000000357627869,
-      "rewards/chosen": -4.5182695388793945,
-      "rewards/margins": 2.907891035079956,
-      "rewards/rejected": -7.426159858703613,
-      "step": 4050
-    },
-    {
-      "epoch": 0.32447552447552447,
-      "grad_norm": 22.85382843017578,
-      "learning_rate": 4.271216086066985e-06,
-      "logits/chosen": -11.041818618774414,
-      "logits/rejected": -11.002632141113281,
-      "logps/chosen": -126.6878890991211,
-      "logps/rejected": -150.86404418945312,
-      "loss": 0.4158,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -5.188539981842041,
-      "rewards/margins": 2.4230780601501465,
-      "rewards/rejected": -7.6116180419921875,
-      "step": 4060
-    },
-    {
-      "epoch": 0.3252747252747253,
-      "grad_norm": 4.903548717498779,
-      "learning_rate": 4.2662866881583275e-06,
-      "logits/chosen": -11.058959007263184,
-      "logits/rejected": -11.033500671386719,
-      "logps/chosen": -118.62371063232422,
-      "logps/rejected": -143.31996154785156,
-      "loss": 0.3557,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -4.8328166007995605,
-      "rewards/margins": 2.1517515182495117,
-      "rewards/rejected": -6.984567165374756,
-      "step": 4070
-    },
-    {
-      "epoch": 0.3260739260739261,
-      "grad_norm": 11.830398559570312,
-      "learning_rate": 4.261343540845498e-06,
-      "logits/chosen": -10.90851879119873,
-      "logits/rejected": -10.939547538757324,
-      "logps/chosen": -123.4830322265625,
-      "logps/rejected": -161.68307495117188,
-      "loss": 0.3113,
-      "rewards/accuracies": 0.8375000357627869,
-      "rewards/chosen": -5.122803688049316,
-      "rewards/margins": 3.1490235328674316,
-      "rewards/rejected": -8.27182674407959,
-      "step": 4080
-    },
-    {
-      "epoch": 0.32687312687312686,
-      "grad_norm": 23.44713592529297,
-      "learning_rate": 4.2563866826077175e-06,
-      "logits/chosen": -11.068094253540039,
-      "logits/rejected": -10.973896026611328,
-      "logps/chosen": -124.6981201171875,
-      "logps/rejected": -151.4980926513672,
-      "loss": 0.5578,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -5.152663230895996,
-      "rewards/margins": 2.0551607608795166,
-      "rewards/rejected": -7.207824230194092,
-      "step": 4090
-    },
-    {
-      "epoch": 0.3276723276723277,
-      "grad_norm": 17.28362274169922,
-      "learning_rate": 4.251416152030931e-06,
-      "logits/chosen": -10.780828475952148,
-      "logits/rejected": -11.083086967468262,
-      "logps/chosen": -123.12934875488281,
-      "logps/rejected": -172.20925903320312,
-      "loss": 0.4164,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": -5.263695240020752,
-      "rewards/margins": 3.3489677906036377,
-      "rewards/rejected": -8.612663269042969,
-      "step": 4100
-    },
-    {
-      "epoch": 0.3284715284715285,
-      "grad_norm": 18.150461196899414,
-      "learning_rate": 4.246431987807519e-06,
-      "logits/chosen": -11.221844673156738,
-      "logits/rejected": -11.119993209838867,
-      "logps/chosen": -123.62873840332031,
-      "logps/rejected": -153.4154510498047,
-      "loss": 0.3419,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -5.4470086097717285,
-      "rewards/margins": 2.5880322456359863,
-      "rewards/rejected": -8.035040855407715,
-      "step": 4110
-    },
-    {
-      "epoch": 0.32927072927072926,
-      "grad_norm": 16.682004928588867,
-      "learning_rate": 4.24143422873599e-06,
-      "logits/chosen": -11.134651184082031,
-      "logits/rejected": -11.10763931274414,
-      "logps/chosen": -120.5302734375,
-      "logps/rejected": -150.9028778076172,
-      "loss": 0.5242,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -5.124821662902832,
-      "rewards/margins": 2.336998701095581,
-      "rewards/rejected": -7.461819648742676,
-      "step": 4120
-    },
-    {
-      "epoch": 0.3300699300699301,
-      "grad_norm": 17.645545959472656,
-      "learning_rate": 4.2364229137206796e-06,
-      "logits/chosen": -11.182649612426758,
-      "logits/rejected": -11.129904747009277,
-      "logps/chosen": -127.3556137084961,
-      "logps/rejected": -155.54188537597656,
-      "loss": 0.4077,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -5.47670316696167,
-      "rewards/margins": 2.3103981018066406,
-      "rewards/rejected": -7.787101745605469,
-      "step": 4130
-    },
-    {
-      "epoch": 0.3308691308691309,
-      "grad_norm": 12.898338317871094,
-      "learning_rate": 4.231398081771445e-06,
-      "logits/chosen": -11.124942779541016,
-      "logits/rejected": -11.109739303588867,
-      "logps/chosen": -123.5134048461914,
-      "logps/rejected": -149.8275146484375,
-      "loss": 0.4371,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -5.445046901702881,
-      "rewards/margins": 1.987865686416626,
-      "rewards/rejected": -7.432912349700928,
-      "step": 4140
-    },
-    {
-      "epoch": 0.33166833166833165,
-      "grad_norm": 11.512399673461914,
-      "learning_rate": 4.226359772003369e-06,
-      "logits/chosen": -11.16440486907959,
-      "logits/rejected": -11.079081535339355,
-      "logps/chosen": -125.5136489868164,
-      "logps/rejected": -150.6031494140625,
-      "loss": 0.3776,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -5.277799606323242,
-      "rewards/margins": 2.2058112621307373,
-      "rewards/rejected": -7.483611583709717,
-      "step": 4150
-    },
-    {
-      "epoch": 0.33246753246753247,
-      "grad_norm": 9.638328552246094,
-      "learning_rate": 4.221308023636446e-06,
-      "logits/chosen": -11.09822940826416,
-      "logits/rejected": -11.125946998596191,
-      "logps/chosen": -120.97274017333984,
-      "logps/rejected": -159.4351806640625,
-      "loss": 0.3834,
-      "rewards/accuracies": 0.8375000357627869,
-      "rewards/chosen": -4.742707252502441,
-      "rewards/margins": 3.2720115184783936,
-      "rewards/rejected": -8.014719009399414,
-      "step": 4160
-    },
-    {
-      "epoch": 0.3332667332667333,
-      "grad_norm": 20.312545776367188,
-      "learning_rate": 4.216242875995284e-06,
-      "logits/chosen": -11.171857833862305,
-      "logits/rejected": -11.13670825958252,
-      "logps/chosen": -126.69731903076172,
-      "logps/rejected": -150.0036163330078,
-      "loss": 0.4831,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -5.390435695648193,
-      "rewards/margins": 1.8986259698867798,
-      "rewards/rejected": -7.2890625,
-      "step": 4170
-    },
-    {
-      "epoch": 0.33406593406593404,
-      "grad_norm": 11.57194995880127,
-      "learning_rate": 4.211164368508794e-06,
-      "logits/chosen": -11.295496940612793,
-      "logits/rejected": -11.227758407592773,
-      "logps/chosen": -118.57830047607422,
-      "logps/rejected": -144.6667938232422,
-      "loss": 0.3698,
-      "rewards/accuracies": 0.8375000357627869,
-      "rewards/chosen": -4.726405143737793,
-      "rewards/margins": 2.247100353240967,
-      "rewards/rejected": -6.973504543304443,
-      "step": 4180
-    },
-    {
-      "epoch": 0.33486513486513486,
-      "grad_norm": 12.002553939819336,
-      "learning_rate": 4.2060725407098875e-06,
-      "logits/chosen": -11.431483268737793,
-      "logits/rejected": -11.363672256469727,
-      "logps/chosen": -115.46284484863281,
-      "logps/rejected": -146.312255859375,
-      "loss": 0.2613,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -4.346168518066406,
-      "rewards/margins": 2.9150230884552,
-      "rewards/rejected": -7.2611918449401855,
-      "step": 4190
-    },
-    {
-      "epoch": 0.3356643356643357,
-      "grad_norm": 10.64266300201416,
-      "learning_rate": 4.200967432235162e-06,
-      "logits/chosen": -11.2706880569458,
-      "logits/rejected": -11.260504722595215,
-      "logps/chosen": -120.92909240722656,
-      "logps/rejected": -154.65122985839844,
-      "loss": 0.5676,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": -4.9186110496521,
-      "rewards/margins": 2.9859673976898193,
-      "rewards/rejected": -7.904578685760498,
-      "step": 4200
-    },
-    {
-      "epoch": 0.33646353646353644,
-      "grad_norm": 7.697137832641602,
-      "learning_rate": 4.195849082824601e-06,
-      "logits/chosen": -11.207712173461914,
-      "logits/rejected": -11.335150718688965,
-      "logps/chosen": -115.43328094482422,
-      "logps/rejected": -149.12472534179688,
-      "loss": 0.4429,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -4.178666591644287,
-      "rewards/margins": 2.7227389812469482,
-      "rewards/rejected": -6.901405334472656,
-      "step": 4210
-    },
-    {
-      "epoch": 0.33726273726273726,
-      "grad_norm": 17.4494571685791,
-      "learning_rate": 4.190717532321256e-06,
-      "logits/chosen": -11.330622673034668,
-      "logits/rejected": -11.254408836364746,
-      "logps/chosen": -121.87444305419922,
-      "logps/rejected": -143.3373260498047,
-      "loss": 0.5077,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -4.501119613647461,
-      "rewards/margins": 2.4991137981414795,
-      "rewards/rejected": -7.0002336502075195,
-      "step": 4220
-    },
-    {
-      "epoch": 0.33806193806193807,
-      "grad_norm": 12.449810028076172,
-      "learning_rate": 4.185572820670942e-06,
-      "logits/chosen": -11.208684921264648,
-      "logits/rejected": -11.281462669372559,
-      "logps/chosen": -118.39754486083984,
-      "logps/rejected": -154.25552368164062,
-      "loss": 0.4068,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -4.382394313812256,
-      "rewards/margins": 2.924964666366577,
-      "rewards/rejected": -7.307359218597412,
-      "step": 4230
-    },
-    {
-      "epoch": 0.3388611388611389,
-      "grad_norm": 20.6654109954834,
-      "learning_rate": 4.180414987921927e-06,
-      "logits/chosen": -11.40068531036377,
-      "logits/rejected": -11.266721725463867,
-      "logps/chosen": -124.9693374633789,
-      "logps/rejected": -145.28414916992188,
-      "loss": 0.3385,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -4.666263103485107,
-      "rewards/margins": 2.40299654006958,
-      "rewards/rejected": -7.0692596435546875,
-      "step": 4240
-    },
-    {
-      "epoch": 0.33966033966033965,
-      "grad_norm": 18.578027725219727,
-      "learning_rate": 4.175244074224615e-06,
-      "logits/chosen": -11.308198928833008,
-      "logits/rejected": -11.219670295715332,
-      "logps/chosen": -119.7624740600586,
-      "logps/rejected": -141.3571014404297,
-      "loss": 0.4127,
-      "rewards/accuracies": 0.7750000357627869,
-      "rewards/chosen": -4.593161582946777,
-      "rewards/margins": 2.236616373062134,
-      "rewards/rejected": -6.82977819442749,
-      "step": 4250
-    },
-    {
-      "epoch": 0.34045954045954047,
-      "grad_norm": 13.050376892089844,
-      "learning_rate": 4.170060119831241e-06,
-      "logits/chosen": -11.333887100219727,
-      "logits/rejected": -11.232104301452637,
-      "logps/chosen": -116.44659423828125,
-      "logps/rejected": -140.7203826904297,
-      "loss": 0.4837,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -4.468829154968262,
-      "rewards/margins": 2.4944465160369873,
-      "rewards/rejected": -6.96327543258667,
-      "step": 4260
-    },
-    {
-      "epoch": 0.3412587412587413,
-      "grad_norm": 5.407907962799072,
-      "learning_rate": 4.1648631650955485e-06,
-      "logits/chosen": -11.151518821716309,
-      "logits/rejected": -11.235893249511719,
-      "logps/chosen": -112.93353271484375,
-      "logps/rejected": -146.8842010498047,
-      "loss": 0.2791,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -4.181356906890869,
-      "rewards/margins": 2.905186414718628,
-      "rewards/rejected": -7.086543560028076,
-      "step": 4270
-    },
-    {
-      "epoch": 0.34205794205794204,
-      "grad_norm": 8.217437744140625,
-      "learning_rate": 4.159653250472483e-06,
-      "logits/chosen": -11.110060691833496,
-      "logits/rejected": -11.240333557128906,
-      "logps/chosen": -114.7879638671875,
-      "logps/rejected": -152.38987731933594,
-      "loss": 0.2782,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -4.323099136352539,
-      "rewards/margins": 3.018723964691162,
-      "rewards/rejected": -7.341823101043701,
-      "step": 4280
-    },
-    {
-      "epoch": 0.34285714285714286,
-      "grad_norm": 8.444278717041016,
-      "learning_rate": 4.154430416517875e-06,
-      "logits/chosen": -11.357917785644531,
-      "logits/rejected": -11.337267875671387,
-      "logps/chosen": -112.3887939453125,
-      "logps/rejected": -136.92361450195312,
-      "loss": 0.4482,
-      "rewards/accuracies": 0.7750000357627869,
-      "rewards/chosen": -4.3228278160095215,
-      "rewards/margins": 2.1373088359832764,
-      "rewards/rejected": -6.460137367248535,
-      "step": 4290
-    },
-    {
-      "epoch": 0.3436563436563437,
-      "grad_norm": 10.270621299743652,
-      "learning_rate": 4.149194703888122e-06,
-      "logits/chosen": -11.301165580749512,
-      "logits/rejected": -11.313618659973145,
-      "logps/chosen": -118.78813934326172,
-      "logps/rejected": -150.01101684570312,
-      "loss": 0.3698,
-      "rewards/accuracies": 0.8375000357627869,
-      "rewards/chosen": -4.814661979675293,
-      "rewards/margins": 2.520723342895508,
-      "rewards/rejected": -7.335385322570801,
-      "step": 4300
-    },
-    {
-      "epoch": 0.34445554445554444,
-      "grad_norm": 22.580326080322266,
-      "learning_rate": 4.143946153339875e-06,
-      "logits/chosen": -11.363855361938477,
-      "logits/rejected": -11.317022323608398,
-      "logps/chosen": -126.20161437988281,
-      "logps/rejected": -152.88064575195312,
-      "loss": 0.6188,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -5.686009407043457,
-      "rewards/margins": 1.9269342422485352,
-      "rewards/rejected": -7.61294412612915,
-      "step": 4310
-    },
-    {
-      "epoch": 0.34525474525474525,
-      "grad_norm": 19.201047897338867,
-      "learning_rate": 4.138684805729719e-06,
-      "logits/chosen": -11.067610740661621,
-      "logits/rejected": -11.298110008239746,
-      "logps/chosen": -113.84529113769531,
-      "logps/rejected": -162.5439910888672,
-      "loss": 0.42,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -4.261666774749756,
-      "rewards/margins": 3.38800048828125,
-      "rewards/rejected": -7.649667263031006,
-      "step": 4320
-    },
-    {
-      "epoch": 0.34605394605394607,
-      "grad_norm": 8.46175479888916,
-      "learning_rate": 4.133410702013855e-06,
-      "logits/chosen": -11.514082908630371,
-      "logits/rejected": -11.415556907653809,
-      "logps/chosen": -110.06465148925781,
-      "logps/rejected": -140.99667358398438,
-      "loss": 0.3945,
-      "rewards/accuracies": 0.8375000357627869,
-      "rewards/chosen": -4.200688362121582,
-      "rewards/margins": 2.7206223011016846,
-      "rewards/rejected": -6.9213104248046875,
-      "step": 4330
-    },
-    {
-      "epoch": 0.34685314685314683,
-      "grad_norm": 25.71700668334961,
-      "learning_rate": 4.128123883247786e-06,
-      "logits/chosen": -11.356071472167969,
-      "logits/rejected": -11.355371475219727,
-      "logps/chosen": -119.1384506225586,
-      "logps/rejected": -140.12051391601562,
-      "loss": 0.4809,
-      "rewards/accuracies": 0.7750000357627869,
-      "rewards/chosen": -4.494596004486084,
-      "rewards/margins": 1.959856390953064,
-      "rewards/rejected": -6.4544525146484375,
-      "step": 4340
-    },
-    {
-      "epoch": 0.34765234765234765,
-      "grad_norm": 11.823657035827637,
-      "learning_rate": 4.122824390585988e-06,
-      "logits/chosen": -11.464831352233887,
-      "logits/rejected": -11.415430068969727,
-      "logps/chosen": -108.30140686035156,
-      "logps/rejected": -143.12405395507812,
-      "loss": 0.2687,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -3.6615757942199707,
-      "rewards/margins": 2.9635941982269287,
-      "rewards/rejected": -6.6251702308654785,
-      "step": 4350
-    },
-    {
-      "epoch": 0.34845154845154847,
-      "grad_norm": 25.495084762573242,
-      "learning_rate": 4.1175122652816e-06,
-      "logits/chosen": -11.501279830932617,
-      "logits/rejected": -11.41281795501709,
-      "logps/chosen": -113.78448486328125,
-      "logps/rejected": -141.14385986328125,
-      "loss": 0.3055,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -3.721360921859741,
-      "rewards/margins": 2.756010055541992,
-      "rewards/rejected": -6.4773712158203125,
-      "step": 4360
-    },
-    {
-      "epoch": 0.3492507492507492,
-      "grad_norm": 26.29249382019043,
-      "learning_rate": 4.112187548686094e-06,
-      "logits/chosen": -11.504768371582031,
-      "logits/rejected": -11.525805473327637,
-      "logps/chosen": -111.2749252319336,
-      "logps/rejected": -143.11293029785156,
-      "loss": 0.5219,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -3.907602071762085,
-      "rewards/margins": 2.458745241165161,
-      "rewards/rejected": -6.366347789764404,
-      "step": 4370
-    },
-    {
-      "epoch": 0.35004995004995004,
-      "grad_norm": 14.89150619506836,
-      "learning_rate": 4.106850282248961e-06,
-      "logits/chosen": -11.601164817810059,
-      "logits/rejected": -11.53286361694336,
-      "logps/chosen": -115.29581451416016,
-      "logps/rejected": -150.5718536376953,
-      "loss": 0.1859,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": -4.150198459625244,
-      "rewards/margins": 3.3452630043029785,
-      "rewards/rejected": -7.495460510253906,
-      "step": 4380
-    },
-    {
-      "epoch": 0.35084915084915086,
-      "grad_norm": 17.596555709838867,
-      "learning_rate": 4.101500507517381e-06,
-      "logits/chosen": -11.60312557220459,
-      "logits/rejected": -11.63845443725586,
-      "logps/chosen": -112.3973617553711,
-      "logps/rejected": -147.88040161132812,
-      "loss": 0.2726,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": -4.04245138168335,
-      "rewards/margins": 3.080420732498169,
-      "rewards/rejected": -7.122871398925781,
-      "step": 4390
-    },
-    {
-      "epoch": 0.3516483516483517,
-      "grad_norm": 11.617938995361328,
-      "learning_rate": 4.096138266135904e-06,
-      "logits/chosen": -11.560687065124512,
-      "logits/rejected": -11.6991548538208,
-      "logps/chosen": -117.25611877441406,
-      "logps/rejected": -165.168701171875,
-      "loss": 0.4202,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -4.600301265716553,
-      "rewards/margins": 3.3603456020355225,
-      "rewards/rejected": -7.960647106170654,
-      "step": 4400
-    },
-    {
-      "epoch": 0.35244755244755244,
-      "grad_norm": 6.783060073852539,
-      "learning_rate": 4.090763599846127e-06,
-      "logits/chosen": -11.430221557617188,
-      "logits/rejected": -11.619598388671875,
-      "logps/chosen": -120.48500061035156,
-      "logps/rejected": -170.92672729492188,
-      "loss": 0.3461,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -5.167738437652588,
-      "rewards/margins": 3.3222923278808594,
-      "rewards/rejected": -8.490030288696289,
-      "step": 4410
-    },
-    {
-      "epoch": 0.35324675324675325,
-      "grad_norm": 22.18805503845215,
-      "learning_rate": 4.0853765504863635e-06,
-      "logits/chosen": -11.484739303588867,
-      "logits/rejected": -11.608406066894531,
-      "logps/chosen": -117.68867492675781,
-      "logps/rejected": -151.27940368652344,
-      "loss": 0.3868,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -4.721958160400391,
-      "rewards/margins": 2.7591640949249268,
-      "rewards/rejected": -7.481122016906738,
-      "step": 4420
-    },
-    {
-      "epoch": 0.35404595404595407,
-      "grad_norm": 2.1606101989746094,
-      "learning_rate": 4.079977159991323e-06,
-      "logits/chosen": -11.673685073852539,
-      "logits/rejected": -11.621269226074219,
-      "logps/chosen": -127.982666015625,
-      "logps/rejected": -160.44656372070312,
-      "loss": 0.4388,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -5.3804521560668945,
-      "rewards/margins": 2.960033655166626,
-      "rewards/rejected": -8.340486526489258,
-      "step": 4430
-    },
-    {
-      "epoch": 0.35484515484515483,
-      "grad_norm": 20.455896377563477,
-      "learning_rate": 4.0745654703917835e-06,
-      "logits/chosen": -11.716124534606934,
-      "logits/rejected": -11.59318733215332,
-      "logps/chosen": -115.47342681884766,
-      "logps/rejected": -143.23660278320312,
-      "loss": 0.4243,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -4.6938300132751465,
-      "rewards/margins": 2.578789710998535,
-      "rewards/rejected": -7.272619724273682,
-      "step": 4440
-    },
-    {
-      "epoch": 0.35564435564435565,
-      "grad_norm": 14.87857723236084,
-      "learning_rate": 4.069141523814263e-06,
-      "logits/chosen": -11.559187889099121,
-      "logits/rejected": -11.59622859954834,
-      "logps/chosen": -122.6277084350586,
-      "logps/rejected": -159.32485961914062,
-      "loss": 0.3081,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": -5.105940341949463,
-      "rewards/margins": 3.1680169105529785,
-      "rewards/rejected": -8.273957252502441,
-      "step": 4450
-    },
-    {
-      "epoch": 0.35644355644355646,
-      "grad_norm": 7.010595321655273,
-      "learning_rate": 4.063705362480691e-06,
-      "logits/chosen": -11.150156021118164,
-      "logits/rejected": -11.421269416809082,
-      "logps/chosen": -126.80523681640625,
-      "logps/rejected": -180.635498046875,
-      "loss": 0.3262,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -5.099090099334717,
-      "rewards/margins": 4.0352678298950195,
-      "rewards/rejected": -9.134358406066895,
-      "step": 4460
-    },
-    {
-      "epoch": 0.3572427572427572,
-      "grad_norm": 14.296852111816406,
-      "learning_rate": 4.0582570287080816e-06,
-      "logits/chosen": -11.594368934631348,
-      "logits/rejected": -11.572097778320312,
-      "logps/chosen": -120.9228515625,
-      "logps/rejected": -152.0857391357422,
-      "loss": 0.4866,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -4.9669365882873535,
-      "rewards/margins": 2.442479133605957,
-      "rewards/rejected": -7.4094157218933105,
-      "step": 4470
-    },
-    {
-      "epoch": 0.35804195804195804,
-      "grad_norm": 6.3187103271484375,
-      "learning_rate": 4.052796564908205e-06,
-      "logits/chosen": -11.478571891784668,
-      "logits/rejected": -11.614974975585938,
-      "logps/chosen": -119.039794921875,
-      "logps/rejected": -159.69773864746094,
-      "loss": 0.3964,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -4.636753082275391,
-      "rewards/margins": 3.4076201915740967,
-      "rewards/rejected": -8.044374465942383,
-      "step": 4480
-    },
-    {
-      "epoch": 0.35884115884115886,
-      "grad_norm": 13.332737922668457,
-      "learning_rate": 4.047324013587254e-06,
-      "logits/chosen": -11.532028198242188,
-      "logits/rejected": -11.49357795715332,
-      "logps/chosen": -121.5921401977539,
-      "logps/rejected": -152.46669006347656,
-      "loss": 0.3726,
-      "rewards/accuracies": 0.8375000357627869,
-      "rewards/chosen": -4.891890525817871,
-      "rewards/margins": 2.8532474040985107,
-      "rewards/rejected": -7.745137691497803,
-      "step": 4490
-    },
-    {
-      "epoch": 0.3596403596403596,
-      "grad_norm": 24.78531837463379,
-      "learning_rate": 4.041839417345517e-06,
-      "logits/chosen": -11.47486400604248,
-      "logits/rejected": -11.41224193572998,
-      "logps/chosen": -118.3624267578125,
-      "logps/rejected": -141.11795043945312,
-      "loss": 0.3697,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -4.2375874519348145,
-      "rewards/margins": 2.763920545578003,
-      "rewards/rejected": -7.001507759094238,
-      "step": 4500
-    },
-    {
-      "epoch": 0.36043956043956044,
-      "grad_norm": 13.082833290100098,
-      "learning_rate": 4.036342818877041e-06,
-      "logits/chosen": -11.220852851867676,
-      "logits/rejected": -11.30420207977295,
-      "logps/chosen": -117.0436019897461,
-      "logps/rejected": -160.73245239257812,
-      "loss": 0.4752,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -4.536223888397217,
-      "rewards/margins": 3.0389859676361084,
-      "rewards/rejected": -7.575210094451904,
-      "step": 4510
-    },
-    {
-      "epoch": 0.36123876123876125,
-      "grad_norm": 21.5020751953125,
-      "learning_rate": 4.030834260969305e-06,
-      "logits/chosen": -11.401605606079102,
-      "logits/rejected": -11.340346336364746,
-      "logps/chosen": -118.60782623291016,
-      "logps/rejected": -141.3742218017578,
-      "loss": 0.4625,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -4.4115118980407715,
-      "rewards/margins": 2.5425033569335938,
-      "rewards/rejected": -6.954014778137207,
-      "step": 4520
-    },
-    {
-      "epoch": 0.362037962037962,
-      "grad_norm": 27.329389572143555,
-      "learning_rate": 4.025313786502886e-06,
-      "logits/chosen": -11.24223804473877,
-      "logits/rejected": -11.305307388305664,
-      "logps/chosen": -115.70033264160156,
-      "logps/rejected": -141.8595428466797,
-      "loss": 0.621,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": -4.739328384399414,
-      "rewards/margins": 2.052288770675659,
-      "rewards/rejected": -6.791616916656494,
-      "step": 4530
-    },
-    {
-      "epoch": 0.36283716283716283,
-      "grad_norm": 6.511563777923584,
-      "learning_rate": 4.019781438451119e-06,
-      "logits/chosen": -11.269216537475586,
-      "logits/rejected": -11.179797172546387,
-      "logps/chosen": -125.05785369873047,
-      "logps/rejected": -148.96380615234375,
-      "loss": 0.4504,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -5.256083965301514,
-      "rewards/margins": 2.2666358947753906,
-      "rewards/rejected": -7.522718906402588,
-      "step": 4540
-    },
-    {
-      "epoch": 0.36363636363636365,
-      "grad_norm": 6.137597560882568,
-      "learning_rate": 4.014237259879773e-06,
-      "logits/chosen": -11.090441703796387,
-      "logits/rejected": -11.186795234680176,
-      "logps/chosen": -117.2938461303711,
-      "logps/rejected": -147.14126586914062,
-      "loss": 0.3896,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -4.62833309173584,
-      "rewards/margins": 2.258892774581909,
-      "rewards/rejected": -6.88722562789917,
-      "step": 4550
-    },
-    {
-      "epoch": 0.36443556443556446,
-      "grad_norm": 12.973394393920898,
-      "learning_rate": 4.008681293946706e-06,
-      "logits/chosen": -11.175661087036133,
-      "logits/rejected": -11.153895378112793,
-      "logps/chosen": -121.31689453125,
-      "logps/rejected": -145.9535369873047,
-      "loss": 0.434,
-      "rewards/accuracies": 0.7750000357627869,
-      "rewards/chosen": -5.121529579162598,
-      "rewards/margins": 2.1015841960906982,
-      "rewards/rejected": -7.223114013671875,
-      "step": 4560
-    },
-    {
-      "epoch": 0.3652347652347652,
-      "grad_norm": 7.249019145965576,
-      "learning_rate": 4.003113583901532e-06,
-      "logits/chosen": -11.06865406036377,
-      "logits/rejected": -11.218696594238281,
-      "logps/chosen": -117.9729232788086,
-      "logps/rejected": -148.4829559326172,
-      "loss": 0.4238,
-      "rewards/accuracies": 0.8375000357627869,
-      "rewards/chosen": -4.702051162719727,
-      "rewards/margins": 2.404320478439331,
-      "rewards/rejected": -7.1063714027404785,
-      "step": 4570
-    },
-    {
-      "epoch": 0.36603396603396604,
-      "grad_norm": 15.080263137817383,
-      "learning_rate": 3.997534173085292e-06,
-      "logits/chosen": -11.193455696105957,
-      "logits/rejected": -11.162653923034668,
-      "logps/chosen": -122.90283203125,
-      "logps/rejected": -151.48036193847656,
-      "loss": 0.3001,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -5.452084541320801,
-      "rewards/margins": 2.473672866821289,
-      "rewards/rejected": -7.925757884979248,
-      "step": 4580
-    },
-    {
-      "epoch": 0.36683316683316686,
-      "grad_norm": 20.21657371520996,
-      "learning_rate": 3.9919431049301e-06,
-      "logits/chosen": -11.141977310180664,
-      "logits/rejected": -11.1973295211792,
-      "logps/chosen": -122.59673309326172,
-      "logps/rejected": -156.5644073486328,
-      "loss": 0.3748,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -4.8686442375183105,
-      "rewards/margins": 2.727036952972412,
-      "rewards/rejected": -7.595680236816406,
-      "step": 4590
-    },
-    {
-      "epoch": 0.3676323676323676,
-      "grad_norm": 1.3978800773620605,
-      "learning_rate": 3.986340422958824e-06,
-      "logits/chosen": -11.233321189880371,
-      "logits/rejected": -11.143681526184082,
-      "logps/chosen": -127.3147201538086,
-      "logps/rejected": -157.35667419433594,
-      "loss": 0.316,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -5.440598964691162,
-      "rewards/margins": 3.0376851558685303,
-      "rewards/rejected": -8.478283882141113,
-      "step": 4600
-    },
-    {
-      "epoch": 0.36843156843156843,
-      "grad_norm": 11.348470687866211,
-      "learning_rate": 3.980726170784734e-06,
-      "logits/chosen": -11.158247947692871,
-      "logits/rejected": -11.260726928710938,
-      "logps/chosen": -129.05722045898438,
-      "logps/rejected": -158.4536590576172,
-      "loss": 0.6352,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": -5.957630634307861,
-      "rewards/margins": 2.1807467937469482,
-      "rewards/rejected": -8.13837718963623,
-      "step": 4610
-    },
-    {
-      "epoch": 0.36923076923076925,
-      "grad_norm": 16.005407333374023,
-      "learning_rate": 3.9751003921111655e-06,
-      "logits/chosen": -11.198158264160156,
-      "logits/rejected": -11.222737312316895,
-      "logps/chosen": -126.79888153076172,
-      "logps/rejected": -159.94309997558594,
-      "loss": 0.4448,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -5.239553928375244,
-      "rewards/margins": 2.853848934173584,
-      "rewards/rejected": -8.093402862548828,
-      "step": 4620
-    },
-    {
-      "epoch": 0.37002997002997,
-      "grad_norm": 19.36884307861328,
-      "learning_rate": 3.969463130731183e-06,
-      "logits/chosen": -11.164327621459961,
-      "logits/rejected": -11.125060081481934,
-      "logps/chosen": -129.86553955078125,
-      "logps/rejected": -157.03219604492188,
-      "loss": 0.3899,
-      "rewards/accuracies": 0.7750000357627869,
-      "rewards/chosen": -5.334031105041504,
-      "rewards/margins": 2.6206605434417725,
-      "rewards/rejected": -7.954692363739014,
-      "step": 4630
-    },
-    {
-      "epoch": 0.37082917082917083,
-      "grad_norm": 20.747709274291992,
-      "learning_rate": 3.963814430527235e-06,
-      "logits/chosen": -11.014444351196289,
-      "logits/rejected": -11.040888786315918,
-      "logps/chosen": -127.09044647216797,
-      "logps/rejected": -154.0817108154297,
-      "loss": 0.4537,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": -5.097658634185791,
-      "rewards/margins": 2.664771318435669,
-      "rewards/rejected": -7.762429714202881,
-      "step": 4640
-    },
-    {
-      "epoch": 0.37162837162837165,
-      "grad_norm": 17.88429832458496,
-      "learning_rate": 3.9581543354708145e-06,
-      "logits/chosen": -11.222525596618652,
-      "logits/rejected": -11.093754768371582,
-      "logps/chosen": -117.86934661865234,
-      "logps/rejected": -147.2012176513672,
-      "loss": 0.2824,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -4.7861504554748535,
-      "rewards/margins": 2.5496411323547363,
-      "rewards/rejected": -7.335791110992432,
-      "step": 4650
-    },
-    {
-      "epoch": 0.3724275724275724,
-      "grad_norm": 19.804134368896484,
-      "learning_rate": 3.952482889622114e-06,
-      "logits/chosen": -11.249043464660645,
-      "logits/rejected": -11.122297286987305,
-      "logps/chosen": -115.67021942138672,
-      "logps/rejected": -141.0105438232422,
-      "loss": 0.4123,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -4.613503932952881,
-      "rewards/margins": 2.592719078063965,
-      "rewards/rejected": -7.2062225341796875,
-      "step": 4660
-    },
-    {
-      "epoch": 0.3732267732267732,
-      "grad_norm": 7.843384265899658,
-      "learning_rate": 3.946800137129688e-06,
-      "logits/chosen": -11.166597366333008,
-      "logits/rejected": -11.1419038772583,
-      "logps/chosen": -120.34991455078125,
-      "logps/rejected": -145.1756591796875,
-      "loss": 0.5373,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -4.934848308563232,
-      "rewards/margins": 2.18345308303833,
-      "rewards/rejected": -7.1183013916015625,
-      "step": 4670
-    },
-    {
-      "epoch": 0.37402597402597404,
-      "grad_norm": 4.25451135635376,
-      "learning_rate": 3.9411061222301035e-06,
-      "logits/chosen": -11.108311653137207,
-      "logits/rejected": -11.061227798461914,
-      "logps/chosen": -120.75566864013672,
-      "logps/rejected": -158.69973754882812,
-      "loss": 0.3985,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -5.261629104614258,
-      "rewards/margins": 2.6066806316375732,
-      "rewards/rejected": -7.86830997467041,
-      "step": 4680
-    },
-    {
-      "epoch": 0.3748251748251748,
-      "grad_norm": 10.511228561401367,
-      "learning_rate": 3.9354008892476e-06,
-      "logits/chosen": -11.127629280090332,
-      "logits/rejected": -11.041041374206543,
-      "logps/chosen": -121.32744598388672,
-      "logps/rejected": -148.3810272216797,
-      "loss": 0.3539,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -4.853220462799072,
-      "rewards/margins": 2.6031506061553955,
-      "rewards/rejected": -7.456370830535889,
-      "step": 4690
-    },
-    {
-      "epoch": 0.3756243756243756,
-      "grad_norm": 9.442309379577637,
-      "learning_rate": 3.929684482593742e-06,
-      "logits/chosen": -11.048380851745605,
-      "logits/rejected": -11.12149715423584,
-      "logps/chosen": -118.2668228149414,
-      "logps/rejected": -154.8514862060547,
-      "loss": 0.4454,
-      "rewards/accuracies": 0.8375000357627869,
-      "rewards/chosen": -4.703024387359619,
-      "rewards/margins": 2.622896194458008,
-      "rewards/rejected": -7.325921058654785,
-      "step": 4700
-    },
-    {
-      "epoch": 0.37642357642357643,
-      "grad_norm": 14.986795425415039,
-      "learning_rate": 3.92395694676707e-06,
-      "logits/chosen": -11.268658638000488,
-      "logits/rejected": -11.239967346191406,
-      "logps/chosen": -116.8264389038086,
-      "logps/rejected": -143.57620239257812,
-      "loss": 0.3981,
-      "rewards/accuracies": 0.8375000357627869,
-      "rewards/chosen": -4.67512321472168,
-      "rewards/margins": 2.1703128814697266,
-      "rewards/rejected": -6.845436096191406,
-      "step": 4710
-    },
-    {
-      "epoch": 0.37722277722277725,
-      "grad_norm": 16.482078552246094,
-      "learning_rate": 3.918218326352764e-06,
-      "logits/chosen": -11.016940116882324,
-      "logits/rejected": -11.044912338256836,
-      "logps/chosen": -124.56678771972656,
-      "logps/rejected": -157.47552490234375,
-      "loss": 0.4496,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -5.093387603759766,
-      "rewards/margins": 2.5396931171417236,
-      "rewards/rejected": -7.63308048248291,
-      "step": 4720
-    },
-    {
-      "epoch": 0.378021978021978,
-      "grad_norm": 11.260992050170898,
-      "learning_rate": 3.912468666022287e-06,
-      "logits/chosen": -11.040938377380371,
-      "logits/rejected": -11.1337308883667,
-      "logps/chosen": -119.56011199951172,
-      "logps/rejected": -158.40647888183594,
-      "loss": 0.3245,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -4.928431034088135,
-      "rewards/margins": 2.9643335342407227,
-      "rewards/rejected": -7.892764568328857,
-      "step": 4730
-    },
-    {
-      "epoch": 0.37882117882117883,
-      "grad_norm": 16.03196907043457,
-      "learning_rate": 3.90670801053304e-06,
-      "logits/chosen": -11.2951021194458,
-      "logits/rejected": -11.170924186706543,
-      "logps/chosen": -116.28707122802734,
-      "logps/rejected": -144.5259552001953,
-      "loss": 0.3258,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -4.481791973114014,
-      "rewards/margins": 2.7760486602783203,
-      "rewards/rejected": -7.257840156555176,
-      "step": 4740
-    },
-    {
-      "epoch": 0.37962037962037964,
-      "grad_norm": 17.72314453125,
-      "learning_rate": 3.900936404728018e-06,
-      "logits/chosen": -11.106257438659668,
-      "logits/rejected": -11.098315238952637,
-      "logps/chosen": -129.11656188964844,
-      "logps/rejected": -154.08767700195312,
-      "loss": 0.3494,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -5.452017307281494,
-      "rewards/margins": 2.3660800457000732,
-      "rewards/rejected": -7.818097114562988,
-      "step": 4750
-    },
-    {
-      "epoch": 0.3804195804195804,
-      "grad_norm": 16.079050064086914,
-      "learning_rate": 3.895153893535452e-06,
-      "logits/chosen": -11.122893333435059,
-      "logits/rejected": -11.181493759155273,
-      "logps/chosen": -124.00019836425781,
-      "logps/rejected": -156.09375,
-      "loss": 0.4524,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": -5.165709972381592,
-      "rewards/margins": 2.5191805362701416,
-      "rewards/rejected": -7.684890270233154,
-      "step": 4760
-    },
-    {
-      "epoch": 0.3812187812187812,
-      "grad_norm": 7.4565911293029785,
-      "learning_rate": 3.889360521968467e-06,
-      "logits/chosen": -11.263586044311523,
-      "logits/rejected": -11.189993858337402,
-      "logps/chosen": -135.36058044433594,
-      "logps/rejected": -168.34458923339844,
-      "loss": 0.4241,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -6.454078674316406,
-      "rewards/margins": 2.627779006958008,
-      "rewards/rejected": -9.081857681274414,
-      "step": 4770
-    },
-    {
-      "epoch": 0.38201798201798204,
-      "grad_norm": 23.963319778442383,
-      "learning_rate": 3.883556335124732e-06,
-      "logits/chosen": -11.29660415649414,
-      "logits/rejected": -11.245856285095215,
-      "logps/chosen": -139.04180908203125,
-      "logps/rejected": -162.57029724121094,
-      "loss": 0.3954,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -6.254592418670654,
-      "rewards/margins": 2.4864120483398438,
-      "rewards/rejected": -8.74100399017334,
-      "step": 4780
-    },
-    {
-      "epoch": 0.3828171828171828,
-      "grad_norm": 15.6788969039917,
-      "learning_rate": 3.8777413781861e-06,
-      "logits/chosen": -11.270599365234375,
-      "logits/rejected": -11.229459762573242,
-      "logps/chosen": -122.94129943847656,
-      "logps/rejected": -154.72312927246094,
-      "loss": 0.3626,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -5.2801127433776855,
-      "rewards/margins": 2.899285078048706,
-      "rewards/rejected": -8.179398536682129,
-      "step": 4790
-    },
-    {
-      "epoch": 0.3836163836163836,
-      "grad_norm": 2.2395031452178955,
-      "learning_rate": 3.871915696418267e-06,
-      "logits/chosen": -11.323569297790527,
-      "logits/rejected": -11.225008964538574,
-      "logps/chosen": -127.18331146240234,
-      "logps/rejected": -154.90211486816406,
-      "loss": 0.3966,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": -5.40226936340332,
-      "rewards/margins": 2.542661666870117,
-      "rewards/rejected": -7.9449310302734375,
-      "step": 4800
-    },
-    {
-      "epoch": 0.38441558441558443,
-      "grad_norm": 26.42649269104004,
-      "learning_rate": 3.866079335170411e-06,
-      "logits/chosen": -11.17712688446045,
-      "logits/rejected": -11.268791198730469,
-      "logps/chosen": -132.24778747558594,
-      "logps/rejected": -166.3517608642578,
-      "loss": 0.4171,
-      "rewards/accuracies": 0.8375000357627869,
-      "rewards/chosen": -5.665971279144287,
-      "rewards/margins": 2.8105599880218506,
-      "rewards/rejected": -8.476531982421875,
-      "step": 4810
-    },
-    {
-      "epoch": 0.3852147852147852,
-      "grad_norm": 6.1242547035217285,
-      "learning_rate": 3.860232339874849e-06,
-      "logits/chosen": -11.225537300109863,
-      "logits/rejected": -11.094820022583008,
-      "logps/chosen": -129.32852172851562,
-      "logps/rejected": -163.7541961669922,
-      "loss": 0.2218,
-      "rewards/accuracies": 0.9000000357627869,
-      "rewards/chosen": -5.649970531463623,
-      "rewards/margins": 3.4977223873138428,
-      "rewards/rejected": -9.147692680358887,
-      "step": 4820
-    },
-    {
-      "epoch": 0.386013986013986,
-      "grad_norm": 21.188064575195312,
-      "learning_rate": 3.854374756046668e-06,
-      "logits/chosen": -11.252513885498047,
-      "logits/rejected": -11.363961219787598,
-      "logps/chosen": -130.01182556152344,
-      "logps/rejected": -173.8732147216797,
-      "loss": 0.4836,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": -5.7001261711120605,
-      "rewards/margins": 3.215656042098999,
-      "rewards/rejected": -8.91578197479248,
-      "step": 4830
-    },
-    {
-      "epoch": 0.3868131868131868,
-      "grad_norm": 22.897891998291016,
-      "learning_rate": 3.84850662928339e-06,
-      "logits/chosen": -11.36469841003418,
-      "logits/rejected": -11.308235168457031,
-      "logps/chosen": -124.7119369506836,
-      "logps/rejected": -153.7621612548828,
-      "loss": 0.3979,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -5.341979503631592,
-      "rewards/margins": 2.4145772457122803,
-      "rewards/rejected": -7.756556987762451,
-      "step": 4840
-    },
-    {
-      "epoch": 0.3876123876123876,
-      "grad_norm": 16.597904205322266,
-      "learning_rate": 3.842628005264599e-06,
-      "logits/chosen": -11.33863353729248,
-      "logits/rejected": -11.120137214660645,
-      "logps/chosen": -128.24586486816406,
-      "logps/rejected": -158.46397399902344,
-      "loss": 0.2603,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -5.695564270019531,
-      "rewards/margins": 2.878471851348877,
-      "rewards/rejected": -8.57403564453125,
-      "step": 4850
-    },
-    {
-      "epoch": 0.3884115884115884,
-      "grad_norm": 23.30820083618164,
-      "learning_rate": 3.836738929751598e-06,
-      "logits/chosen": -11.38867473602295,
-      "logits/rejected": -11.331558227539062,
-      "logps/chosen": -131.3330535888672,
-      "logps/rejected": -161.74876403808594,
-      "loss": 0.4301,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -6.176599979400635,
-      "rewards/margins": 2.7395026683807373,
-      "rewards/rejected": -8.916102409362793,
-      "step": 4860
-    },
-    {
-      "epoch": 0.3892107892107892,
-      "grad_norm": 16.99883460998535,
-      "learning_rate": 3.830839448587044e-06,
-      "logits/chosen": -11.340460777282715,
-      "logits/rejected": -11.230690002441406,
-      "logps/chosen": -131.34194946289062,
-      "logps/rejected": -167.99429321289062,
-      "loss": 0.2272,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": -6.064395427703857,
-      "rewards/margins": 3.4520843029022217,
-      "rewards/rejected": -9.5164794921875,
-      "step": 4870
-    },
-    {
-      "epoch": 0.39000999000999004,
-      "grad_norm": 12.724471092224121,
-      "learning_rate": 3.824929607694603e-06,
-      "logits/chosen": -11.239911079406738,
-      "logits/rejected": -11.31291389465332,
-      "logps/chosen": -133.40721130371094,
-      "logps/rejected": -177.3763885498047,
-      "loss": 0.267,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -6.169772624969482,
-      "rewards/margins": 3.4861183166503906,
-      "rewards/rejected": -9.655891418457031,
-      "step": 4880
-    },
-    {
-      "epoch": 0.3908091908091908,
-      "grad_norm": 4.819546222686768,
-      "learning_rate": 3.819009453078575e-06,
-      "logits/chosen": -11.466912269592285,
-      "logits/rejected": -11.483650207519531,
-      "logps/chosen": -140.72911071777344,
-      "logps/rejected": -161.90838623046875,
-      "loss": 0.664,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -6.880612850189209,
-      "rewards/margins": 1.9591782093048096,
-      "rewards/rejected": -8.839791297912598,
-      "step": 4890
-    },
-    {
-      "epoch": 0.3916083916083916,
-      "grad_norm": 9.755080223083496,
-      "learning_rate": 3.8130790308235514e-06,
-      "logits/chosen": -11.490347862243652,
-      "logits/rejected": -11.463488578796387,
-      "logps/chosen": -138.8660125732422,
-      "logps/rejected": -163.330078125,
-      "loss": 0.2698,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -6.24899435043335,
-      "rewards/margins": 2.7623746395111084,
-      "rewards/rejected": -9.011369705200195,
-      "step": 4900
-    },
-    {
-      "epoch": 0.39240759240759243,
-      "grad_norm": 12.476890563964844,
-      "learning_rate": 3.807138387094051e-06,
-      "logits/chosen": -11.449475288391113,
-      "logits/rejected": -11.378355026245117,
-      "logps/chosen": -136.75514221191406,
-      "logps/rejected": -166.49583435058594,
-      "loss": 0.3061,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -6.289257049560547,
-      "rewards/margins": 2.8188440799713135,
-      "rewards/rejected": -9.108100891113281,
-      "step": 4910
-    },
-    {
-      "epoch": 0.3932067932067932,
-      "grad_norm": 18.56893539428711,
-      "learning_rate": 3.801187568134158e-06,
-      "logits/chosen": -11.466303825378418,
-      "logits/rejected": -11.479487419128418,
-      "logps/chosen": -139.21115112304688,
-      "logps/rejected": -167.7974090576172,
-      "loss": 0.4465,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -6.593389987945557,
-      "rewards/margins": 2.4661831855773926,
-      "rewards/rejected": -9.05957317352295,
-      "step": 4920
-    },
-    {
-      "epoch": 0.394005994005994,
-      "grad_norm": 20.420665740966797,
-      "learning_rate": 3.795226620267164e-06,
-      "logits/chosen": -11.526203155517578,
-      "logits/rejected": -11.497588157653809,
-      "logps/chosen": -129.42332458496094,
-      "logps/rejected": -165.5619659423828,
-      "loss": 0.4104,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -6.029610633850098,
-      "rewards/margins": 2.836603879928589,
-      "rewards/rejected": -8.86621379852295,
-      "step": 4930
-    },
-    {
-      "epoch": 0.3948051948051948,
-      "grad_norm": 26.027118682861328,
-      "learning_rate": 3.789255589895211e-06,
-      "logits/chosen": -11.534173965454102,
-      "logits/rejected": -11.530200004577637,
-      "logps/chosen": -135.2562713623047,
-      "logps/rejected": -164.6595916748047,
-      "loss": 0.4494,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -6.352004051208496,
-      "rewards/margins": 2.6263926029205322,
-      "rewards/rejected": -8.97839641571045,
-      "step": 4940
-    },
-    {
-      "epoch": 0.3956043956043956,
-      "grad_norm": 16.840450286865234,
-      "learning_rate": 3.783274523498922e-06,
-      "logits/chosen": -11.555611610412598,
-      "logits/rejected": -11.451794624328613,
-      "logps/chosen": -133.27464294433594,
-      "logps/rejected": -160.38633728027344,
-      "loss": 0.3939,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -5.937171459197998,
-      "rewards/margins": 2.600982189178467,
-      "rewards/rejected": -8.538153648376465,
-      "step": 4950
-    },
-    {
-      "epoch": 0.3964035964035964,
-      "grad_norm": 20.077451705932617,
-      "learning_rate": 3.7772834676370472e-06,
-      "logits/chosen": -11.302371978759766,
-      "logits/rejected": -11.412243843078613,
-      "logps/chosen": -137.33421325683594,
-      "logps/rejected": -169.89122009277344,
-      "loss": 0.5306,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": -6.458264350891113,
-      "rewards/margins": 2.7358386516571045,
-      "rewards/rejected": -9.19410228729248,
-      "step": 4960
-    },
-    {
-      "epoch": 0.3972027972027972,
-      "grad_norm": 11.95050048828125,
-      "learning_rate": 3.7712824689460983e-06,
-      "logits/chosen": -11.424470901489258,
-      "logits/rejected": -11.386736869812012,
-      "logps/chosen": -123.45647430419922,
-      "logps/rejected": -159.3142852783203,
-      "loss": 0.3746,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -5.280496120452881,
-      "rewards/margins": 3.0739736557006836,
-      "rewards/rejected": -8.354469299316406,
-      "step": 4970
-    },
-    {
-      "epoch": 0.398001998001998,
-      "grad_norm": 3.365035057067871,
-      "learning_rate": 3.765271574139984e-06,
-      "logits/chosen": -11.364173889160156,
-      "logits/rejected": -11.299944877624512,
-      "logps/chosen": -136.6787567138672,
-      "logps/rejected": -157.73204040527344,
-      "loss": 0.4474,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -6.267431735992432,
-      "rewards/margins": 2.2008512020111084,
-      "rewards/rejected": -8.468283653259277,
-      "step": 4980
-    },
-    {
-      "epoch": 0.3988011988011988,
-      "grad_norm": 17.4781436920166,
-      "learning_rate": 3.7592508300096484e-06,
-      "logits/chosen": -11.538864135742188,
-      "logits/rejected": -11.433642387390137,
-      "logps/chosen": -129.91366577148438,
-      "logps/rejected": -157.15330505371094,
-      "loss": 0.4185,
-      "rewards/accuracies": 0.7750000357627869,
-      "rewards/chosen": -5.601563930511475,
-      "rewards/margins": 2.885087728500366,
-      "rewards/rejected": -8.486651420593262,
-      "step": 4990
-    },
-    {
-      "epoch": 0.3996003996003996,
-      "grad_norm": 15.64525318145752,
-      "learning_rate": 3.753220283422707e-06,
-      "logits/chosen": -11.302849769592285,
-      "logits/rejected": -11.33233642578125,
-      "logps/chosen": -128.71824645996094,
-      "logps/rejected": -167.21182250976562,
-      "loss": 0.434,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -5.4165167808532715,
-      "rewards/margins": 3.409837484359741,
-      "rewards/rejected": -8.826354026794434,
-      "step": 5000
-    },
-    {
-      "epoch": 0.4003996003996004,
-      "grad_norm": 11.32697868347168,
-      "learning_rate": 3.7471799813230804e-06,
-      "logits/chosen": -11.393603324890137,
-      "logits/rejected": -11.337724685668945,
-      "logps/chosen": -132.21694946289062,
-      "logps/rejected": -156.4403533935547,
-      "loss": 0.3707,
-      "rewards/accuracies": 0.8375000357627869,
-      "rewards/chosen": -5.787557125091553,
-      "rewards/margins": 2.73476243019104,
-      "rewards/rejected": -8.522319793701172,
-      "step": 5010
-    },
-    {
-      "epoch": 0.4011988011988012,
-      "grad_norm": 29.624269485473633,
-      "learning_rate": 3.741129970730629e-06,
-      "logits/chosen": -11.327362060546875,
-      "logits/rejected": -11.262354850769043,
-      "logps/chosen": -134.0426788330078,
-      "logps/rejected": -158.7528533935547,
-      "loss": 0.6308,
-      "rewards/accuracies": 0.7125000357627869,
-      "rewards/chosen": -6.222131252288818,
-      "rewards/margins": 2.2467002868652344,
-      "rewards/rejected": -8.468831062316895,
-      "step": 5020
-    },
-    {
-      "epoch": 0.401998001998002,
-      "grad_norm": 17.55906867980957,
-      "learning_rate": 3.7350702987407905e-06,
-      "logits/chosen": -11.174518585205078,
-      "logits/rejected": -11.17292594909668,
-      "logps/chosen": -131.39266967773438,
-      "logps/rejected": -164.64566040039062,
-      "loss": 0.3337,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -6.027444362640381,
-      "rewards/margins": 2.907334089279175,
-      "rewards/rejected": -8.934779167175293,
-      "step": 5030
-    },
-    {
-      "epoch": 0.4027972027972028,
-      "grad_norm": 16.413555145263672,
-      "learning_rate": 3.7290010125242067e-06,
-      "logits/chosen": -11.280322074890137,
-      "logits/rejected": -11.202780723571777,
-      "logps/chosen": -131.39865112304688,
-      "logps/rejected": -163.6033477783203,
-      "loss": 0.2517,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": -5.911177635192871,
-      "rewards/margins": 3.122466802597046,
-      "rewards/rejected": -9.033644676208496,
-      "step": 5040
-    },
-    {
-      "epoch": 0.4035964035964036,
-      "grad_norm": 13.44735050201416,
-      "learning_rate": 3.722922159326361e-06,
-      "logits/chosen": -11.311308860778809,
-      "logits/rejected": -11.27857494354248,
-      "logps/chosen": -120.613037109375,
-      "logps/rejected": -158.1907501220703,
-      "loss": 0.2883,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -5.26397705078125,
-      "rewards/margins": 3.0146336555480957,
-      "rewards/rejected": -8.278611183166504,
-      "step": 5050
-    },
-    {
-      "epoch": 0.4043956043956044,
-      "grad_norm": 14.18871021270752,
-      "learning_rate": 3.716833786467211e-06,
-      "logits/chosen": -11.254302024841309,
-      "logits/rejected": -11.289687156677246,
-      "logps/chosen": -135.5138702392578,
-      "logps/rejected": -175.453857421875,
-      "loss": 0.3296,
-      "rewards/accuracies": 0.8375000357627869,
-      "rewards/chosen": -6.355984687805176,
-      "rewards/margins": 3.1152279376983643,
-      "rewards/rejected": -9.471213340759277,
-      "step": 5060
-    },
-    {
-      "epoch": 0.4051948051948052,
-      "grad_norm": 11.13774585723877,
-      "learning_rate": 3.7107359413408174e-06,
-      "logits/chosen": -11.420717239379883,
-      "logits/rejected": -11.386115074157715,
-      "logps/chosen": -126.4744873046875,
-      "logps/rejected": -147.27798461914062,
-      "loss": 0.4327,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -5.2654266357421875,
-      "rewards/margins": 2.395294427871704,
-      "rewards/rejected": -7.6607208251953125,
-      "step": 5070
-    },
-    {
-      "epoch": 0.405994005994006,
-      "grad_norm": 2.9058945178985596,
-      "learning_rate": 3.704628671414977e-06,
-      "logits/chosen": -11.163016319274902,
-      "logits/rejected": -11.267110824584961,
-      "logps/chosen": -123.90020751953125,
-      "logps/rejected": -171.50375366210938,
-      "loss": 0.277,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -5.394163608551025,
-      "rewards/margins": 3.496960163116455,
-      "rewards/rejected": -8.891122817993164,
-      "step": 5080
-    },
-    {
-      "epoch": 0.4067932067932068,
-      "grad_norm": 7.419495582580566,
-      "learning_rate": 3.698512024230852e-06,
-      "logits/chosen": -11.335700988769531,
-      "logits/rejected": -11.425955772399902,
-      "logps/chosen": -134.76046752929688,
-      "logps/rejected": -169.32139587402344,
-      "loss": 0.3385,
-      "rewards/accuracies": 0.8375000357627869,
-      "rewards/chosen": -6.120731353759766,
-      "rewards/margins": 3.1183226108551025,
-      "rewards/rejected": -9.239054679870605,
-      "step": 5090
-    },
-    {
-      "epoch": 0.4075924075924076,
-      "grad_norm": 13.70363712310791,
-      "learning_rate": 3.6923860474026006e-06,
-      "logits/chosen": -11.150275230407715,
-      "logits/rejected": -11.342555046081543,
-      "logps/chosen": -132.2725067138672,
-      "logps/rejected": -174.48876953125,
-      "loss": 0.3964,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -5.876949310302734,
-      "rewards/margins": 3.0079543590545654,
-      "rewards/rejected": -8.884903907775879,
-      "step": 5100
-    },
-    {
-      "epoch": 0.4083916083916084,
-      "grad_norm": 11.280318260192871,
-      "learning_rate": 3.6862507886170076e-06,
-      "logits/chosen": -11.437921524047852,
-      "logits/rejected": -11.367156982421875,
-      "logps/chosen": -127.62154388427734,
-      "logps/rejected": -153.3055877685547,
-      "loss": 0.4352,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -5.57307767868042,
-      "rewards/margins": 2.4606423377990723,
-      "rewards/rejected": -8.033720016479492,
-      "step": 5110
-    },
-    {
-      "epoch": 0.4091908091908092,
-      "grad_norm": 22.68574333190918,
-      "learning_rate": 3.6801062956331107e-06,
-      "logits/chosen": -11.349067687988281,
-      "logits/rejected": -11.321033477783203,
-      "logps/chosen": -124.84337615966797,
-      "logps/rejected": -149.53057861328125,
-      "loss": 0.4011,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -5.402184963226318,
-      "rewards/margins": 2.327807664871216,
-      "rewards/rejected": -7.729992866516113,
-      "step": 5120
-    },
-    {
-      "epoch": 0.40999000999001,
-      "grad_norm": 15.720293998718262,
-      "learning_rate": 3.673952616281829e-06,
-      "logits/chosen": -11.356735229492188,
-      "logits/rejected": -11.330194473266602,
-      "logps/chosen": -124.01165008544922,
-      "logps/rejected": -152.3444366455078,
-      "loss": 0.4892,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -5.425565242767334,
-      "rewards/margins": 2.4649224281311035,
-      "rewards/rejected": -7.890487194061279,
-      "step": 5130
-    },
-    {
-      "epoch": 0.41078921078921077,
-      "grad_norm": 19.84290313720703,
-      "learning_rate": 3.667789798465593e-06,
-      "logits/chosen": -11.16023063659668,
-      "logits/rejected": -11.322819709777832,
-      "logps/chosen": -133.19427490234375,
-      "logps/rejected": -161.7868194580078,
-      "loss": 0.4032,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -6.037976264953613,
-      "rewards/margins": 2.626598834991455,
-      "rewards/rejected": -8.664575576782227,
-      "step": 5140
-    },
-    {
-      "epoch": 0.4115884115884116,
-      "grad_norm": 14.076642036437988,
-      "learning_rate": 3.66161789015797e-06,
-      "logits/chosen": -11.39356517791748,
-      "logits/rejected": -11.287891387939453,
-      "logps/chosen": -125.5867691040039,
-      "logps/rejected": -156.72073364257812,
-      "loss": 0.2652,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": -5.437306880950928,
-      "rewards/margins": 2.7909939289093018,
-      "rewards/rejected": -8.228301048278809,
-      "step": 5150
-    },
-    {
-      "epoch": 0.4123876123876124,
-      "grad_norm": 23.01809310913086,
-      "learning_rate": 3.655436939403291e-06,
-      "logits/chosen": -11.316798210144043,
-      "logits/rejected": -11.310867309570312,
-      "logps/chosen": -122.05809783935547,
-      "logps/rejected": -152.26194763183594,
-      "loss": 0.4263,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": -5.106055736541748,
-      "rewards/margins": 2.926095962524414,
-      "rewards/rejected": -8.03215217590332,
-      "step": 5160
-    },
-    {
-      "epoch": 0.41318681318681316,
-      "grad_norm": 20.18770408630371,
-      "learning_rate": 3.6492469943162763e-06,
-      "logits/chosen": -11.234484672546387,
-      "logits/rejected": -11.252148628234863,
-      "logps/chosen": -133.88201904296875,
-      "logps/rejected": -170.78016662597656,
-      "loss": 0.343,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": -6.114971160888672,
-      "rewards/margins": 3.3324711322784424,
-      "rewards/rejected": -9.447442054748535,
-      "step": 5170
-    },
-    {
-      "epoch": 0.413986013986014,
-      "grad_norm": 3.1725199222564697,
-      "learning_rate": 3.6430481030816605e-06,
-      "logits/chosen": -11.296338081359863,
-      "logits/rejected": -11.215032577514648,
-      "logps/chosen": -130.7757568359375,
-      "logps/rejected": -162.8612823486328,
-      "loss": 0.2902,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": -5.711872100830078,
-      "rewards/margins": 3.095353841781616,
-      "rewards/rejected": -8.80722713470459,
-      "step": 5180
-    },
-    {
-      "epoch": 0.4147852147852148,
-      "grad_norm": 16.667728424072266,
-      "learning_rate": 3.63684031395382e-06,
-      "logits/chosen": -11.247970581054688,
-      "logits/rejected": -11.210436820983887,
-      "logps/chosen": -137.0149688720703,
-      "logps/rejected": -168.78842163085938,
-      "loss": 0.4346,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -6.682591438293457,
-      "rewards/margins": 2.4868977069854736,
-      "rewards/rejected": -9.169488906860352,
-      "step": 5190
-    },
-    {
-      "epoch": 0.4155844155844156,
-      "grad_norm": 20.200645446777344,
-      "learning_rate": 3.6306236752563944e-06,
-      "logits/chosen": -11.168639183044434,
-      "logits/rejected": -11.210589408874512,
-      "logps/chosen": -146.05776977539062,
-      "logps/rejected": -176.04727172851562,
-      "loss": 0.5641,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -7.1779398918151855,
-      "rewards/margins": 2.3040833473205566,
-      "rewards/rejected": -9.482022285461426,
-      "step": 5200
-    },
-    {
-      "epoch": 0.4163836163836164,
-      "grad_norm": 21.512962341308594,
-      "learning_rate": 3.6243982353819107e-06,
-      "logits/chosen": -11.174321174621582,
-      "logits/rejected": -11.221858024597168,
-      "logps/chosen": -143.70730590820312,
-      "logps/rejected": -182.22032165527344,
-      "loss": 0.3803,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -7.2408294677734375,
-      "rewards/margins": 3.031430959701538,
-      "rewards/rejected": -10.272259712219238,
-      "step": 5210
-    },
-    {
-      "epoch": 0.4171828171828172,
-      "grad_norm": 11.553345680236816,
-      "learning_rate": 3.61816404279141e-06,
-      "logits/chosen": -11.24735164642334,
-      "logits/rejected": -11.294947624206543,
-      "logps/chosen": -141.84124755859375,
-      "logps/rejected": -179.4265594482422,
-      "loss": 0.2679,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -7.167459011077881,
-      "rewards/margins": 3.133382558822632,
-      "rewards/rejected": -10.300841331481934,
-      "step": 5220
-    },
-    {
-      "epoch": 0.417982017982018,
-      "grad_norm": 5.386078357696533,
-      "learning_rate": 3.6119211460140653e-06,
-      "logits/chosen": -11.38837718963623,
-      "logits/rejected": -11.318389892578125,
-      "logps/chosen": -145.8816375732422,
-      "logps/rejected": -175.28038024902344,
-      "loss": 0.3684,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -7.498332977294922,
-      "rewards/margins": 2.6997792720794678,
-      "rewards/rejected": -10.198111534118652,
-      "step": 5230
-    },
-    {
-      "epoch": 0.41878121878121877,
-      "grad_norm": 11.861577033996582,
-      "learning_rate": 3.605669593646807e-06,
-      "logits/chosen": -11.43593978881836,
-      "logits/rejected": -11.400115013122559,
-      "logps/chosen": -149.88784790039062,
-      "logps/rejected": -176.93736267089844,
-      "loss": 0.4127,
-      "rewards/accuracies": 0.8375000357627869,
-      "rewards/chosen": -6.870369911193848,
-      "rewards/margins": 2.8564064502716064,
-      "rewards/rejected": -9.726776123046875,
-      "step": 5240
-    },
-    {
-      "epoch": 0.4195804195804196,
-      "grad_norm": 21.493452072143555,
-      "learning_rate": 3.5994094343539432e-06,
-      "logits/chosen": -11.365499496459961,
-      "logits/rejected": -11.282806396484375,
-      "logps/chosen": -143.357666015625,
-      "logps/rejected": -168.38922119140625,
-      "loss": 0.427,
-      "rewards/accuracies": 0.8375000357627869,
-      "rewards/chosen": -7.295146465301514,
-      "rewards/margins": 2.4684741497039795,
-      "rewards/rejected": -9.76362133026123,
-      "step": 5250
-    },
-    {
-      "epoch": 0.4203796203796204,
-      "grad_norm": 0.4495425820350647,
-      "learning_rate": 3.593140716866782e-06,
-      "logits/chosen": -11.218469619750977,
-      "logits/rejected": -11.267443656921387,
-      "logps/chosen": -146.51470947265625,
-      "logps/rejected": -177.70867919921875,
-      "loss": 0.6352,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -7.302960395812988,
-      "rewards/margins": 2.6098790168762207,
-      "rewards/rejected": -9.91283893585205,
-      "step": 5260
-    },
-    {
-      "epoch": 0.42117882117882116,
-      "grad_norm": 13.078190803527832,
-      "learning_rate": 3.586863489983251e-06,
-      "logits/chosen": -11.30057430267334,
-      "logits/rejected": -11.281638145446777,
-      "logps/chosen": -146.62454223632812,
-      "logps/rejected": -178.44712829589844,
-      "loss": 0.3762,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -7.451076030731201,
-      "rewards/margins": 2.838658094406128,
-      "rewards/rejected": -10.28973388671875,
-      "step": 5270
-    },
-    {
-      "epoch": 0.421978021978022,
-      "grad_norm": 5.35089111328125,
-      "learning_rate": 3.5805778025675187e-06,
-      "logits/chosen": -11.239503860473633,
-      "logits/rejected": -11.252326011657715,
-      "logps/chosen": -144.1239776611328,
-      "logps/rejected": -175.24636840820312,
-      "loss": 0.3267,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -7.1252336502075195,
-      "rewards/margins": 2.922801971435547,
-      "rewards/rejected": -10.048035621643066,
-      "step": 5280
-    },
-    {
-      "epoch": 0.4227772227772228,
-      "grad_norm": 18.745399475097656,
-      "learning_rate": 3.574283703549613e-06,
-      "logits/chosen": -11.312861442565918,
-      "logits/rejected": -11.271149635314941,
-      "logps/chosen": -141.7392120361328,
-      "logps/rejected": -170.530029296875,
-      "loss": 0.369,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -6.952455043792725,
-      "rewards/margins": 2.7100565433502197,
-      "rewards/rejected": -9.662511825561523,
-      "step": 5290
-    },
-    {
-      "epoch": 0.42357642357642356,
-      "grad_norm": 17.1224365234375,
-      "learning_rate": 3.567981241925039e-06,
-      "logits/chosen": -11.326899528503418,
-      "logits/rejected": -11.237112998962402,
-      "logps/chosen": -130.76602172851562,
-      "logps/rejected": -160.88723754882812,
-      "loss": 0.3678,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -6.116973400115967,
-      "rewards/margins": 2.7395730018615723,
-      "rewards/rejected": -8.856546401977539,
-      "step": 5300
-    },
-    {
-      "epoch": 0.42437562437562437,
-      "grad_norm": 9.71540641784668,
-      "learning_rate": 3.561670466754404e-06,
-      "logits/chosen": -11.243242263793945,
-      "logits/rejected": -11.395706176757812,
-      "logps/chosen": -131.95762634277344,
-      "logps/rejected": -170.0781707763672,
-      "loss": 0.3609,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -6.066464900970459,
-      "rewards/margins": 2.9204325675964355,
-      "rewards/rejected": -8.986898422241211,
-      "step": 5310
-    },
-    {
-      "epoch": 0.4251748251748252,
-      "grad_norm": 25.07352066040039,
-      "learning_rate": 3.5553514271630246e-06,
-      "logits/chosen": -11.4105806350708,
-      "logits/rejected": -11.331195831298828,
-      "logps/chosen": -135.40977478027344,
-      "logps/rejected": -172.0722198486328,
-      "loss": 0.3262,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -6.2659735679626465,
-      "rewards/margins": 3.272118330001831,
-      "rewards/rejected": -9.538092613220215,
-      "step": 5320
-    },
-    {
-      "epoch": 0.42597402597402595,
-      "grad_norm": 3.465949058532715,
-      "learning_rate": 3.5490241723405554e-06,
-      "logits/chosen": -11.407347679138184,
-      "logits/rejected": -11.300511360168457,
-      "logps/chosen": -131.75108337402344,
-      "logps/rejected": -164.5821533203125,
-      "loss": 0.3111,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": -6.055675029754639,
-      "rewards/margins": 3.0339181423187256,
-      "rewards/rejected": -9.089591979980469,
-      "step": 5330
-    },
-    {
-      "epoch": 0.42677322677322677,
-      "grad_norm": 15.745570182800293,
-      "learning_rate": 3.5426887515405992e-06,
-      "logits/chosen": -11.424102783203125,
-      "logits/rejected": -11.38721752166748,
-      "logps/chosen": -136.82215881347656,
-      "logps/rejected": -168.42901611328125,
-      "loss": 0.3984,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": -6.02234411239624,
-      "rewards/margins": 3.1885621547698975,
-      "rewards/rejected": -9.210906028747559,
-      "step": 5340
-    },
-    {
-      "epoch": 0.4275724275724276,
-      "grad_norm": 29.318758010864258,
-      "learning_rate": 3.5363452140803278e-06,
-      "logits/chosen": -11.261390686035156,
-      "logits/rejected": -11.335871696472168,
-      "logps/chosen": -135.13772583007812,
-      "logps/rejected": -174.3563995361328,
-      "loss": 0.408,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -6.623199462890625,
-      "rewards/margins": 2.7855746746063232,
-      "rewards/rejected": -9.408774375915527,
-      "step": 5350
-    },
-    {
-      "epoch": 0.4283716283716284,
-      "grad_norm": 9.279765129089355,
-      "learning_rate": 3.5299936093400923e-06,
-      "logits/chosen": -11.37162971496582,
-      "logits/rejected": -11.379800796508789,
-      "logps/chosen": -128.99534606933594,
-      "logps/rejected": -169.96551513671875,
-      "loss": 0.2843,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": -5.9273362159729,
-      "rewards/margins": 3.295164108276367,
-      "rewards/rejected": -9.222500801086426,
-      "step": 5360
-    },
-    {
-      "epoch": 0.42917082917082916,
-      "grad_norm": 18.056049346923828,
-      "learning_rate": 3.523633986763045e-06,
-      "logits/chosen": -11.416415214538574,
-      "logits/rejected": -11.34842586517334,
-      "logps/chosen": -133.43345642089844,
-      "logps/rejected": -172.94705200195312,
-      "loss": 0.3892,
-      "rewards/accuracies": 0.8375000357627869,
-      "rewards/chosen": -6.3930888175964355,
-      "rewards/margins": 3.1415815353393555,
-      "rewards/rejected": -9.53467082977295,
-      "step": 5370
-    },
-    {
-      "epoch": 0.42997002997003,
-      "grad_norm": 17.734703063964844,
-      "learning_rate": 3.5172663958547515e-06,
-      "logits/chosen": -11.419660568237305,
-      "logits/rejected": -11.477546691894531,
-      "logps/chosen": -140.54173278808594,
-      "logps/rejected": -177.56910705566406,
-      "loss": 0.4549,
-      "rewards/accuracies": 0.7750000357627869,
-      "rewards/chosen": -6.954007148742676,
-      "rewards/margins": 3.2621498107910156,
-      "rewards/rejected": -10.216156959533691,
-      "step": 5380
-    },
-    {
-      "epoch": 0.4307692307692308,
-      "grad_norm": 16.20943832397461,
-      "learning_rate": 3.510890886182805e-06,
-      "logits/chosen": -11.570549011230469,
-      "logits/rejected": -11.518790245056152,
-      "logps/chosen": -136.420166015625,
-      "logps/rejected": -168.0776824951172,
-      "loss": 0.3758,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -6.656927585601807,
-      "rewards/margins": 2.7592663764953613,
-      "rewards/rejected": -9.416193962097168,
-      "step": 5390
-    },
-    {
-      "epoch": 0.43156843156843155,
-      "grad_norm": 20.32877540588379,
-      "learning_rate": 3.504507507376443e-06,
-      "logits/chosen": -11.393777847290039,
-      "logits/rejected": -11.544252395629883,
-      "logps/chosen": -136.61734008789062,
-      "logps/rejected": -182.57798767089844,
-      "loss": 0.3016,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": -6.323850154876709,
-      "rewards/margins": 3.562857151031494,
-      "rewards/rejected": -9.88670825958252,
-      "step": 5400
-    },
-    {
-      "epoch": 0.43236763236763237,
-      "grad_norm": 19.763639450073242,
-      "learning_rate": 3.4981163091261564e-06,
-      "logits/chosen": -11.555663108825684,
-      "logits/rejected": -11.4998197555542,
-      "logps/chosen": -133.34974670410156,
-      "logps/rejected": -170.10853576660156,
-      "loss": 0.3189,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -6.394133567810059,
-      "rewards/margins": 3.111896514892578,
-      "rewards/rejected": -9.506030082702637,
-      "step": 5410
-    },
-    {
-      "epoch": 0.4331668331668332,
-      "grad_norm": 15.836555480957031,
-      "learning_rate": 3.491717341183308e-06,
-      "logits/chosen": -11.50979995727539,
-      "logits/rejected": -11.49087142944336,
-      "logps/chosen": -137.8575439453125,
-      "logps/rejected": -172.9957733154297,
-      "loss": 0.3918,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -6.803933143615723,
-      "rewards/margins": 3.098384380340576,
-      "rewards/rejected": -9.90231704711914,
-      "step": 5420
-    },
-    {
-      "epoch": 0.43396603396603395,
-      "grad_norm": 20.438100814819336,
-      "learning_rate": 3.4853106533597407e-06,
-      "logits/chosen": -11.659512519836426,
-      "logits/rejected": -11.545653343200684,
-      "logps/chosen": -139.26284790039062,
-      "logps/rejected": -166.8800811767578,
-      "loss": 0.3952,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": -6.549859523773193,
-      "rewards/margins": 2.782632827758789,
-      "rewards/rejected": -9.332491874694824,
-      "step": 5430
-    },
-    {
-      "epoch": 0.43476523476523476,
-      "grad_norm": 11.529703140258789,
-      "learning_rate": 3.4788962955273937e-06,
-      "logits/chosen": -11.542508125305176,
-      "logits/rejected": -11.533997535705566,
-      "logps/chosen": -137.6142578125,
-      "logps/rejected": -171.8604278564453,
-      "loss": 0.3696,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -6.552028179168701,
-      "rewards/margins": 3.198517084121704,
-      "rewards/rejected": -9.750544548034668,
-      "step": 5440
-    },
-    {
-      "epoch": 0.4355644355644356,
-      "grad_norm": 30.6385555267334,
-      "learning_rate": 3.47247431761791e-06,
-      "logits/chosen": -11.666610717773438,
-      "logits/rejected": -11.568266868591309,
-      "logps/chosen": -135.29673767089844,
-      "logps/rejected": -170.51951599121094,
-      "loss": 0.2996,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": -6.135403633117676,
-      "rewards/margins": 3.248842239379883,
-      "rewards/rejected": -9.384245872497559,
-      "step": 5450
-    },
-    {
-      "epoch": 0.43636363636363634,
-      "grad_norm": 18.952804565429688,
-      "learning_rate": 3.466044769622251e-06,
-      "logits/chosen": -11.497401237487793,
-      "logits/rejected": -11.602614402770996,
-      "logps/chosen": -137.4938201904297,
-      "logps/rejected": -175.70530700683594,
-      "loss": 0.3578,
-      "rewards/accuracies": 0.8375000357627869,
-      "rewards/chosen": -6.638597011566162,
-      "rewards/margins": 3.1292226314544678,
-      "rewards/rejected": -9.767820358276367,
-      "step": 5460
-    },
-    {
-      "epoch": 0.43716283716283716,
-      "grad_norm": 13.136646270751953,
-      "learning_rate": 3.459607701590307e-06,
-      "logits/chosen": -11.773407936096191,
-      "logits/rejected": -11.67356014251709,
-      "logps/chosen": -133.32456970214844,
-      "logps/rejected": -166.8814239501953,
-      "loss": 0.1966,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": -5.986332416534424,
-      "rewards/margins": 3.3608269691467285,
-      "rewards/rejected": -9.347159385681152,
-      "step": 5470
-    },
-    {
-      "epoch": 0.437962037962038,
-      "grad_norm": 26.669445037841797,
-      "learning_rate": 3.4531631636305073e-06,
-      "logits/chosen": -11.608343124389648,
-      "logits/rejected": -11.653532981872559,
-      "logps/chosen": -131.6614227294922,
-      "logps/rejected": -168.92247009277344,
-      "loss": 0.498,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": -6.01355504989624,
-      "rewards/margins": 3.241530656814575,
-      "rewards/rejected": -9.255085945129395,
-      "step": 5480
-    },
-    {
-      "epoch": 0.43876123876123874,
-      "grad_norm": 18.060199737548828,
-      "learning_rate": 3.4467112059094278e-06,
-      "logits/chosen": -11.787359237670898,
-      "logits/rejected": -11.68607234954834,
-      "logps/chosen": -138.62278747558594,
-      "logps/rejected": -175.707763671875,
-      "loss": 0.2875,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -6.7202582359313965,
-      "rewards/margins": 3.28520131111145,
-      "rewards/rejected": -10.005459785461426,
-      "step": 5490
-    },
-    {
-      "epoch": 0.43956043956043955,
-      "grad_norm": 17.64265251159668,
-      "learning_rate": 3.440251878651403e-06,
-      "logits/chosen": -11.573155403137207,
-      "logits/rejected": -11.616162300109863,
-      "logps/chosen": -133.86123657226562,
-      "logps/rejected": -178.83773803710938,
-      "loss": 0.2896,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -6.422027587890625,
-      "rewards/margins": 3.547304630279541,
-      "rewards/rejected": -9.969332695007324,
-      "step": 5500
-    },
-    {
-      "epoch": 0.44035964035964037,
-      "grad_norm": 29.483295440673828,
-      "learning_rate": 3.433785232138136e-06,
-      "logits/chosen": -11.753422737121582,
-      "logits/rejected": -11.673492431640625,
-      "logps/chosen": -145.7272186279297,
-      "logps/rejected": -174.14427185058594,
-      "loss": 0.4392,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -7.115805149078369,
-      "rewards/margins": 2.8145713806152344,
-      "rewards/rejected": -9.930377006530762,
-      "step": 5510
-    },
-    {
-      "epoch": 0.4411588411588412,
-      "grad_norm": 12.080334663391113,
-      "learning_rate": 3.4273113167083044e-06,
-      "logits/chosen": -11.56371784210205,
-      "logits/rejected": -11.701234817504883,
-      "logps/chosen": -146.6568145751953,
-      "logps/rejected": -193.4566650390625,
-      "loss": 0.3658,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -7.4243621826171875,
-      "rewards/margins": 3.485074758529663,
-      "rewards/rejected": -10.90943717956543,
-      "step": 5520
-    },
-    {
-      "epoch": 0.44195804195804195,
-      "grad_norm": 15.53564167022705,
-      "learning_rate": 3.42083018275717e-06,
-      "logits/chosen": -11.542001724243164,
-      "logits/rejected": -11.681236267089844,
-      "logps/chosen": -146.4755096435547,
-      "logps/rejected": -176.1334686279297,
-      "loss": 0.4656,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -6.761958599090576,
-      "rewards/margins": 2.8060147762298584,
-      "rewards/rejected": -9.567973136901855,
-      "step": 5530
-    },
-    {
-      "epoch": 0.44275724275724276,
-      "grad_norm": 15.031530380249023,
-      "learning_rate": 3.4143418807361855e-06,
-      "logits/chosen": -11.692107200622559,
-      "logits/rejected": -11.600741386413574,
-      "logps/chosen": -140.62460327148438,
-      "logps/rejected": -172.9504852294922,
-      "loss": 0.4792,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -7.037936687469482,
-      "rewards/margins": 2.9279301166534424,
-      "rewards/rejected": -9.965866088867188,
-      "step": 5540
-    },
-    {
-      "epoch": 0.4435564435564436,
-      "grad_norm": 20.444416046142578,
-      "learning_rate": 3.407846461152603e-06,
-      "logits/chosen": -11.632269859313965,
-      "logits/rejected": -11.471343040466309,
-      "logps/chosen": -142.7677459716797,
-      "logps/rejected": -171.82293701171875,
-      "loss": 0.4842,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -6.891223430633545,
-      "rewards/margins": 2.928086519241333,
-      "rewards/rejected": -9.81930923461914,
-      "step": 5550
-    },
-    {
-      "epoch": 0.44435564435564434,
-      "grad_norm": 22.404922485351562,
-      "learning_rate": 3.4013439745690808e-06,
-      "logits/chosen": -11.596283912658691,
-      "logits/rejected": -11.566835403442383,
-      "logps/chosen": -134.34170532226562,
-      "logps/rejected": -161.03402709960938,
-      "loss": 0.4384,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -6.086970329284668,
-      "rewards/margins": 2.6648194789886475,
-      "rewards/rejected": -8.751790046691895,
-      "step": 5560
-    },
-    {
-      "epoch": 0.44515484515484516,
-      "grad_norm": 19.768524169921875,
-      "learning_rate": 3.394834471603288e-06,
-      "logits/chosen": -11.539773941040039,
-      "logits/rejected": -11.523975372314453,
-      "logps/chosen": -127.50848388671875,
-      "logps/rejected": -155.53262329101562,
-      "loss": 0.3969,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": -5.676835536956787,
-      "rewards/margins": 2.6125049591064453,
-      "rewards/rejected": -8.28934097290039,
-      "step": 5570
-    },
-    {
-      "epoch": 0.445954045954046,
-      "grad_norm": 7.4425368309021,
-      "learning_rate": 3.388318002927512e-06,
-      "logits/chosen": -11.550580024719238,
-      "logits/rejected": -11.480045318603516,
-      "logps/chosen": -119.912109375,
-      "logps/rejected": -154.88641357421875,
-      "loss": 0.2913,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -5.05513334274292,
-      "rewards/margins": 2.9182076454162598,
-      "rewards/rejected": -7.9733405113220215,
-      "step": 5580
-    },
-    {
-      "epoch": 0.44675324675324674,
-      "grad_norm": 13.496734619140625,
-      "learning_rate": 3.381794619268265e-06,
-      "logits/chosen": -11.431795120239258,
-      "logits/rejected": -11.407143592834473,
-      "logps/chosen": -125.38933563232422,
-      "logps/rejected": -165.07102966308594,
-      "loss": 0.351,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -5.567803859710693,
-      "rewards/margins": 3.2488465309143066,
-      "rewards/rejected": -8.816650390625,
-      "step": 5590
-    },
-    {
-      "epoch": 0.44755244755244755,
-      "grad_norm": 4.2875542640686035,
-      "learning_rate": 3.3752643714058865e-06,
-      "logits/chosen": -11.522619247436523,
-      "logits/rejected": -11.534686088562012,
-      "logps/chosen": -131.5494842529297,
-      "logps/rejected": -165.81948852539062,
-      "loss": 0.4184,
-      "rewards/accuracies": 0.8375000357627869,
-      "rewards/chosen": -5.848127365112305,
-      "rewards/margins": 2.6813228130340576,
-      "rewards/rejected": -8.529451370239258,
-      "step": 5600
-    },
-    {
-      "epoch": 0.44835164835164837,
-      "grad_norm": 9.015324592590332,
-      "learning_rate": 3.368727310174149e-06,
-      "logits/chosen": -11.630866050720215,
-      "logits/rejected": -11.585362434387207,
-      "logps/chosen": -128.38352966308594,
-      "logps/rejected": -166.4827423095703,
-      "loss": 0.2426,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": -5.695451259613037,
-      "rewards/margins": 3.1963601112365723,
-      "rewards/rejected": -8.891810417175293,
-      "step": 5610
-    },
-    {
-      "epoch": 0.44915084915084913,
-      "grad_norm": 13.984660148620605,
-      "learning_rate": 3.362183486459866e-06,
-      "logits/chosen": -11.568634986877441,
-      "logits/rejected": -11.601783752441406,
-      "logps/chosen": -120.8480224609375,
-      "logps/rejected": -155.35992431640625,
-      "loss": 0.4462,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -5.112626552581787,
-      "rewards/margins": 2.8295915126800537,
-      "rewards/rejected": -7.94221830368042,
-      "step": 5620
-    },
-    {
-      "epoch": 0.44995004995004995,
-      "grad_norm": 17.67071533203125,
-      "learning_rate": 3.355632951202487e-06,
-      "logits/chosen": -11.756998062133789,
-      "logits/rejected": -11.699213981628418,
-      "logps/chosen": -126.94670867919922,
-      "logps/rejected": -154.5124969482422,
-      "loss": 0.4142,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -5.655637264251709,
-      "rewards/margins": 2.714101791381836,
-      "rewards/rejected": -8.369739532470703,
-      "step": 5630
-    },
-    {
-      "epoch": 0.45074925074925076,
-      "grad_norm": 8.57283878326416,
-      "learning_rate": 3.3490757553937116e-06,
-      "logits/chosen": -11.72691822052002,
-      "logits/rejected": -11.641200065612793,
-      "logps/chosen": -133.0535888671875,
-      "logps/rejected": -165.2667694091797,
-      "loss": 0.2749,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": -6.048217296600342,
-      "rewards/margins": 2.853140354156494,
-      "rewards/rejected": -8.901357650756836,
-      "step": 5640
-    },
-    {
-      "epoch": 0.4515484515484515,
-      "grad_norm": 18.986122131347656,
-      "learning_rate": 3.342511950077085e-06,
-      "logits/chosen": -11.60710620880127,
-      "logits/rejected": -11.67328929901123,
-      "logps/chosen": -126.88002014160156,
-      "logps/rejected": -170.1356964111328,
-      "loss": 0.3749,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -5.616649627685547,
-      "rewards/margins": 3.3346574306488037,
-      "rewards/rejected": -8.95130729675293,
-      "step": 5650
-    },
-    {
-      "epoch": 0.45234765234765234,
-      "grad_norm": 25.572860717773438,
-      "learning_rate": 3.335941586347604e-06,
-      "logits/chosen": -11.555695533752441,
-      "logits/rejected": -11.696009635925293,
-      "logps/chosen": -138.34214782714844,
-      "logps/rejected": -189.22885131835938,
-      "loss": 0.4135,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -6.813581943511963,
-      "rewards/margins": 3.562899112701416,
-      "rewards/rejected": -10.376480102539062,
-      "step": 5660
-    },
-    {
-      "epoch": 0.45314685314685316,
-      "grad_norm": 10.874223709106445,
-      "learning_rate": 3.3293647153513186e-06,
-      "logits/chosen": -11.666109085083008,
-      "logits/rejected": -11.598344802856445,
-      "logps/chosen": -142.4362030029297,
-      "logps/rejected": -175.0700225830078,
-      "loss": 0.3762,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": -7.118638515472412,
-      "rewards/margins": 2.685850143432617,
-      "rewards/rejected": -9.804489135742188,
-      "step": 5670
-    },
-    {
-      "epoch": 0.453946053946054,
-      "grad_norm": 18.52907371520996,
-      "learning_rate": 3.3227813882849317e-06,
-      "logits/chosen": -11.668466567993164,
-      "logits/rejected": -11.669447898864746,
-      "logps/chosen": -134.0895233154297,
-      "logps/rejected": -173.8521270751953,
-      "loss": 0.366,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -5.887041091918945,
-      "rewards/margins": 3.568530321121216,
-      "rewards/rejected": -9.455571174621582,
-      "step": 5680
-    },
-    {
-      "epoch": 0.45474525474525473,
-      "grad_norm": 24.41785430908203,
-      "learning_rate": 3.3161916563954055e-06,
-      "logits/chosen": -11.74404239654541,
-      "logits/rejected": -11.673611640930176,
-      "logps/chosen": -134.0076141357422,
-      "logps/rejected": -159.10769653320312,
-      "loss": 0.4059,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": -6.283403396606445,
-      "rewards/margins": 2.2803585529327393,
-      "rewards/rejected": -8.563761711120605,
-      "step": 5690
-    },
-    {
-      "epoch": 0.45554445554445555,
-      "grad_norm": 22.08030891418457,
-      "learning_rate": 3.309595570979558e-06,
-      "logits/chosen": -11.776788711547852,
-      "logits/rejected": -11.712291717529297,
-      "logps/chosen": -130.04751586914062,
-      "logps/rejected": -164.14674377441406,
-      "loss": 0.3205,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -5.803739070892334,
-      "rewards/margins": 3.337026357650757,
-      "rewards/rejected": -9.140765190124512,
-      "step": 5700
-    },
-    {
-      "epoch": 0.45634365634365637,
-      "grad_norm": 22.82491683959961,
-      "learning_rate": 3.302993183383664e-06,
-      "logits/chosen": -11.634814262390137,
-      "logits/rejected": -11.573974609375,
-      "logps/chosen": -133.70005798339844,
-      "logps/rejected": -168.7314910888672,
-      "loss": 0.3072,
-      "rewards/accuracies": 0.8375000357627869,
-      "rewards/chosen": -6.228097438812256,
-      "rewards/margins": 3.287574052810669,
-      "rewards/rejected": -9.515671730041504,
-      "step": 5710
-    },
-    {
-      "epoch": 0.45714285714285713,
-      "grad_norm": 25.32102394104004,
-      "learning_rate": 3.296384545003061e-06,
-      "logits/chosen": -11.479426383972168,
-      "logits/rejected": -11.564851760864258,
-      "logps/chosen": -143.55029296875,
-      "logps/rejected": -177.17578125,
-      "loss": 0.4755,
-      "rewards/accuracies": 0.7750000357627869,
-      "rewards/chosen": -7.07968282699585,
-      "rewards/margins": 2.969311475753784,
-      "rewards/rejected": -10.048995018005371,
-      "step": 5720
-    },
-    {
-      "epoch": 0.45794205794205795,
-      "grad_norm": 22.186708450317383,
-      "learning_rate": 3.289769707281739e-06,
-      "logits/chosen": -11.619668960571289,
-      "logits/rejected": -11.503392219543457,
-      "logps/chosen": -136.2913360595703,
-      "logps/rejected": -165.82847595214844,
-      "loss": 0.3837,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -6.5318603515625,
-      "rewards/margins": 2.7948224544525146,
-      "rewards/rejected": -9.326683044433594,
-      "step": 5730
-    },
-    {
-      "epoch": 0.45874125874125876,
-      "grad_norm": 12.99971866607666,
-      "learning_rate": 3.283148721711951e-06,
-      "logits/chosen": -11.65119457244873,
-      "logits/rejected": -11.584024429321289,
-      "logps/chosen": -136.58804321289062,
-      "logps/rejected": -158.6575164794922,
-      "loss": 0.5453,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": -6.229686260223389,
-      "rewards/margins": 2.2587311267852783,
-      "rewards/rejected": -8.488417625427246,
-      "step": 5740
-    },
-    {
-      "epoch": 0.4595404595404595,
-      "grad_norm": 9.734941482543945,
-      "learning_rate": 3.2765216398338045e-06,
-      "logits/chosen": -11.430429458618164,
-      "logits/rejected": -11.579851150512695,
-      "logps/chosen": -130.63340759277344,
-      "logps/rejected": -172.06951904296875,
-      "loss": 0.4769,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -5.979673862457275,
-      "rewards/margins": 3.3236942291259766,
-      "rewards/rejected": -9.303369522094727,
-      "step": 5750
-    },
-    {
-      "epoch": 0.46033966033966034,
-      "grad_norm": 25.108633041381836,
-      "learning_rate": 3.2698885132348636e-06,
-      "logits/chosen": -11.56406021118164,
-      "logits/rejected": -11.590652465820312,
-      "logps/chosen": -126.61653900146484,
-      "logps/rejected": -160.206787109375,
-      "loss": 0.3961,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -5.716445446014404,
-      "rewards/margins": 2.6987292766571045,
-      "rewards/rejected": -8.415175437927246,
-      "step": 5760
-    },
-    {
-      "epoch": 0.46113886113886116,
-      "grad_norm": 9.939102172851562,
-      "learning_rate": 3.263249393549745e-06,
-      "logits/chosen": -11.49644947052002,
-      "logits/rejected": -11.518269538879395,
-      "logps/chosen": -122.70234680175781,
-      "logps/rejected": -158.04835510253906,
-      "loss": 0.5105,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -5.228819370269775,
-      "rewards/margins": 2.753589391708374,
-      "rewards/rejected": -7.9824090003967285,
-      "step": 5770
-    },
-    {
-      "epoch": 0.4619380619380619,
-      "grad_norm": 13.974944114685059,
-      "learning_rate": 3.2566043324597207e-06,
-      "logits/chosen": -11.622356414794922,
-      "logits/rejected": -11.50833797454834,
-      "logps/chosen": -129.7686767578125,
-      "logps/rejected": -153.8483428955078,
-      "loss": 0.3405,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -5.369255065917969,
-      "rewards/margins": 2.421156167984009,
-      "rewards/rejected": -7.790411472320557,
-      "step": 5780
-    },
-    {
-      "epoch": 0.46273726273726273,
-      "grad_norm": 26.6611385345459,
-      "learning_rate": 3.24995338169231e-06,
-      "logits/chosen": -11.474855422973633,
-      "logits/rejected": -11.517420768737793,
-      "logps/chosen": -123.2922592163086,
-      "logps/rejected": -161.94032287597656,
-      "loss": 0.2899,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -5.161477565765381,
-      "rewards/margins": 3.1793460845947266,
-      "rewards/rejected": -8.34082317352295,
-      "step": 5790
-    },
-    {
-      "epoch": 0.46353646353646355,
-      "grad_norm": 9.813111305236816,
-      "learning_rate": 3.243296593020881e-06,
-      "logits/chosen": -11.567530632019043,
-      "logits/rejected": -11.515045166015625,
-      "logps/chosen": -117.55345916748047,
-      "logps/rejected": -151.91372680664062,
-      "loss": 0.2833,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -4.333185195922852,
-      "rewards/margins": 3.3671486377716064,
-      "rewards/rejected": -7.700333595275879,
-      "step": 5800
-    },
-    {
-      "epoch": 0.4643356643356643,
-      "grad_norm": 13.48766803741455,
-      "learning_rate": 3.2366340182642463e-06,
-      "logits/chosen": -11.541363716125488,
-      "logits/rejected": -11.488557815551758,
-      "logps/chosen": -120.5531234741211,
-      "logps/rejected": -146.57481384277344,
-      "loss": 0.5048,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -5.052888870239258,
-      "rewards/margins": 2.1482205390930176,
-      "rewards/rejected": -7.201109409332275,
-      "step": 5810
-    },
-    {
-      "epoch": 0.4651348651348651,
-      "grad_norm": 4.32265043258667,
-      "learning_rate": 3.2299657092862578e-06,
-      "logits/chosen": -11.515625,
-      "logits/rejected": -11.42825698852539,
-      "logps/chosen": -124.0775375366211,
-      "logps/rejected": -149.80300903320312,
-      "loss": 0.4282,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -5.136544704437256,
-      "rewards/margins": 2.7891533374786377,
-      "rewards/rejected": -7.925698280334473,
-      "step": 5820
-    },
-    {
-      "epoch": 0.46593406593406594,
-      "grad_norm": 11.962272644042969,
-      "learning_rate": 3.2232917179954065e-06,
-      "logits/chosen": -11.495707511901855,
-      "logits/rejected": -11.510721206665039,
-      "logps/chosen": -119.21192932128906,
-      "logps/rejected": -146.3975372314453,
-      "loss": 0.3292,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -4.805904388427734,
-      "rewards/margins": 2.5667319297790527,
-      "rewards/rejected": -7.372636318206787,
-      "step": 5830
-    },
-    {
-      "epoch": 0.46673326673326676,
-      "grad_norm": 28.148340225219727,
-      "learning_rate": 3.216612096344416e-06,
-      "logits/chosen": -11.531808853149414,
-      "logits/rejected": -11.46023178100586,
-      "logps/chosen": -125.92998504638672,
-      "logps/rejected": -153.1452178955078,
-      "loss": 0.4162,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -5.375319957733154,
-      "rewards/margins": 2.6422712802886963,
-      "rewards/rejected": -8.017592430114746,
-      "step": 5840
-    },
-    {
-      "epoch": 0.4675324675324675,
-      "grad_norm": 12.14610481262207,
-      "learning_rate": 3.2099268963298373e-06,
-      "logits/chosen": -11.485904693603516,
-      "logits/rejected": -11.439776420593262,
-      "logps/chosen": -126.36785125732422,
-      "logps/rejected": -154.28050231933594,
-      "loss": 0.4197,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -5.592311859130859,
-      "rewards/margins": 2.688056230545044,
-      "rewards/rejected": -8.280367851257324,
-      "step": 5850
-    },
-    {
-      "epoch": 0.46833166833166834,
-      "grad_norm": 28.95405387878418,
-      "learning_rate": 3.2032361699916468e-06,
-      "logits/chosen": -11.596104621887207,
-      "logits/rejected": -11.53996467590332,
-      "logps/chosen": -128.96401977539062,
-      "logps/rejected": -162.4442596435547,
-      "loss": 0.3903,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -5.833486080169678,
-      "rewards/margins": 2.660830020904541,
-      "rewards/rejected": -8.494316101074219,
-      "step": 5860
-    },
-    {
-      "epoch": 0.46913086913086915,
-      "grad_norm": 27.126829147338867,
-      "learning_rate": 3.1965399694128407e-06,
-      "logits/chosen": -11.568537712097168,
-      "logits/rejected": -11.536808967590332,
-      "logps/chosen": -126.20818328857422,
-      "logps/rejected": -153.99098205566406,
-      "loss": 0.417,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -5.186219215393066,
-      "rewards/margins": 2.5134482383728027,
-      "rewards/rejected": -7.699667453765869,
-      "step": 5870
-    },
-    {
-      "epoch": 0.4699300699300699,
-      "grad_norm": 22.773910522460938,
-      "learning_rate": 3.189838346719026e-06,
-      "logits/chosen": -11.48618221282959,
-      "logits/rejected": -11.561136245727539,
-      "logps/chosen": -120.80643463134766,
-      "logps/rejected": -151.2483673095703,
-      "loss": 0.4354,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -5.008599758148193,
-      "rewards/margins": 2.5712697505950928,
-      "rewards/rejected": -7.579869270324707,
-      "step": 5880
-    },
-    {
-      "epoch": 0.47072927072927073,
-      "grad_norm": 11.728520393371582,
-      "learning_rate": 3.1831313540780194e-06,
-      "logits/chosen": -11.446146965026855,
-      "logits/rejected": -11.34732723236084,
-      "logps/chosen": -126.05049133300781,
-      "logps/rejected": -156.72842407226562,
-      "loss": 0.2727,
-      "rewards/accuracies": 0.8375000357627869,
-      "rewards/chosen": -5.173259735107422,
-      "rewards/margins": 2.996213436126709,
-      "rewards/rejected": -8.169473648071289,
-      "step": 5890
-    },
-    {
-      "epoch": 0.47152847152847155,
-      "grad_norm": 24.382061004638672,
-      "learning_rate": 3.176419043699438e-06,
-      "logits/chosen": -11.504803657531738,
-      "logits/rejected": -11.451822280883789,
-      "logps/chosen": -139.47645568847656,
-      "logps/rejected": -169.45704650878906,
-      "loss": 0.3647,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -6.237081050872803,
-      "rewards/margins": 3.121647596359253,
-      "rewards/rejected": -9.358729362487793,
-      "step": 5900
-    },
-    {
-      "epoch": 0.4723276723276723,
-      "grad_norm": 4.873622417449951,
-      "learning_rate": 3.1697014678342947e-06,
-      "logits/chosen": -11.487327575683594,
-      "logits/rejected": -11.406160354614258,
-      "logps/chosen": -126.65863037109375,
-      "logps/rejected": -162.76718139648438,
-      "loss": 0.409,
-      "rewards/accuracies": 0.8375000357627869,
-      "rewards/chosen": -5.274600028991699,
-      "rewards/margins": 3.505866289138794,
-      "rewards/rejected": -8.780466079711914,
-      "step": 5910
-    },
-    {
-      "epoch": 0.4731268731268731,
-      "grad_norm": 19.963346481323242,
-      "learning_rate": 3.1629786787745913e-06,
-      "logits/chosen": -11.518810272216797,
-      "logits/rejected": -11.414355278015137,
-      "logps/chosen": -126.4761962890625,
-      "logps/rejected": -150.93936157226562,
-      "loss": 0.3393,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -5.367621898651123,
-      "rewards/margins": 2.3547561168670654,
-      "rewards/rejected": -7.722377777099609,
-      "step": 5920
-    },
-    {
-      "epoch": 0.47392607392607394,
-      "grad_norm": 7.520387649536133,
-      "learning_rate": 3.1562507288529086e-06,
-      "logits/chosen": -11.432220458984375,
-      "logits/rejected": -11.422904968261719,
-      "logps/chosen": -128.3930206298828,
-      "logps/rejected": -162.5411376953125,
-      "loss": 0.2754,
-      "rewards/accuracies": 0.9000000357627869,
-      "rewards/chosen": -5.4026618003845215,
-      "rewards/margins": 2.8992373943328857,
-      "rewards/rejected": -8.301898956298828,
-      "step": 5930
-    },
-    {
-      "epoch": 0.4747252747252747,
-      "grad_norm": 16.576828002929688,
-      "learning_rate": 3.149517670442006e-06,
-      "logits/chosen": -11.4193754196167,
-      "logits/rejected": -11.340259552001953,
-      "logps/chosen": -132.0810546875,
-      "logps/rejected": -154.9105224609375,
-      "loss": 0.4457,
-      "rewards/accuracies": 0.8375000357627869,
-      "rewards/chosen": -6.109832763671875,
-      "rewards/margins": 2.2515816688537598,
-      "rewards/rejected": -8.361414909362793,
-      "step": 5940
-    },
-    {
-      "epoch": 0.4755244755244755,
-      "grad_norm": 14.507401466369629,
-      "learning_rate": 3.142779555954404e-06,
-      "logits/chosen": -11.47339916229248,
-      "logits/rejected": -11.411892890930176,
-      "logps/chosen": -134.33644104003906,
-      "logps/rejected": -165.84617614746094,
-      "loss": 0.4462,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": -6.668950080871582,
-      "rewards/margins": 2.714482545852661,
-      "rewards/rejected": -9.383432388305664,
-      "step": 5950
-    },
-    {
-      "epoch": 0.47632367632367634,
-      "grad_norm": 12.509074211120605,
-      "learning_rate": 3.136036437841985e-06,
-      "logits/chosen": -11.514371871948242,
-      "logits/rejected": -11.443360328674316,
-      "logps/chosen": -129.04168701171875,
-      "logps/rejected": -160.53907775878906,
-      "loss": 0.3236,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": -5.836568832397461,
-      "rewards/margins": 3.0077452659606934,
-      "rewards/rejected": -8.844313621520996,
-      "step": 5960
-    },
-    {
-      "epoch": 0.4771228771228771,
-      "grad_norm": 4.581553936004639,
-      "learning_rate": 3.129288368595579e-06,
-      "logits/chosen": -11.53625202178955,
-      "logits/rejected": -11.540243148803711,
-      "logps/chosen": -132.4757843017578,
-      "logps/rejected": -163.11990356445312,
-      "loss": 0.383,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -5.838006019592285,
-      "rewards/margins": 3.061767578125,
-      "rewards/rejected": -8.899773597717285,
-      "step": 5970
-    },
-    {
-      "epoch": 0.4779220779220779,
-      "grad_norm": 9.759702682495117,
-      "learning_rate": 3.12253540074456e-06,
-      "logits/chosen": -11.46550464630127,
-      "logits/rejected": -11.35322093963623,
-      "logps/chosen": -132.01536560058594,
-      "logps/rejected": -156.2576141357422,
-      "loss": 0.3063,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": -5.380997180938721,
-      "rewards/margins": 2.9995877742767334,
-      "rewards/rejected": -8.380584716796875,
-      "step": 5980
-    },
-    {
-      "epoch": 0.47872127872127873,
-      "grad_norm": 10.778613090515137,
-      "learning_rate": 3.115777586856431e-06,
-      "logits/chosen": -11.414281845092773,
-      "logits/rejected": -11.433594703674316,
-      "logps/chosen": -134.04603576660156,
-      "logps/rejected": -162.22964477539062,
-      "loss": 0.4015,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -6.125640869140625,
-      "rewards/margins": 2.7001750469207764,
-      "rewards/rejected": -8.82581615447998,
-      "step": 5990
-    },
-    {
-      "epoch": 0.47952047952047955,
-      "grad_norm": 3.2692790031433105,
-      "learning_rate": 3.1090149795364235e-06,
-      "logits/chosen": -11.468214988708496,
-      "logits/rejected": -11.398176193237305,
-      "logps/chosen": -125.1272201538086,
-      "logps/rejected": -152.47537231445312,
-      "loss": 0.2722,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": -5.043107509613037,
-      "rewards/margins": 2.932727575302124,
-      "rewards/rejected": -7.975834846496582,
-      "step": 6000
-    },
-    {
-      "epoch": 0.4803196803196803,
-      "grad_norm": 12.835345268249512,
-      "learning_rate": 3.102247631427076e-06,
-      "logits/chosen": -11.353800773620605,
-      "logits/rejected": -11.416449546813965,
-      "logps/chosen": -123.83309173583984,
-      "logps/rejected": -162.6663055419922,
-      "loss": 0.3042,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": -5.1951117515563965,
-      "rewards/margins": 3.4802663326263428,
-      "rewards/rejected": -8.67537784576416,
-      "step": 6010
-    },
-    {
-      "epoch": 0.4811188811188811,
-      "grad_norm": 9.277067184448242,
-      "learning_rate": 3.0954755952078364e-06,
-      "logits/chosen": -11.306561470031738,
-      "logits/rejected": -11.516226768493652,
-      "logps/chosen": -133.26333618164062,
-      "logps/rejected": -178.9595184326172,
-      "loss": 0.4447,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -5.789787292480469,
-      "rewards/margins": 3.703688383102417,
-      "rewards/rejected": -9.493475914001465,
-      "step": 6020
-    },
-    {
-      "epoch": 0.48191808191808194,
-      "grad_norm": 7.591968536376953,
-      "learning_rate": 3.0886989235946453e-06,
-      "logits/chosen": -11.364740371704102,
-      "logits/rejected": -11.323538780212402,
-      "logps/chosen": -135.7610626220703,
-      "logps/rejected": -174.10716247558594,
-      "loss": 0.4027,
-      "rewards/accuracies": 0.8375000357627869,
-      "rewards/chosen": -6.542807102203369,
-      "rewards/margins": 3.1208431720733643,
-      "rewards/rejected": -9.663650512695312,
-      "step": 6030
-    },
-    {
-      "epoch": 0.4827172827172827,
-      "grad_norm": 18.552133560180664,
-      "learning_rate": 3.0819176693395244e-06,
-      "logits/chosen": -11.329437255859375,
-      "logits/rejected": -11.398274421691895,
-      "logps/chosen": -131.01780700683594,
-      "logps/rejected": -163.99034118652344,
-      "loss": 0.3373,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": -5.752480983734131,
-      "rewards/margins": 2.9649791717529297,
-      "rewards/rejected": -8.717459678649902,
-      "step": 6040
-    },
-    {
-      "epoch": 0.4835164835164835,
-      "grad_norm": 10.821385383605957,
-      "learning_rate": 3.0751318852301714e-06,
-      "logits/chosen": -11.517924308776855,
-      "logits/rejected": -11.394759178161621,
-      "logps/chosen": -129.92469787597656,
-      "logps/rejected": -162.8036651611328,
-      "loss": 0.3801,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -5.685582637786865,
-      "rewards/margins": 3.0326192378997803,
-      "rewards/rejected": -8.718202590942383,
-      "step": 6050
-    },
-    {
-      "epoch": 0.48431568431568434,
-      "grad_norm": 14.22142219543457,
-      "learning_rate": 3.0683416240895442e-06,
-      "logits/chosen": -11.438823699951172,
-      "logits/rejected": -11.391785621643066,
-      "logps/chosen": -128.8540496826172,
-      "logps/rejected": -154.49069213867188,
-      "loss": 0.4022,
-      "rewards/accuracies": 0.8375000357627869,
-      "rewards/chosen": -5.778228282928467,
-      "rewards/margins": 2.4159903526306152,
-      "rewards/rejected": -8.194218635559082,
-      "step": 6060
-    },
-    {
-      "epoch": 0.4851148851148851,
-      "grad_norm": 16.488258361816406,
-      "learning_rate": 3.061546938775453e-06,
-      "logits/chosen": -11.339021682739258,
-      "logits/rejected": -11.365957260131836,
-      "logps/chosen": -132.5143585205078,
-      "logps/rejected": -163.8883056640625,
-      "loss": 0.4347,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -5.749459743499756,
-      "rewards/margins": 2.8744282722473145,
-      "rewards/rejected": -8.62388801574707,
-      "step": 6070
-    },
-    {
-      "epoch": 0.4859140859140859,
-      "grad_norm": 19.89655876159668,
-      "learning_rate": 3.0547478821801453e-06,
-      "logits/chosen": -11.328526496887207,
-      "logits/rejected": -11.221301078796387,
-      "logps/chosen": -132.3710174560547,
-      "logps/rejected": -159.27967834472656,
-      "loss": 0.3679,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": -5.77177095413208,
-      "rewards/margins": 2.7920846939086914,
-      "rewards/rejected": -8.56385612487793,
-      "step": 6080
-    },
-    {
-      "epoch": 0.48671328671328673,
-      "grad_norm": 16.053306579589844,
-      "learning_rate": 3.047944507229898e-06,
-      "logits/chosen": -11.368062973022461,
-      "logits/rejected": -11.316546440124512,
-      "logps/chosen": -126.5253677368164,
-      "logps/rejected": -160.58131408691406,
-      "loss": 0.3345,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -5.919539451599121,
-      "rewards/margins": 2.943220615386963,
-      "rewards/rejected": -8.862760543823242,
-      "step": 6090
-    },
-    {
-      "epoch": 0.4875124875124875,
-      "grad_norm": 8.105134963989258,
-      "learning_rate": 3.041136866884602e-06,
-      "logits/chosen": -11.351892471313477,
-      "logits/rejected": -11.312132835388184,
-      "logps/chosen": -126.24224090576172,
-      "logps/rejected": -148.96542358398438,
-      "loss": 0.4109,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -5.046599864959717,
-      "rewards/margins": 2.666109085083008,
-      "rewards/rejected": -7.712708473205566,
-      "step": 6100
-    },
-    {
-      "epoch": 0.4883116883116883,
-      "grad_norm": 17.759172439575195,
-      "learning_rate": 3.034325014137354e-06,
-      "logits/chosen": -11.400179862976074,
-      "logits/rejected": -11.338107109069824,
-      "logps/chosen": -123.9681625366211,
-      "logps/rejected": -157.7777557373047,
-      "loss": 0.3122,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -5.135572910308838,
-      "rewards/margins": 3.009586811065674,
-      "rewards/rejected": -8.145159721374512,
-      "step": 6110
-    },
-    {
-      "epoch": 0.4891108891108891,
-      "grad_norm": 8.84757137298584,
-      "learning_rate": 3.027509002014038e-06,
-      "logits/chosen": -11.294565200805664,
-      "logits/rejected": -11.251248359680176,
-      "logps/chosen": -115.3251953125,
-      "logps/rejected": -144.0845489501953,
-      "loss": 0.3833,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -4.976505279541016,
-      "rewards/margins": 2.265813112258911,
-      "rewards/rejected": -7.242318630218506,
-      "step": 6120
-    },
-    {
-      "epoch": 0.4899100899100899,
-      "grad_norm": 10.700142860412598,
-      "learning_rate": 3.0206888835729205e-06,
-      "logits/chosen": -11.136385917663574,
-      "logits/rejected": -11.254197120666504,
-      "logps/chosen": -123.32000732421875,
-      "logps/rejected": -160.7563018798828,
-      "loss": 0.403,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -4.967443466186523,
-      "rewards/margins": 2.6428089141845703,
-      "rewards/rejected": -7.610252380371094,
-      "step": 6130
-    },
-    {
-      "epoch": 0.4907092907092907,
-      "grad_norm": 17.164878845214844,
-      "learning_rate": 3.0138647119042285e-06,
-      "logits/chosen": -11.266057014465332,
-      "logits/rejected": -11.28874397277832,
-      "logps/chosen": -124.63701629638672,
-      "logps/rejected": -164.44822692871094,
-      "loss": 0.4058,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -5.296164035797119,
-      "rewards/margins": 3.023517370223999,
-      "rewards/rejected": -8.319682121276855,
-      "step": 6140
-    },
-    {
-      "epoch": 0.4915084915084915,
-      "grad_norm": 23.604183197021484,
-      "learning_rate": 3.0070365401297436e-06,
-      "logits/chosen": -11.161355018615723,
-      "logits/rejected": -11.324044227600098,
-      "logps/chosen": -121.20892333984375,
-      "logps/rejected": -156.708251953125,
-      "loss": 0.3666,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -4.975308895111084,
-      "rewards/margins": 2.7478647232055664,
-      "rewards/rejected": -7.72317361831665,
-      "step": 6150
-    },
-    {
-      "epoch": 0.49230769230769234,
-      "grad_norm": 11.527639389038086,
-      "learning_rate": 3.0002044214023855e-06,
-      "logits/chosen": -11.327208518981934,
-      "logits/rejected": -11.194961547851562,
-      "logps/chosen": -133.3356170654297,
-      "logps/rejected": -158.64599609375,
-      "loss": 0.3586,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -5.831835746765137,
-      "rewards/margins": 2.5763776302337646,
-      "rewards/rejected": -8.408212661743164,
-      "step": 6160
-    },
-    {
-      "epoch": 0.4931068931068931,
-      "grad_norm": 13.63038158416748,
-      "learning_rate": 2.993368408905797e-06,
-      "logits/chosen": -11.36072063446045,
-      "logits/rejected": -11.294100761413574,
-      "logps/chosen": -123.3004150390625,
-      "logps/rejected": -157.74998474121094,
-      "loss": 0.4092,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -4.961327075958252,
-      "rewards/margins": 3.066984176635742,
-      "rewards/rejected": -8.028311729431152,
-      "step": 6170
-    },
-    {
-      "epoch": 0.4939060939060939,
-      "grad_norm": 23.36896514892578,
-      "learning_rate": 2.986528555853932e-06,
-      "logits/chosen": -11.407052040100098,
-      "logits/rejected": -11.317217826843262,
-      "logps/chosen": -129.75979614257812,
-      "logps/rejected": -156.02842712402344,
-      "loss": 0.3605,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -5.522770404815674,
-      "rewards/margins": 2.877673864364624,
-      "rewards/rejected": -8.400444030761719,
-      "step": 6180
-    },
-    {
-      "epoch": 0.49470529470529473,
-      "grad_norm": 9.922686576843262,
-      "learning_rate": 2.9796849154906413e-06,
-      "logits/chosen": -11.337298393249512,
-      "logits/rejected": -11.34604549407959,
-      "logps/chosen": -125.0017318725586,
-      "logps/rejected": -157.4385986328125,
-      "loss": 0.3078,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -5.264926433563232,
-      "rewards/margins": 3.1293437480926514,
-      "rewards/rejected": -8.394269943237305,
-      "step": 6190
-    },
-    {
-      "epoch": 0.4955044955044955,
-      "grad_norm": 10.044878959655762,
-      "learning_rate": 2.972837541089258e-06,
-      "logits/chosen": -11.287920951843262,
-      "logits/rejected": -11.290532112121582,
-      "logps/chosen": -123.71797180175781,
-      "logps/rejected": -151.79458618164062,
-      "loss": 0.3276,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": -4.984632968902588,
-      "rewards/margins": 2.7066895961761475,
-      "rewards/rejected": -7.691322326660156,
-      "step": 6200
-    },
-    {
-      "epoch": 0.4963036963036963,
-      "grad_norm": 15.848016738891602,
-      "learning_rate": 2.965986485952179e-06,
-      "logits/chosen": -11.337532997131348,
-      "logits/rejected": -11.282081604003906,
-      "logps/chosen": -123.0903549194336,
-      "logps/rejected": -153.668701171875,
-      "loss": 0.324,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -5.402825832366943,
-      "rewards/margins": 2.583712339401245,
-      "rewards/rejected": -7.986538887023926,
-      "step": 6210
-    },
-    {
-      "epoch": 0.4971028971028971,
-      "grad_norm": 6.485332012176514,
-      "learning_rate": 2.9591318034104582e-06,
-      "logits/chosen": -11.205342292785645,
-      "logits/rejected": -11.266764640808105,
-      "logps/chosen": -123.7608642578125,
-      "logps/rejected": -173.7487335205078,
-      "loss": 0.2616,
-      "rewards/accuracies": 0.9000000357627869,
-      "rewards/chosen": -5.240143299102783,
-      "rewards/margins": 3.775834798812866,
-      "rewards/rejected": -9.01597785949707,
-      "step": 6220
-    },
-    {
-      "epoch": 0.4979020979020979,
-      "grad_norm": 25.655223846435547,
-      "learning_rate": 2.952273546823382e-06,
-      "logits/chosen": -11.000259399414062,
-      "logits/rejected": -11.31278133392334,
-      "logps/chosen": -122.6508560180664,
-      "logps/rejected": -170.8686981201172,
-      "loss": 0.4322,
-      "rewards/accuracies": 0.8375000357627869,
-      "rewards/chosen": -5.438472747802734,
-      "rewards/margins": 3.2245330810546875,
-      "rewards/rejected": -8.663005828857422,
-      "step": 6230
-    },
-    {
-      "epoch": 0.4987012987012987,
-      "grad_norm": 2.9546709060668945,
-      "learning_rate": 2.945411769578061e-06,
-      "logits/chosen": -11.244526863098145,
-      "logits/rejected": -11.344860076904297,
-      "logps/chosen": -119.2520523071289,
-      "logps/rejected": -170.0446014404297,
-      "loss": 0.2259,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -4.752492904663086,
-      "rewards/margins": 4.267310619354248,
-      "rewards/rejected": -9.019803047180176,
-      "step": 6240
-    },
-    {
-      "epoch": 0.4995004995004995,
-      "grad_norm": 3.8100578784942627,
-      "learning_rate": 2.9385465250890117e-06,
-      "logits/chosen": -11.432222366333008,
-      "logits/rejected": -11.338223457336426,
-      "logps/chosen": -129.2135009765625,
-      "logps/rejected": -162.3838348388672,
-      "loss": 0.3603,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -5.67446231842041,
-      "rewards/margins": 3.3238778114318848,
-      "rewards/rejected": -8.998339653015137,
-      "step": 6250
-    },
-    {
-      "epoch": 0.5002997002997003,
-      "grad_norm": 9.692636489868164,
-      "learning_rate": 2.931677866797741e-06,
-      "logits/chosen": -11.506033897399902,
-      "logits/rejected": -11.438011169433594,
-      "logps/chosen": -128.01248168945312,
-      "logps/rejected": -158.8745574951172,
-      "loss": 0.3812,
-      "rewards/accuracies": 0.8375000357627869,
-      "rewards/chosen": -5.921238422393799,
-      "rewards/margins": 2.5627658367156982,
-      "rewards/rejected": -8.484004020690918,
-      "step": 6260
-    },
-    {
-      "epoch": 0.5010989010989011,
-      "grad_norm": 5.371511936187744,
-      "learning_rate": 2.9248058481723276e-06,
-      "logits/chosen": -11.460619926452637,
-      "logits/rejected": -11.407272338867188,
-      "logps/chosen": -123.73529052734375,
-      "logps/rejected": -152.5821075439453,
-      "loss": 0.3659,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -5.121123790740967,
-      "rewards/margins": 2.8517537117004395,
-      "rewards/rejected": -7.972877502441406,
-      "step": 6270
-    },
-    {
-      "epoch": 0.5018981018981019,
-      "grad_norm": 16.965383529663086,
-      "learning_rate": 2.917930522707011e-06,
-      "logits/chosen": -11.50102424621582,
-      "logits/rejected": -11.463568687438965,
-      "logps/chosen": -135.0405731201172,
-      "logps/rejected": -156.81704711914062,
-      "loss": 0.4087,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -6.007693767547607,
-      "rewards/margins": 2.3972384929656982,
-      "rewards/rejected": -8.404932022094727,
-      "step": 6280
-    },
-    {
-      "epoch": 0.5026973026973027,
-      "grad_norm": 20.136611938476562,
-      "learning_rate": 2.9110519439217716e-06,
-      "logits/chosen": -11.443613052368164,
-      "logits/rejected": -11.3901948928833,
-      "logps/chosen": -131.59017944335938,
-      "logps/rejected": -157.96385192871094,
-      "loss": 0.3206,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -5.728478908538818,
-      "rewards/margins": 2.8057968616485596,
-      "rewards/rejected": -8.53427505493164,
-      "step": 6290
-    },
-    {
-      "epoch": 0.5034965034965035,
-      "grad_norm": 13.07650375366211,
-      "learning_rate": 2.904170165361915e-06,
-      "logits/chosen": -11.365015983581543,
-      "logits/rejected": -11.318251609802246,
-      "logps/chosen": -132.44960021972656,
-      "logps/rejected": -159.26023864746094,
-      "loss": 0.3993,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -5.908681392669678,
-      "rewards/margins": 2.480193853378296,
-      "rewards/rejected": -8.388875007629395,
-      "step": 6300
-    },
-    {
-      "epoch": 0.5042957042957043,
-      "grad_norm": 7.5846266746521,
-      "learning_rate": 2.897285240597654e-06,
-      "logits/chosen": -11.41088581085205,
-      "logits/rejected": -11.317362785339355,
-      "logps/chosen": -129.2015838623047,
-      "logps/rejected": -160.67935180664062,
-      "loss": 0.2749,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": -5.745380401611328,
-      "rewards/margins": 3.1319942474365234,
-      "rewards/rejected": -8.877374649047852,
-      "step": 6310
-    },
-    {
-      "epoch": 0.5050949050949051,
-      "grad_norm": 25.293514251708984,
-      "learning_rate": 2.890397223223693e-06,
-      "logits/chosen": -11.397605895996094,
-      "logits/rejected": -11.41882610321045,
-      "logps/chosen": -133.62168884277344,
-      "logps/rejected": -161.03211975097656,
-      "loss": 0.4385,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -6.165483474731445,
-      "rewards/margins": 2.430185079574585,
-      "rewards/rejected": -8.59566879272461,
-      "step": 6320
-    },
-    {
-      "epoch": 0.5058941058941059,
-      "grad_norm": 8.16553783416748,
-      "learning_rate": 2.8835061668588115e-06,
-      "logits/chosen": -11.45673942565918,
-      "logits/rejected": -11.374701499938965,
-      "logps/chosen": -124.32869720458984,
-      "logps/rejected": -161.6531219482422,
-      "loss": 0.2911,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -5.535003185272217,
-      "rewards/margins": 3.165271043777466,
-      "rewards/rejected": -8.700274467468262,
-      "step": 6330
-    },
-    {
-      "epoch": 0.5066933066933067,
-      "grad_norm": 20.802236557006836,
-      "learning_rate": 2.8766121251454436e-06,
-      "logits/chosen": -11.275466918945312,
-      "logits/rejected": -11.344443321228027,
-      "logps/chosen": -140.5220489501953,
-      "logps/rejected": -168.2504425048828,
-      "loss": 0.5411,
-      "rewards/accuracies": 0.7750000357627869,
-      "rewards/chosen": -6.457730293273926,
-      "rewards/margins": 2.4598119258880615,
-      "rewards/rejected": -8.917542457580566,
-      "step": 6340
-    },
-    {
-      "epoch": 0.5074925074925075,
-      "grad_norm": 22.857755661010742,
-      "learning_rate": 2.869715151749265e-06,
-      "logits/chosen": -11.370877265930176,
-      "logits/rejected": -11.313145637512207,
-      "logps/chosen": -127.39593505859375,
-      "logps/rejected": -159.90689086914062,
-      "loss": 0.3446,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -5.570003032684326,
-      "rewards/margins": 2.9114511013031006,
-      "rewards/rejected": -8.481454849243164,
-      "step": 6350
-    },
-    {
-      "epoch": 0.5082917082917083,
-      "grad_norm": 18.165931701660156,
-      "learning_rate": 2.862815300358769e-06,
-      "logits/chosen": -11.411456108093262,
-      "logits/rejected": -11.31737232208252,
-      "logps/chosen": -134.9620361328125,
-      "logps/rejected": -166.88096618652344,
-      "loss": 0.2869,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -6.136592388153076,
-      "rewards/margins": 2.9589595794677734,
-      "rewards/rejected": -9.095552444458008,
-      "step": 6360
-    },
-    {
-      "epoch": 0.509090909090909,
-      "grad_norm": 6.038599014282227,
-      "learning_rate": 2.8559126246848555e-06,
-      "logits/chosen": -11.491700172424316,
-      "logits/rejected": -11.428311347961426,
-      "logps/chosen": -131.22360229492188,
-      "logps/rejected": -162.4156036376953,
-      "loss": 0.3434,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -5.813791751861572,
-      "rewards/margins": 3.208665609359741,
-      "rewards/rejected": -9.02245807647705,
-      "step": 6370
-    },
-    {
-      "epoch": 0.5098901098901099,
-      "grad_norm": 17.950714111328125,
-      "learning_rate": 2.849007178460409e-06,
-      "logits/chosen": -11.359423637390137,
-      "logits/rejected": -11.383310317993164,
-      "logps/chosen": -135.36038208007812,
-      "logps/rejected": -170.8843994140625,
-      "loss": 0.3516,
-      "rewards/accuracies": 0.8375000357627869,
-      "rewards/chosen": -6.4963860511779785,
-      "rewards/margins": 2.9759793281555176,
-      "rewards/rejected": -9.472365379333496,
-      "step": 6380
-    },
-    {
-      "epoch": 0.5106893106893107,
-      "grad_norm": 18.40981101989746,
-      "learning_rate": 2.8420990154398797e-06,
-      "logits/chosen": -11.430685043334961,
-      "logits/rejected": -11.38786506652832,
-      "logps/chosen": -126.0456314086914,
-      "logps/rejected": -164.4066162109375,
-      "loss": 0.2168,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": -5.840683937072754,
-      "rewards/margins": 3.352201223373413,
-      "rewards/rejected": -9.192885398864746,
-      "step": 6390
-    },
-    {
-      "epoch": 0.5114885114885115,
-      "grad_norm": 13.405781745910645,
-      "learning_rate": 2.8351881893988676e-06,
-      "logits/chosen": -11.44830322265625,
-      "logits/rejected": -11.458935737609863,
-      "logps/chosen": -128.2720184326172,
-      "logps/rejected": -164.4755096435547,
-      "loss": 0.3764,
-      "rewards/accuracies": 0.8375000357627869,
-      "rewards/chosen": -5.7464118003845215,
-      "rewards/margins": 3.048938274383545,
-      "rewards/rejected": -8.795351028442383,
-      "step": 6400
-    },
-    {
-      "epoch": 0.5122877122877123,
-      "grad_norm": 6.940419673919678,
-      "learning_rate": 2.828274754133701e-06,
-      "logits/chosen": -11.531062126159668,
-      "logits/rejected": -11.452791213989258,
-      "logps/chosen": -133.2300262451172,
-      "logps/rejected": -162.8380889892578,
-      "loss": 0.3959,
-      "rewards/accuracies": 0.8375000357627869,
-      "rewards/chosen": -5.945829391479492,
-      "rewards/margins": 2.948850393295288,
-      "rewards/rejected": -8.89468002319336,
-      "step": 6410
-    },
-    {
-      "epoch": 0.5130869130869131,
-      "grad_norm": 8.350247383117676,
-      "learning_rate": 2.8213587634610208e-06,
-      "logits/chosen": -11.350533485412598,
-      "logits/rejected": -11.415249824523926,
-      "logps/chosen": -137.113037109375,
-      "logps/rejected": -170.0409698486328,
-      "loss": 0.3759,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -6.5920329093933105,
-      "rewards/margins": 2.7595016956329346,
-      "rewards/rejected": -9.351534843444824,
-      "step": 6420
-    },
-    {
-      "epoch": 0.5138861138861139,
-      "grad_norm": 17.83112907409668,
-      "learning_rate": 2.8144402712173603e-06,
-      "logits/chosen": -11.405962944030762,
-      "logits/rejected": -11.444286346435547,
-      "logps/chosen": -138.73373413085938,
-      "logps/rejected": -173.86257934570312,
-      "loss": 0.3382,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -6.600230693817139,
-      "rewards/margins": 3.010009527206421,
-      "rewards/rejected": -9.61023998260498,
-      "step": 6430
-    },
-    {
-      "epoch": 0.5146853146853146,
-      "grad_norm": 19.871795654296875,
-      "learning_rate": 2.807519331258725e-06,
-      "logits/chosen": -11.595879554748535,
-      "logits/rejected": -11.460305213928223,
-      "logps/chosen": -122.62462615966797,
-      "logps/rejected": -162.06069946289062,
-      "loss": 0.2427,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": -5.370727062225342,
-      "rewards/margins": 3.411602735519409,
-      "rewards/rejected": -8.782328605651855,
-      "step": 6440
-    },
-    {
-      "epoch": 0.5154845154845155,
-      "grad_norm": 28.14447021484375,
-      "learning_rate": 2.8005959974601745e-06,
-      "logits/chosen": -11.591659545898438,
-      "logits/rejected": -11.570056915283203,
-      "logps/chosen": -131.94337463378906,
-      "logps/rejected": -155.0570831298828,
-      "loss": 0.6203,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": -5.858651161193848,
-      "rewards/margins": 2.373267889022827,
-      "rewards/rejected": -8.231918334960938,
-      "step": 6450
-    },
-    {
-      "epoch": 0.5162837162837163,
-      "grad_norm": 3.5957095623016357,
-      "learning_rate": 2.793670323715403e-06,
-      "logits/chosen": -11.566207885742188,
-      "logits/rejected": -11.564242362976074,
-      "logps/chosen": -124.3234634399414,
-      "logps/rejected": -158.995361328125,
-      "loss": 0.2785,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -5.2411088943481445,
-      "rewards/margins": 2.958801746368408,
-      "rewards/rejected": -8.199910163879395,
-      "step": 6460
-    },
-    {
-      "epoch": 0.5170829170829171,
-      "grad_norm": 16.131362915039062,
-      "learning_rate": 2.786742363936319e-06,
-      "logits/chosen": -11.578590393066406,
-      "logits/rejected": -11.549878120422363,
-      "logps/chosen": -129.6881561279297,
-      "logps/rejected": -157.4022674560547,
-      "loss": 0.4613,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": -5.846691131591797,
-      "rewards/margins": 2.518402576446533,
-      "rewards/rejected": -8.365094184875488,
-      "step": 6470
-    },
-    {
-      "epoch": 0.5178821178821179,
-      "grad_norm": 15.283625602722168,
-      "learning_rate": 2.7798121720526284e-06,
-      "logits/chosen": -11.606233596801758,
-      "logits/rejected": -11.500770568847656,
-      "logps/chosen": -127.3968276977539,
-      "logps/rejected": -164.02928161621094,
-      "loss": 0.2688,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": -5.440892219543457,
-      "rewards/margins": 3.132579803466797,
-      "rewards/rejected": -8.57347297668457,
-      "step": 6480
-    },
-    {
-      "epoch": 0.5186813186813187,
-      "grad_norm": 14.48303508758545,
-      "learning_rate": 2.772879802011409e-06,
-      "logits/chosen": -11.520255088806152,
-      "logits/rejected": -11.596087455749512,
-      "logps/chosen": -127.57610321044922,
-      "logps/rejected": -164.66795349121094,
-      "loss": 0.3047,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -5.130354881286621,
-      "rewards/margins": 3.3222453594207764,
-      "rewards/rejected": -8.452600479125977,
-      "step": 6490
-    },
-    {
-      "epoch": 0.5194805194805194,
-      "grad_norm": 9.490450859069824,
-      "learning_rate": 2.765945307776699e-06,
-      "logits/chosen": -11.591975212097168,
-      "logits/rejected": -11.577244758605957,
-      "logps/chosen": -126.11113739013672,
-      "logps/rejected": -156.4290313720703,
-      "loss": 0.3091,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -5.523040771484375,
-      "rewards/margins": 2.5617921352386475,
-      "rewards/rejected": -8.084833145141602,
-      "step": 6500
-    },
-    {
-      "epoch": 0.5202797202797202,
-      "grad_norm": 9.939345359802246,
-      "learning_rate": 2.7590087433290675e-06,
-      "logits/chosen": -11.697029113769531,
-      "logits/rejected": -11.669296264648438,
-      "logps/chosen": -133.2665557861328,
-      "logps/rejected": -156.92132568359375,
-      "loss": 0.508,
-      "rewards/accuracies": 0.7750000357627869,
-      "rewards/chosen": -5.727529525756836,
-      "rewards/margins": 2.31671404838562,
-      "rewards/rejected": -8.044242858886719,
-      "step": 6510
-    },
-    {
-      "epoch": 0.5210789210789211,
-      "grad_norm": 9.655838966369629,
-      "learning_rate": 2.7520701626652015e-06,
-      "logits/chosen": -11.617982864379883,
-      "logits/rejected": -11.650958061218262,
-      "logps/chosen": -126.43855285644531,
-      "logps/rejected": -154.1655731201172,
-      "loss": 0.4552,
-      "rewards/accuracies": 0.7750000357627869,
-      "rewards/chosen": -5.498693466186523,
-      "rewards/margins": 2.4159810543060303,
-      "rewards/rejected": -7.914674282073975,
-      "step": 6520
-    },
-    {
-      "epoch": 0.5218781218781219,
-      "grad_norm": 6.4705915451049805,
-      "learning_rate": 2.7451296197974813e-06,
-      "logits/chosen": -11.647347450256348,
-      "logits/rejected": -11.611047744750977,
-      "logps/chosen": -124.4271469116211,
-      "logps/rejected": -153.71011352539062,
-      "loss": 0.3712,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -5.522699356079102,
-      "rewards/margins": 2.498748302459717,
-      "rewards/rejected": -8.021448135375977,
-      "step": 6530
-    },
-    {
-      "epoch": 0.5226773226773227,
-      "grad_norm": 21.493528366088867,
-      "learning_rate": 2.7381871687535645e-06,
-      "logits/chosen": -11.727682113647461,
-      "logits/rejected": -11.6151704788208,
-      "logps/chosen": -127.32549285888672,
-      "logps/rejected": -152.15147399902344,
-      "loss": 0.4652,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -5.30194091796875,
-      "rewards/margins": 2.3877198696136475,
-      "rewards/rejected": -7.689660549163818,
-      "step": 6540
-    },
-    {
-      "epoch": 0.5234765234765235,
-      "grad_norm": 17.98436737060547,
-      "learning_rate": 2.7312428635759586e-06,
-      "logits/chosen": -11.713229179382324,
-      "logits/rejected": -11.695450782775879,
-      "logps/chosen": -122.22379302978516,
-      "logps/rejected": -151.58152770996094,
-      "loss": 0.3433,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -4.994765281677246,
-      "rewards/margins": 2.8997604846954346,
-      "rewards/rejected": -7.894525051116943,
-      "step": 6550
-    },
-    {
-      "epoch": 0.5242757242757243,
-      "grad_norm": 28.182954788208008,
-      "learning_rate": 2.724296758321607e-06,
-      "logits/chosen": -11.636377334594727,
-      "logits/rejected": -11.652270317077637,
-      "logps/chosen": -134.98965454101562,
-      "logps/rejected": -167.31524658203125,
-      "loss": 0.4186,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -6.121760368347168,
-      "rewards/margins": 2.65596079826355,
-      "rewards/rejected": -8.77772045135498,
-      "step": 6560
-    },
-    {
-      "epoch": 0.525074925074925,
-      "grad_norm": 11.316851615905762,
-      "learning_rate": 2.7173489070614665e-06,
-      "logits/chosen": -11.504924774169922,
-      "logits/rejected": -11.642782211303711,
-      "logps/chosen": -127.2563247680664,
-      "logps/rejected": -163.58242797851562,
-      "loss": 0.3449,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -5.3001837730407715,
-      "rewards/margins": 3.24763560295105,
-      "rewards/rejected": -8.547820091247559,
-      "step": 6570
-    },
-    {
-      "epoch": 0.5258741258741259,
-      "grad_norm": 20.439794540405273,
-      "learning_rate": 2.710399363880082e-06,
-      "logits/chosen": -11.691489219665527,
-      "logits/rejected": -11.638246536254883,
-      "logps/chosen": -137.6809539794922,
-      "logps/rejected": -165.55506896972656,
-      "loss": 0.4359,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -6.262702465057373,
-      "rewards/margins": 2.647810220718384,
-      "rewards/rejected": -8.910513877868652,
-      "step": 6580
-    },
-    {
-      "epoch": 0.5266733266733267,
-      "grad_norm": 7.232861042022705,
-      "learning_rate": 2.7034481828751725e-06,
-      "logits/chosen": -11.58561897277832,
-      "logits/rejected": -11.567968368530273,
-      "logps/chosen": -135.5861358642578,
-      "logps/rejected": -165.37832641601562,
-      "loss": 0.37,
-      "rewards/accuracies": 0.8375000357627869,
-      "rewards/chosen": -6.533517360687256,
-      "rewards/margins": 2.9390041828155518,
-      "rewards/rejected": -9.472521781921387,
-      "step": 6590
-    },
-    {
-      "epoch": 0.5274725274725275,
-      "grad_norm": 4.421411037445068,
-      "learning_rate": 2.696495418157203e-06,
-      "logits/chosen": -11.66301441192627,
-      "logits/rejected": -11.595341682434082,
-      "logps/chosen": -123.23015594482422,
-      "logps/rejected": -156.77691650390625,
-      "loss": 0.2654,
-      "rewards/accuracies": 0.9000000357627869,
-      "rewards/chosen": -5.253417491912842,
-      "rewards/margins": 3.1225807666778564,
-      "rewards/rejected": -8.375998497009277,
-      "step": 6600
-    },
-    {
-      "epoch": 0.5282717282717283,
-      "grad_norm": 17.922216415405273,
-      "learning_rate": 2.6895411238489693e-06,
-      "logits/chosen": -11.644619941711426,
-      "logits/rejected": -11.56207275390625,
-      "logps/chosen": -137.2550811767578,
-      "logps/rejected": -170.13377380371094,
-      "loss": 0.3874,
-      "rewards/accuracies": 0.8375000357627869,
-      "rewards/chosen": -6.341902732849121,
-      "rewards/margins": 3.211812734603882,
-      "rewards/rejected": -9.553715705871582,
-      "step": 6610
-    },
-    {
-      "epoch": 0.5290709290709291,
-      "grad_norm": 20.525747299194336,
-      "learning_rate": 2.6825853540851734e-06,
-      "logits/chosen": -11.454477310180664,
-      "logits/rejected": -11.510601997375488,
-      "logps/chosen": -131.1232452392578,
-      "logps/rejected": -173.05613708496094,
-      "loss": 0.4646,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": -6.018099308013916,
-      "rewards/margins": 3.082097291946411,
-      "rewards/rejected": -9.100196838378906,
-      "step": 6620
-    },
-    {
-      "epoch": 0.5298701298701298,
-      "grad_norm": 7.949403762817383,
-      "learning_rate": 2.675628163012003e-06,
-      "logits/chosen": -11.606698989868164,
-      "logits/rejected": -11.567008018493652,
-      "logps/chosen": -127.6317367553711,
-      "logps/rejected": -164.38133239746094,
-      "loss": 0.3936,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": -5.411200046539307,
-      "rewards/margins": 3.3422775268554688,
-      "rewards/rejected": -8.753478050231934,
-      "step": 6630
-    },
-    {
-      "epoch": 0.5306693306693306,
-      "grad_norm": 14.905630111694336,
-      "learning_rate": 2.668669604786708e-06,
-      "logits/chosen": -11.588334083557129,
-      "logits/rejected": -11.603109359741211,
-      "logps/chosen": -118.76777648925781,
-      "logps/rejected": -154.91506958007812,
-      "loss": 0.4069,
-      "rewards/accuracies": 0.7750000357627869,
-      "rewards/chosen": -5.042624473571777,
-      "rewards/margins": 2.8027021884918213,
-      "rewards/rejected": -7.8453264236450195,
-      "step": 6640
-    },
-    {
-      "epoch": 0.5314685314685315,
-      "grad_norm": 4.4029765129089355,
-      "learning_rate": 2.661709733577183e-06,
-      "logits/chosen": -11.654472351074219,
-      "logits/rejected": -11.480487823486328,
-      "logps/chosen": -122.21766662597656,
-      "logps/rejected": -149.0961151123047,
-      "loss": 0.4225,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -4.87084436416626,
-      "rewards/margins": 2.7185890674591064,
-      "rewards/rejected": -7.589433193206787,
-      "step": 6650
-    },
-    {
-      "epoch": 0.5322677322677323,
-      "grad_norm": 14.659449577331543,
-      "learning_rate": 2.6547486035615417e-06,
-      "logits/chosen": -11.516745567321777,
-      "logits/rejected": -11.544442176818848,
-      "logps/chosen": -123.1968765258789,
-      "logps/rejected": -161.07862854003906,
-      "loss": 0.2719,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": -4.940864086151123,
-      "rewards/margins": 3.3532333374023438,
-      "rewards/rejected": -8.294096946716309,
-      "step": 6660
-    },
-    {
-      "epoch": 0.5330669330669331,
-      "grad_norm": 12.307703018188477,
-      "learning_rate": 2.647786268927699e-06,
-      "logits/chosen": -11.513728141784668,
-      "logits/rejected": -11.616964340209961,
-      "logps/chosen": -122.0667724609375,
-      "logps/rejected": -160.54408264160156,
-      "loss": 0.3413,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -5.145030975341797,
-      "rewards/margins": 3.0670719146728516,
-      "rewards/rejected": -8.212103843688965,
-      "step": 6670
-    },
-    {
-      "epoch": 0.5338661338661339,
-      "grad_norm": 17.356006622314453,
-      "learning_rate": 2.640822783872944e-06,
-      "logits/chosen": -11.620468139648438,
-      "logits/rejected": -11.610020637512207,
-      "logps/chosen": -117.32698822021484,
-      "logps/rejected": -155.15017700195312,
-      "loss": 0.3875,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -4.829188823699951,
-      "rewards/margins": 2.9089419841766357,
-      "rewards/rejected": -7.738131999969482,
-      "step": 6680
-    },
-    {
-      "epoch": 0.5346653346653346,
-      "grad_norm": 13.765165328979492,
-      "learning_rate": 2.6338582026035225e-06,
-      "logits/chosen": -11.588213920593262,
-      "logits/rejected": -11.664743423461914,
-      "logps/chosen": -127.8271713256836,
-      "logps/rejected": -161.5267333984375,
-      "loss": 0.3936,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -5.350343227386475,
-      "rewards/margins": 2.847808599472046,
-      "rewards/rejected": -8.198151588439941,
-      "step": 6690
-    },
-    {
-      "epoch": 0.5354645354645354,
-      "grad_norm": 12.285625457763672,
-      "learning_rate": 2.6268925793342156e-06,
-      "logits/chosen": -11.603546142578125,
-      "logits/rejected": -11.589448928833008,
-      "logps/chosen": -122.1004409790039,
-      "logps/rejected": -166.7223663330078,
-      "loss": 0.2309,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": -4.7422685623168945,
-      "rewards/margins": 3.8949966430664062,
-      "rewards/rejected": -8.6372652053833,
-      "step": 6700
-    },
-    {
-      "epoch": 0.5362637362637362,
-      "grad_norm": 14.499353408813477,
-      "learning_rate": 2.61992596828791e-06,
-      "logits/chosen": -11.676197052001953,
-      "logits/rejected": -11.625629425048828,
-      "logps/chosen": -126.79286193847656,
-      "logps/rejected": -166.4468994140625,
-      "loss": 0.3508,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -5.603641510009766,
-      "rewards/margins": 3.058976173400879,
-      "rewards/rejected": -8.662618637084961,
-      "step": 6710
-    },
-    {
-      "epoch": 0.5370629370629371,
-      "grad_norm": 7.24888801574707,
-      "learning_rate": 2.61295842369519e-06,
-      "logits/chosen": -11.607515335083008,
-      "logits/rejected": -11.604525566101074,
-      "logps/chosen": -126.09590148925781,
-      "logps/rejected": -160.10606384277344,
-      "loss": 0.3292,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -5.387364387512207,
-      "rewards/margins": 2.9844861030578613,
-      "rewards/rejected": -8.371850967407227,
-      "step": 6720
-    },
-    {
-      "epoch": 0.5378621378621379,
-      "grad_norm": 17.12464714050293,
-      "learning_rate": 2.6059899997938985e-06,
-      "logits/chosen": -11.621406555175781,
-      "logits/rejected": -11.764171600341797,
-      "logps/chosen": -123.8239974975586,
-      "logps/rejected": -162.6329803466797,
-      "loss": 0.3555,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -5.230836391448975,
-      "rewards/margins": 3.0304319858551025,
-      "rewards/rejected": -8.261268615722656,
-      "step": 6730
-    },
-    {
-      "epoch": 0.5386613386613387,
-      "grad_norm": 3.6876800060272217,
-      "learning_rate": 2.599020750828729e-06,
-      "logits/chosen": -11.572379112243652,
-      "logits/rejected": -11.728179931640625,
-      "logps/chosen": -129.61288452148438,
-      "logps/rejected": -175.3890380859375,
-      "loss": 0.289,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": -5.580990791320801,
-      "rewards/margins": 3.503598928451538,
-      "rewards/rejected": -9.084589004516602,
-      "step": 6740
-    },
-    {
-      "epoch": 0.5394605394605395,
-      "grad_norm": 12.355541229248047,
-      "learning_rate": 2.592050731050796e-06,
-      "logits/chosen": -11.755597114562988,
-      "logits/rejected": -11.703054428100586,
-      "logps/chosen": -128.87600708007812,
-      "logps/rejected": -159.89944458007812,
-      "loss": 0.3536,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -5.637346267700195,
-      "rewards/margins": 3.03788685798645,
-      "rewards/rejected": -8.675232887268066,
-      "step": 6750
-    },
-    {
-      "epoch": 0.5402597402597402,
-      "grad_norm": 5.701478481292725,
-      "learning_rate": 2.585079994717213e-06,
-      "logits/chosen": -11.542892456054688,
-      "logits/rejected": -11.755084037780762,
-      "logps/chosen": -130.22605895996094,
-      "logps/rejected": -173.2449951171875,
-      "loss": 0.3475,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -6.152543544769287,
-      "rewards/margins": 2.9172680377960205,
-      "rewards/rejected": -9.06981086730957,
-      "step": 6760
-    },
-    {
-      "epoch": 0.541058941058941,
-      "grad_norm": 20.41600227355957,
-      "learning_rate": 2.5781085960906717e-06,
-      "logits/chosen": -11.833487510681152,
-      "logits/rejected": -11.74298095703125,
-      "logps/chosen": -136.2042236328125,
-      "logps/rejected": -166.535888671875,
-      "loss": 0.3894,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -6.5142717361450195,
-      "rewards/margins": 2.6732680797576904,
-      "rewards/rejected": -9.187541007995605,
-      "step": 6770
-    },
-    {
-      "epoch": 0.5418581418581419,
-      "grad_norm": 17.795629501342773,
-      "learning_rate": 2.571136589439022e-06,
-      "logits/chosen": -11.85097885131836,
-      "logits/rejected": -11.796053886413574,
-      "logps/chosen": -131.79345703125,
-      "logps/rejected": -167.21141052246094,
-      "loss": 0.3484,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -6.373319149017334,
-      "rewards/margins": 3.0190727710723877,
-      "rewards/rejected": -9.3923921585083,
-      "step": 6780
-    },
-    {
-      "epoch": 0.5426573426573427,
-      "grad_norm": 17.895790100097656,
-      "learning_rate": 2.5641640290348433e-06,
-      "logits/chosen": -11.890719413757324,
-      "logits/rejected": -11.878389358520508,
-      "logps/chosen": -123.0273666381836,
-      "logps/rejected": -149.88409423828125,
-      "loss": 0.4736,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -5.400292873382568,
-      "rewards/margins": 2.478921890258789,
-      "rewards/rejected": -7.879214763641357,
-      "step": 6790
-    },
-    {
-      "epoch": 0.5434565434565435,
-      "grad_norm": 14.481913566589355,
-      "learning_rate": 2.557190969155028e-06,
-      "logits/chosen": -11.737726211547852,
-      "logits/rejected": -11.736207962036133,
-      "logps/chosen": -138.1727294921875,
-      "logps/rejected": -166.3176727294922,
-      "loss": 0.6222,
-      "rewards/accuracies": 0.7125000357627869,
-      "rewards/chosen": -6.413295745849609,
-      "rewards/margins": 2.5197951793670654,
-      "rewards/rejected": -8.93309211730957,
-      "step": 6800
-    },
-    {
-      "epoch": 0.5442557442557443,
-      "grad_norm": 14.624731063842773,
-      "learning_rate": 2.5502174640803563e-06,
-      "logits/chosen": -11.793267250061035,
-      "logits/rejected": -11.738395690917969,
-      "logps/chosen": -135.68045043945312,
-      "logps/rejected": -165.3558807373047,
-      "loss": 0.3013,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -6.361385822296143,
-      "rewards/margins": 2.90442156791687,
-      "rewards/rejected": -9.265807151794434,
-      "step": 6810
-    },
-    {
-      "epoch": 0.545054945054945,
-      "grad_norm": 19.45505714416504,
-      "learning_rate": 2.543243568095071e-06,
-      "logits/chosen": -11.760348320007324,
-      "logits/rejected": -11.717044830322266,
-      "logps/chosen": -129.8686981201172,
-      "logps/rejected": -160.18338012695312,
-      "loss": 0.3429,
-      "rewards/accuracies": 0.8375000357627869,
-      "rewards/chosen": -5.758988380432129,
-      "rewards/margins": 2.9481396675109863,
-      "rewards/rejected": -8.707127571105957,
-      "step": 6820
-    },
-    {
-      "epoch": 0.5458541458541458,
-      "grad_norm": 12.527294158935547,
-      "learning_rate": 2.5362693354864633e-06,
-      "logits/chosen": -11.742194175720215,
-      "logits/rejected": -11.664824485778809,
-      "logps/chosen": -133.44761657714844,
-      "logps/rejected": -164.05215454101562,
-      "loss": 0.3524,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -6.2009992599487305,
-      "rewards/margins": 2.690530300140381,
-      "rewards/rejected": -8.89153003692627,
-      "step": 6830
-    },
-    {
-      "epoch": 0.5466533466533466,
-      "grad_norm": 11.662212371826172,
-      "learning_rate": 2.52929482054444e-06,
-      "logits/chosen": -11.61274242401123,
-      "logits/rejected": -11.704111099243164,
-      "logps/chosen": -134.6204833984375,
-      "logps/rejected": -168.80142211914062,
-      "loss": 0.3746,
-      "rewards/accuracies": 0.8375000357627869,
-      "rewards/chosen": -6.361228942871094,
-      "rewards/margins": 2.601390838623047,
-      "rewards/rejected": -8.962618827819824,
-      "step": 6840
-    },
-    {
-      "epoch": 0.5474525474525475,
-      "grad_norm": 3.937527656555176,
-      "learning_rate": 2.5223200775611088e-06,
-      "logits/chosen": -11.758200645446777,
-      "logits/rejected": -11.731139183044434,
-      "logps/chosen": -135.62171936035156,
-      "logps/rejected": -163.55113220214844,
-      "loss": 0.4414,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -6.344542980194092,
-      "rewards/margins": 2.751279354095459,
-      "rewards/rejected": -9.09582233428955,
-      "step": 6850
-    },
-    {
-      "epoch": 0.5482517482517483,
-      "grad_norm": 26.53714942932129,
-      "learning_rate": 2.5153451608303496e-06,
-      "logits/chosen": -11.395251274108887,
-      "logits/rejected": -11.532169342041016,
-      "logps/chosen": -134.3628387451172,
-      "logps/rejected": -174.730224609375,
-      "loss": 0.3843,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -6.299798488616943,
-      "rewards/margins": 2.9694130420684814,
-      "rewards/rejected": -9.26921272277832,
-      "step": 6860
-    },
-    {
-      "epoch": 0.5490509490509491,
-      "grad_norm": 5.707951068878174,
-      "learning_rate": 2.508370124647398e-06,
-      "logits/chosen": -11.675009727478027,
-      "logits/rejected": -11.641143798828125,
-      "logps/chosen": -131.61106872558594,
-      "logps/rejected": -169.80726623535156,
-      "loss": 0.3232,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -5.924302577972412,
-      "rewards/margins": 3.168771505355835,
-      "rewards/rejected": -9.093073844909668,
-      "step": 6870
-    },
-    {
-      "epoch": 0.5498501498501499,
-      "grad_norm": 10.27609634399414,
-      "learning_rate": 2.501395023308417e-06,
-      "logits/chosen": -11.786933898925781,
-      "logits/rejected": -11.572066307067871,
-      "logps/chosen": -137.42251586914062,
-      "logps/rejected": -165.23170471191406,
-      "loss": 0.2985,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": -6.281071662902832,
-      "rewards/margins": 3.019702911376953,
-      "rewards/rejected": -9.300774574279785,
-      "step": 6880
-    },
-    {
-      "epoch": 0.5506493506493506,
-      "grad_norm": 14.359272956848145,
-      "learning_rate": 2.494419911110078e-06,
-      "logits/chosen": -11.483604431152344,
-      "logits/rejected": -11.650732040405273,
-      "logps/chosen": -133.93240356445312,
-      "logps/rejected": -182.27554321289062,
-      "loss": 0.3212,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -6.0398077964782715,
-      "rewards/margins": 3.8503170013427734,
-      "rewards/rejected": -9.890124320983887,
-      "step": 6890
-    },
-    {
-      "epoch": 0.5514485514485514,
-      "grad_norm": 6.619143009185791,
-      "learning_rate": 2.487444842349137e-06,
-      "logits/chosen": -11.382094383239746,
-      "logits/rejected": -11.701674461364746,
-      "logps/chosen": -140.65806579589844,
-      "logps/rejected": -192.93771362304688,
-      "loss": 0.2937,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -6.495396614074707,
-      "rewards/margins": 4.221354961395264,
-      "rewards/rejected": -10.716750144958496,
-      "step": 6900
-    },
-    {
-      "epoch": 0.5522477522477522,
-      "grad_norm": 8.495231628417969,
-      "learning_rate": 2.480469871322012e-06,
-      "logits/chosen": -11.646524429321289,
-      "logits/rejected": -11.510838508605957,
-      "logps/chosen": -137.0931854248047,
-      "logps/rejected": -164.97488403320312,
-      "loss": 0.2923,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -6.126364231109619,
-      "rewards/margins": 3.2065727710723877,
-      "rewards/rejected": -9.332937240600586,
-      "step": 6910
-    },
-    {
-      "epoch": 0.5530469530469531,
-      "grad_norm": 13.335329055786133,
-      "learning_rate": 2.4734950523243585e-06,
-      "logits/chosen": -11.614421844482422,
-      "logits/rejected": -11.578577041625977,
-      "logps/chosen": -138.21836853027344,
-      "logps/rejected": -176.8801727294922,
-      "loss": 0.4081,
-      "rewards/accuracies": 0.8375000357627869,
-      "rewards/chosen": -6.809669494628906,
-      "rewards/margins": 3.36152720451355,
-      "rewards/rejected": -10.171197891235352,
-      "step": 6920
-    },
-    {
-      "epoch": 0.5538461538461539,
-      "grad_norm": 12.142842292785645,
-      "learning_rate": 2.4665204396506506e-06,
-      "logits/chosen": -11.637190818786621,
-      "logits/rejected": -11.552616119384766,
-      "logps/chosen": -132.9210205078125,
-      "logps/rejected": -162.3968505859375,
-      "loss": 0.361,
-      "rewards/accuracies": 0.7750000357627869,
-      "rewards/chosen": -6.1483588218688965,
-      "rewards/margins": 2.8956549167633057,
-      "rewards/rejected": -9.044013977050781,
-      "step": 6930
-    },
-    {
-      "epoch": 0.5546453546453547,
-      "grad_norm": 16.278545379638672,
-      "learning_rate": 2.459546087593754e-06,
-      "logits/chosen": -11.451531410217285,
-      "logits/rejected": -11.571043014526367,
-      "logps/chosen": -137.22300720214844,
-      "logps/rejected": -174.10533142089844,
-      "loss": 0.3569,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -6.477262020111084,
-      "rewards/margins": 3.052589178085327,
-      "rewards/rejected": -9.529850959777832,
-      "step": 6940
-    },
-    {
-      "epoch": 0.5554445554445554,
-      "grad_norm": 19.218730926513672,
-      "learning_rate": 2.4525720504445094e-06,
-      "logits/chosen": -11.449682235717773,
-      "logits/rejected": -11.61042594909668,
-      "logps/chosen": -135.9434814453125,
-      "logps/rejected": -180.7333221435547,
-      "loss": 0.3883,
-      "rewards/accuracies": 0.8375000357627869,
-      "rewards/chosen": -6.44287109375,
-      "rewards/margins": 3.520261764526367,
-      "rewards/rejected": -9.963133811950684,
-      "step": 6950
-    },
-    {
-      "epoch": 0.5562437562437562,
-      "grad_norm": 13.073826789855957,
-      "learning_rate": 2.445598382491303e-06,
-      "logits/chosen": -11.61707592010498,
-      "logits/rejected": -11.575032234191895,
-      "logps/chosen": -138.22373962402344,
-      "logps/rejected": -168.16453552246094,
-      "loss": 0.3805,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -6.686540126800537,
-      "rewards/margins": 2.725484848022461,
-      "rewards/rejected": -9.412025451660156,
-      "step": 6960
-    },
-    {
-      "epoch": 0.557042957042957,
-      "grad_norm": 15.138775825500488,
-      "learning_rate": 2.438625138019648e-06,
-      "logits/chosen": -11.641753196716309,
-      "logits/rejected": -11.598915100097656,
-      "logps/chosen": -136.12643432617188,
-      "logps/rejected": -172.71237182617188,
-      "loss": 0.2387,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": -6.391035556793213,
-      "rewards/margins": 3.441020965576172,
-      "rewards/rejected": -9.832056045532227,
-      "step": 6970
-    },
-    {
-      "epoch": 0.5578421578421578,
-      "grad_norm": 14.95441722869873,
-      "learning_rate": 2.431652371311761e-06,
-      "logits/chosen": -11.644039154052734,
-      "logits/rejected": -11.552374839782715,
-      "logps/chosen": -131.80564880371094,
-      "logps/rejected": -168.03323364257812,
-      "loss": 0.1461,
-      "rewards/accuracies": 0.9625000357627869,
-      "rewards/chosen": -6.098605632781982,
-      "rewards/margins": 3.4380176067352295,
-      "rewards/rejected": -9.536623001098633,
-      "step": 6980
-    },
-    {
-      "epoch": 0.5586413586413587,
-      "grad_norm": 32.34636688232422,
-      "learning_rate": 2.424680136646139e-06,
-      "logits/chosen": -11.327258110046387,
-      "logits/rejected": -11.61541748046875,
-      "logps/chosen": -135.111328125,
-      "logps/rejected": -198.33401489257812,
-      "loss": 0.2801,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -6.237724304199219,
-      "rewards/margins": 4.712153911590576,
-      "rewards/rejected": -10.949877738952637,
-      "step": 6990
-    },
-    {
-      "epoch": 0.5594405594405595,
-      "grad_norm": 15.639080047607422,
-      "learning_rate": 2.4177084882971383e-06,
-      "logits/chosen": -11.775609016418457,
-      "logits/rejected": -11.67325496673584,
-      "logps/chosen": -135.2359161376953,
-      "logps/rejected": -176.1598358154297,
-      "loss": 0.1936,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": -6.017864227294922,
-      "rewards/margins": 3.718736410140991,
-      "rewards/rejected": -9.736600875854492,
-      "step": 7000
-    },
-    {
-      "epoch": 0.5602397602397602,
-      "grad_norm": 20.159929275512695,
-      "learning_rate": 2.4107374805345503e-06,
-      "logits/chosen": -11.6835298538208,
-      "logits/rejected": -11.644970893859863,
-      "logps/chosen": -137.1764373779297,
-      "logps/rejected": -166.14791870117188,
-      "loss": 0.4313,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -6.661825656890869,
-      "rewards/margins": 2.7629051208496094,
-      "rewards/rejected": -9.424731254577637,
-      "step": 7010
-    },
-    {
-      "epoch": 0.561038961038961,
-      "grad_norm": 21.4606990814209,
-      "learning_rate": 2.403767167623182e-06,
-      "logits/chosen": -11.821564674377441,
-      "logits/rejected": -11.74991512298584,
-      "logps/chosen": -133.5713348388672,
-      "logps/rejected": -171.05905151367188,
-      "loss": 0.2655,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -6.1799540519714355,
-      "rewards/margins": 3.628549337387085,
-      "rewards/rejected": -9.808504104614258,
-      "step": 7020
-    },
-    {
-      "epoch": 0.5618381618381618,
-      "grad_norm": 5.711723804473877,
-      "learning_rate": 2.396797603822427e-06,
-      "logits/chosen": -11.704976081848145,
-      "logits/rejected": -11.744967460632324,
-      "logps/chosen": -142.4958038330078,
-      "logps/rejected": -183.40602111816406,
-      "loss": 0.2698,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": -6.731109619140625,
-      "rewards/margins": 3.8384969234466553,
-      "rewards/rejected": -10.56960678100586,
-      "step": 7030
-    },
-    {
-      "epoch": 0.5626373626373626,
-      "grad_norm": 6.017128944396973,
-      "learning_rate": 2.3898288433858533e-06,
-      "logits/chosen": -11.8095064163208,
-      "logits/rejected": -11.813130378723145,
-      "logps/chosen": -134.5272674560547,
-      "logps/rejected": -168.5955047607422,
-      "loss": 0.4139,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -6.269063472747803,
-      "rewards/margins": 3.3533921241760254,
-      "rewards/rejected": -9.622456550598145,
-      "step": 7040
-    },
-    {
-      "epoch": 0.5634365634365635,
-      "grad_norm": 6.567190647125244,
-      "learning_rate": 2.3828609405607695e-06,
-      "logits/chosen": -11.842195510864258,
-      "logits/rejected": -11.711915969848633,
-      "logps/chosen": -139.6970977783203,
-      "logps/rejected": -174.705322265625,
-      "loss": 0.3947,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -6.268485069274902,
-      "rewards/margins": 3.5105018615722656,
-      "rewards/rejected": -9.778986930847168,
-      "step": 7050
-    },
-    {
-      "epoch": 0.5642357642357643,
-      "grad_norm": 17.429113388061523,
-      "learning_rate": 2.3758939495878133e-06,
-      "logits/chosen": -11.822334289550781,
-      "logits/rejected": -11.771531105041504,
-      "logps/chosen": -139.36573791503906,
-      "logps/rejected": -168.2772674560547,
-      "loss": 0.4196,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -6.192044734954834,
-      "rewards/margins": 3.0493838787078857,
-      "rewards/rejected": -9.241429328918457,
-      "step": 7060
-    },
-    {
-      "epoch": 0.5650349650349651,
-      "grad_norm": 29.083621978759766,
-      "learning_rate": 2.368927924700519e-06,
-      "logits/chosen": -11.639960289001465,
-      "logits/rejected": -11.651318550109863,
-      "logps/chosen": -136.7073211669922,
-      "logps/rejected": -168.293212890625,
-      "loss": 0.5914,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -6.656527996063232,
-      "rewards/margins": 2.682825803756714,
-      "rewards/rejected": -9.339354515075684,
-      "step": 7070
-    },
-    {
-      "epoch": 0.5658341658341658,
-      "grad_norm": 23.236740112304688,
-      "learning_rate": 2.361962920124906e-06,
-      "logits/chosen": -11.70814037322998,
-      "logits/rejected": -11.660468101501465,
-      "logps/chosen": -137.8658905029297,
-      "logps/rejected": -167.1419219970703,
-      "loss": 0.4809,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -6.641336917877197,
-      "rewards/margins": 2.169726610183716,
-      "rewards/rejected": -8.811062812805176,
-      "step": 7080
-    },
-    {
-      "epoch": 0.5666333666333666,
-      "grad_norm": 11.2471923828125,
-      "learning_rate": 2.3549989900790483e-06,
-      "logits/chosen": -11.647383689880371,
-      "logits/rejected": -11.612262725830078,
-      "logps/chosen": -129.31448364257812,
-      "logps/rejected": -161.3763427734375,
-      "loss": 0.2951,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -5.5792717933654785,
-      "rewards/margins": 3.1086349487304688,
-      "rewards/rejected": -8.687907218933105,
-      "step": 7090
-    },
-    {
-      "epoch": 0.5674325674325674,
-      "grad_norm": 20.049741744995117,
-      "learning_rate": 2.3480361887726543e-06,
-      "logits/chosen": -11.605853080749512,
-      "logits/rejected": -11.535835266113281,
-      "logps/chosen": -133.00050354003906,
-      "logps/rejected": -170.5866241455078,
-      "loss": 0.2957,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -5.958683490753174,
-      "rewards/margins": 3.6147267818450928,
-      "rewards/rejected": -9.573410034179688,
-      "step": 7100
-    },
-    {
-      "epoch": 0.5682317682317682,
-      "grad_norm": 5.025212287902832,
-      "learning_rate": 2.341074570406649e-06,
-      "logits/chosen": -11.57744312286377,
-      "logits/rejected": -11.561944007873535,
-      "logps/chosen": -129.87149047851562,
-      "logps/rejected": -160.2803192138672,
-      "loss": 0.4334,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": -5.91774320602417,
-      "rewards/margins": 2.6352014541625977,
-      "rewards/rejected": -8.55294418334961,
-      "step": 7110
-    },
-    {
-      "epoch": 0.5690309690309691,
-      "grad_norm": 29.120363235473633,
-      "learning_rate": 2.3341141891727455e-06,
-      "logits/chosen": -11.528254508972168,
-      "logits/rejected": -11.633169174194336,
-      "logps/chosen": -136.9733123779297,
-      "logps/rejected": -179.38023376464844,
-      "loss": 0.3707,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -6.5856757164001465,
-      "rewards/margins": 3.35172438621521,
-      "rewards/rejected": -9.937399864196777,
-      "step": 7120
-    },
-    {
-      "epoch": 0.5698301698301699,
-      "grad_norm": 29.150190353393555,
-      "learning_rate": 2.327155099253029e-06,
-      "logits/chosen": -11.719690322875977,
-      "logits/rejected": -11.611499786376953,
-      "logps/chosen": -139.12258911132812,
-      "logps/rejected": -171.97805786132812,
-      "loss": 0.5197,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -6.766092777252197,
-      "rewards/margins": 2.636690855026245,
-      "rewards/rejected": -9.402783393859863,
-      "step": 7130
-    },
-    {
-      "epoch": 0.5706293706293706,
-      "grad_norm": 20.327125549316406,
-      "learning_rate": 2.320197354819533e-06,
-      "logits/chosen": -11.680355072021484,
-      "logits/rejected": -11.572413444519043,
-      "logps/chosen": -130.0739288330078,
-      "logps/rejected": -167.05996704101562,
-      "loss": 0.2657,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -5.997226238250732,
-      "rewards/margins": 3.4760901927948,
-      "rewards/rejected": -9.473316192626953,
-      "step": 7140
-    },
-    {
-      "epoch": 0.5714285714285714,
-      "grad_norm": 4.4441609382629395,
-      "learning_rate": 2.313241010033816e-06,
-      "logits/chosen": -11.671097755432129,
-      "logits/rejected": -11.7160062789917,
-      "logps/chosen": -136.84556579589844,
-      "logps/rejected": -171.6093292236328,
-      "loss": 0.2819,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": -6.346888065338135,
-      "rewards/margins": 3.2735469341278076,
-      "rewards/rejected": -9.62043571472168,
-      "step": 7150
-    },
-    {
-      "epoch": 0.5722277722277722,
-      "grad_norm": 17.469314575195312,
-      "learning_rate": 2.3062861190465406e-06,
-      "logits/chosen": -11.597847938537598,
-      "logits/rejected": -11.6502046585083,
-      "logps/chosen": -141.5410614013672,
-      "logps/rejected": -185.66099548339844,
-      "loss": 0.2744,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -6.66350793838501,
-      "rewards/margins": 3.9067444801330566,
-      "rewards/rejected": -10.570252418518066,
-      "step": 7160
-    },
-    {
-      "epoch": 0.573026973026973,
-      "grad_norm": 28.554418563842773,
-      "learning_rate": 2.2993327359970543e-06,
-      "logits/chosen": -11.666646003723145,
-      "logits/rejected": -11.741533279418945,
-      "logps/chosen": -135.6744842529297,
-      "logps/rejected": -182.89093017578125,
-      "loss": 0.2811,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -6.476232051849365,
-      "rewards/margins": 3.928942918777466,
-      "rewards/rejected": -10.40517520904541,
-      "step": 7170
-    },
-    {
-      "epoch": 0.5738261738261738,
-      "grad_norm": 22.679075241088867,
-      "learning_rate": 2.292380915012965e-06,
-      "logits/chosen": -11.87631893157959,
-      "logits/rejected": -11.814967155456543,
-      "logps/chosen": -145.776611328125,
-      "logps/rejected": -179.8468780517578,
-      "loss": 0.431,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -7.040016174316406,
-      "rewards/margins": 3.4664227962493896,
-      "rewards/rejected": -10.506439208984375,
-      "step": 7180
-    },
-    {
-      "epoch": 0.5746253746253747,
-      "grad_norm": 14.531339645385742,
-      "learning_rate": 2.2854307102097203e-06,
-      "logits/chosen": -11.632336616516113,
-      "logits/rejected": -11.706696510314941,
-      "logps/chosen": -145.1896514892578,
-      "logps/rejected": -180.5661163330078,
-      "loss": 0.3684,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -7.195313453674316,
-      "rewards/margins": 3.159334897994995,
-      "rewards/rejected": -10.354647636413574,
-      "step": 7190
-    },
-    {
-      "epoch": 0.5754245754245755,
-      "grad_norm": 18.591238021850586,
-      "learning_rate": 2.2784821756901897e-06,
-      "logits/chosen": -11.817484855651855,
-      "logits/rejected": -11.724958419799805,
-      "logps/chosen": -137.12351989746094,
-      "logps/rejected": -165.34730529785156,
-      "loss": 0.3226,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -6.445562839508057,
-      "rewards/margins": 2.863682985305786,
-      "rewards/rejected": -9.309245109558105,
-      "step": 7200
-    },
-    {
-      "epoch": 0.5762237762237762,
-      "grad_norm": 14.905205726623535,
-      "learning_rate": 2.271535365544237e-06,
-      "logits/chosen": -11.813508033752441,
-      "logits/rejected": -11.71204662322998,
-      "logps/chosen": -131.49957275390625,
-      "logps/rejected": -165.07249450683594,
-      "loss": 0.4205,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -5.962726593017578,
-      "rewards/margins": 2.8882148265838623,
-      "rewards/rejected": -8.85094165802002,
-      "step": 7210
-    },
-    {
-      "epoch": 0.577022977022977,
-      "grad_norm": 28.541269302368164,
-      "learning_rate": 2.2645903338483067e-06,
-      "logits/chosen": -11.709794998168945,
-      "logits/rejected": -11.729889869689941,
-      "logps/chosen": -134.0455322265625,
-      "logps/rejected": -171.5481414794922,
-      "loss": 0.3351,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -6.145474910736084,
-      "rewards/margins": 3.5605645179748535,
-      "rewards/rejected": -9.706038475036621,
-      "step": 7220
-    },
-    {
-      "epoch": 0.5778221778221778,
-      "grad_norm": 6.713076114654541,
-      "learning_rate": 2.257647134664995e-06,
-      "logits/chosen": -11.831107139587402,
-      "logits/rejected": -11.820475578308105,
-      "logps/chosen": -141.7353973388672,
-      "logps/rejected": -175.28956604003906,
-      "loss": 0.4453,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -7.017711162567139,
-      "rewards/margins": 2.9292123317718506,
-      "rewards/rejected": -9.94692325592041,
-      "step": 7230
-    },
-    {
-      "epoch": 0.5786213786213786,
-      "grad_norm": 24.139158248901367,
-      "learning_rate": 2.2507058220426363e-06,
-      "logits/chosen": -11.819141387939453,
-      "logits/rejected": -11.812275886535645,
-      "logps/chosen": -136.2567138671875,
-      "logps/rejected": -166.5044708251953,
-      "loss": 0.2744,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": -6.4751787185668945,
-      "rewards/margins": 3.2244858741760254,
-      "rewards/rejected": -9.699664115905762,
-      "step": 7240
-    },
-    {
-      "epoch": 0.5794205794205795,
-      "grad_norm": 28.02431869506836,
-      "learning_rate": 2.243766450014878e-06,
-      "logits/chosen": -11.851669311523438,
-      "logits/rejected": -11.71220588684082,
-      "logps/chosen": -130.9696044921875,
-      "logps/rejected": -162.55459594726562,
-      "loss": 0.3657,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -6.227396488189697,
-      "rewards/margins": 2.9397330284118652,
-      "rewards/rejected": -9.167129516601562,
-      "step": 7250
-    },
-    {
-      "epoch": 0.5802197802197803,
-      "grad_norm": 23.626102447509766,
-      "learning_rate": 2.2368290726002612e-06,
-      "logits/chosen": -11.816548347473145,
-      "logits/rejected": -11.808770179748535,
-      "logps/chosen": -143.23057556152344,
-      "logps/rejected": -174.32748413085938,
-      "loss": 0.359,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -6.896770000457764,
-      "rewards/margins": 2.985402822494507,
-      "rewards/rejected": -9.882172584533691,
-      "step": 7260
-    },
-    {
-      "epoch": 0.581018981018981,
-      "grad_norm": 18.646875381469727,
-      "learning_rate": 2.229893743801802e-06,
-      "logits/chosen": -11.771193504333496,
-      "logits/rejected": -11.754602432250977,
-      "logps/chosen": -134.02288818359375,
-      "logps/rejected": -174.69639587402344,
-      "loss": 0.297,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -6.309868812561035,
-      "rewards/margins": 3.3649346828460693,
-      "rewards/rejected": -9.674803733825684,
-      "step": 7270
-    },
-    {
-      "epoch": 0.5818181818181818,
-      "grad_norm": 12.013740539550781,
-      "learning_rate": 2.222960517606568e-06,
-      "logits/chosen": -11.723421096801758,
-      "logits/rejected": -11.799936294555664,
-      "logps/chosen": -129.4722900390625,
-      "logps/rejected": -172.6265411376953,
-      "loss": 0.2266,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": -5.641164302825928,
-      "rewards/margins": 3.7413856983184814,
-      "rewards/rejected": -9.382550239562988,
-      "step": 7280
-    },
-    {
-      "epoch": 0.5826173826173826,
-      "grad_norm": 19.346921920776367,
-      "learning_rate": 2.216029447985257e-06,
-      "logits/chosen": -11.976000785827637,
-      "logits/rejected": -11.845633506774902,
-      "logps/chosen": -142.5968017578125,
-      "logps/rejected": -173.55303955078125,
-      "loss": 0.3285,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -6.888112545013428,
-      "rewards/margins": 2.92460298538208,
-      "rewards/rejected": -9.812715530395508,
-      "step": 7290
-    },
-    {
-      "epoch": 0.5834165834165834,
-      "grad_norm": 20.232454299926758,
-      "learning_rate": 2.209100588891784e-06,
-      "logits/chosen": -11.874612808227539,
-      "logits/rejected": -11.819443702697754,
-      "logps/chosen": -140.20346069335938,
-      "logps/rejected": -168.8477783203125,
-      "loss": 0.4696,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -6.774542331695557,
-      "rewards/margins": 2.758819818496704,
-      "rewards/rejected": -9.53336238861084,
-      "step": 7300
-    },
-    {
-      "epoch": 0.5842157842157842,
-      "grad_norm": 25.89459228515625,
-      "learning_rate": 2.2021739942628523e-06,
-      "logits/chosen": -11.719486236572266,
-      "logits/rejected": -11.66235065460205,
-      "logps/chosen": -142.28126525878906,
-      "logps/rejected": -169.52713012695312,
-      "loss": 0.4287,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -6.982347011566162,
-      "rewards/margins": 3.118687391281128,
-      "rewards/rejected": -10.101035118103027,
-      "step": 7310
-    },
-    {
-      "epoch": 0.5850149850149851,
-      "grad_norm": 14.937485694885254,
-      "learning_rate": 2.19524971801754e-06,
-      "logits/chosen": -11.89107894897461,
-      "logits/rejected": -11.849514961242676,
-      "logps/chosen": -138.71609497070312,
-      "logps/rejected": -166.32655334472656,
-      "loss": 0.3933,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -6.49770975112915,
-      "rewards/margins": 2.7181451320648193,
-      "rewards/rejected": -9.215853691101074,
-      "step": 7320
-    },
-    {
-      "epoch": 0.5858141858141858,
-      "grad_norm": 21.55057716369629,
-      "learning_rate": 2.188327814056879e-06,
-      "logits/chosen": -11.80604076385498,
-      "logits/rejected": -11.749602317810059,
-      "logps/chosen": -135.71804809570312,
-      "logps/rejected": -173.85256958007812,
-      "loss": 0.3159,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -6.362946033477783,
-      "rewards/margins": 3.4700915813446045,
-      "rewards/rejected": -9.833038330078125,
-      "step": 7330
-    },
-    {
-      "epoch": 0.5866133866133866,
-      "grad_norm": 24.583791732788086,
-      "learning_rate": 2.181408336263431e-06,
-      "logits/chosen": -11.832521438598633,
-      "logits/rejected": -11.709059715270996,
-      "logps/chosen": -129.7410430908203,
-      "logps/rejected": -165.6168975830078,
-      "loss": 0.2412,
-      "rewards/accuracies": 0.9000000357627869,
-      "rewards/chosen": -5.796443939208984,
-      "rewards/margins": 3.4826927185058594,
-      "rewards/rejected": -9.279136657714844,
-      "step": 7340
-    },
-    {
-      "epoch": 0.5874125874125874,
-      "grad_norm": 15.538948059082031,
-      "learning_rate": 2.1744913385008754e-06,
-      "logits/chosen": -11.968062400817871,
-      "logits/rejected": -11.912883758544922,
-      "logps/chosen": -133.07554626464844,
-      "logps/rejected": -172.6363983154297,
-      "loss": 0.2909,
-      "rewards/accuracies": 0.9000000357627869,
-      "rewards/chosen": -6.1756696701049805,
-      "rewards/margins": 3.65207839012146,
-      "rewards/rejected": -9.827747344970703,
-      "step": 7350
-    },
-    {
-      "epoch": 0.5882117882117882,
-      "grad_norm": 6.073270797729492,
-      "learning_rate": 2.1675768746135834e-06,
-      "logits/chosen": -11.79679012298584,
-      "logits/rejected": -11.776970863342285,
-      "logps/chosen": -144.96511840820312,
-      "logps/rejected": -182.63917541503906,
-      "loss": 0.3296,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -6.888614177703857,
-      "rewards/margins": 3.526427984237671,
-      "rewards/rejected": -10.41504192352295,
-      "step": 7360
-    },
-    {
-      "epoch": 0.589010989010989,
-      "grad_norm": 9.234392166137695,
-      "learning_rate": 2.1606649984262026e-06,
-      "logits/chosen": -11.908934593200684,
-      "logits/rejected": -11.895230293273926,
-      "logps/chosen": -138.29873657226562,
-      "logps/rejected": -171.9561309814453,
-      "loss": 0.433,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -6.586507320404053,
-      "rewards/margins": 3.242464542388916,
-      "rewards/rejected": -9.828972816467285,
-      "step": 7370
-    },
-    {
-      "epoch": 0.5898101898101898,
-      "grad_norm": 23.896827697753906,
-      "learning_rate": 2.1537557637432373e-06,
-      "logits/chosen": -11.898290634155273,
-      "logits/rejected": -11.851018905639648,
-      "logps/chosen": -136.0522918701172,
-      "logps/rejected": -167.02296447753906,
-      "loss": 0.3718,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -6.579265117645264,
-      "rewards/margins": 2.880708694458008,
-      "rewards/rejected": -9.45997428894043,
-      "step": 7380
-    },
-    {
-      "epoch": 0.5906093906093907,
-      "grad_norm": 14.000650405883789,
-      "learning_rate": 2.14684922434863e-06,
-      "logits/chosen": -11.770608901977539,
-      "logits/rejected": -11.747450828552246,
-      "logps/chosen": -145.64353942871094,
-      "logps/rejected": -195.82298278808594,
-      "loss": 0.2345,
-      "rewards/accuracies": 0.9000000357627869,
-      "rewards/chosen": -7.322200775146484,
-      "rewards/margins": 4.2243781089782715,
-      "rewards/rejected": -11.546579360961914,
-      "step": 7390
-    },
-    {
-      "epoch": 0.5914085914085914,
-      "grad_norm": 23.516277313232422,
-      "learning_rate": 2.13994543400534e-06,
-      "logits/chosen": -11.839532852172852,
-      "logits/rejected": -11.901089668273926,
-      "logps/chosen": -130.7752227783203,
-      "logps/rejected": -164.57212829589844,
-      "loss": 0.3316,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -5.908977031707764,
-      "rewards/margins": 3.042396306991577,
-      "rewards/rejected": -8.951373100280762,
-      "step": 7400
-    },
-    {
-      "epoch": 0.5922077922077922,
-      "grad_norm": 8.04878044128418,
-      "learning_rate": 2.1330444464549294e-06,
-      "logits/chosen": -11.905807495117188,
-      "logits/rejected": -11.82955265045166,
-      "logps/chosen": -136.5023956298828,
-      "logps/rejected": -167.64791870117188,
-      "loss": 0.3431,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -6.316127300262451,
-      "rewards/margins": 3.2832679748535156,
-      "rewards/rejected": -9.599395751953125,
-      "step": 7410
-    },
-    {
-      "epoch": 0.593006993006993,
-      "grad_norm": 15.6613187789917,
-      "learning_rate": 2.12614631541714e-06,
-      "logits/chosen": -11.857256889343262,
-      "logits/rejected": -11.838902473449707,
-      "logps/chosen": -140.44126892089844,
-      "logps/rejected": -169.483154296875,
-      "loss": 0.51,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -6.7558135986328125,
-      "rewards/margins": 2.6962313652038574,
-      "rewards/rejected": -9.452044486999512,
-      "step": 7420
-    },
-    {
-      "epoch": 0.5938061938061938,
-      "grad_norm": 8.334183692932129,
-      "learning_rate": 2.11925109458948e-06,
-      "logits/chosen": -11.655596733093262,
-      "logits/rejected": -11.752328872680664,
-      "logps/chosen": -143.51341247558594,
-      "logps/rejected": -174.9862823486328,
-      "loss": 0.5089,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -6.837812900543213,
-      "rewards/margins": 2.854212522506714,
-      "rewards/rejected": -9.692026138305664,
-      "step": 7430
-    },
-    {
-      "epoch": 0.5946053946053946,
-      "grad_norm": 11.822402954101562,
-      "learning_rate": 2.1123588376468002e-06,
-      "logits/chosen": -11.773293495178223,
-      "logits/rejected": -11.767990112304688,
-      "logps/chosen": -132.2900848388672,
-      "logps/rejected": -157.36392211914062,
-      "loss": 0.342,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -6.004176139831543,
-      "rewards/margins": 2.783604145050049,
-      "rewards/rejected": -8.78778076171875,
-      "step": 7440
-    },
-    {
-      "epoch": 0.5954045954045954,
-      "grad_norm": 21.490421295166016,
-      "learning_rate": 2.1054695982408835e-06,
-      "logits/chosen": -11.79708480834961,
-      "logits/rejected": -11.739686965942383,
-      "logps/chosen": -129.34506225585938,
-      "logps/rejected": -160.83860778808594,
-      "loss": 0.3376,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -5.65243673324585,
-      "rewards/margins": 3.1864545345306396,
-      "rewards/rejected": -8.83889102935791,
-      "step": 7450
-    },
-    {
-      "epoch": 0.5962037962037962,
-      "grad_norm": 19.79187774658203,
-      "learning_rate": 2.098583430000021e-06,
-      "logits/chosen": -11.772259712219238,
-      "logits/rejected": -11.739084243774414,
-      "logps/chosen": -134.60317993164062,
-      "logps/rejected": -166.33535766601562,
-      "loss": 0.3493,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -6.131229400634766,
-      "rewards/margins": 2.9408345222473145,
-      "rewards/rejected": -9.072063446044922,
-      "step": 7460
-    },
-    {
-      "epoch": 0.597002997002997,
-      "grad_norm": 12.900521278381348,
-      "learning_rate": 2.0917003865285957e-06,
-      "logits/chosen": -11.751112937927246,
-      "logits/rejected": -11.60622787475586,
-      "logps/chosen": -131.0332489013672,
-      "logps/rejected": -163.39964294433594,
-      "loss": 0.2372,
-      "rewards/accuracies": 0.9000000357627869,
-      "rewards/chosen": -5.856359004974365,
-      "rewards/margins": 3.285870313644409,
-      "rewards/rejected": -9.142229080200195,
-      "step": 7470
-    },
-    {
-      "epoch": 0.5978021978021978,
-      "grad_norm": 5.033387184143066,
-      "learning_rate": 2.0848205214066696e-06,
-      "logits/chosen": -11.830767631530762,
-      "logits/rejected": -11.734435081481934,
-      "logps/chosen": -135.92637634277344,
-      "logps/rejected": -172.6446075439453,
-      "loss": 0.2797,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": -6.440777778625488,
-      "rewards/margins": 3.401015520095825,
-      "rewards/rejected": -9.84179401397705,
-      "step": 7480
-    },
-    {
-      "epoch": 0.5986013986013986,
-      "grad_norm": 19.945152282714844,
-      "learning_rate": 2.0779438881895588e-06,
-      "logits/chosen": -11.79990005493164,
-      "logits/rejected": -11.725126266479492,
-      "logps/chosen": -135.30685424804688,
-      "logps/rejected": -164.91278076171875,
-      "loss": 0.4466,
-      "rewards/accuracies": 0.8375000357627869,
-      "rewards/chosen": -6.066940784454346,
-      "rewards/margins": 3.07889986038208,
-      "rewards/rejected": -9.145840644836426,
-      "step": 7490
-    },
-    {
-      "epoch": 0.5994005994005994,
-      "grad_norm": 8.44441032409668,
-      "learning_rate": 2.0710705404074248e-06,
-      "logits/chosen": -11.647852897644043,
-      "logits/rejected": -11.723174095153809,
-      "logps/chosen": -134.59754943847656,
-      "logps/rejected": -174.5525665283203,
-      "loss": 0.3067,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -5.988490581512451,
-      "rewards/margins": 3.6994380950927734,
-      "rewards/rejected": -9.687929153442383,
-      "step": 7500
-    },
-    {
-      "epoch": 0.6001998001998002,
-      "grad_norm": 9.705737113952637,
-      "learning_rate": 2.064200531564852e-06,
-      "logits/chosen": -11.800305366516113,
-      "logits/rejected": -11.705087661743164,
-      "logps/chosen": -134.26145935058594,
-      "logps/rejected": -159.4337158203125,
-      "loss": 0.42,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -6.30759859085083,
-      "rewards/margins": 2.557569980621338,
-      "rewards/rejected": -8.865168571472168,
-      "step": 7510
-    },
-    {
-      "epoch": 0.600999000999001,
-      "grad_norm": 17.974916458129883,
-      "learning_rate": 2.057333915140436e-06,
-      "logits/chosen": -11.721623420715332,
-      "logits/rejected": -11.687102317810059,
-      "logps/chosen": -132.43795776367188,
-      "logps/rejected": -160.4076385498047,
-      "loss": 0.3576,
-      "rewards/accuracies": 0.7750000357627869,
-      "rewards/chosen": -6.114051818847656,
-      "rewards/margins": 2.8029837608337402,
-      "rewards/rejected": -8.917036056518555,
-      "step": 7520
-    },
-    {
-      "epoch": 0.6017982017982018,
-      "grad_norm": 8.903719902038574,
-      "learning_rate": 2.050470744586361e-06,
-      "logits/chosen": -11.7128267288208,
-      "logits/rejected": -11.649970054626465,
-      "logps/chosen": -131.73008728027344,
-      "logps/rejected": -168.0287322998047,
-      "loss": 0.3465,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -6.430423736572266,
-      "rewards/margins": 3.116658926010132,
-      "rewards/rejected": -9.547082901000977,
-      "step": 7530
-    },
-    {
-      "epoch": 0.6025974025974026,
-      "grad_norm": 28.28753089904785,
-      "learning_rate": 2.0436110733279903e-06,
-      "logits/chosen": -11.736186981201172,
-      "logits/rejected": -11.664861679077148,
-      "logps/chosen": -140.7863006591797,
-      "logps/rejected": -173.4311981201172,
-      "loss": 0.4711,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -6.536956310272217,
-      "rewards/margins": 3.3173999786376953,
-      "rewards/rejected": -9.85435676574707,
-      "step": 7540
-    },
-    {
-      "epoch": 0.6033966033966034,
-      "grad_norm": 5.7631378173828125,
-      "learning_rate": 2.036754954763445e-06,
-      "logits/chosen": -11.690491676330566,
-      "logits/rejected": -11.641989707946777,
-      "logps/chosen": -126.3610610961914,
-      "logps/rejected": -155.80697631835938,
-      "loss": 0.3711,
-      "rewards/accuracies": 0.8375000357627869,
-      "rewards/chosen": -5.4952712059021,
-      "rewards/margins": 2.84647798538208,
-      "rewards/rejected": -8.341748237609863,
-      "step": 7550
-    },
-    {
-      "epoch": 0.6041958041958042,
-      "grad_norm": 15.714640617370605,
-      "learning_rate": 2.0299024422631935e-06,
-      "logits/chosen": -11.448824882507324,
-      "logits/rejected": -11.5723876953125,
-      "logps/chosen": -138.2605438232422,
-      "logps/rejected": -191.513427734375,
-      "loss": 0.2363,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": -6.519647121429443,
-      "rewards/margins": 4.100674629211426,
-      "rewards/rejected": -10.620322227478027,
-      "step": 7560
-    },
-    {
-      "epoch": 0.604995004995005,
-      "grad_norm": 6.421223163604736,
-      "learning_rate": 2.023053589169629e-06,
-      "logits/chosen": -11.518157005310059,
-      "logits/rejected": -11.726150512695312,
-      "logps/chosen": -145.86155700683594,
-      "logps/rejected": -190.57566833496094,
-      "loss": 0.4637,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": -7.412997722625732,
-      "rewards/margins": 3.1876895427703857,
-      "rewards/rejected": -10.600687026977539,
-      "step": 7570
-    },
-    {
-      "epoch": 0.6057942057942058,
-      "grad_norm": 17.49582862854004,
-      "learning_rate": 2.016208448796664e-06,
-      "logits/chosen": -11.685585975646973,
-      "logits/rejected": -11.665175437927246,
-      "logps/chosen": -134.1170196533203,
-      "logps/rejected": -178.17271423339844,
-      "loss": 0.2991,
-      "rewards/accuracies": 0.8375000357627869,
-      "rewards/chosen": -6.360551357269287,
-      "rewards/margins": 3.50065541267395,
-      "rewards/rejected": -9.8612060546875,
-      "step": 7580
-    },
-    {
-      "epoch": 0.6065934065934065,
-      "grad_norm": 10.478436470031738,
-      "learning_rate": 2.0093670744293076e-06,
-      "logits/chosen": -11.785795211791992,
-      "logits/rejected": -11.638191223144531,
-      "logps/chosen": -139.64341735839844,
-      "logps/rejected": -176.122802734375,
-      "loss": 0.235,
-      "rewards/accuracies": 0.9000000357627869,
-      "rewards/chosen": -6.726115703582764,
-      "rewards/margins": 3.5022995471954346,
-      "rewards/rejected": -10.228415489196777,
-      "step": 7590
-    },
-    {
-      "epoch": 0.6073926073926074,
-      "grad_norm": 10.910778999328613,
-      "learning_rate": 2.00252951932325e-06,
-      "logits/chosen": -11.830448150634766,
-      "logits/rejected": -11.769044876098633,
-      "logps/chosen": -141.6128692626953,
-      "logps/rejected": -177.5415496826172,
-      "loss": 0.3714,
-      "rewards/accuracies": 0.8375000357627869,
-      "rewards/chosen": -7.083285808563232,
-      "rewards/margins": 3.236823320388794,
-      "rewards/rejected": -10.320109367370605,
-      "step": 7600
-    },
-    {
-      "epoch": 0.6081918081918082,
-      "grad_norm": 5.193593502044678,
-      "learning_rate": 1.9956958367044556e-06,
-      "logits/chosen": -11.796545028686523,
-      "logits/rejected": -11.719943046569824,
-      "logps/chosen": -136.192138671875,
-      "logps/rejected": -175.61273193359375,
-      "loss": 0.3025,
-      "rewards/accuracies": 0.9000000357627869,
-      "rewards/chosen": -6.514850616455078,
-      "rewards/margins": 3.5937154293060303,
-      "rewards/rejected": -10.108565330505371,
-      "step": 7610
-    },
-    {
-      "epoch": 0.608991008991009,
-      "grad_norm": 7.215260982513428,
-      "learning_rate": 1.988866079768742e-06,
-      "logits/chosen": -11.846189498901367,
-      "logits/rejected": -11.764154434204102,
-      "logps/chosen": -140.27761840820312,
-      "logps/rejected": -181.82554626464844,
-      "loss": 0.1825,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": -6.6040191650390625,
-      "rewards/margins": 3.760995626449585,
-      "rewards/rejected": -10.365015029907227,
-      "step": 7620
-    },
-    {
-      "epoch": 0.6097902097902098,
-      "grad_norm": 24.5975399017334,
-      "learning_rate": 1.9820403016813654e-06,
-      "logits/chosen": -11.917876243591309,
-      "logits/rejected": -11.835023880004883,
-      "logps/chosen": -145.0996856689453,
-      "logps/rejected": -175.0860595703125,
-      "loss": 0.4181,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -7.362113952636719,
-      "rewards/margins": 3.0609323978424072,
-      "rewards/rejected": -10.423047065734863,
-      "step": 7630
-    },
-    {
-      "epoch": 0.6105894105894106,
-      "grad_norm": 10.365233421325684,
-      "learning_rate": 1.975218555576614e-06,
-      "logits/chosen": -11.834147453308105,
-      "logits/rejected": -11.762421607971191,
-      "logps/chosen": -154.51683044433594,
-      "logps/rejected": -179.79820251464844,
-      "loss": 0.4891,
-      "rewards/accuracies": 0.7750000357627869,
-      "rewards/chosen": -8.335761070251465,
-      "rewards/margins": 2.4658353328704834,
-      "rewards/rejected": -10.801596641540527,
-      "step": 7640
-    },
-    {
-      "epoch": 0.6113886113886113,
-      "grad_norm": 27.7049617767334,
-      "learning_rate": 1.9684008945573868e-06,
-      "logits/chosen": -11.799275398254395,
-      "logits/rejected": -11.89445972442627,
-      "logps/chosen": -137.6669921875,
-      "logps/rejected": -180.056396484375,
-      "loss": 0.3949,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -6.349592685699463,
-      "rewards/margins": 3.630722761154175,
-      "rewards/rejected": -9.980315208435059,
-      "step": 7650
-    },
-    {
-      "epoch": 0.6121878121878122,
-      "grad_norm": 21.286104202270508,
-      "learning_rate": 1.9615873716947826e-06,
-      "logits/chosen": -11.911754608154297,
-      "logits/rejected": -11.789848327636719,
-      "logps/chosen": -144.7218475341797,
-      "logps/rejected": -183.15301513671875,
-      "loss": 0.2749,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -7.164471626281738,
-      "rewards/margins": 3.6273086071014404,
-      "rewards/rejected": -10.791780471801758,
-      "step": 7660
-    },
-    {
-      "epoch": 0.612987012987013,
-      "grad_norm": 12.904945373535156,
-      "learning_rate": 1.9547780400276894e-06,
-      "logits/chosen": -11.631792068481445,
-      "logits/rejected": -11.907357215881348,
-      "logps/chosen": -140.32591247558594,
-      "logps/rejected": -208.23159790039062,
-      "loss": 0.226,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": -7.169688701629639,
-      "rewards/margins": 4.8264594078063965,
-      "rewards/rejected": -11.996147155761719,
-      "step": 7670
-    },
-    {
-      "epoch": 0.6137862137862138,
-      "grad_norm": 9.08845329284668,
-      "learning_rate": 1.947972952562367e-06,
-      "logits/chosen": -11.924591064453125,
-      "logits/rejected": -11.830028533935547,
-      "logps/chosen": -146.22962951660156,
-      "logps/rejected": -177.83482360839844,
-      "loss": 0.3168,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -7.2979044914245605,
-      "rewards/margins": 3.254610776901245,
-      "rewards/rejected": -10.552515029907227,
-      "step": 7680
-    },
-    {
-      "epoch": 0.6145854145854146,
-      "grad_norm": 24.466474533081055,
-      "learning_rate": 1.941172162272038e-06,
-      "logits/chosen": -11.928359985351562,
-      "logits/rejected": -11.892593383789062,
-      "logps/chosen": -140.37429809570312,
-      "logps/rejected": -175.9022674560547,
-      "loss": 0.4531,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -6.549317836761475,
-      "rewards/margins": 3.375227451324463,
-      "rewards/rejected": -9.924545288085938,
-      "step": 7690
-    },
-    {
-      "epoch": 0.6153846153846154,
-      "grad_norm": 22.421144485473633,
-      "learning_rate": 1.934375722096476e-06,
-      "logits/chosen": -11.918984413146973,
-      "logits/rejected": -11.808549880981445,
-      "logps/chosen": -149.1699676513672,
-      "logps/rejected": -190.60423278808594,
-      "loss": 0.3033,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -7.526586055755615,
-      "rewards/margins": 3.572465658187866,
-      "rewards/rejected": -11.099051475524902,
-      "step": 7700
-    },
-    {
-      "epoch": 0.6161838161838162,
-      "grad_norm": 21.116573333740234,
-      "learning_rate": 1.927583684941588e-06,
-      "logits/chosen": -11.983835220336914,
-      "logits/rejected": -11.894403457641602,
-      "logps/chosen": -140.6542510986328,
-      "logps/rejected": -161.8164825439453,
-      "loss": 0.4075,
-      "rewards/accuracies": 0.8375000357627869,
-      "rewards/chosen": -6.755140781402588,
-      "rewards/margins": 2.6090028285980225,
-      "rewards/rejected": -9.364143371582031,
-      "step": 7710
-    },
-    {
-      "epoch": 0.6169830169830169,
-      "grad_norm": 21.25105094909668,
-      "learning_rate": 1.9207961036790102e-06,
-      "logits/chosen": -11.8851957321167,
-      "logits/rejected": -11.874221801757812,
-      "logps/chosen": -141.8541717529297,
-      "logps/rejected": -179.7147216796875,
-      "loss": 0.3484,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -6.89147424697876,
-      "rewards/margins": 3.592167615890503,
-      "rewards/rejected": -10.483642578125,
-      "step": 7720
-    },
-    {
-      "epoch": 0.6177822177822178,
-      "grad_norm": 34.456199645996094,
-      "learning_rate": 1.914013031145689e-06,
-      "logits/chosen": -11.994946479797363,
-      "logits/rejected": -11.91721248626709,
-      "logps/chosen": -144.0264434814453,
-      "logps/rejected": -168.73768615722656,
-      "loss": 0.5265,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -7.199904918670654,
-      "rewards/margins": 2.4753315448760986,
-      "rewards/rejected": -9.675236701965332,
-      "step": 7730
-    },
-    {
-      "epoch": 0.6185814185814186,
-      "grad_norm": 20.07630157470703,
-      "learning_rate": 1.9072345201434762e-06,
-      "logits/chosen": -11.889498710632324,
-      "logits/rejected": -11.834100723266602,
-      "logps/chosen": -138.1482391357422,
-      "logps/rejected": -167.1501922607422,
-      "loss": 0.3476,
-      "rewards/accuracies": 0.8375000357627869,
-      "rewards/chosen": -6.664390563964844,
-      "rewards/margins": 3.0665018558502197,
-      "rewards/rejected": -9.730891227722168,
-      "step": 7740
-    },
-    {
-      "epoch": 0.6193806193806194,
-      "grad_norm": 30.202720642089844,
-      "learning_rate": 1.9004606234387136e-06,
-      "logits/chosen": -11.906639099121094,
-      "logits/rejected": -11.892326354980469,
-      "logps/chosen": -141.24717712402344,
-      "logps/rejected": -165.0991973876953,
-      "loss": 0.511,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -6.966446876525879,
-      "rewards/margins": 2.272266149520874,
-      "rewards/rejected": -9.238713264465332,
-      "step": 7750
-    },
-    {
-      "epoch": 0.6201798201798202,
-      "grad_norm": 24.96922492980957,
-      "learning_rate": 1.8936913937618223e-06,
-      "logits/chosen": -11.73468017578125,
-      "logits/rejected": -11.824886322021484,
-      "logps/chosen": -134.58274841308594,
-      "logps/rejected": -169.0125732421875,
-      "loss": 0.3117,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -6.045121192932129,
-      "rewards/margins": 2.8591301441192627,
-      "rewards/rejected": -8.904251098632812,
-      "step": 7760
-    },
-    {
-      "epoch": 0.620979020979021,
-      "grad_norm": 5.264580249786377,
-      "learning_rate": 1.8869268838068974e-06,
-      "logits/chosen": -11.776805877685547,
-      "logits/rejected": -11.741408348083496,
-      "logps/chosen": -131.12840270996094,
-      "logps/rejected": -173.3931884765625,
-      "loss": 0.3123,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -5.892524242401123,
-      "rewards/margins": 3.6981360912323,
-      "rewards/rejected": -9.590662002563477,
-      "step": 7770
-    },
-    {
-      "epoch": 0.6217782217782217,
-      "grad_norm": 15.255341529846191,
-      "learning_rate": 1.8801671462312915e-06,
-      "logits/chosen": -11.7980375289917,
-      "logits/rejected": -11.830161094665527,
-      "logps/chosen": -139.07627868652344,
-      "logps/rejected": -171.51541137695312,
-      "loss": 0.4382,
-      "rewards/accuracies": 0.8375000357627869,
-      "rewards/chosen": -6.2788543701171875,
-      "rewards/margins": 3.0182015895843506,
-      "rewards/rejected": -9.297057151794434,
-      "step": 7780
-    },
-    {
-      "epoch": 0.6225774225774225,
-      "grad_norm": 11.021414756774902,
-      "learning_rate": 1.8734122336552064e-06,
-      "logits/chosen": -11.884256362915039,
-      "logits/rejected": -11.807990074157715,
-      "logps/chosen": -134.9803924560547,
-      "logps/rejected": -168.32986450195312,
-      "loss": 0.4141,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -6.500608921051025,
-      "rewards/margins": 3.0347039699554443,
-      "rewards/rejected": -9.53531265258789,
-      "step": 7790
-    },
-    {
-      "epoch": 0.6233766233766234,
-      "grad_norm": 16.86857032775879,
-      "learning_rate": 1.8666621986612862e-06,
-      "logits/chosen": -11.899696350097656,
-      "logits/rejected": -11.776812553405762,
-      "logps/chosen": -136.2554168701172,
-      "logps/rejected": -166.63600158691406,
-      "loss": 0.4472,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -5.7103095054626465,
-      "rewards/margins": 3.368211507797241,
-      "rewards/rejected": -9.078520774841309,
-      "step": 7800
-    },
-    {
-      "epoch": 0.6241758241758242,
-      "grad_norm": 13.539883613586426,
-      "learning_rate": 1.8599170937942052e-06,
-      "logits/chosen": -11.772984504699707,
-      "logits/rejected": -11.795088768005371,
-      "logps/chosen": -132.6141815185547,
-      "logps/rejected": -172.2427978515625,
-      "loss": 0.2926,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -6.096679210662842,
-      "rewards/margins": 3.412428617477417,
-      "rewards/rejected": -9.509108543395996,
-      "step": 7810
-    },
-    {
-      "epoch": 0.624975024975025,
-      "grad_norm": 9.423611640930176,
-      "learning_rate": 1.8531769715602592e-06,
-      "logits/chosen": -11.731941223144531,
-      "logits/rejected": -11.785901069641113,
-      "logps/chosen": -129.5788116455078,
-      "logps/rejected": -174.1603240966797,
-      "loss": 0.2647,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": -5.790372371673584,
-      "rewards/margins": 3.897062063217163,
-      "rewards/rejected": -9.687434196472168,
-      "step": 7820
-    },
-    {
-      "epoch": 0.6257742257742258,
-      "grad_norm": 20.105205535888672,
-      "learning_rate": 1.8464418844269613e-06,
-      "logits/chosen": -11.899773597717285,
-      "logits/rejected": -11.88874340057373,
-      "logps/chosen": -135.35647583007812,
-      "logps/rejected": -163.97122192382812,
-      "loss": 0.3659,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -6.088498592376709,
-      "rewards/margins": 2.9287052154541016,
-      "rewards/rejected": -9.017203330993652,
-      "step": 7830
-    },
-    {
-      "epoch": 0.6265734265734266,
-      "grad_norm": 19.505035400390625,
-      "learning_rate": 1.8397118848226243e-06,
-      "logits/chosen": -11.527631759643555,
-      "logits/rejected": -11.852806091308594,
-      "logps/chosen": -125.40199279785156,
-      "logps/rejected": -172.7395477294922,
-      "loss": 0.4511,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -5.882230281829834,
-      "rewards/margins": 3.1205761432647705,
-      "rewards/rejected": -9.002806663513184,
-      "step": 7840
-    },
-    {
-      "epoch": 0.6273726273726273,
-      "grad_norm": 20.985919952392578,
-      "learning_rate": 1.83298702513596e-06,
-      "logits/chosen": -11.820273399353027,
-      "logits/rejected": -11.838905334472656,
-      "logps/chosen": -133.88182067871094,
-      "logps/rejected": -167.5359649658203,
-      "loss": 0.4026,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -5.9572062492370605,
-      "rewards/margins": 3.2605140209198,
-      "rewards/rejected": -9.217720031738281,
-      "step": 7850
-    },
-    {
-      "epoch": 0.6281718281718282,
-      "grad_norm": 22.85596466064453,
-      "learning_rate": 1.8262673577156715e-06,
-      "logits/chosen": -11.791523933410645,
-      "logits/rejected": -11.800186157226562,
-      "logps/chosen": -129.60888671875,
-      "logps/rejected": -170.03489685058594,
-      "loss": 0.3262,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -5.9811692237854,
-      "rewards/margins": 3.4138295650482178,
-      "rewards/rejected": -9.394999504089355,
-      "step": 7860
-    },
-    {
-      "epoch": 0.628971028971029,
-      "grad_norm": 22.263011932373047,
-      "learning_rate": 1.819552934870039e-06,
-      "logits/chosen": -11.816177368164062,
-      "logits/rejected": -11.766512870788574,
-      "logps/chosen": -134.4183349609375,
-      "logps/rejected": -160.70606994628906,
-      "loss": 0.3487,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -6.456480503082275,
-      "rewards/margins": 2.6257550716400146,
-      "rewards/rejected": -9.082236289978027,
-      "step": 7870
-    },
-    {
-      "epoch": 0.6297702297702298,
-      "grad_norm": 16.189403533935547,
-      "learning_rate": 1.8128438088665203e-06,
-      "logits/chosen": -11.913474082946777,
-      "logits/rejected": -11.84186840057373,
-      "logps/chosen": -130.8267364501953,
-      "logps/rejected": -166.58152770996094,
-      "loss": 0.3789,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -5.943449974060059,
-      "rewards/margins": 3.0290629863739014,
-      "rewards/rejected": -8.972512245178223,
-      "step": 7880
-    },
-    {
-      "epoch": 0.6305694305694306,
-      "grad_norm": 22.53179359436035,
-      "learning_rate": 1.8061400319313413e-06,
-      "logits/chosen": -11.964155197143555,
-      "logits/rejected": -11.797599792480469,
-      "logps/chosen": -135.60662841796875,
-      "logps/rejected": -156.34146118164062,
-      "loss": 0.398,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": -5.974491119384766,
-      "rewards/margins": 2.4024009704589844,
-      "rewards/rejected": -8.37689208984375,
-      "step": 7890
-    },
-    {
-      "epoch": 0.6313686313686314,
-      "grad_norm": 4.610691547393799,
-      "learning_rate": 1.7994416562490857e-06,
-      "logits/chosen": -11.861818313598633,
-      "logits/rejected": -11.840315818786621,
-      "logps/chosen": -138.08766174316406,
-      "logps/rejected": -165.96096801757812,
-      "loss": 0.438,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -6.488312721252441,
-      "rewards/margins": 2.513715982437134,
-      "rewards/rejected": -9.002028465270996,
-      "step": 7900
-    },
-    {
-      "epoch": 0.6321678321678321,
-      "grad_norm": 20.50584602355957,
-      "learning_rate": 1.7927487339622945e-06,
-      "logits/chosen": -11.852715492248535,
-      "logits/rejected": -11.7929048538208,
-      "logps/chosen": -134.0271453857422,
-      "logps/rejected": -173.9302520751953,
-      "loss": 0.2253,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": -5.759385585784912,
-      "rewards/margins": 3.7614006996154785,
-      "rewards/rejected": -9.52078628540039,
-      "step": 7910
-    },
-    {
-      "epoch": 0.6329670329670329,
-      "grad_norm": 25.841930389404297,
-      "learning_rate": 1.786061317171056e-06,
-      "logits/chosen": -11.94718074798584,
-      "logits/rejected": -11.8970947265625,
-      "logps/chosen": -132.42471313476562,
-      "logps/rejected": -159.99021911621094,
-      "loss": 0.3374,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": -5.739545822143555,
-      "rewards/margins": 2.903136730194092,
-      "rewards/rejected": -8.642683029174805,
-      "step": 7920
-    },
-    {
-      "epoch": 0.6337662337662338,
-      "grad_norm": 22.893617630004883,
-      "learning_rate": 1.7793794579326034e-06,
-      "logits/chosen": -11.949989318847656,
-      "logits/rejected": -11.922906875610352,
-      "logps/chosen": -132.06028747558594,
-      "logps/rejected": -160.3749237060547,
-      "loss": 0.4116,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -6.025626182556152,
-      "rewards/margins": 2.6233956813812256,
-      "rewards/rejected": -8.649022102355957,
-      "step": 7930
-    },
-    {
-      "epoch": 0.6345654345654346,
-      "grad_norm": 26.917198181152344,
-      "learning_rate": 1.7727032082609056e-06,
-      "logits/chosen": -11.717082977294922,
-      "logits/rejected": -11.747480392456055,
-      "logps/chosen": -131.19947814941406,
-      "logps/rejected": -168.3651885986328,
-      "loss": 0.5501,
-      "rewards/accuracies": 0.7750000357627869,
-      "rewards/chosen": -6.003649711608887,
-      "rewards/margins": 2.971651792526245,
-      "rewards/rejected": -8.975301742553711,
-      "step": 7940
-    },
-    {
-      "epoch": 0.6353646353646354,
-      "grad_norm": 18.45380210876465,
-      "learning_rate": 1.766032620126267e-06,
-      "logits/chosen": -11.985687255859375,
-      "logits/rejected": -11.93552303314209,
-      "logps/chosen": -129.75242614746094,
-      "logps/rejected": -165.28660583496094,
-      "loss": 0.4615,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -5.860755443572998,
-      "rewards/margins": 3.0425896644592285,
-      "rewards/rejected": -8.903345108032227,
-      "step": 7950
-    },
-    {
-      "epoch": 0.6361638361638362,
-      "grad_norm": 17.263208389282227,
-      "learning_rate": 1.7593677454549196e-06,
-      "logits/chosen": -11.919219970703125,
-      "logits/rejected": -11.939974784851074,
-      "logps/chosen": -122.06037902832031,
-      "logps/rejected": -156.06805419921875,
-      "loss": 0.3701,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -5.043397426605225,
-      "rewards/margins": 2.970799207687378,
-      "rewards/rejected": -8.01419734954834,
-      "step": 7960
-    },
-    {
-      "epoch": 0.6369630369630369,
-      "grad_norm": 4.554449081420898,
-      "learning_rate": 1.75270863612862e-06,
-      "logits/chosen": -11.795648574829102,
-      "logits/rejected": -11.84864330291748,
-      "logps/chosen": -133.35818481445312,
-      "logps/rejected": -165.5010223388672,
-      "loss": 0.435,
-      "rewards/accuracies": 0.8375000357627869,
-      "rewards/chosen": -5.868055820465088,
-      "rewards/margins": 2.9226765632629395,
-      "rewards/rejected": -8.790732383728027,
-      "step": 7970
-    },
-    {
-      "epoch": 0.6377622377622377,
-      "grad_norm": 21.28313446044922,
-      "learning_rate": 1.7460553439842436e-06,
-      "logits/chosen": -11.524486541748047,
-      "logits/rejected": -11.776488304138184,
-      "logps/chosen": -127.97064208984375,
-      "logps/rejected": -174.8679962158203,
-      "loss": 0.3301,
-      "rewards/accuracies": 0.8375000357627869,
-      "rewards/chosen": -5.636492729187012,
-      "rewards/margins": 3.5929839611053467,
-      "rewards/rejected": -9.229476928710938,
-      "step": 7980
-    },
-    {
-      "epoch": 0.6385614385614385,
-      "grad_norm": 17.33797836303711,
-      "learning_rate": 1.7394079208133862e-06,
-      "logits/chosen": -11.877965927124023,
-      "logits/rejected": -11.819464683532715,
-      "logps/chosen": -128.1938018798828,
-      "logps/rejected": -160.5552978515625,
-      "loss": 0.4107,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": -6.0180277824401855,
-      "rewards/margins": 2.84769606590271,
-      "rewards/rejected": -8.865723609924316,
-      "step": 7990
-    },
-    {
-      "epoch": 0.6393606393606394,
-      "grad_norm": 14.697345733642578,
-      "learning_rate": 1.7327664183619542e-06,
-      "logits/chosen": -11.768370628356934,
-      "logits/rejected": -11.756645202636719,
-      "logps/chosen": -128.9132537841797,
-      "logps/rejected": -168.78407287597656,
-      "loss": 0.4746,
-      "rewards/accuracies": 0.8375000357627869,
-      "rewards/chosen": -5.855597972869873,
-      "rewards/margins": 3.4122471809387207,
-      "rewards/rejected": -9.267845153808594,
-      "step": 8000
-    },
-    {
-      "epoch": 0.6401598401598402,
-      "grad_norm": 17.21199607849121,
-      "learning_rate": 1.7261308883297657e-06,
-      "logits/chosen": -11.842738151550293,
-      "logits/rejected": -11.809967994689941,
-      "logps/chosen": -122.29509735107422,
-      "logps/rejected": -156.7180633544922,
-      "loss": 0.3429,
-      "rewards/accuracies": 0.8375000357627869,
-      "rewards/chosen": -5.321641445159912,
-      "rewards/margins": 3.0324690341949463,
-      "rewards/rejected": -8.354109764099121,
-      "step": 8010
-    },
-    {
-      "epoch": 0.640959040959041,
-      "grad_norm": 4.18531608581543,
-      "learning_rate": 1.7195013823701502e-06,
-      "logits/chosen": -11.789680480957031,
-      "logits/rejected": -11.803601264953613,
-      "logps/chosen": -130.67324829101562,
-      "logps/rejected": -178.9420928955078,
-      "loss": 0.2404,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -5.621804237365723,
-      "rewards/margins": 4.316339015960693,
-      "rewards/rejected": -9.938143730163574,
-      "step": 8020
-    },
-    {
-      "epoch": 0.6417582417582418,
-      "grad_norm": 18.576751708984375,
-      "learning_rate": 1.7128779520895388e-06,
-      "logits/chosen": -11.933005332946777,
-      "logits/rejected": -11.829184532165527,
-      "logps/chosen": -138.2496795654297,
-      "logps/rejected": -162.3791961669922,
-      "loss": 0.489,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": -6.467626094818115,
-      "rewards/margins": 2.623823642730713,
-      "rewards/rejected": -9.091450691223145,
-      "step": 8030
-    },
-    {
-      "epoch": 0.6425574425574425,
-      "grad_norm": 12.980820655822754,
-      "learning_rate": 1.7062606490470723e-06,
-      "logits/chosen": -11.933849334716797,
-      "logits/rejected": -11.795561790466309,
-      "logps/chosen": -124.96809387207031,
-      "logps/rejected": -159.16551208496094,
-      "loss": 0.2555,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -5.338871002197266,
-      "rewards/margins": 3.3996238708496094,
-      "rewards/rejected": -8.738494873046875,
-      "step": 8040
-    },
-    {
-      "epoch": 0.6433566433566433,
-      "grad_norm": 13.886013984680176,
-      "learning_rate": 1.699649524754191e-06,
-      "logits/chosen": -11.79322624206543,
-      "logits/rejected": -11.8323392868042,
-      "logps/chosen": -132.85533142089844,
-      "logps/rejected": -181.29251098632812,
-      "loss": 0.3334,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -6.280879497528076,
-      "rewards/margins": 3.7641408443450928,
-      "rewards/rejected": -10.04502010345459,
-      "step": 8050
-    },
-    {
-      "epoch": 0.6441558441558441,
-      "grad_norm": 9.62502384185791,
-      "learning_rate": 1.69304463067424e-06,
-      "logits/chosen": -11.845576286315918,
-      "logits/rejected": -11.832905769348145,
-      "logps/chosen": -133.00015258789062,
-      "logps/rejected": -170.3971405029297,
-      "loss": 0.3076,
-      "rewards/accuracies": 0.9000000357627869,
-      "rewards/chosen": -6.159894943237305,
-      "rewards/margins": 3.499089479446411,
-      "rewards/rejected": -9.658984184265137,
-      "step": 8060
-    },
-    {
-      "epoch": 0.644955044955045,
-      "grad_norm": 15.236932754516602,
-      "learning_rate": 1.6864460182220638e-06,
-      "logits/chosen": -11.90943431854248,
-      "logits/rejected": -11.870123863220215,
-      "logps/chosen": -127.06009674072266,
-      "logps/rejected": -156.89779663085938,
-      "loss": 0.2198,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": -5.702993392944336,
-      "rewards/margins": 3.039123296737671,
-      "rewards/rejected": -8.74211597442627,
-      "step": 8070
-    },
-    {
-      "epoch": 0.6457542457542458,
-      "grad_norm": 13.849470138549805,
-      "learning_rate": 1.6798537387636117e-06,
-      "logits/chosen": -11.978996276855469,
-      "logits/rejected": -11.893428802490234,
-      "logps/chosen": -134.58424377441406,
-      "logps/rejected": -166.0704803466797,
-      "loss": 0.3493,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -6.158796787261963,
-      "rewards/margins": 2.917485475540161,
-      "rewards/rejected": -9.076281547546387,
-      "step": 8080
-    },
-    {
-      "epoch": 0.6465534465534466,
-      "grad_norm": 7.262673854827881,
-      "learning_rate": 1.673267843615533e-06,
-      "logits/chosen": -11.90329647064209,
-      "logits/rejected": -11.8521089553833,
-      "logps/chosen": -127.5135498046875,
-      "logps/rejected": -168.3212890625,
-      "loss": 0.2334,
-      "rewards/accuracies": 0.9000000357627869,
-      "rewards/chosen": -5.749518871307373,
-      "rewards/margins": 3.7164947986602783,
-      "rewards/rejected": -9.46601390838623,
-      "step": 8090
-    },
-    {
-      "epoch": 0.6473526473526473,
-      "grad_norm": 26.374208450317383,
-      "learning_rate": 1.6666883840447795e-06,
-      "logits/chosen": -11.87460994720459,
-      "logits/rejected": -11.98270034790039,
-      "logps/chosen": -127.64669036865234,
-      "logps/rejected": -169.7811737060547,
-      "loss": 0.3447,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -5.7494425773620605,
-      "rewards/margins": 3.531590223312378,
-      "rewards/rejected": -9.281033515930176,
-      "step": 8100
-    },
-    {
-      "epoch": 0.6481518481518481,
-      "grad_norm": 19.140628814697266,
-      "learning_rate": 1.660115411268205e-06,
-      "logits/chosen": -12.037627220153809,
-      "logits/rejected": -12.00280475616455,
-      "logps/chosen": -131.36767578125,
-      "logps/rejected": -163.42430114746094,
-      "loss": 0.3456,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -5.937842845916748,
-      "rewards/margins": 3.060131072998047,
-      "rewards/rejected": -8.997973442077637,
-      "step": 8110
-    },
-    {
-      "epoch": 0.6489510489510489,
-      "grad_norm": 18.36117935180664,
-      "learning_rate": 1.6535489764521694e-06,
-      "logits/chosen": -11.952110290527344,
-      "logits/rejected": -11.9171142578125,
-      "logps/chosen": -134.99124145507812,
-      "logps/rejected": -164.7380828857422,
-      "loss": 0.3421,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -6.250131607055664,
-      "rewards/margins": 2.9784958362579346,
-      "rewards/rejected": -9.228628158569336,
-      "step": 8120
-    },
-    {
-      "epoch": 0.6497502497502498,
-      "grad_norm": 7.19674825668335,
-      "learning_rate": 1.6469891307121375e-06,
-      "logits/chosen": -11.941972732543945,
-      "logits/rejected": -11.91457462310791,
-      "logps/chosen": -140.79519653320312,
-      "logps/rejected": -175.3110809326172,
-      "loss": 0.2734,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": -6.770716190338135,
-      "rewards/margins": 3.318540334701538,
-      "rewards/rejected": -10.089256286621094,
-      "step": 8130
-    },
-    {
-      "epoch": 0.6505494505494506,
-      "grad_norm": 23.839767456054688,
-      "learning_rate": 1.6404359251122841e-06,
-      "logits/chosen": -12.008670806884766,
-      "logits/rejected": -12.029498100280762,
-      "logps/chosen": -139.68283081054688,
-      "logps/rejected": -173.7126007080078,
-      "loss": 0.3948,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -6.6642351150512695,
-      "rewards/margins": 3.3310203552246094,
-      "rewards/rejected": -9.995255470275879,
-      "step": 8140
-    },
-    {
-      "epoch": 0.6513486513486514,
-      "grad_norm": 5.914085865020752,
-      "learning_rate": 1.6338894106650937e-06,
-      "logits/chosen": -12.029558181762695,
-      "logits/rejected": -11.95956039428711,
-      "logps/chosen": -137.5056610107422,
-      "logps/rejected": -167.1519317626953,
-      "loss": 0.3481,
-      "rewards/accuracies": 0.8375000357627869,
-      "rewards/chosen": -6.586909770965576,
-      "rewards/margins": 3.0631840229034424,
-      "rewards/rejected": -9.650093078613281,
-      "step": 8150
-    },
-    {
-      "epoch": 0.6521478521478522,
-      "grad_norm": 16.490812301635742,
-      "learning_rate": 1.6273496383309639e-06,
-      "logits/chosen": -12.051712036132812,
-      "logits/rejected": -11.95470905303955,
-      "logps/chosen": -145.7891082763672,
-      "logps/rejected": -173.90988159179688,
-      "loss": 0.3861,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -7.219048976898193,
-      "rewards/margins": 2.925100326538086,
-      "rewards/rejected": -10.144148826599121,
-      "step": 8160
-    },
-    {
-      "epoch": 0.6529470529470529,
-      "grad_norm": 23.95977210998535,
-      "learning_rate": 1.6208166590178103e-06,
-      "logits/chosen": -11.942181587219238,
-      "logits/rejected": -11.906472206115723,
-      "logps/chosen": -137.82131958007812,
-      "logps/rejected": -174.8419952392578,
-      "loss": 0.2974,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -6.748953342437744,
-      "rewards/margins": 3.4214248657226562,
-      "rewards/rejected": -10.170378684997559,
-      "step": 8170
-    },
-    {
-      "epoch": 0.6537462537462537,
-      "grad_norm": 11.647270202636719,
-      "learning_rate": 1.6142905235806686e-06,
-      "logits/chosen": -11.824019432067871,
-      "logits/rejected": -11.93186092376709,
-      "logps/chosen": -140.55181884765625,
-      "logps/rejected": -189.73968505859375,
-      "loss": 0.264,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": -7.006324291229248,
-      "rewards/margins": 3.950300693511963,
-      "rewards/rejected": -10.956624984741211,
-      "step": 8180
-    },
-    {
-      "epoch": 0.6545454545454545,
-      "grad_norm": 26.083467483520508,
-      "learning_rate": 1.6077712828212991e-06,
-      "logits/chosen": -12.013392448425293,
-      "logits/rejected": -11.914697647094727,
-      "logps/chosen": -129.43295288085938,
-      "logps/rejected": -167.26235961914062,
-      "loss": 0.4253,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -6.127293586730957,
-      "rewards/margins": 3.643151044845581,
-      "rewards/rejected": -9.770444869995117,
-      "step": 8190
-    },
-    {
-      "epoch": 0.6553446553446554,
-      "grad_norm": 11.834199905395508,
-      "learning_rate": 1.6012589874877932e-06,
-      "logits/chosen": -11.91779613494873,
-      "logits/rejected": -11.839629173278809,
-      "logps/chosen": -143.1315155029297,
-      "logps/rejected": -175.9871826171875,
-      "loss": 0.4131,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -6.787700176239014,
-      "rewards/margins": 3.1637423038482666,
-      "rewards/rejected": -9.951441764831543,
-      "step": 8200
-    },
-    {
-      "epoch": 0.6561438561438562,
-      "grad_norm": 17.905786514282227,
-      "learning_rate": 1.594753688274175e-06,
-      "logits/chosen": -11.74980354309082,
-      "logits/rejected": -11.829841613769531,
-      "logps/chosen": -137.26463317871094,
-      "logps/rejected": -186.15435791015625,
-      "loss": 0.2388,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -6.492833614349365,
-      "rewards/margins": 4.322066783905029,
-      "rewards/rejected": -10.814901351928711,
-      "step": 8210
-    },
-    {
-      "epoch": 0.656943056943057,
-      "grad_norm": 9.472769737243652,
-      "learning_rate": 1.588255435820008e-06,
-      "logits/chosen": -11.901835441589355,
-      "logits/rejected": -11.882403373718262,
-      "logps/chosen": -133.53775024414062,
-      "logps/rejected": -170.35520935058594,
-      "loss": 0.2352,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -6.069746971130371,
-      "rewards/margins": 3.205721616744995,
-      "rewards/rejected": -9.275468826293945,
-      "step": 8220
-    },
-    {
-      "epoch": 0.6577422577422577,
-      "grad_norm": 17.477588653564453,
-      "learning_rate": 1.5817642807100032e-06,
-      "logits/chosen": -11.738460540771484,
-      "logits/rejected": -11.921717643737793,
-      "logps/chosen": -140.16307067871094,
-      "logps/rejected": -179.6772918701172,
-      "loss": 0.3529,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -6.902975559234619,
-      "rewards/margins": 3.342000961303711,
-      "rewards/rejected": -10.244976997375488,
-      "step": 8230
-    },
-    {
-      "epoch": 0.6585414585414585,
-      "grad_norm": 18.44438934326172,
-      "learning_rate": 1.5752802734736205e-06,
-      "logits/chosen": -12.039575576782227,
-      "logits/rejected": -11.903204917907715,
-      "logps/chosen": -144.63621520996094,
-      "logps/rejected": -172.9490203857422,
-      "loss": 0.3552,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -6.591945171356201,
-      "rewards/margins": 3.317340850830078,
-      "rewards/rejected": -9.909285545349121,
-      "step": 8240
-    },
-    {
-      "epoch": 0.6593406593406593,
-      "grad_norm": 15.810492515563965,
-      "learning_rate": 1.5688034645846812e-06,
-      "logits/chosen": -12.04300594329834,
-      "logits/rejected": -11.972992897033691,
-      "logps/chosen": -142.68243408203125,
-      "logps/rejected": -173.29159545898438,
-      "loss": 0.2877,
-      "rewards/accuracies": 0.8375000357627869,
-      "rewards/chosen": -6.7158203125,
-      "rewards/margins": 3.4133870601654053,
-      "rewards/rejected": -10.129207611083984,
-      "step": 8250
-    },
-    {
-      "epoch": 0.6601398601398601,
-      "grad_norm": 19.852882385253906,
-      "learning_rate": 1.562333904460969e-06,
-      "logits/chosen": -12.049633979797363,
-      "logits/rejected": -11.976860046386719,
-      "logps/chosen": -138.35752868652344,
-      "logps/rejected": -172.098876953125,
-      "loss": 0.2891,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -6.675235271453857,
-      "rewards/margins": 3.0819966793060303,
-      "rewards/rejected": -9.757231712341309,
-      "step": 8260
-    },
-    {
-      "epoch": 0.660939060939061,
-      "grad_norm": 12.621918678283691,
-      "learning_rate": 1.5558716434638439e-06,
-      "logits/chosen": -12.09384536743164,
-      "logits/rejected": -12.005844116210938,
-      "logps/chosen": -136.9380645751953,
-      "logps/rejected": -162.90650939941406,
-      "loss": 0.4806,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": -6.58151388168335,
-      "rewards/margins": 2.9374194145202637,
-      "rewards/rejected": -9.518933296203613,
-      "step": 8270
-    },
-    {
-      "epoch": 0.6617382617382618,
-      "grad_norm": 17.852094650268555,
-      "learning_rate": 1.5494167318978447e-06,
-      "logits/chosen": -12.019203186035156,
-      "logits/rejected": -11.953641891479492,
-      "logps/chosen": -139.24766540527344,
-      "logps/rejected": -171.17405700683594,
-      "loss": 0.3713,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -6.647255897521973,
-      "rewards/margins": 3.0848212242126465,
-      "rewards/rejected": -9.732077598571777,
-      "step": 8280
-    },
-    {
-      "epoch": 0.6625374625374625,
-      "grad_norm": 13.454997062683105,
-      "learning_rate": 1.5429692200103002e-06,
-      "logits/chosen": -12.048730850219727,
-      "logits/rejected": -11.99394702911377,
-      "logps/chosen": -142.8029327392578,
-      "logps/rejected": -180.88462829589844,
-      "loss": 0.4453,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -7.108801364898682,
-      "rewards/margins": 3.500664472579956,
-      "rewards/rejected": -10.609465599060059,
-      "step": 8290
-    },
-    {
-      "epoch": 0.6633366633366633,
-      "grad_norm": 4.650618076324463,
-      "learning_rate": 1.5365291579909381e-06,
-      "logits/chosen": -12.007485389709473,
-      "logits/rejected": -11.939274787902832,
-      "logps/chosen": -144.82249450683594,
-      "logps/rejected": -179.46641540527344,
-      "loss": 0.3484,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -7.1744842529296875,
-      "rewards/margins": 3.2270514965057373,
-      "rewards/rejected": -10.40153694152832,
-      "step": 8300
-    },
-    {
-      "epoch": 0.6641358641358641,
-      "grad_norm": 9.876091957092285,
-      "learning_rate": 1.5300965959714913e-06,
-      "logits/chosen": -12.025744438171387,
-      "logits/rejected": -11.93729019165039,
-      "logps/chosen": -139.57839965820312,
-      "logps/rejected": -179.4062042236328,
-      "loss": 0.2826,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": -6.6735758781433105,
-      "rewards/margins": 3.564296007156372,
-      "rewards/rejected": -10.237872123718262,
-      "step": 8310
-    },
-    {
-      "epoch": 0.6649350649350649,
-      "grad_norm": 23.834379196166992,
-      "learning_rate": 1.5236715840253133e-06,
-      "logits/chosen": -11.981059074401855,
-      "logits/rejected": -11.987380981445312,
-      "logps/chosen": -139.60533142089844,
-      "logps/rejected": -172.072998046875,
-      "loss": 0.3353,
-      "rewards/accuracies": 0.8375000357627869,
-      "rewards/chosen": -6.68464994430542,
-      "rewards/margins": 3.2275238037109375,
-      "rewards/rejected": -9.9121732711792,
-      "step": 8320
-    },
-    {
-      "epoch": 0.6657342657342658,
-      "grad_norm": 7.897576332092285,
-      "learning_rate": 1.5172541721669835e-06,
-      "logits/chosen": -12.034445762634277,
-      "logits/rejected": -11.971217155456543,
-      "logps/chosen": -136.6117401123047,
-      "logps/rejected": -169.5500946044922,
-      "loss": 0.3007,
-      "rewards/accuracies": 0.8375000357627869,
-      "rewards/chosen": -6.41780948638916,
-      "rewards/margins": 3.3305251598358154,
-      "rewards/rejected": -9.748333930969238,
-      "step": 8330
-    },
-    {
-      "epoch": 0.6665334665334666,
-      "grad_norm": 23.81068229675293,
-      "learning_rate": 1.51084441035192e-06,
-      "logits/chosen": -11.942031860351562,
-      "logits/rejected": -11.921050071716309,
-      "logps/chosen": -144.6532440185547,
-      "logps/rejected": -177.57672119140625,
-      "loss": 0.4247,
-      "rewards/accuracies": 0.8375000357627869,
-      "rewards/chosen": -7.154791355133057,
-      "rewards/margins": 2.8586318492889404,
-      "rewards/rejected": -10.013422966003418,
-      "step": 8340
-    },
-    {
-      "epoch": 0.6673326673326674,
-      "grad_norm": 20.714122772216797,
-      "learning_rate": 1.5044423484759885e-06,
-      "logits/chosen": -11.91887378692627,
-      "logits/rejected": -11.879464149475098,
-      "logps/chosen": -133.75344848632812,
-      "logps/rejected": -170.0335235595703,
-      "loss": 0.2709,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": -6.0853271484375,
-      "rewards/margins": 3.601853132247925,
-      "rewards/rejected": -9.68718147277832,
-      "step": 8350
-    },
-    {
-      "epoch": 0.6681318681318681,
-      "grad_norm": 16.654499053955078,
-      "learning_rate": 1.4980480363751188e-06,
-      "logits/chosen": -11.80296802520752,
-      "logits/rejected": -11.813720703125,
-      "logps/chosen": -127.4434814453125,
-      "logps/rejected": -165.0455780029297,
-      "loss": 0.3481,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -5.857985973358154,
-      "rewards/margins": 3.3033204078674316,
-      "rewards/rejected": -9.161307334899902,
-      "step": 8360
-    },
-    {
-      "epoch": 0.6689310689310689,
-      "grad_norm": 16.267879486083984,
-      "learning_rate": 1.4916615238249105e-06,
-      "logits/chosen": -11.99771785736084,
-      "logits/rejected": -11.967117309570312,
-      "logps/chosen": -134.845703125,
-      "logps/rejected": -163.13528442382812,
-      "loss": 0.3352,
-      "rewards/accuracies": 0.8375000357627869,
-      "rewards/chosen": -6.337829113006592,
-      "rewards/margins": 2.708822250366211,
-      "rewards/rejected": -9.046650886535645,
-      "step": 8370
-    },
-    {
-      "epoch": 0.6697302697302697,
-      "grad_norm": 0.8992837071418762,
-      "learning_rate": 1.4852828605402498e-06,
-      "logits/chosen": -11.961249351501465,
-      "logits/rejected": -11.922606468200684,
-      "logps/chosen": -129.33297729492188,
-      "logps/rejected": -176.64752197265625,
-      "loss": 0.2072,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": -5.882615566253662,
-      "rewards/margins": 4.4001336097717285,
-      "rewards/rejected": -10.28274917602539,
-      "step": 8380
-    },
-    {
-      "epoch": 0.6705294705294705,
-      "grad_norm": 24.916736602783203,
-      "learning_rate": 1.4789120961749237e-06,
-      "logits/chosen": -12.017481803894043,
-      "logits/rejected": -11.904123306274414,
-      "logps/chosen": -133.84495544433594,
-      "logps/rejected": -171.244140625,
-      "loss": 0.2808,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -6.4952392578125,
-      "rewards/margins": 3.2577569484710693,
-      "rewards/rejected": -9.752996444702148,
-      "step": 8390
-    },
-    {
-      "epoch": 0.6713286713286714,
-      "grad_norm": 8.651235580444336,
-      "learning_rate": 1.4725492803212276e-06,
-      "logits/chosen": -11.901087760925293,
-      "logits/rejected": -11.888206481933594,
-      "logps/chosen": -139.06187438964844,
-      "logps/rejected": -170.82212829589844,
-      "loss": 0.3629,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -6.631328105926514,
-      "rewards/margins": 3.2681057453155518,
-      "rewards/rejected": -9.899434089660645,
-      "step": 8400
-    },
-    {
-      "epoch": 0.6721278721278722,
-      "grad_norm": 28.10076904296875,
-      "learning_rate": 1.4661944625095859e-06,
-      "logits/chosen": -11.990219116210938,
-      "logits/rejected": -11.961593627929688,
-      "logps/chosen": -132.63717651367188,
-      "logps/rejected": -173.53184509277344,
-      "loss": 0.3157,
-      "rewards/accuracies": 0.8375000357627869,
-      "rewards/chosen": -6.050393581390381,
-      "rewards/margins": 3.7682533264160156,
-      "rewards/rejected": -9.818647384643555,
-      "step": 8410
-    },
-    {
-      "epoch": 0.6729270729270729,
-      "grad_norm": 15.581920623779297,
-      "learning_rate": 1.4598476922081602e-06,
-      "logits/chosen": -11.983906745910645,
-      "logits/rejected": -11.960977554321289,
-      "logps/chosen": -141.9501495361328,
-      "logps/rejected": -177.84751892089844,
-      "loss": 0.4203,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -6.768718719482422,
-      "rewards/margins": 3.4412667751312256,
-      "rewards/rejected": -10.209985733032227,
-      "step": 8420
-    },
-    {
-      "epoch": 0.6737262737262737,
-      "grad_norm": 12.788844108581543,
-      "learning_rate": 1.453509018822471e-06,
-      "logits/chosen": -11.961883544921875,
-      "logits/rejected": -11.877273559570312,
-      "logps/chosen": -134.1496124267578,
-      "logps/rejected": -171.31613159179688,
-      "loss": 0.3052,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": -6.019261837005615,
-      "rewards/margins": 3.596672773361206,
-      "rewards/rejected": -9.615935325622559,
-      "step": 8430
-    },
-    {
-      "epoch": 0.6745254745254745,
-      "grad_norm": 9.919685363769531,
-      "learning_rate": 1.447178491695006e-06,
-      "logits/chosen": -11.880547523498535,
-      "logits/rejected": -11.865416526794434,
-      "logps/chosen": -132.86553955078125,
-      "logps/rejected": -170.8365478515625,
-      "loss": 0.1769,
-      "rewards/accuracies": 0.9000000357627869,
-      "rewards/chosen": -6.088500499725342,
-      "rewards/margins": 3.6103732585906982,
-      "rewards/rejected": -9.698874473571777,
-      "step": 8440
-    },
-    {
-      "epoch": 0.6753246753246753,
-      "grad_norm": 18.776813507080078,
-      "learning_rate": 1.4408561601048424e-06,
-      "logits/chosen": -11.933755874633789,
-      "logits/rejected": -11.8805570602417,
-      "logps/chosen": -134.97341918945312,
-      "logps/rejected": -165.8595733642578,
-      "loss": 0.2969,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -6.317468166351318,
-      "rewards/margins": 3.1471211910247803,
-      "rewards/rejected": -9.46458911895752,
-      "step": 8450
-    },
-    {
-      "epoch": 0.6761238761238761,
-      "grad_norm": 9.240545272827148,
-      "learning_rate": 1.43454207326726e-06,
-      "logits/chosen": -11.75614070892334,
-      "logits/rejected": -11.854555130004883,
-      "logps/chosen": -140.373779296875,
-      "logps/rejected": -183.63133239746094,
-      "loss": 0.2824,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": -6.380944728851318,
-      "rewards/margins": 3.887887716293335,
-      "rewards/rejected": -10.268832206726074,
-      "step": 8460
-    },
-    {
-      "epoch": 0.676923076923077,
-      "grad_norm": 15.975516319274902,
-      "learning_rate": 1.4282362803333568e-06,
-      "logits/chosen": -11.95441722869873,
-      "logits/rejected": -11.85694408416748,
-      "logps/chosen": -145.7323760986328,
-      "logps/rejected": -173.59068298339844,
-      "loss": 0.3025,
-      "rewards/accuracies": 0.8375000357627869,
-      "rewards/chosen": -6.761271953582764,
-      "rewards/margins": 3.1796350479125977,
-      "rewards/rejected": -9.940905570983887,
-      "step": 8470
-    },
-    {
-      "epoch": 0.6777222777222778,
-      "grad_norm": 27.438404083251953,
-      "learning_rate": 1.4219388303896694e-06,
-      "logits/chosen": -11.915318489074707,
-      "logits/rejected": -11.81243896484375,
-      "logps/chosen": -139.9368896484375,
-      "logps/rejected": -175.39682006835938,
-      "loss": 0.2454,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": -6.785569667816162,
-      "rewards/margins": 3.490673065185547,
-      "rewards/rejected": -10.276243209838867,
-      "step": 8480
-    },
-    {
-      "epoch": 0.6785214785214785,
-      "grad_norm": 7.901242733001709,
-      "learning_rate": 1.4156497724577882e-06,
-      "logits/chosen": -11.949751853942871,
-      "logits/rejected": -11.887277603149414,
-      "logps/chosen": -142.1164093017578,
-      "logps/rejected": -184.327392578125,
-      "loss": 0.2677,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": -7.090362071990967,
-      "rewards/margins": 3.9313347339630127,
-      "rewards/rejected": -11.021697044372559,
-      "step": 8490
-    },
-    {
-      "epoch": 0.6793206793206793,
-      "grad_norm": 22.44212532043457,
-      "learning_rate": 1.4093691554939803e-06,
-      "logits/chosen": -11.91172981262207,
-      "logits/rejected": -11.868974685668945,
-      "logps/chosen": -145.0400848388672,
-      "logps/rejected": -179.8319854736328,
-      "loss": 0.5325,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": -7.323304653167725,
-      "rewards/margins": 2.8826370239257812,
-      "rewards/rejected": -10.205942153930664,
-      "step": 8500
-    },
-    {
-      "epoch": 0.6801198801198801,
-      "grad_norm": 19.31974220275879,
-      "learning_rate": 1.4030970283888019e-06,
-      "logits/chosen": -11.890034675598145,
-      "logits/rejected": -11.861732482910156,
-      "logps/chosen": -141.76638793945312,
-      "logps/rejected": -181.8256072998047,
-      "loss": 0.215,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": -6.82415771484375,
-      "rewards/margins": 3.7997186183929443,
-      "rewards/rejected": -10.623876571655273,
-      "step": 8510
-    },
-    {
-      "epoch": 0.6809190809190809,
-      "grad_norm": 25.229759216308594,
-      "learning_rate": 1.3968334399667233e-06,
-      "logits/chosen": -11.853147506713867,
-      "logits/rejected": -11.905603408813477,
-      "logps/chosen": -143.1599578857422,
-      "logps/rejected": -184.19448852539062,
-      "loss": 0.3467,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -7.183055400848389,
-      "rewards/margins": 3.6329782009124756,
-      "rewards/rejected": -10.816032409667969,
-      "step": 8520
-    },
-    {
-      "epoch": 0.6817182817182817,
-      "grad_norm": 17.510889053344727,
-      "learning_rate": 1.3905784389857454e-06,
-      "logits/chosen": -11.911269187927246,
-      "logits/rejected": -12.005571365356445,
-      "logps/chosen": -136.019287109375,
-      "logps/rejected": -190.45565795898438,
-      "loss": 0.3107,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -6.7939348220825195,
-      "rewards/margins": 4.104278087615967,
-      "rewards/rejected": -10.898213386535645,
-      "step": 8530
-    },
-    {
-      "epoch": 0.6825174825174826,
-      "grad_norm": 19.730716705322266,
-      "learning_rate": 1.3843320741370215e-06,
-      "logits/chosen": -11.923260688781738,
-      "logits/rejected": -11.941040992736816,
-      "logps/chosen": -148.4237823486328,
-      "logps/rejected": -183.8870391845703,
-      "loss": 0.3389,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -7.617877960205078,
-      "rewards/margins": 3.186866283416748,
-      "rewards/rejected": -10.804743766784668,
-      "step": 8540
-    },
-    {
-      "epoch": 0.6833166833166833,
-      "grad_norm": 16.90422821044922,
-      "learning_rate": 1.3780943940444796e-06,
-      "logits/chosen": -11.971269607543945,
-      "logits/rejected": -11.968070983886719,
-      "logps/chosen": -143.80210876464844,
-      "logps/rejected": -187.85350036621094,
-      "loss": 0.3598,
-      "rewards/accuracies": 0.8375000357627869,
-      "rewards/chosen": -7.351616859436035,
-      "rewards/margins": 4.00991678237915,
-      "rewards/rejected": -11.361533164978027,
-      "step": 8550
-    },
-    {
-      "epoch": 0.6841158841158841,
-      "grad_norm": 22.93766975402832,
-      "learning_rate": 1.371865447264441e-06,
-      "logits/chosen": -11.840710639953613,
-      "logits/rejected": -11.903143882751465,
-      "logps/chosen": -148.31610107421875,
-      "logps/rejected": -190.86203002929688,
-      "loss": 0.3712,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -7.375509738922119,
-      "rewards/margins": 3.749105453491211,
-      "rewards/rejected": -11.124615669250488,
-      "step": 8560
-    },
-    {
-      "epoch": 0.6849150849150849,
-      "grad_norm": 20.963476181030273,
-      "learning_rate": 1.3656452822852435e-06,
-      "logits/chosen": -11.893011093139648,
-      "logits/rejected": -11.907843589782715,
-      "logps/chosen": -148.926513671875,
-      "logps/rejected": -194.2201385498047,
-      "loss": 0.3125,
-      "rewards/accuracies": 0.8375000357627869,
-      "rewards/chosen": -7.471877574920654,
-      "rewards/margins": 3.704897403717041,
-      "rewards/rejected": -11.176774978637695,
-      "step": 8570
-    },
-    {
-      "epoch": 0.6857142857142857,
-      "grad_norm": 11.871672630310059,
-      "learning_rate": 1.359433947526865e-06,
-      "logits/chosen": -12.030394554138184,
-      "logits/rejected": -11.94849681854248,
-      "logps/chosen": -142.36770629882812,
-      "logps/rejected": -162.66616821289062,
-      "loss": 0.4577,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -6.938450813293457,
-      "rewards/margins": 2.4852776527404785,
-      "rewards/rejected": -9.423727989196777,
-      "step": 8580
-    },
-    {
-      "epoch": 0.6865134865134865,
-      "grad_norm": 11.037942886352539,
-      "learning_rate": 1.3532314913405475e-06,
-      "logits/chosen": -11.955025672912598,
-      "logits/rejected": -11.948328018188477,
-      "logps/chosen": -137.1912384033203,
-      "logps/rejected": -169.99908447265625,
-      "loss": 0.3992,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -6.782596588134766,
-      "rewards/margins": 2.9637951850891113,
-      "rewards/rejected": -9.746392250061035,
-      "step": 8590
-    },
-    {
-      "epoch": 0.6873126873126874,
-      "grad_norm": 18.294801712036133,
-      "learning_rate": 1.3470379620084162e-06,
-      "logits/chosen": -11.998222351074219,
-      "logits/rejected": -11.910609245300293,
-      "logps/chosen": -144.5846405029297,
-      "logps/rejected": -171.8914337158203,
-      "loss": 0.5622,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -7.4668402671813965,
-      "rewards/margins": 2.5146892070770264,
-      "rewards/rejected": -9.98153018951416,
-      "step": 8600
-    },
-    {
-      "epoch": 0.6881118881118881,
-      "grad_norm": 33.774845123291016,
-      "learning_rate": 1.3408534077431073e-06,
-      "logits/chosen": -11.941729545593262,
-      "logits/rejected": -11.9019136428833,
-      "logps/chosen": -149.40721130371094,
-      "logps/rejected": -183.9861602783203,
-      "loss": 0.3689,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -7.600094795227051,
-      "rewards/margins": 3.5185508728027344,
-      "rewards/rejected": -11.118644714355469,
-      "step": 8610
-    },
-    {
-      "epoch": 0.6889110889110889,
-      "grad_norm": 20.866195678710938,
-      "learning_rate": 1.3346778766873913e-06,
-      "logits/chosen": -11.994695663452148,
-      "logits/rejected": -11.934279441833496,
-      "logps/chosen": -141.11233520507812,
-      "logps/rejected": -169.05845642089844,
-      "loss": 0.3318,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -6.689768314361572,
-      "rewards/margins": 3.0001330375671387,
-      "rewards/rejected": -9.689901351928711,
-      "step": 8620
-    },
-    {
-      "epoch": 0.6897102897102897,
-      "grad_norm": 9.034646987915039,
-      "learning_rate": 1.3285114169138014e-06,
-      "logits/chosen": -11.752184867858887,
-      "logits/rejected": -11.826130867004395,
-      "logps/chosen": -135.8509063720703,
-      "logps/rejected": -178.4853973388672,
-      "loss": 0.3433,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -6.685969829559326,
-      "rewards/margins": 3.47393798828125,
-      "rewards/rejected": -10.159908294677734,
-      "step": 8630
-    },
-    {
-      "epoch": 0.6905094905094905,
-      "grad_norm": 17.035112380981445,
-      "learning_rate": 1.3223540764242548e-06,
-      "logits/chosen": -11.80763053894043,
-      "logits/rejected": -11.887674331665039,
-      "logps/chosen": -140.6165771484375,
-      "logps/rejected": -188.7302703857422,
-      "loss": 0.2802,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -6.798198223114014,
-      "rewards/margins": 3.803972005844116,
-      "rewards/rejected": -10.602169036865234,
-      "step": 8640
-    },
-    {
-      "epoch": 0.6913086913086913,
-      "grad_norm": 5.593992710113525,
-      "learning_rate": 1.3162059031496808e-06,
-      "logits/chosen": -12.052057266235352,
-      "logits/rejected": -11.946160316467285,
-      "logps/chosen": -137.8290252685547,
-      "logps/rejected": -173.44752502441406,
-      "loss": 0.4096,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -6.423327922821045,
-      "rewards/margins": 3.2692582607269287,
-      "rewards/rejected": -9.692585945129395,
-      "step": 8650
-    },
-    {
-      "epoch": 0.6921078921078921,
-      "grad_norm": 7.153141021728516,
-      "learning_rate": 1.3100669449496477e-06,
-      "logits/chosen": -11.97681999206543,
-      "logits/rejected": -11.896674156188965,
-      "logps/chosen": -139.6688232421875,
-      "logps/rejected": -171.5281982421875,
-      "loss": 0.4408,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -6.9902472496032715,
-      "rewards/margins": 2.9182491302490234,
-      "rewards/rejected": -9.908495903015137,
-      "step": 8660
-    },
-    {
-      "epoch": 0.692907092907093,
-      "grad_norm": 6.881078243255615,
-      "learning_rate": 1.3039372496119913e-06,
-      "logits/chosen": -11.875187873840332,
-      "logits/rejected": -11.845479011535645,
-      "logps/chosen": -137.83982849121094,
-      "logps/rejected": -174.3751220703125,
-      "loss": 0.3343,
-      "rewards/accuracies": 0.8375000357627869,
-      "rewards/chosen": -6.501437664031982,
-      "rewards/margins": 3.5338332653045654,
-      "rewards/rejected": -10.035271644592285,
-      "step": 8670
-    },
-    {
-      "epoch": 0.6937062937062937,
-      "grad_norm": 11.295818328857422,
-      "learning_rate": 1.297816864852442e-06,
-      "logits/chosen": -11.932998657226562,
-      "logits/rejected": -11.892457962036133,
-      "logps/chosen": -132.5271759033203,
-      "logps/rejected": -176.42637634277344,
-      "loss": 0.2364,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": -6.290658473968506,
-      "rewards/margins": 3.558929443359375,
-      "rewards/rejected": -9.849588394165039,
-      "step": 8680
-    },
-    {
-      "epoch": 0.6945054945054945,
-      "grad_norm": 17.45670509338379,
-      "learning_rate": 1.2917058383142512e-06,
-      "logits/chosen": -11.967516899108887,
-      "logits/rejected": -11.983444213867188,
-      "logps/chosen": -136.74710083007812,
-      "logps/rejected": -164.64846801757812,
-      "loss": 0.3881,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -6.710936069488525,
-      "rewards/margins": 2.4481399059295654,
-      "rewards/rejected": -9.159075736999512,
-      "step": 8690
-    },
-    {
-      "epoch": 0.6953046953046953,
-      "grad_norm": 24.012779235839844,
-      "learning_rate": 1.285604217567827e-06,
-      "logits/chosen": -12.027499198913574,
-      "logits/rejected": -11.98870849609375,
-      "logps/chosen": -132.03857421875,
-      "logps/rejected": -163.7149200439453,
-      "loss": 0.385,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -6.256402015686035,
-      "rewards/margins": 2.849329710006714,
-      "rewards/rejected": -9.105731010437012,
-      "step": 8700
-    },
-    {
-      "epoch": 0.6961038961038961,
-      "grad_norm": 18.590831756591797,
-      "learning_rate": 1.279512050110354e-06,
-      "logits/chosen": -11.987296104431152,
-      "logits/rejected": -11.934345245361328,
-      "logps/chosen": -136.5035858154297,
-      "logps/rejected": -177.7553253173828,
-      "loss": 0.3114,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -6.4167070388793945,
-      "rewards/margins": 3.879009962081909,
-      "rewards/rejected": -10.295717239379883,
-      "step": 8710
-    },
-    {
-      "epoch": 0.6969030969030969,
-      "grad_norm": 27.830778121948242,
-      "learning_rate": 1.2734293833654343e-06,
-      "logits/chosen": -11.944058418273926,
-      "logits/rejected": -11.921808242797852,
-      "logps/chosen": -126.79027557373047,
-      "logps/rejected": -165.22268676757812,
-      "loss": 0.3681,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -5.9452948570251465,
-      "rewards/margins": 3.4183101654052734,
-      "rewards/rejected": -9.363604545593262,
-      "step": 8720
-    },
-    {
-      "epoch": 0.6977022977022977,
-      "grad_norm": 23.463634490966797,
-      "learning_rate": 1.2673562646827093e-06,
-      "logits/chosen": -12.001285552978516,
-      "logits/rejected": -11.909459114074707,
-      "logps/chosen": -145.12686157226562,
-      "logps/rejected": -176.2911834716797,
-      "loss": 0.4613,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -7.490067958831787,
-      "rewards/margins": 2.621905565261841,
-      "rewards/rejected": -10.11197280883789,
-      "step": 8730
-    },
-    {
-      "epoch": 0.6985014985014985,
-      "grad_norm": 15.69598388671875,
-      "learning_rate": 1.261292741337496e-06,
-      "logits/chosen": -11.881102561950684,
-      "logits/rejected": -11.994211196899414,
-      "logps/chosen": -141.6178741455078,
-      "logps/rejected": -180.69947814941406,
-      "loss": 0.2886,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -6.818112373352051,
-      "rewards/margins": 3.4645633697509766,
-      "rewards/rejected": -10.282675743103027,
-      "step": 8740
-    },
-    {
-      "epoch": 0.6993006993006993,
-      "grad_norm": 21.76118278503418,
-      "learning_rate": 1.2552388605304171e-06,
-      "logits/chosen": -12.187832832336426,
-      "logits/rejected": -12.12454891204834,
-      "logps/chosen": -143.12188720703125,
-      "logps/rejected": -167.5365447998047,
-      "loss": 0.4977,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -7.027975559234619,
-      "rewards/margins": 2.5431363582611084,
-      "rewards/rejected": -9.571112632751465,
-      "step": 8750
-    },
-    {
-      "epoch": 0.7000999000999001,
-      "grad_norm": 23.841753005981445,
-      "learning_rate": 1.249194669387037e-06,
-      "logits/chosen": -11.964838981628418,
-      "logits/rejected": -11.99802303314209,
-      "logps/chosen": -140.7192840576172,
-      "logps/rejected": -190.56285095214844,
-      "loss": 0.2418,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": -6.834800720214844,
-      "rewards/margins": 4.413144588470459,
-      "rewards/rejected": -11.247944831848145,
-      "step": 8760
-    },
-    {
-      "epoch": 0.7008991008991009,
-      "grad_norm": 16.165857315063477,
-      "learning_rate": 1.2431602149574904e-06,
-      "logits/chosen": -11.971301078796387,
-      "logits/rejected": -11.9610595703125,
-      "logps/chosen": -142.71690368652344,
-      "logps/rejected": -179.90980529785156,
-      "loss": 0.3262,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -7.132534027099609,
-      "rewards/margins": 3.2041168212890625,
-      "rewards/rejected": -10.336649894714355,
-      "step": 8770
-    },
-    {
-      "epoch": 0.7016983016983017,
-      "grad_norm": 7.840427875518799,
-      "learning_rate": 1.2371355442161177e-06,
-      "logits/chosen": -11.834911346435547,
-      "logits/rejected": -11.934123039245605,
-      "logps/chosen": -132.8182373046875,
-      "logps/rejected": -183.05284118652344,
-      "loss": 0.2908,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -6.253757476806641,
-      "rewards/margins": 4.034875392913818,
-      "rewards/rejected": -10.2886323928833,
-      "step": 8780
-    },
-    {
-      "epoch": 0.7024975024975025,
-      "grad_norm": 9.38103199005127,
-      "learning_rate": 1.2311207040611e-06,
-      "logits/chosen": -12.060912132263184,
-      "logits/rejected": -12.053423881530762,
-      "logps/chosen": -135.8765106201172,
-      "logps/rejected": -176.725341796875,
-      "loss": 0.2891,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -6.833681583404541,
-      "rewards/margins": 3.2996063232421875,
-      "rewards/rejected": -10.13328742980957,
-      "step": 8790
-    },
-    {
-      "epoch": 0.7032967032967034,
-      "grad_norm": 18.98407745361328,
-      "learning_rate": 1.225115741314093e-06,
-      "logits/chosen": -11.9566650390625,
-      "logits/rejected": -12.026636123657227,
-      "logps/chosen": -142.5130615234375,
-      "logps/rejected": -191.20530700683594,
-      "loss": 0.3048,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -7.1057000160217285,
-      "rewards/margins": 3.827117919921875,
-      "rewards/rejected": -10.932818412780762,
-      "step": 8800
-    },
-    {
-      "epoch": 0.7040959040959041,
-      "grad_norm": 23.714433670043945,
-      "learning_rate": 1.2191207027198652e-06,
-      "logits/chosen": -12.08755111694336,
-      "logits/rejected": -12.043774604797363,
-      "logps/chosen": -137.6053466796875,
-      "logps/rejected": -172.8417205810547,
-      "loss": 0.3975,
-      "rewards/accuracies": 0.8375000357627869,
-      "rewards/chosen": -6.714520454406738,
-      "rewards/margins": 3.4424521923065186,
-      "rewards/rejected": -10.156972885131836,
-      "step": 8810
-    },
-    {
-      "epoch": 0.7048951048951049,
-      "grad_norm": 7.635548114776611,
-      "learning_rate": 1.2131356349459289e-06,
-      "logits/chosen": -12.12052059173584,
-      "logits/rejected": -12.106313705444336,
-      "logps/chosen": -148.7930908203125,
-      "logps/rejected": -181.5776824951172,
-      "loss": 0.3263,
-      "rewards/accuracies": 0.8375000357627869,
-      "rewards/chosen": -7.428526401519775,
-      "rewards/margins": 2.8995888233184814,
-      "rewards/rejected": -10.328116416931152,
-      "step": 8820
-    },
-    {
-      "epoch": 0.7056943056943057,
-      "grad_norm": 10.742875099182129,
-      "learning_rate": 1.2071605845821831e-06,
-      "logits/chosen": -12.148566246032715,
-      "logits/rejected": -12.06815242767334,
-      "logps/chosen": -143.27427673339844,
-      "logps/rejected": -177.5906219482422,
-      "loss": 0.3652,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -7.30825662612915,
-      "rewards/margins": 3.222393751144409,
-      "rewards/rejected": -10.53065013885498,
-      "step": 8830
-    },
-    {
-      "epoch": 0.7064935064935065,
-      "grad_norm": 3.153169870376587,
-      "learning_rate": 1.2011955981405452e-06,
-      "logits/chosen": -12.066043853759766,
-      "logits/rejected": -12.125990867614746,
-      "logps/chosen": -140.4459228515625,
-      "logps/rejected": -174.4756317138672,
-      "loss": 0.2911,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -6.634981632232666,
-      "rewards/margins": 3.069305181503296,
-      "rewards/rejected": -9.7042875289917,
-      "step": 8840
-    },
-    {
-      "epoch": 0.7072927072927073,
-      "grad_norm": 20.059146881103516,
-      "learning_rate": 1.1952407220545935e-06,
-      "logits/chosen": -11.848055839538574,
-      "logits/rejected": -12.043110847473145,
-      "logps/chosen": -141.62196350097656,
-      "logps/rejected": -195.0041046142578,
-      "loss": 0.2868,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": -6.92336893081665,
-      "rewards/margins": 4.187471389770508,
-      "rewards/rejected": -11.110841751098633,
-      "step": 8850
-    },
-    {
-      "epoch": 0.7080919080919081,
-      "grad_norm": 6.406337738037109,
-      "learning_rate": 1.1892960026792028e-06,
-      "logits/chosen": -11.949435234069824,
-      "logits/rejected": -12.110575675964355,
-      "logps/chosen": -142.2572479248047,
-      "logps/rejected": -191.9774932861328,
-      "loss": 0.3554,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": -7.03297758102417,
-      "rewards/margins": 3.8227384090423584,
-      "rewards/rejected": -10.85571575164795,
-      "step": 8860
-    },
-    {
-      "epoch": 0.7088911088911088,
-      "grad_norm": 13.234110832214355,
-      "learning_rate": 1.1833614862901831e-06,
-      "logits/chosen": -12.01836109161377,
-      "logits/rejected": -12.124737739562988,
-      "logps/chosen": -144.43814086914062,
-      "logps/rejected": -179.77439880371094,
-      "loss": 0.4641,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -7.332951545715332,
-      "rewards/margins": 3.058926582336426,
-      "rewards/rejected": -10.391879081726074,
-      "step": 8870
-    },
-    {
-      "epoch": 0.7096903096903097,
-      "grad_norm": 15.513260841369629,
-      "learning_rate": 1.1774372190839217e-06,
-      "logits/chosen": -11.77405071258545,
-      "logits/rejected": -12.004359245300293,
-      "logps/chosen": -142.2852020263672,
-      "logps/rejected": -183.36306762695312,
-      "loss": 0.4402,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -6.909853458404541,
-      "rewards/margins": 3.2997658252716064,
-      "rewards/rejected": -10.209619522094727,
-      "step": 8880
-    },
-    {
-      "epoch": 0.7104895104895105,
-      "grad_norm": 21.058481216430664,
-      "learning_rate": 1.1715232471770238e-06,
-      "logits/chosen": -12.074751853942871,
-      "logits/rejected": -12.054216384887695,
-      "logps/chosen": -138.72988891601562,
-      "logps/rejected": -168.6265869140625,
-      "loss": 0.362,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -6.738202095031738,
-      "rewards/margins": 2.762160062789917,
-      "rewards/rejected": -9.500362396240234,
-      "step": 8890
-    },
-    {
-      "epoch": 0.7112887112887113,
-      "grad_norm": 16.561132431030273,
-      "learning_rate": 1.1656196166059522e-06,
-      "logits/chosen": -12.030017852783203,
-      "logits/rejected": -12.005465507507324,
-      "logps/chosen": -128.9953155517578,
-      "logps/rejected": -172.272216796875,
-      "loss": 0.3627,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -6.146731853485107,
-      "rewards/margins": 3.88200044631958,
-      "rewards/rejected": -10.028731346130371,
-      "step": 8900
-    },
-    {
-      "epoch": 0.7120879120879121,
-      "grad_norm": 21.26597785949707,
-      "learning_rate": 1.1597263733266675e-06,
-      "logits/chosen": -12.106926918029785,
-      "logits/rejected": -12.040375709533691,
-      "logps/chosen": -132.04551696777344,
-      "logps/rejected": -166.60655212402344,
-      "loss": 0.2782,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": -6.095043659210205,
-      "rewards/margins": 3.67785382270813,
-      "rewards/rejected": -9.772897720336914,
-      "step": 8910
-    },
-    {
-      "epoch": 0.7128871128871129,
-      "grad_norm": 27.104345321655273,
-      "learning_rate": 1.1538435632142725e-06,
-      "logits/chosen": -11.983980178833008,
-      "logits/rejected": -12.028865814208984,
-      "logps/chosen": -143.6648406982422,
-      "logps/rejected": -189.95579528808594,
-      "loss": 0.3007,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -6.832543849945068,
-      "rewards/margins": 4.263246536254883,
-      "rewards/rejected": -11.09579086303711,
-      "step": 8920
-    },
-    {
-      "epoch": 0.7136863136863136,
-      "grad_norm": 5.344789505004883,
-      "learning_rate": 1.1479712320626539e-06,
-      "logits/chosen": -12.143596649169922,
-      "logits/rejected": -12.0750150680542,
-      "logps/chosen": -139.9822235107422,
-      "logps/rejected": -168.68690490722656,
-      "loss": 0.425,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -6.855285167694092,
-      "rewards/margins": 2.723268508911133,
-      "rewards/rejected": -9.578554153442383,
-      "step": 8930
-    },
-    {
-      "epoch": 0.7144855144855145,
-      "grad_norm": 15.3145112991333,
-      "learning_rate": 1.142109425584129e-06,
-      "logits/chosen": -12.17399787902832,
-      "logits/rejected": -12.131495475769043,
-      "logps/chosen": -145.2841339111328,
-      "logps/rejected": -177.79811096191406,
-      "loss": 0.2866,
-      "rewards/accuracies": 0.8375000357627869,
-      "rewards/chosen": -7.038047790527344,
-      "rewards/margins": 3.1866939067840576,
-      "rewards/rejected": -10.22474193572998,
-      "step": 8940
-    },
-    {
-      "epoch": 0.7152847152847153,
-      "grad_norm": 7.397363185882568,
-      "learning_rate": 1.1362581894090838e-06,
-      "logits/chosen": -12.136129379272461,
-      "logits/rejected": -12.08305549621582,
-      "logps/chosen": -134.76052856445312,
-      "logps/rejected": -170.01675415039062,
-      "loss": 0.3493,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": -6.409527778625488,
-      "rewards/margins": 3.3055386543273926,
-      "rewards/rejected": -9.715067863464355,
-      "step": 8950
-    },
-    {
-      "epoch": 0.7160839160839161,
-      "grad_norm": 11.436649322509766,
-      "learning_rate": 1.1304175690856248e-06,
-      "logits/chosen": -12.094843864440918,
-      "logits/rejected": -12.051828384399414,
-      "logps/chosen": -143.3557586669922,
-      "logps/rejected": -185.1837615966797,
-      "loss": 0.2889,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -6.966352939605713,
-      "rewards/margins": 3.7756309509277344,
-      "rewards/rejected": -10.741984367370605,
-      "step": 8960
-    },
-    {
-      "epoch": 0.7168831168831169,
-      "grad_norm": 14.971203804016113,
-      "learning_rate": 1.1245876100792172e-06,
-      "logits/chosen": -12.04066276550293,
-      "logits/rejected": -12.061993598937988,
-      "logps/chosen": -136.90672302246094,
-      "logps/rejected": -179.86668395996094,
-      "loss": 0.3857,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -6.587124824523926,
-      "rewards/margins": 3.634998321533203,
-      "rewards/rejected": -10.222123146057129,
-      "step": 8970
-    },
-    {
-      "epoch": 0.7176823176823177,
-      "grad_norm": 18.0239315032959,
-      "learning_rate": 1.1187683577723387e-06,
-      "logits/chosen": -12.104411125183105,
-      "logits/rejected": -12.02660083770752,
-      "logps/chosen": -146.8551025390625,
-      "logps/rejected": -180.09315490722656,
-      "loss": 0.3966,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -7.065395355224609,
-      "rewards/margins": 3.219470739364624,
-      "rewards/rejected": -10.284865379333496,
-      "step": 8980
-    },
-    {
-      "epoch": 0.7184815184815185,
-      "grad_norm": 6.462433815002441,
-      "learning_rate": 1.1129598574641196e-06,
-      "logits/chosen": -12.141711235046387,
-      "logits/rejected": -12.030007362365723,
-      "logps/chosen": -147.1630859375,
-      "logps/rejected": -185.5413360595703,
-      "loss": 0.2629,
-      "rewards/accuracies": 0.9000000357627869,
-      "rewards/chosen": -7.326399326324463,
-      "rewards/margins": 3.6373565196990967,
-      "rewards/rejected": -10.96375560760498,
-      "step": 8990
-    },
-    {
-      "epoch": 0.7192807192807192,
-      "grad_norm": 26.467262268066406,
-      "learning_rate": 1.1071621543699928e-06,
-      "logits/chosen": -12.08127498626709,
-      "logits/rejected": -12.010610580444336,
-      "logps/chosen": -137.03489685058594,
-      "logps/rejected": -174.58682250976562,
-      "loss": 0.2862,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -6.4601731300354,
-      "rewards/margins": 3.4933907985687256,
-      "rewards/rejected": -9.95356273651123,
-      "step": 9000
-    },
-    {
-      "epoch": 0.72007992007992,
-      "grad_norm": 13.444426536560059,
-      "learning_rate": 1.101375293621342e-06,
-      "logits/chosen": -12.075004577636719,
-      "logits/rejected": -12.021206855773926,
-      "logps/chosen": -140.4081573486328,
-      "logps/rejected": -180.05389404296875,
-      "loss": 0.2852,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -6.969844341278076,
-      "rewards/margins": 3.6408824920654297,
-      "rewards/rejected": -10.61072826385498,
-      "step": 9010
-    },
-    {
-      "epoch": 0.7208791208791209,
-      "grad_norm": 8.855780601501465,
-      "learning_rate": 1.0955993202651516e-06,
-      "logits/chosen": -12.019062995910645,
-      "logits/rejected": -12.06286334991455,
-      "logps/chosen": -142.978515625,
-      "logps/rejected": -181.03060913085938,
-      "loss": 0.4451,
-      "rewards/accuracies": 0.8375000357627869,
-      "rewards/chosen": -7.313838958740234,
-      "rewards/margins": 3.2950217723846436,
-      "rewards/rejected": -10.608860969543457,
-      "step": 9020
-    },
-    {
-      "epoch": 0.7216783216783217,
-      "grad_norm": 17.150039672851562,
-      "learning_rate": 1.0898342792636545e-06,
-      "logits/chosen": -12.045552253723145,
-      "logits/rejected": -11.971229553222656,
-      "logps/chosen": -139.53346252441406,
-      "logps/rejected": -175.6201629638672,
-      "loss": 0.2941,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": -6.614267826080322,
-      "rewards/margins": 3.574150562286377,
-      "rewards/rejected": -10.1884183883667,
-      "step": 9030
-    },
-    {
-      "epoch": 0.7224775224775225,
-      "grad_norm": 18.159971237182617,
-      "learning_rate": 1.0840802154939808e-06,
-      "logits/chosen": -12.094442367553711,
-      "logits/rejected": -11.974318504333496,
-      "logps/chosen": -145.1159210205078,
-      "logps/rejected": -177.1186065673828,
-      "loss": 0.3183,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -7.227726936340332,
-      "rewards/margins": 3.012040376663208,
-      "rewards/rejected": -10.239768028259277,
-      "step": 9040
-    },
-    {
-      "epoch": 0.7232767232767233,
-      "grad_norm": 4.617658615112305,
-      "learning_rate": 1.0783371737478096e-06,
-      "logits/chosen": -12.134450912475586,
-      "logits/rejected": -12.029266357421875,
-      "logps/chosen": -148.46795654296875,
-      "logps/rejected": -183.1780242919922,
-      "loss": 0.3814,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -7.2909979820251465,
-      "rewards/margins": 3.348233461380005,
-      "rewards/rejected": -10.63923168182373,
-      "step": 9050
-    },
-    {
-      "epoch": 0.724075924075924,
-      "grad_norm": 18.927030563354492,
-      "learning_rate": 1.0726051987310234e-06,
-      "logits/chosen": -12.074101448059082,
-      "logits/rejected": -12.02935791015625,
-      "logps/chosen": -142.89334106445312,
-      "logps/rejected": -177.9833984375,
-      "loss": 0.3096,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -7.062876224517822,
-      "rewards/margins": 3.2246220111846924,
-      "rewards/rejected": -10.287498474121094,
-      "step": 9060
-    },
-    {
-      "epoch": 0.7248751248751248,
-      "grad_norm": 5.6172776222229,
-      "learning_rate": 1.0668843350633546e-06,
-      "logits/chosen": -12.099202156066895,
-      "logits/rejected": -11.991988182067871,
-      "logps/chosen": -140.8617401123047,
-      "logps/rejected": -171.303466796875,
-      "loss": 0.3356,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -6.896264553070068,
-      "rewards/margins": 3.1805150508880615,
-      "rewards/rejected": -10.07677936553955,
-      "step": 9070
-    },
-    {
-      "epoch": 0.7256743256743257,
-      "grad_norm": 20.949323654174805,
-      "learning_rate": 1.0611746272780427e-06,
-      "logits/chosen": -11.929264068603516,
-      "logits/rejected": -11.94435977935791,
-      "logps/chosen": -144.0133819580078,
-      "logps/rejected": -175.9869842529297,
-      "loss": 0.4879,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -6.666404724121094,
-      "rewards/margins": 3.1060521602630615,
-      "rewards/rejected": -9.772457122802734,
-      "step": 9080
-    },
-    {
-      "epoch": 0.7264735264735265,
-      "grad_norm": 4.174493789672852,
-      "learning_rate": 1.0554761198214847e-06,
-      "logits/chosen": -12.062529563903809,
-      "logits/rejected": -12.054919242858887,
-      "logps/chosen": -146.851806640625,
-      "logps/rejected": -187.8809814453125,
-      "loss": 0.2554,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -7.321177959442139,
-      "rewards/margins": 3.880296468734741,
-      "rewards/rejected": -11.201473236083984,
-      "step": 9090
-    },
-    {
-      "epoch": 0.7272727272727273,
-      "grad_norm": 22.710742950439453,
-      "learning_rate": 1.0497888570528896e-06,
-      "logits/chosen": -11.8251371383667,
-      "logits/rejected": -11.967094421386719,
-      "logps/chosen": -141.9603271484375,
-      "logps/rejected": -195.25697326660156,
-      "loss": 0.3168,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -6.738444805145264,
-      "rewards/margins": 3.8849964141845703,
-      "rewards/rejected": -10.623441696166992,
-      "step": 9100
-    },
-    {
-      "epoch": 0.7280719280719281,
-      "grad_norm": 5.7591633796691895,
-      "learning_rate": 1.0441128832439367e-06,
-      "logits/chosen": -12.042027473449707,
-      "logits/rejected": -12.05923843383789,
-      "logps/chosen": -139.1199951171875,
-      "logps/rejected": -185.91397094726562,
-      "loss": 0.2971,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -6.593803405761719,
-      "rewards/margins": 3.8986644744873047,
-      "rewards/rejected": -10.49246883392334,
-      "step": 9110
-    },
-    {
-      "epoch": 0.7288711288711289,
-      "grad_norm": 10.990606307983398,
-      "learning_rate": 1.0384482425784254e-06,
-      "logits/chosen": -12.108098030090332,
-      "logits/rejected": -12.069998741149902,
-      "logps/chosen": -147.50961303710938,
-      "logps/rejected": -180.3170623779297,
-      "loss": 0.2576,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": -7.34895658493042,
-      "rewards/margins": 3.5233139991760254,
-      "rewards/rejected": -10.872270584106445,
-      "step": 9120
-    },
-    {
-      "epoch": 0.7296703296703296,
-      "grad_norm": 19.3331241607666,
-      "learning_rate": 1.0327949791519333e-06,
-      "logits/chosen": -12.132037162780762,
-      "logits/rejected": -12.075540542602539,
-      "logps/chosen": -143.96888732910156,
-      "logps/rejected": -182.4113006591797,
-      "loss": 0.2734,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -7.375763893127441,
-      "rewards/margins": 3.3979785442352295,
-      "rewards/rejected": -10.77374267578125,
-      "step": 9130
-    },
-    {
-      "epoch": 0.7304695304695304,
-      "grad_norm": 14.094701766967773,
-      "learning_rate": 1.0271531369714768e-06,
-      "logits/chosen": -12.086943626403809,
-      "logits/rejected": -12.090289115905762,
-      "logps/chosen": -146.3450927734375,
-      "logps/rejected": -192.55758666992188,
-      "loss": 0.2234,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": -7.3004913330078125,
-      "rewards/margins": 4.146739959716797,
-      "rewards/rejected": -11.447232246398926,
-      "step": 9140
-    },
-    {
-      "epoch": 0.7312687312687313,
-      "grad_norm": 17.656003952026367,
-      "learning_rate": 1.0215227599551617e-06,
-      "logits/chosen": -11.908515930175781,
-      "logits/rejected": -11.969069480895996,
-      "logps/chosen": -136.28677368164062,
-      "logps/rejected": -177.72303771972656,
-      "loss": 0.292,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -6.698323726654053,
-      "rewards/margins": 3.431936264038086,
-      "rewards/rejected": -10.13025951385498,
-      "step": 9150
-    },
-    {
-      "epoch": 0.7320679320679321,
-      "grad_norm": 17.50604248046875,
-      "learning_rate": 1.0159038919318475e-06,
-      "logits/chosen": -11.928923606872559,
-      "logits/rejected": -12.080354690551758,
-      "logps/chosen": -141.27496337890625,
-      "logps/rejected": -197.4722442626953,
-      "loss": 0.2261,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -6.990950107574463,
-      "rewards/margins": 4.4784111976623535,
-      "rewards/rejected": -11.469362258911133,
-      "step": 9160
-    },
-    {
-      "epoch": 0.7328671328671329,
-      "grad_norm": 17.071834564208984,
-      "learning_rate": 1.0102965766408024e-06,
-      "logits/chosen": -12.118680000305176,
-      "logits/rejected": -12.06589412689209,
-      "logps/chosen": -143.53135681152344,
-      "logps/rejected": -174.1621551513672,
-      "loss": 0.3915,
-      "rewards/accuracies": 0.8375000357627869,
-      "rewards/chosen": -7.412281036376953,
-      "rewards/margins": 2.7176148891448975,
-      "rewards/rejected": -10.12989616394043,
-      "step": 9170
-    },
-    {
-      "epoch": 0.7336663336663337,
-      "grad_norm": 26.264467239379883,
-      "learning_rate": 1.004700857731363e-06,
-      "logits/chosen": -12.215692520141602,
-      "logits/rejected": -12.088415145874023,
-      "logps/chosen": -147.3823699951172,
-      "logps/rejected": -180.8734893798828,
-      "loss": 0.4314,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -7.172257900238037,
-      "rewards/margins": 3.250685930252075,
-      "rewards/rejected": -10.422944068908691,
-      "step": 9180
-    },
-    {
-      "epoch": 0.7344655344655344,
-      "grad_norm": 13.467389106750488,
-      "learning_rate": 9.991167787625986e-07,
-      "logits/chosen": -12.089383125305176,
-      "logits/rejected": -12.085790634155273,
-      "logps/chosen": -139.9268798828125,
-      "logps/rejected": -170.32952880859375,
-      "loss": 0.3885,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -6.753329753875732,
-      "rewards/margins": 2.873528003692627,
-      "rewards/rejected": -9.626856803894043,
-      "step": 9190
-    },
-    {
-      "epoch": 0.7352647352647352,
-      "grad_norm": 12.627513885498047,
-      "learning_rate": 9.93544383202965e-07,
-      "logits/chosen": -12.144837379455566,
-      "logits/rejected": -12.10294246673584,
-      "logps/chosen": -134.33380126953125,
-      "logps/rejected": -171.6858367919922,
-      "loss": 0.3938,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -6.5074615478515625,
-      "rewards/margins": 3.308091878890991,
-      "rewards/rejected": -9.815553665161133,
-      "step": 9200
-    },
-    {
-      "epoch": 0.736063936063936,
-      "grad_norm": 16.804401397705078,
-      "learning_rate": 9.879837144299748e-07,
-      "logits/chosen": -12.007431030273438,
-      "logits/rejected": -12.111856460571289,
-      "logps/chosen": -140.52272033691406,
-      "logps/rejected": -183.57952880859375,
-      "loss": 0.2648,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": -6.721423625946045,
-      "rewards/margins": 3.799896001815796,
-      "rewards/rejected": -10.521319389343262,
-      "step": 9210
-    },
-    {
-      "epoch": 0.7368631368631369,
-      "grad_norm": 4.39699125289917,
-      "learning_rate": 9.824348157298513e-07,
-      "logits/chosen": -12.106515884399414,
-      "logits/rejected": -12.116042137145996,
-      "logps/chosen": -138.69273376464844,
-      "logps/rejected": -177.86119079589844,
-      "loss": 0.2122,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": -6.870777130126953,
-      "rewards/margins": 3.3793563842773438,
-      "rewards/rejected": -10.250133514404297,
-      "step": 9220
-    },
-    {
-      "epoch": 0.7376623376623377,
-      "grad_norm": 15.644159317016602,
-      "learning_rate": 9.768977302971958e-07,
-      "logits/chosen": -12.205962181091309,
-      "logits/rejected": -12.170328140258789,
-      "logps/chosen": -138.8316192626953,
-      "logps/rejected": -173.80870056152344,
-      "loss": 0.2908,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -6.71931791305542,
-      "rewards/margins": 3.319683790206909,
-      "rewards/rejected": -10.03900146484375,
-      "step": 9230
-    },
-    {
-      "epoch": 0.7384615384615385,
-      "grad_norm": 11.687942504882812,
-      "learning_rate": 9.71372501234654e-07,
-      "logits/chosen": -11.921524047851562,
-      "logits/rejected": -12.134785652160645,
-      "logps/chosen": -147.5392303466797,
-      "logps/rejected": -200.3516082763672,
-      "loss": 0.2618,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -7.338183879852295,
-      "rewards/margins": 4.458912372589111,
-      "rewards/rejected": -11.797096252441406,
-      "step": 9240
-    },
-    {
-      "epoch": 0.7392607392607392,
-      "grad_norm": 30.63665008544922,
-      "learning_rate": 9.65859171552574e-07,
-      "logits/chosen": -12.055826187133789,
-      "logits/rejected": -12.116484642028809,
-      "logps/chosen": -148.4337615966797,
-      "logps/rejected": -192.30284118652344,
-      "loss": 0.3506,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -7.428224086761475,
-      "rewards/margins": 3.7830750942230225,
-      "rewards/rejected": -11.211298942565918,
-      "step": 9250
-    },
-    {
-      "epoch": 0.74005994005994,
-      "grad_norm": 16.43461799621582,
-      "learning_rate": 9.603577841686753e-07,
-      "logits/chosen": -12.26970100402832,
-      "logits/rejected": -12.166749000549316,
-      "logps/chosen": -143.5255126953125,
-      "logps/rejected": -178.06578063964844,
-      "loss": 0.4378,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -7.392195224761963,
-      "rewards/margins": 3.0472142696380615,
-      "rewards/rejected": -10.439410209655762,
-      "step": 9260
-    },
-    {
-      "epoch": 0.7408591408591408,
-      "grad_norm": 25.912494659423828,
-      "learning_rate": 9.548683819077173e-07,
-      "logits/chosen": -12.251623153686523,
-      "logits/rejected": -12.168976783752441,
-      "logps/chosen": -143.90147399902344,
-      "logps/rejected": -187.81251525878906,
-      "loss": 0.2799,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -7.194044589996338,
-      "rewards/margins": 4.0475568771362305,
-      "rewards/rejected": -11.24160099029541,
-      "step": 9270
-    },
-    {
-      "epoch": 0.7416583416583417,
-      "grad_norm": 25.348522186279297,
-      "learning_rate": 9.493910075011586e-07,
-      "logits/chosen": -12.282008171081543,
-      "logits/rejected": -12.1243257522583,
-      "logps/chosen": -138.51602172851562,
-      "logps/rejected": -172.79367065429688,
-      "loss": 0.3618,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -6.6430583000183105,
-      "rewards/margins": 3.337941884994507,
-      "rewards/rejected": -9.980999946594238,
-      "step": 9280
-    },
-    {
-      "epoch": 0.7424575424575425,
-      "grad_norm": 21.15696144104004,
-      "learning_rate": 9.439257035868329e-07,
-      "logits/chosen": -12.23343563079834,
-      "logits/rejected": -12.241632461547852,
-      "logps/chosen": -146.10997009277344,
-      "logps/rejected": -180.65330505371094,
-      "loss": 0.3843,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -7.304535865783691,
-      "rewards/margins": 2.8212978839874268,
-      "rewards/rejected": -10.125833511352539,
-      "step": 9290
-    },
-    {
-      "epoch": 0.7432567432567433,
-      "grad_norm": 26.879541397094727,
-      "learning_rate": 9.384725127086097e-07,
-      "logits/chosen": -12.251128196716309,
-      "logits/rejected": -12.189742088317871,
-      "logps/chosen": -145.11155700683594,
-      "logps/rejected": -167.321533203125,
-      "loss": 0.4695,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": -7.0795488357543945,
-      "rewards/margins": 2.4740891456604004,
-      "rewards/rejected": -9.553638458251953,
-      "step": 9300
-    },
-    {
-      "epoch": 0.7440559440559441,
-      "grad_norm": 15.358477592468262,
-      "learning_rate": 9.33031477316067e-07,
-      "logits/chosen": -12.167581558227539,
-      "logits/rejected": -12.135851860046387,
-      "logps/chosen": -143.1781005859375,
-      "logps/rejected": -178.07818603515625,
-      "loss": 0.3024,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": -6.998317241668701,
-      "rewards/margins": 3.064319372177124,
-      "rewards/rejected": -10.062636375427246,
-      "step": 9310
-    },
-    {
-      "epoch": 0.7448551448551448,
-      "grad_norm": 19.7071533203125,
-      "learning_rate": 9.276026397641613e-07,
-      "logits/chosen": -12.193705558776855,
-      "logits/rejected": -12.153387069702148,
-      "logps/chosen": -143.9748077392578,
-      "logps/rejected": -186.61892700195312,
-      "loss": 0.3386,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -7.485788822174072,
-      "rewards/margins": 3.6492555141448975,
-      "rewards/rejected": -11.13504409790039,
-      "step": 9320
-    },
-    {
-      "epoch": 0.7456543456543456,
-      "grad_norm": 17.71928596496582,
-      "learning_rate": 9.22186042312897e-07,
-      "logits/chosen": -12.080641746520996,
-      "logits/rejected": -12.146191596984863,
-      "logps/chosen": -143.53831481933594,
-      "logps/rejected": -181.89451599121094,
-      "loss": 0.4066,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -7.032649993896484,
-      "rewards/margins": 3.341438055038452,
-      "rewards/rejected": -10.374088287353516,
-      "step": 9330
-    },
-    {
-      "epoch": 0.7464535464535464,
-      "grad_norm": 21.585674285888672,
-      "learning_rate": 9.167817271269955e-07,
-      "logits/chosen": -12.091574668884277,
-      "logits/rejected": -12.047171592712402,
-      "logps/chosen": -141.76171875,
-      "logps/rejected": -177.56228637695312,
-      "loss": 0.5082,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -6.7036261558532715,
-      "rewards/margins": 3.2643883228302,
-      "rewards/rejected": -9.96801471710205,
-      "step": 9340
-    },
-    {
-      "epoch": 0.7472527472527473,
-      "grad_norm": 5.732624053955078,
-      "learning_rate": 9.113897362755695e-07,
-      "logits/chosen": -12.26594066619873,
-      "logits/rejected": -12.120298385620117,
-      "logps/chosen": -142.8588409423828,
-      "logps/rejected": -178.0065460205078,
-      "loss": 0.251,
-      "rewards/accuracies": 0.9000000357627869,
-      "rewards/chosen": -7.191504955291748,
-      "rewards/margins": 3.2799580097198486,
-      "rewards/rejected": -10.471463203430176,
-      "step": 9350
-    },
-    {
-      "epoch": 0.7480519480519481,
-      "grad_norm": 1.6368052959442139,
-      "learning_rate": 9.060101117317937e-07,
-      "logits/chosen": -12.139222145080566,
-      "logits/rejected": -12.207667350769043,
-      "logps/chosen": -137.1557159423828,
-      "logps/rejected": -187.65736389160156,
-      "loss": 0.1842,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": -6.830145359039307,
-      "rewards/margins": 4.090631484985352,
-      "rewards/rejected": -10.9207763671875,
-      "step": 9360
-    },
-    {
-      "epoch": 0.7488511488511489,
-      "grad_norm": 12.057502746582031,
-      "learning_rate": 9.006428953725821e-07,
-      "logits/chosen": -12.18132209777832,
-      "logits/rejected": -12.152626037597656,
-      "logps/chosen": -140.4557342529297,
-      "logps/rejected": -176.78677368164062,
-      "loss": 0.3208,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -7.000676155090332,
-      "rewards/margins": 3.409414052963257,
-      "rewards/rejected": -10.410090446472168,
-      "step": 9370
-    },
-    {
-      "epoch": 0.7496503496503496,
-      "grad_norm": 16.389795303344727,
-      "learning_rate": 8.95288128978255e-07,
-      "logits/chosen": -12.259406089782715,
-      "logits/rejected": -12.169455528259277,
-      "logps/chosen": -150.5641632080078,
-      "logps/rejected": -182.72840881347656,
-      "loss": 0.3896,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -7.585774898529053,
-      "rewards/margins": 3.1430985927581787,
-      "rewards/rejected": -10.728873252868652,
-      "step": 9380
-    },
-    {
-      "epoch": 0.7504495504495504,
-      "grad_norm": 24.197708129882812,
-      "learning_rate": 8.899458542322214e-07,
-      "logits/chosen": -12.070429801940918,
-      "logits/rejected": -12.212790489196777,
-      "logps/chosen": -146.36294555664062,
-      "logps/rejected": -186.1180877685547,
-      "loss": 0.3653,
-      "rewards/accuracies": 0.8375000357627869,
-      "rewards/chosen": -7.398922920227051,
-      "rewards/margins": 3.5901920795440674,
-      "rewards/rejected": -10.989115715026855,
-      "step": 9390
-    },
-    {
-      "epoch": 0.7512487512487512,
-      "grad_norm": 21.409324645996094,
-      "learning_rate": 8.846161127206477e-07,
-      "logits/chosen": -12.170221328735352,
-      "logits/rejected": -12.146059036254883,
-      "logps/chosen": -145.71340942382812,
-      "logps/rejected": -177.34117126464844,
-      "loss": 0.3983,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -7.039772033691406,
-      "rewards/margins": 3.1227729320526123,
-      "rewards/rejected": -10.162545204162598,
-      "step": 9400
-    },
-    {
-      "epoch": 0.752047952047952,
-      "grad_norm": 5.432770252227783,
-      "learning_rate": 8.792989459321399e-07,
-      "logits/chosen": -12.037500381469727,
-      "logits/rejected": -12.09168529510498,
-      "logps/chosen": -145.8544464111328,
-      "logps/rejected": -185.66140747070312,
-      "loss": 0.249,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -7.2284698486328125,
-      "rewards/margins": 3.3508777618408203,
-      "rewards/rejected": -10.579347610473633,
-      "step": 9410
-    },
-    {
-      "epoch": 0.7528471528471529,
-      "grad_norm": 25.123559951782227,
-      "learning_rate": 8.739943952574159e-07,
-      "logits/chosen": -12.261021614074707,
-      "logits/rejected": -12.223187446594238,
-      "logps/chosen": -139.6779022216797,
-      "logps/rejected": -170.65255737304688,
-      "loss": 0.4073,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -6.672074794769287,
-      "rewards/margins": 2.9437406063079834,
-      "rewards/rejected": -9.615816116333008,
-      "step": 9420
-    },
-    {
-      "epoch": 0.7536463536463537,
-      "grad_norm": 31.40964698791504,
-      "learning_rate": 8.68702501988986e-07,
-      "logits/chosen": -12.135108947753906,
-      "logits/rejected": -12.11870002746582,
-      "logps/chosen": -144.87762451171875,
-      "logps/rejected": -181.89866638183594,
-      "loss": 0.5055,
-      "rewards/accuracies": 0.8375000357627869,
-      "rewards/chosen": -7.044806003570557,
-      "rewards/margins": 3.2895405292510986,
-      "rewards/rejected": -10.33434772491455,
-      "step": 9430
-    },
-    {
-      "epoch": 0.7544455544455545,
-      "grad_norm": 29.706546783447266,
-      "learning_rate": 8.634233073208298e-07,
-      "logits/chosen": -12.178078651428223,
-      "logits/rejected": -12.21338176727295,
-      "logps/chosen": -137.90139770507812,
-      "logps/rejected": -171.4556884765625,
-      "loss": 0.4634,
-      "rewards/accuracies": 0.7750000357627869,
-      "rewards/chosen": -6.509404182434082,
-      "rewards/margins": 3.0218255519866943,
-      "rewards/rejected": -9.531229019165039,
-      "step": 9440
-    },
-    {
-      "epoch": 0.7552447552447552,
-      "grad_norm": 12.484968185424805,
-      "learning_rate": 8.581568523480782e-07,
-      "logits/chosen": -11.835305213928223,
-      "logits/rejected": -12.079849243164062,
-      "logps/chosen": -147.91847229003906,
-      "logps/rejected": -193.1023406982422,
-      "loss": 0.5511,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": -7.32641077041626,
-      "rewards/margins": 3.2503297328948975,
-      "rewards/rejected": -10.576741218566895,
-      "step": 9450
-    },
-    {
-      "epoch": 0.756043956043956,
-      "grad_norm": 12.35820198059082,
-      "learning_rate": 8.529031780666916e-07,
-      "logits/chosen": -12.199759483337402,
-      "logits/rejected": -12.08620548248291,
-      "logps/chosen": -135.8587188720703,
-      "logps/rejected": -164.94857788085938,
-      "loss": 0.3512,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -6.230266094207764,
-      "rewards/margins": 3.0740859508514404,
-      "rewards/rejected": -9.304351806640625,
-      "step": 9460
-    },
-    {
-      "epoch": 0.7568431568431568,
-      "grad_norm": 16.328044891357422,
-      "learning_rate": 8.476623253731398e-07,
-      "logits/chosen": -12.183242797851562,
-      "logits/rejected": -12.148956298828125,
-      "logps/chosen": -140.8086700439453,
-      "logps/rejected": -183.7784881591797,
-      "loss": 0.4349,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": -6.907830238342285,
-      "rewards/margins": 3.4587128162384033,
-      "rewards/rejected": -10.36654281616211,
-      "step": 9470
-    },
-    {
-      "epoch": 0.7576423576423577,
-      "grad_norm": 24.032363891601562,
-      "learning_rate": 8.424343350640851e-07,
-      "logits/chosen": -12.169767379760742,
-      "logits/rejected": -12.077974319458008,
-      "logps/chosen": -145.02978515625,
-      "logps/rejected": -176.01589965820312,
-      "loss": 0.4232,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -6.929136753082275,
-      "rewards/margins": 3.1232104301452637,
-      "rewards/rejected": -10.052347183227539,
-      "step": 9480
-    },
-    {
-      "epoch": 0.7584415584415585,
-      "grad_norm": 28.697765350341797,
-      "learning_rate": 8.372192478360638e-07,
-      "logits/chosen": -12.231818199157715,
-      "logits/rejected": -12.12775707244873,
-      "logps/chosen": -139.65875244140625,
-      "logps/rejected": -182.7017364501953,
-      "loss": 0.2807,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -6.625915050506592,
-      "rewards/margins": 3.8090245723724365,
-      "rewards/rejected": -10.434938430786133,
-      "step": 9490
-    },
-    {
-      "epoch": 0.7592407592407593,
-      "grad_norm": 27.807371139526367,
-      "learning_rate": 8.320171042851726e-07,
-      "logits/chosen": -12.16058349609375,
-      "logits/rejected": -12.050048828125,
-      "logps/chosen": -139.90960693359375,
-      "logps/rejected": -166.60195922851562,
-      "loss": 0.3813,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -6.837401866912842,
-      "rewards/margins": 2.82214617729187,
-      "rewards/rejected": -9.65954875946045,
-      "step": 9500
-    },
-    {
-      "epoch": 0.76003996003996,
-      "grad_norm": 9.570755958557129,
-      "learning_rate": 8.268279449067466e-07,
-      "logits/chosen": -12.190852165222168,
-      "logits/rejected": -12.130860328674316,
-      "logps/chosen": -146.51673889160156,
-      "logps/rejected": -175.041015625,
-      "loss": 0.4391,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -7.120797634124756,
-      "rewards/margins": 2.683896780014038,
-      "rewards/rejected": -9.804695129394531,
-      "step": 9510
-    },
-    {
-      "epoch": 0.7608391608391608,
-      "grad_norm": 20.496923446655273,
-      "learning_rate": 8.216518100950504e-07,
-      "logits/chosen": -12.104537010192871,
-      "logits/rejected": -12.122392654418945,
-      "logps/chosen": -146.18064880371094,
-      "logps/rejected": -178.27735900878906,
-      "loss": 0.4707,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -7.32900857925415,
-      "rewards/margins": 3.099689245223999,
-      "rewards/rejected": -10.42869758605957,
-      "step": 9520
-    },
-    {
-      "epoch": 0.7616383616383616,
-      "grad_norm": 15.134071350097656,
-      "learning_rate": 8.164887401429584e-07,
-      "logits/chosen": -12.044901847839355,
-      "logits/rejected": -12.104376792907715,
-      "logps/chosen": -138.58233642578125,
-      "logps/rejected": -178.3093719482422,
-      "loss": 0.2723,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": -6.6672234535217285,
-      "rewards/margins": 3.596958875656128,
-      "rewards/rejected": -10.264182090759277,
-      "step": 9530
-    },
-    {
-      "epoch": 0.7624375624375624,
-      "grad_norm": 25.646800994873047,
-      "learning_rate": 8.113387752416457e-07,
-      "logits/chosen": -12.149320602416992,
-      "logits/rejected": -12.149224281311035,
-      "logps/chosen": -143.04383850097656,
-      "logps/rejected": -172.9322967529297,
-      "loss": 0.5022,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -7.168498516082764,
-      "rewards/margins": 2.843183994293213,
-      "rewards/rejected": -10.011682510375977,
-      "step": 9540
-    },
-    {
-      "epoch": 0.7632367632367633,
-      "grad_norm": 22.260087966918945,
-      "learning_rate": 8.062019554802714e-07,
-      "logits/chosen": -12.193267822265625,
-      "logits/rejected": -12.10947322845459,
-      "logps/chosen": -131.3878936767578,
-      "logps/rejected": -170.19956970214844,
-      "loss": 0.2735,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -5.5966477394104,
-      "rewards/margins": 4.071971416473389,
-      "rewards/rejected": -9.668619155883789,
-      "step": 9550
-    },
-    {
-      "epoch": 0.7640359640359641,
-      "grad_norm": 19.730064392089844,
-      "learning_rate": 8.010783208456684e-07,
-      "logits/chosen": -12.17712688446045,
-      "logits/rejected": -12.060352325439453,
-      "logps/chosen": -140.58041381835938,
-      "logps/rejected": -164.36985778808594,
-      "loss": 0.5613,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -6.790876865386963,
-      "rewards/margins": 2.490446090698242,
-      "rewards/rejected": -9.281322479248047,
-      "step": 9560
-    },
-    {
-      "epoch": 0.7648351648351648,
-      "grad_norm": 22.50751495361328,
-      "learning_rate": 7.959679112220314e-07,
-      "logits/chosen": -12.113037109375,
-      "logits/rejected": -12.047637939453125,
-      "logps/chosen": -134.7208251953125,
-      "logps/rejected": -164.46522521972656,
-      "loss": 0.3593,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -6.313191890716553,
-      "rewards/margins": 3.0246636867523193,
-      "rewards/rejected": -9.33785629272461,
-      "step": 9570
-    },
-    {
-      "epoch": 0.7656343656343656,
-      "grad_norm": 5.428042411804199,
-      "learning_rate": 7.908707663906087e-07,
-      "logits/chosen": -12.088178634643555,
-      "logits/rejected": -12.075183868408203,
-      "logps/chosen": -137.69187927246094,
-      "logps/rejected": -178.31585693359375,
-      "loss": 0.2981,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -6.6719231605529785,
-      "rewards/margins": 3.5246846675872803,
-      "rewards/rejected": -10.196608543395996,
-      "step": 9580
-    },
-    {
-      "epoch": 0.7664335664335664,
-      "grad_norm": 20.321847915649414,
-      "learning_rate": 7.857869260293899e-07,
-      "logits/chosen": -11.96019172668457,
-      "logits/rejected": -12.067197799682617,
-      "logps/chosen": -140.22085571289062,
-      "logps/rejected": -175.7401123046875,
-      "loss": 0.3875,
-      "rewards/accuracies": 0.8375000357627869,
-      "rewards/chosen": -6.614634037017822,
-      "rewards/margins": 3.3720016479492188,
-      "rewards/rejected": -9.9866361618042,
-      "step": 9590
-    },
-    {
-      "epoch": 0.7672327672327672,
-      "grad_norm": 9.35619831085205,
-      "learning_rate": 7.807164297127973e-07,
-      "logits/chosen": -12.161341667175293,
-      "logits/rejected": -12.166190147399902,
-      "logps/chosen": -137.65902709960938,
-      "logps/rejected": -170.58155822753906,
-      "loss": 0.4852,
-      "rewards/accuracies": 0.7750000357627869,
-      "rewards/chosen": -6.776899814605713,
-      "rewards/margins": 2.8173375129699707,
-      "rewards/rejected": -9.594237327575684,
-      "step": 9600
-    },
-    {
-      "epoch": 0.768031968031968,
-      "grad_norm": 12.084158897399902,
-      "learning_rate": 7.756593169113783e-07,
-      "logits/chosen": -12.133938789367676,
-      "logits/rejected": -12.069931030273438,
-      "logps/chosen": -137.7152862548828,
-      "logps/rejected": -172.8800811767578,
-      "loss": 0.2689,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": -6.58184814453125,
-      "rewards/margins": 3.251387119293213,
-      "rewards/rejected": -9.833235740661621,
-      "step": 9610
-    },
-    {
-      "epoch": 0.7688311688311689,
-      "grad_norm": 19.758743286132812,
-      "learning_rate": 7.70615626991498e-07,
-      "logits/chosen": -12.128284454345703,
-      "logits/rejected": -12.0636625289917,
-      "logps/chosen": -132.8829345703125,
-      "logps/rejected": -169.2217254638672,
-      "loss": 0.3985,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -6.250820159912109,
-      "rewards/margins": 3.20519757270813,
-      "rewards/rejected": -9.456018447875977,
-      "step": 9620
-    },
-    {
-      "epoch": 0.7696303696303697,
-      "grad_norm": 30.162477493286133,
-      "learning_rate": 7.65585399215035e-07,
-      "logits/chosen": -11.996148109436035,
-      "logits/rejected": -12.095602989196777,
-      "logps/chosen": -146.44432067871094,
-      "logps/rejected": -189.7522735595703,
-      "loss": 0.5154,
-      "rewards/accuracies": 0.7750000357627869,
-      "rewards/chosen": -7.2563796043396,
-      "rewards/margins": 3.7690529823303223,
-      "rewards/rejected": -11.025432586669922,
-      "step": 9630
-    },
-    {
-      "epoch": 0.7704295704295704,
-      "grad_norm": 22.42033576965332,
-      "learning_rate": 7.605686727390726e-07,
-      "logits/chosen": -12.08029556274414,
-      "logits/rejected": -12.12644100189209,
-      "logps/chosen": -142.2587127685547,
-      "logps/rejected": -178.220458984375,
-      "loss": 0.4022,
-      "rewards/accuracies": 0.8375000357627869,
-      "rewards/chosen": -7.209232330322266,
-      "rewards/margins": 3.0976860523223877,
-      "rewards/rejected": -10.306918144226074,
-      "step": 9640
-    },
-    {
-      "epoch": 0.7712287712287712,
-      "grad_norm": 21.99079704284668,
-      "learning_rate": 7.555654866155945e-07,
-      "logits/chosen": -12.194561004638672,
-      "logits/rejected": -12.096150398254395,
-      "logps/chosen": -129.3114471435547,
-      "logps/rejected": -172.11154174804688,
-      "loss": 0.2096,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": -6.011118412017822,
-      "rewards/margins": 3.7386741638183594,
-      "rewards/rejected": -9.74979305267334,
-      "step": 9650
-    },
-    {
-      "epoch": 0.772027972027972,
-      "grad_norm": 20.23876190185547,
-      "learning_rate": 7.505758797911813e-07,
-      "logits/chosen": -12.108684539794922,
-      "logits/rejected": -12.02026653289795,
-      "logps/chosen": -139.0235137939453,
-      "logps/rejected": -177.53076171875,
-      "loss": 0.2553,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": -6.655057430267334,
-      "rewards/margins": 3.561795949935913,
-      "rewards/rejected": -10.216853141784668,
-      "step": 9660
-    },
-    {
-      "epoch": 0.7728271728271728,
-      "grad_norm": 15.677886962890625,
-      "learning_rate": 7.4559989110671e-07,
-      "logits/chosen": -11.971091270446777,
-      "logits/rejected": -12.057757377624512,
-      "logps/chosen": -142.4282684326172,
-      "logps/rejected": -188.99403381347656,
-      "loss": 0.228,
-      "rewards/accuracies": 0.9000000357627869,
-      "rewards/chosen": -6.66241979598999,
-      "rewards/margins": 4.3741455078125,
-      "rewards/rejected": -11.036564826965332,
-      "step": 9670
-    },
-    {
-      "epoch": 0.7736263736263737,
-      "grad_norm": 16.64512062072754,
-      "learning_rate": 7.406375592970463e-07,
-      "logits/chosen": -12.08842658996582,
-      "logits/rejected": -12.025527000427246,
-      "logps/chosen": -142.18788146972656,
-      "logps/rejected": -171.26231384277344,
-      "loss": 0.3499,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": -6.8277459144592285,
-      "rewards/margins": 3.1406383514404297,
-      "rewards/rejected": -9.9683837890625,
-      "step": 9680
-    },
-    {
-      "epoch": 0.7744255744255745,
-      "grad_norm": 22.483983993530273,
-      "learning_rate": 7.356889229907469e-07,
-      "logits/chosen": -12.193324089050293,
-      "logits/rejected": -12.0947904586792,
-      "logps/chosen": -141.10386657714844,
-      "logps/rejected": -170.87815856933594,
-      "loss": 0.4599,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -6.796746253967285,
-      "rewards/margins": 2.7952632904052734,
-      "rewards/rejected": -9.592009544372559,
-      "step": 9690
-    },
-    {
-      "epoch": 0.7752247752247752,
-      "grad_norm": 27.179227828979492,
-      "learning_rate": 7.307540207097577e-07,
-      "logits/chosen": -11.823623657226562,
-      "logits/rejected": -12.023360252380371,
-      "logps/chosen": -137.4159393310547,
-      "logps/rejected": -183.09959411621094,
-      "loss": 0.3338,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -6.463667392730713,
-      "rewards/margins": 3.8263802528381348,
-      "rewards/rejected": -10.290048599243164,
-      "step": 9700
-    },
-    {
-      "epoch": 0.776023976023976,
-      "grad_norm": 16.15134620666504,
-      "learning_rate": 7.258328908691146e-07,
-      "logits/chosen": -12.085688591003418,
-      "logits/rejected": -12.043122291564941,
-      "logps/chosen": -146.719970703125,
-      "logps/rejected": -175.64566040039062,
-      "loss": 0.4249,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -7.268960475921631,
-      "rewards/margins": 2.7510926723480225,
-      "rewards/rejected": -10.020052909851074,
-      "step": 9710
-    },
-    {
-      "epoch": 0.7768231768231768,
-      "grad_norm": 10.104687690734863,
-      "learning_rate": 7.209255717766442e-07,
-      "logits/chosen": -12.123698234558105,
-      "logits/rejected": -12.063090324401855,
-      "logps/chosen": -131.28086853027344,
-      "logps/rejected": -170.21096801757812,
-      "loss": 0.3127,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -6.040278434753418,
-      "rewards/margins": 3.5554089546203613,
-      "rewards/rejected": -9.595686912536621,
-      "step": 9720
-    },
-    {
-      "epoch": 0.7776223776223776,
-      "grad_norm": 24.116329193115234,
-      "learning_rate": 7.160321016326647e-07,
-      "logits/chosen": -12.137951850891113,
-      "logits/rejected": -12.023710250854492,
-      "logps/chosen": -146.9043426513672,
-      "logps/rejected": -179.3995819091797,
-      "loss": 0.4506,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -7.208752632141113,
-      "rewards/margins": 2.702526092529297,
-      "rewards/rejected": -9.911277770996094,
-      "step": 9730
-    },
-    {
-      "epoch": 0.7784215784215784,
-      "grad_norm": 22.142757415771484,
-      "learning_rate": 7.111525185296885e-07,
-      "logits/chosen": -12.15946102142334,
-      "logits/rejected": -12.03650188446045,
-      "logps/chosen": -140.1642608642578,
-      "logps/rejected": -166.34169006347656,
-      "loss": 0.2573,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -6.5106353759765625,
-      "rewards/margins": 3.1692912578582764,
-      "rewards/rejected": -9.679926872253418,
-      "step": 9740
-    },
-    {
-      "epoch": 0.7792207792207793,
-      "grad_norm": 12.795488357543945,
-      "learning_rate": 7.062868604521269e-07,
-      "logits/chosen": -12.090128898620605,
-      "logits/rejected": -11.979033470153809,
-      "logps/chosen": -147.46238708496094,
-      "logps/rejected": -187.11485290527344,
-      "loss": 0.2375,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": -7.589272975921631,
-      "rewards/margins": 3.632127046585083,
-      "rewards/rejected": -11.221400260925293,
-      "step": 9750
-    },
-    {
-      "epoch": 0.7800199800199801,
-      "grad_norm": 19.524473190307617,
-      "learning_rate": 7.014351652759943e-07,
-      "logits/chosen": -12.12690258026123,
-      "logits/rejected": -12.100534439086914,
-      "logps/chosen": -143.68600463867188,
-      "logps/rejected": -180.36550903320312,
-      "loss": 0.3881,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -7.622966289520264,
-      "rewards/margins": 3.1440651416778564,
-      "rewards/rejected": -10.767030715942383,
-      "step": 9760
-    },
-    {
-      "epoch": 0.7808191808191808,
-      "grad_norm": 26.3162784576416,
-      "learning_rate": 6.965974707686135e-07,
-      "logits/chosen": -12.084385871887207,
-      "logits/rejected": -11.995930671691895,
-      "logps/chosen": -142.748046875,
-      "logps/rejected": -167.95767211914062,
-      "loss": 0.3584,
-      "rewards/accuracies": 0.8375000357627869,
-      "rewards/chosen": -6.631211280822754,
-      "rewards/margins": 2.9673187732696533,
-      "rewards/rejected": -9.598529815673828,
-      "step": 9770
-    },
-    {
-      "epoch": 0.7816183816183816,
-      "grad_norm": 6.5272650718688965,
-      "learning_rate": 6.917738145883193e-07,
-      "logits/chosen": -12.040802001953125,
-      "logits/rejected": -12.028519630432129,
-      "logps/chosen": -142.88552856445312,
-      "logps/rejected": -170.24624633789062,
-      "loss": 0.4208,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -6.980207920074463,
-      "rewards/margins": 2.5228495597839355,
-      "rewards/rejected": -9.503057479858398,
-      "step": 9780
-    },
-    {
-      "epoch": 0.7824175824175824,
-      "grad_norm": 17.57386016845703,
-      "learning_rate": 6.869642342841676e-07,
-      "logits/chosen": -12.028456687927246,
-      "logits/rejected": -12.003031730651855,
-      "logps/chosen": -142.5583953857422,
-      "logps/rejected": -181.0504150390625,
-      "loss": 0.3639,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -7.004924774169922,
-      "rewards/margins": 3.4568188190460205,
-      "rewards/rejected": -10.461743354797363,
-      "step": 9790
-    },
-    {
-      "epoch": 0.7832167832167832,
-      "grad_norm": 14.398419380187988,
-      "learning_rate": 6.821687672956443e-07,
-      "logits/chosen": -11.970176696777344,
-      "logits/rejected": -12.020694732666016,
-      "logps/chosen": -133.77078247070312,
-      "logps/rejected": -177.79864501953125,
-      "loss": 0.3025,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -6.5025954246521,
-      "rewards/margins": 3.779121160507202,
-      "rewards/rejected": -10.281717300415039,
-      "step": 9800
-    },
-    {
-      "epoch": 0.784015984015984,
-      "grad_norm": 22.83677864074707,
-      "learning_rate": 6.773874509523704e-07,
-      "logits/chosen": -12.11127758026123,
-      "logits/rejected": -12.049725532531738,
-      "logps/chosen": -146.80577087402344,
-      "logps/rejected": -176.0094757080078,
-      "loss": 0.4239,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -7.20656156539917,
-      "rewards/margins": 2.8656184673309326,
-      "rewards/rejected": -10.072178840637207,
-      "step": 9810
-    },
-    {
-      "epoch": 0.7848151848151849,
-      "grad_norm": 36.667903900146484,
-      "learning_rate": 6.726203224738131e-07,
-      "logits/chosen": -11.976351737976074,
-      "logits/rejected": -12.022822380065918,
-      "logps/chosen": -143.89427185058594,
-      "logps/rejected": -186.2271270751953,
-      "loss": 0.4661,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -7.236876964569092,
-      "rewards/margins": 3.4888558387756348,
-      "rewards/rejected": -10.725732803344727,
-      "step": 9820
-    },
-    {
-      "epoch": 0.7856143856143856,
-      "grad_norm": 19.734975814819336,
-      "learning_rate": 6.678674189689988e-07,
-      "logits/chosen": -11.778903007507324,
-      "logits/rejected": -11.971693992614746,
-      "logps/chosen": -144.428955078125,
-      "logps/rejected": -195.58026123046875,
-      "loss": 0.3042,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": -6.7768378257751465,
-      "rewards/margins": 4.174657344818115,
-      "rewards/rejected": -10.951495170593262,
-      "step": 9830
-    },
-    {
-      "epoch": 0.7864135864135864,
-      "grad_norm": 10.465705871582031,
-      "learning_rate": 6.631287774362183e-07,
-      "logits/chosen": -12.12484073638916,
-      "logits/rejected": -12.062652587890625,
-      "logps/chosen": -133.8185577392578,
-      "logps/rejected": -168.7183837890625,
-      "loss": 0.3035,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -6.065422058105469,
-      "rewards/margins": 3.3315017223358154,
-      "rewards/rejected": -9.396924018859863,
-      "step": 9840
-    },
-    {
-      "epoch": 0.7872127872127872,
-      "grad_norm": 10.517167091369629,
-      "learning_rate": 6.584044347627449e-07,
-      "logits/chosen": -11.837389945983887,
-      "logits/rejected": -11.989556312561035,
-      "logps/chosen": -140.94505310058594,
-      "logps/rejected": -196.12718200683594,
-      "loss": 0.3067,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": -6.967642307281494,
-      "rewards/margins": 4.32666540145874,
-      "rewards/rejected": -11.294306755065918,
-      "step": 9850
-    },
-    {
-      "epoch": 0.788011988011988,
-      "grad_norm": 36.782264709472656,
-      "learning_rate": 6.536944277245428e-07,
-      "logits/chosen": -12.064345359802246,
-      "logits/rejected": -12.121374130249023,
-      "logps/chosen": -139.4950714111328,
-      "logps/rejected": -174.64431762695312,
-      "loss": 0.3918,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -6.963471412658691,
-      "rewards/margins": 3.0049195289611816,
-      "rewards/rejected": -9.968390464782715,
-      "step": 9860
-    },
-    {
-      "epoch": 0.7888111888111888,
-      "grad_norm": 18.88641929626465,
-      "learning_rate": 6.489987929859826e-07,
-      "logits/chosen": -11.981590270996094,
-      "logits/rejected": -11.994577407836914,
-      "logps/chosen": -148.50503540039062,
-      "logps/rejected": -184.12901306152344,
-      "loss": 0.3362,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -7.542794704437256,
-      "rewards/margins": 3.0821638107299805,
-      "rewards/rejected": -10.624958992004395,
-      "step": 9870
-    },
-    {
-      "epoch": 0.7896103896103897,
-      "grad_norm": 15.17828369140625,
-      "learning_rate": 6.443175670995553e-07,
-      "logits/chosen": -12.158833503723145,
-      "logits/rejected": -12.074531555175781,
-      "logps/chosen": -148.4818115234375,
-      "logps/rejected": -178.9248809814453,
-      "loss": 0.4821,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -7.234109401702881,
-      "rewards/margins": 3.1974503993988037,
-      "rewards/rejected": -10.431560516357422,
-      "step": 9880
-    },
-    {
-      "epoch": 0.7904095904095904,
-      "grad_norm": 22.32534408569336,
-      "learning_rate": 6.396507865055915e-07,
-      "logits/chosen": -12.169313430786133,
-      "logits/rejected": -12.043606758117676,
-      "logps/chosen": -136.39280700683594,
-      "logps/rejected": -174.1809844970703,
-      "loss": 0.2659,
-      "rewards/accuracies": 0.9000000357627869,
-      "rewards/chosen": -6.499543190002441,
-      "rewards/margins": 3.417070150375366,
-      "rewards/rejected": -9.916613578796387,
-      "step": 9890
-    },
-    {
-      "epoch": 0.7912087912087912,
-      "grad_norm": 32.212120056152344,
-      "learning_rate": 6.349984875319715e-07,
-      "logits/chosen": -12.060479164123535,
-      "logits/rejected": -12.053366661071777,
-      "logps/chosen": -138.3122100830078,
-      "logps/rejected": -160.47293090820312,
-      "loss": 0.5078,
-      "rewards/accuracies": 0.7750000357627869,
-      "rewards/chosen": -6.464173316955566,
-      "rewards/margins": 2.4671502113342285,
-      "rewards/rejected": -8.931323051452637,
-      "step": 9900
-    },
-    {
-      "epoch": 0.792007992007992,
-      "grad_norm": 7.342180252075195,
-      "learning_rate": 6.303607063938452e-07,
-      "logits/chosen": -12.209647178649902,
-      "logits/rejected": -12.158961296081543,
-      "logps/chosen": -146.908935546875,
-      "logps/rejected": -172.33717346191406,
-      "loss": 0.4753,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -7.479587554931641,
-      "rewards/margins": 2.5916097164154053,
-      "rewards/rejected": -10.071197509765625,
-      "step": 9910
-    },
-    {
-      "epoch": 0.7928071928071928,
-      "grad_norm": 23.605545043945312,
-      "learning_rate": 6.257374791933516e-07,
-      "logits/chosen": -11.992355346679688,
-      "logits/rejected": -12.11590576171875,
-      "logps/chosen": -140.69029235839844,
-      "logps/rejected": -183.41162109375,
-      "loss": 0.1958,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": -6.8030595779418945,
-      "rewards/margins": 3.877943515777588,
-      "rewards/rejected": -10.681002616882324,
-      "step": 9920
-    },
-    {
-      "epoch": 0.7936063936063936,
-      "grad_norm": 26.920862197875977,
-      "learning_rate": 6.211288419193376e-07,
-      "logits/chosen": -12.082389831542969,
-      "logits/rejected": -12.017309188842773,
-      "logps/chosen": -131.4438934326172,
-      "logps/rejected": -172.9900665283203,
-      "loss": 0.327,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -6.080530643463135,
-      "rewards/margins": 3.84183669090271,
-      "rewards/rejected": -9.922367095947266,
-      "step": 9930
-    },
-    {
-      "epoch": 0.7944055944055944,
-      "grad_norm": 1.5619314908981323,
-      "learning_rate": 6.16534830447075e-07,
-      "logits/chosen": -11.758840560913086,
-      "logits/rejected": -12.005099296569824,
-      "logps/chosen": -141.88792419433594,
-      "logps/rejected": -204.21011352539062,
-      "loss": 0.4402,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": -7.180381774902344,
-      "rewards/margins": 4.366318225860596,
-      "rewards/rejected": -11.546700477600098,
-      "step": 9940
-    },
-    {
-      "epoch": 0.7952047952047953,
-      "grad_norm": 2.933011054992676,
-      "learning_rate": 6.119554805379838e-07,
-      "logits/chosen": -12.14220905303955,
-      "logits/rejected": -12.060901641845703,
-      "logps/chosen": -136.44869995117188,
-      "logps/rejected": -175.4172821044922,
-      "loss": 0.2271,
-      "rewards/accuracies": 0.9000000357627869,
-      "rewards/chosen": -6.447462558746338,
-      "rewards/margins": 3.6552579402923584,
-      "rewards/rejected": -10.102721214294434,
-      "step": 9950
-    },
-    {
-      "epoch": 0.796003996003996,
-      "grad_norm": 12.679380416870117,
-      "learning_rate": 6.073908278393545e-07,
-      "logits/chosen": -12.124244689941406,
-      "logits/rejected": -12.009539604187012,
-      "logps/chosen": -138.8267059326172,
-      "logps/rejected": -175.90574645996094,
-      "loss": 0.339,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -6.448057651519775,
-      "rewards/margins": 3.742446184158325,
-      "rewards/rejected": -10.190505027770996,
-      "step": 9960
-    },
-    {
-      "epoch": 0.7968031968031968,
-      "grad_norm": 14.418034553527832,
-      "learning_rate": 6.028409078840666e-07,
-      "logits/chosen": -12.108378410339355,
-      "logits/rejected": -12.136550903320312,
-      "logps/chosen": -134.92137145996094,
-      "logps/rejected": -166.4263458251953,
-      "loss": 0.3816,
-      "rewards/accuracies": 0.8375000357627869,
-      "rewards/chosen": -6.45334005355835,
-      "rewards/margins": 2.92667818069458,
-      "rewards/rejected": -9.38001823425293,
-      "step": 9970
-    },
-    {
-      "epoch": 0.7976023976023976,
-      "grad_norm": 11.231529235839844,
-      "learning_rate": 5.983057560903182e-07,
-      "logits/chosen": -12.065553665161133,
-      "logits/rejected": -12.090777397155762,
-      "logps/chosen": -148.79173278808594,
-      "logps/rejected": -186.18435668945312,
-      "loss": 0.3533,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -7.6139116287231445,
-      "rewards/margins": 3.0789458751678467,
-      "rewards/rejected": -10.692858695983887,
-      "step": 9980
-    },
-    {
-      "epoch": 0.7984015984015984,
-      "grad_norm": 11.536249160766602,
-      "learning_rate": 5.937854077613436e-07,
-      "logits/chosen": -12.032034873962402,
-      "logits/rejected": -11.980717658996582,
-      "logps/chosen": -133.6082000732422,
-      "logps/rejected": -177.53001403808594,
-      "loss": 0.2564,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -6.097396373748779,
-      "rewards/margins": 4.0506815910339355,
-      "rewards/rejected": -10.148077964782715,
-      "step": 9990
-    },
-    {
-      "epoch": 0.7992007992007992,
-      "grad_norm": 17.777193069458008,
-      "learning_rate": 5.892798980851433e-07,
-      "logits/chosen": -12.116438865661621,
-      "logits/rejected": -12.072038650512695,
-      "logps/chosen": -130.113037109375,
-      "logps/rejected": -165.0628662109375,
-      "loss": 0.2075,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": -5.809086322784424,
-      "rewards/margins": 3.3257298469543457,
-      "rewards/rejected": -9.13481616973877,
-      "step": 10000
-    },
-    {
-      "epoch": 0.7992007992007992,
-      "eval_logits/chosen": -12.101720809936523,
-      "eval_logits/rejected": -12.082523345947266,
-      "eval_logps/chosen": -136.00057983398438,
-      "eval_logps/rejected": -173.1504669189453,
-      "eval_loss": 0.3416883945465088,
-      "eval_rewards/accuracies": 0.8510294556617737,
-      "eval_rewards/chosen": -6.40966796875,
-      "eval_rewards/margins": 3.35494065284729,
-      "eval_rewards/rejected": -9.764608383178711,
-      "eval_runtime": 5747.7437,
-      "eval_samples_per_second": 1.935,
-      "eval_steps_per_second": 1.935,
-      "step": 10000
-    },
-    {
-      "epoch": 0.8,
-      "grad_norm": 24.77110481262207,
-      "learning_rate": 5.847892621342067e-07,
-      "logits/chosen": -12.075623512268066,
-      "logits/rejected": -12.089860916137695,
-      "logps/chosen": -137.27749633789062,
-      "logps/rejected": -165.76673889160156,
-      "loss": 0.4547,
-      "rewards/accuracies": 0.8375000357627869,
-      "rewards/chosen": -6.621934413909912,
-      "rewards/margins": 2.814884662628174,
-      "rewards/rejected": -9.436820030212402,
-      "step": 10010
-    },
-    {
-      "epoch": 0.8007992007992007,
-      "grad_norm": 16.389404296875,
-      "learning_rate": 5.803135348652444e-07,
-      "logits/chosen": -11.978671073913574,
-      "logits/rejected": -12.097380638122559,
-      "logps/chosen": -142.80758666992188,
-      "logps/rejected": -187.1731414794922,
-      "loss": 0.3793,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -7.017158508300781,
-      "rewards/margins": 3.64750337600708,
-      "rewards/rejected": -10.66466236114502,
-      "step": 10020
-    },
-    {
-      "epoch": 0.8015984015984016,
-      "grad_norm": 21.381385803222656,
-      "learning_rate": 5.758527511189093e-07,
-      "logits/chosen": -12.174432754516602,
-      "logits/rejected": -12.08348560333252,
-      "logps/chosen": -134.58120727539062,
-      "logps/rejected": -162.95948791503906,
-      "loss": 0.4193,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -6.095840930938721,
-      "rewards/margins": 3.165473222732544,
-      "rewards/rejected": -9.261314392089844,
-      "step": 10030
-    },
-    {
-      "epoch": 0.8023976023976024,
-      "grad_norm": 21.245094299316406,
-      "learning_rate": 5.714069456195298e-07,
-      "logits/chosen": -11.906299591064453,
-      "logits/rejected": -12.103926658630371,
-      "logps/chosen": -137.03237915039062,
-      "logps/rejected": -198.3710174560547,
-      "loss": 0.2923,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -6.643685817718506,
-      "rewards/margins": 4.371219158172607,
-      "rewards/rejected": -11.01490592956543,
-      "step": 10040
-    },
-    {
-      "epoch": 0.8031968031968032,
-      "grad_norm": 22.07200813293457,
-      "learning_rate": 5.669761529748369e-07,
-      "logits/chosen": -12.148396492004395,
-      "logits/rejected": -12.064966201782227,
-      "logps/chosen": -136.11058044433594,
-      "logps/rejected": -172.6295623779297,
-      "loss": 0.4112,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -6.241727828979492,
-      "rewards/margins": 3.4949424266815186,
-      "rewards/rejected": -9.73667049407959,
-      "step": 10050
-    },
-    {
-      "epoch": 0.803996003996004,
-      "grad_norm": 14.887276649475098,
-      "learning_rate": 5.625604076756993e-07,
-      "logits/chosen": -11.927736282348633,
-      "logits/rejected": -12.04349422454834,
-      "logps/chosen": -134.58953857421875,
-      "logps/rejected": -171.0834197998047,
-      "loss": 0.4037,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -6.585518836975098,
-      "rewards/margins": 2.7835166454315186,
-      "rewards/rejected": -9.369035720825195,
-      "step": 10060
-    },
-    {
-      "epoch": 0.8047952047952048,
-      "grad_norm": 23.996013641357422,
-      "learning_rate": 5.58159744095848e-07,
-      "logits/chosen": -12.14687442779541,
-      "logits/rejected": -12.094939231872559,
-      "logps/chosen": -134.4910125732422,
-      "logps/rejected": -170.1653594970703,
-      "loss": 0.3855,
-      "rewards/accuracies": 0.8375000357627869,
-      "rewards/chosen": -6.144654750823975,
-      "rewards/margins": 3.518383026123047,
-      "rewards/rejected": -9.663037300109863,
-      "step": 10070
-    },
-    {
-      "epoch": 0.8055944055944056,
-      "grad_norm": 8.238819122314453,
-      "learning_rate": 5.537741964916163e-07,
-      "logits/chosen": -12.098764419555664,
-      "logits/rejected": -12.075196266174316,
-      "logps/chosen": -126.00044250488281,
-      "logps/rejected": -160.56826782226562,
-      "loss": 0.3695,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -5.844523906707764,
-      "rewards/margins": 2.8255226612091064,
-      "rewards/rejected": -8.67004680633545,
-      "step": 10080
-    },
-    {
-      "epoch": 0.8063936063936064,
-      "grad_norm": 21.31419563293457,
-      "learning_rate": 5.494037990016662e-07,
-      "logits/chosen": -12.082022666931152,
-      "logits/rejected": -12.048373222351074,
-      "logps/chosen": -132.8041229248047,
-      "logps/rejected": -177.38174438476562,
-      "loss": 0.3177,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -6.145178318023682,
-      "rewards/margins": 3.741793155670166,
-      "rewards/rejected": -9.886971473693848,
-      "step": 10090
-    },
-    {
-      "epoch": 0.8071928071928072,
-      "grad_norm": 16.712675094604492,
-      "learning_rate": 5.450485856467272e-07,
-      "logits/chosen": -12.198698997497559,
-      "logits/rejected": -12.049723625183105,
-      "logps/chosen": -132.35696411132812,
-      "logps/rejected": -165.849365234375,
-      "loss": 0.313,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": -6.282183647155762,
-      "rewards/margins": 3.38324236869812,
-      "rewards/rejected": -9.665426254272461,
-      "step": 10100
-    },
-    {
-      "epoch": 0.807992007992008,
-      "grad_norm": 27.159934997558594,
-      "learning_rate": 5.407085903293307e-07,
-      "logits/chosen": -12.112894058227539,
-      "logits/rejected": -12.144476890563965,
-      "logps/chosen": -134.45498657226562,
-      "logps/rejected": -166.0764923095703,
-      "loss": 0.4121,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -6.28261137008667,
-      "rewards/margins": 2.749035596847534,
-      "rewards/rejected": -9.031646728515625,
-      "step": 10110
-    },
-    {
-      "epoch": 0.8087912087912088,
-      "grad_norm": 12.262354850769043,
-      "learning_rate": 5.363838468335444e-07,
-      "logits/chosen": -12.156054496765137,
-      "logits/rejected": -12.093586921691895,
-      "logps/chosen": -127.3411636352539,
-      "logps/rejected": -168.14630126953125,
-      "loss": 0.2362,
-      "rewards/accuracies": 0.9000000357627869,
-      "rewards/chosen": -5.510113716125488,
-      "rewards/margins": 3.73405385017395,
-      "rewards/rejected": -9.244168281555176,
-      "step": 10120
-    },
-    {
-      "epoch": 0.8095904095904096,
-      "grad_norm": 4.4673686027526855,
-      "learning_rate": 5.320743888247098e-07,
-      "logits/chosen": -12.116663932800293,
-      "logits/rejected": -11.99610424041748,
-      "logps/chosen": -134.79776000976562,
-      "logps/rejected": -172.5289764404297,
-      "loss": 0.2452,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": -6.235927104949951,
-      "rewards/margins": 3.509941816329956,
-      "rewards/rejected": -9.745868682861328,
-      "step": 10130
-    },
-    {
-      "epoch": 0.8103896103896104,
-      "grad_norm": 8.002169609069824,
-      "learning_rate": 5.27780249849183e-07,
-      "logits/chosen": -12.125099182128906,
-      "logits/rejected": -12.0616455078125,
-      "logps/chosen": -132.7480926513672,
-      "logps/rejected": -164.4807586669922,
-      "loss": 0.2871,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -5.943828105926514,
-      "rewards/margins": 2.9890506267547607,
-      "rewards/rejected": -8.932878494262695,
-      "step": 10140
-    },
-    {
-      "epoch": 0.8111888111888111,
-      "grad_norm": 10.680505752563477,
-      "learning_rate": 5.235014633340702e-07,
-      "logits/chosen": -12.201571464538574,
-      "logits/rejected": -12.188226699829102,
-      "logps/chosen": -136.27940368652344,
-      "logps/rejected": -173.36851501464844,
-      "loss": 0.2702,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": -6.025295734405518,
-      "rewards/margins": 3.6738479137420654,
-      "rewards/rejected": -9.69914436340332,
-      "step": 10150
-    },
-    {
-      "epoch": 0.811988011988012,
-      "grad_norm": 20.33141326904297,
-      "learning_rate": 5.192380625869675e-07,
-      "logits/chosen": -12.140373229980469,
-      "logits/rejected": -12.101428031921387,
-      "logps/chosen": -136.59375,
-      "logps/rejected": -168.2414093017578,
-      "loss": 0.3829,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -6.619575500488281,
-      "rewards/margins": 2.707887649536133,
-      "rewards/rejected": -9.32746410369873,
-      "step": 10160
-    },
-    {
-      "epoch": 0.8127872127872128,
-      "grad_norm": 11.491778373718262,
-      "learning_rate": 5.149900807957042e-07,
-      "logits/chosen": -12.219225883483887,
-      "logits/rejected": -12.102474212646484,
-      "logps/chosen": -139.05274963378906,
-      "logps/rejected": -168.98056030273438,
-      "loss": 0.5197,
-      "rewards/accuracies": 0.7750000357627869,
-      "rewards/chosen": -7.039988994598389,
-      "rewards/margins": 2.9014103412628174,
-      "rewards/rejected": -9.941398620605469,
-      "step": 10170
-    },
-    {
-      "epoch": 0.8135864135864136,
-      "grad_norm": 22.21400260925293,
-      "learning_rate": 5.10757551028081e-07,
-      "logits/chosen": -12.091588973999023,
-      "logits/rejected": -12.064826011657715,
-      "logps/chosen": -138.83665466308594,
-      "logps/rejected": -176.20933532714844,
-      "loss": 0.3501,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": -6.5815958976745605,
-      "rewards/margins": 3.0772833824157715,
-      "rewards/rejected": -9.658879280090332,
-      "step": 10180
-    },
-    {
-      "epoch": 0.8143856143856144,
-      "grad_norm": 23.83111572265625,
-      "learning_rate": 5.065405062316178e-07,
-      "logits/chosen": -12.201834678649902,
-      "logits/rejected": -12.071370124816895,
-      "logps/chosen": -137.3291778564453,
-      "logps/rejected": -169.20510864257812,
-      "loss": 0.4158,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -6.280584812164307,
-      "rewards/margins": 3.077683687210083,
-      "rewards/rejected": -9.358267784118652,
-      "step": 10190
-    },
-    {
-      "epoch": 0.8151848151848152,
-      "grad_norm": 16.33125877380371,
-      "learning_rate": 5.023389792332905e-07,
-      "logits/chosen": -12.089197158813477,
-      "logits/rejected": -12.10118579864502,
-      "logps/chosen": -138.49813842773438,
-      "logps/rejected": -169.3131866455078,
-      "loss": 0.389,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -6.365344524383545,
-      "rewards/margins": 3.1605288982391357,
-      "rewards/rejected": -9.525874137878418,
-      "step": 10200
-    },
-    {
-      "epoch": 0.8159840159840159,
-      "grad_norm": 6.746474742889404,
-      "learning_rate": 4.981530027392819e-07,
-      "logits/chosen": -11.987895965576172,
-      "logits/rejected": -12.172119140625,
-      "logps/chosen": -137.7808380126953,
-      "logps/rejected": -186.75067138671875,
-      "loss": 0.4087,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -6.512866973876953,
-      "rewards/margins": 3.6988563537597656,
-      "rewards/rejected": -10.211723327636719,
-      "step": 10210
-    },
-    {
-      "epoch": 0.8167832167832167,
-      "grad_norm": 14.158825874328613,
-      "learning_rate": 4.939826093347219e-07,
-      "logits/chosen": -12.10173225402832,
-      "logits/rejected": -12.04938793182373,
-      "logps/chosen": -143.2938995361328,
-      "logps/rejected": -180.4727783203125,
-      "loss": 0.287,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -6.548187255859375,
-      "rewards/margins": 3.6537537574768066,
-      "rewards/rejected": -10.20194149017334,
-      "step": 10220
-    },
-    {
-      "epoch": 0.8175824175824176,
-      "grad_norm": 33.68525695800781,
-      "learning_rate": 4.898278314834364e-07,
-      "logits/chosen": -12.106209754943848,
-      "logits/rejected": -12.101561546325684,
-      "logps/chosen": -143.71617126464844,
-      "logps/rejected": -174.1287078857422,
-      "loss": 0.3957,
-      "rewards/accuracies": 0.8375000357627869,
-      "rewards/chosen": -7.241799831390381,
-      "rewards/margins": 2.7012784481048584,
-      "rewards/rejected": -9.94307804107666,
-      "step": 10230
-    },
-    {
-      "epoch": 0.8183816183816184,
-      "grad_norm": 30.694433212280273,
-      "learning_rate": 4.856887015276962e-07,
-      "logits/chosen": -12.139378547668457,
-      "logits/rejected": -12.083256721496582,
-      "logps/chosen": -137.62254333496094,
-      "logps/rejected": -165.1597442626953,
-      "loss": 0.5257,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": -6.3524699211120605,
-      "rewards/margins": 3.091831922531128,
-      "rewards/rejected": -9.444302558898926,
-      "step": 10240
-    },
-    {
-      "epoch": 0.8191808191808192,
-      "grad_norm": 7.444268226623535,
-      "learning_rate": 4.815652516879612e-07,
-      "logits/chosen": -12.124509811401367,
-      "logits/rejected": -12.027182579040527,
-      "logps/chosen": -136.4917449951172,
-      "logps/rejected": -177.56820678710938,
-      "loss": 0.278,
-      "rewards/accuracies": 0.9000000357627869,
-      "rewards/chosen": -6.367459774017334,
-      "rewards/margins": 4.033092021942139,
-      "rewards/rejected": -10.400551795959473,
-      "step": 10250
-    },
-    {
-      "epoch": 0.81998001998002,
-      "grad_norm": 12.406705856323242,
-      "learning_rate": 4.774575140626317e-07,
-      "logits/chosen": -12.133296012878418,
-      "logits/rejected": -12.092856407165527,
-      "logps/chosen": -140.69540405273438,
-      "logps/rejected": -170.78366088867188,
-      "loss": 0.4126,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -6.922964572906494,
-      "rewards/margins": 3.0302162170410156,
-      "rewards/rejected": -9.953181266784668,
-      "step": 10260
-    },
-    {
-      "epoch": 0.8207792207792208,
-      "grad_norm": 1.2486238479614258,
-      "learning_rate": 4.733655206277998e-07,
-      "logits/chosen": -12.079439163208008,
-      "logits/rejected": -11.998786926269531,
-      "logps/chosen": -138.69778442382812,
-      "logps/rejected": -172.0188446044922,
-      "loss": 0.3869,
-      "rewards/accuracies": 0.8375000357627869,
-      "rewards/chosen": -6.606309413909912,
-      "rewards/margins": 2.9970874786376953,
-      "rewards/rejected": -9.60339641571045,
-      "step": 10270
-    },
-    {
-      "epoch": 0.8215784215784215,
-      "grad_norm": 15.977368354797363,
-      "learning_rate": 4.6928930323699966e-07,
-      "logits/chosen": -12.145803451538086,
-      "logits/rejected": -12.092337608337402,
-      "logps/chosen": -135.21353149414062,
-      "logps/rejected": -169.82662963867188,
-      "loss": 0.2114,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": -6.082850933074951,
-      "rewards/margins": 3.3691093921661377,
-      "rewards/rejected": -9.451960563659668,
-      "step": 10280
-    },
-    {
-      "epoch": 0.8223776223776224,
-      "grad_norm": 15.644160270690918,
-      "learning_rate": 4.6522889362095705e-07,
-      "logits/chosen": -11.8373441696167,
-      "logits/rejected": -11.967199325561523,
-      "logps/chosen": -132.7919158935547,
-      "logps/rejected": -182.2775115966797,
-      "loss": 0.3536,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -6.177986145019531,
-      "rewards/margins": 3.8847222328186035,
-      "rewards/rejected": -10.062707901000977,
-      "step": 10290
-    },
-    {
-      "epoch": 0.8231768231768232,
-      "grad_norm": 11.739263534545898,
-      "learning_rate": 4.6118432338734585e-07,
-      "logits/chosen": -12.13614559173584,
-      "logits/rejected": -12.028302192687988,
-      "logps/chosen": -139.42991638183594,
-      "logps/rejected": -167.25608825683594,
-      "loss": 0.3365,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -6.724884033203125,
-      "rewards/margins": 2.830545425415039,
-      "rewards/rejected": -9.555429458618164,
-      "step": 10300
-    },
-    {
-      "epoch": 0.823976023976024,
-      "grad_norm": 4.798367023468018,
-      "learning_rate": 4.5715562402053907e-07,
-      "logits/chosen": -12.067252159118652,
-      "logits/rejected": -12.060690879821777,
-      "logps/chosen": -136.2438201904297,
-      "logps/rejected": -172.13461303710938,
-      "loss": 0.4651,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -6.77243185043335,
-      "rewards/margins": 3.219806432723999,
-      "rewards/rejected": -9.99223804473877,
-      "step": 10310
-    },
-    {
-      "epoch": 0.8247752247752248,
-      "grad_norm": 21.794477462768555,
-      "learning_rate": 4.5314282688136756e-07,
-      "logits/chosen": -12.122682571411133,
-      "logits/rejected": -12.074995994567871,
-      "logps/chosen": -132.51370239257812,
-      "logps/rejected": -167.5258026123047,
-      "loss": 0.3818,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -6.128775119781494,
-      "rewards/margins": 3.507434129714966,
-      "rewards/rejected": -9.636209487915039,
-      "step": 10320
-    },
-    {
-      "epoch": 0.8255744255744256,
-      "grad_norm": 14.419798851013184,
-      "learning_rate": 4.4914596320687233e-07,
-      "logits/chosen": -12.145401000976562,
-      "logits/rejected": -12.102604866027832,
-      "logps/chosen": -138.48948669433594,
-      "logps/rejected": -180.5524139404297,
-      "loss": 0.304,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": -6.420684814453125,
-      "rewards/margins": 3.787900686264038,
-      "rewards/rejected": -10.208584785461426,
-      "step": 10330
-    },
-    {
-      "epoch": 0.8263736263736263,
-      "grad_norm": 5.609336853027344,
-      "learning_rate": 4.4516506411006245e-07,
-      "logits/chosen": -11.9972562789917,
-      "logits/rejected": -12.038344383239746,
-      "logps/chosen": -142.2910919189453,
-      "logps/rejected": -175.0443572998047,
-      "loss": 0.454,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -6.836406230926514,
-      "rewards/margins": 2.868818998336792,
-      "rewards/rejected": -9.705225944519043,
-      "step": 10340
-    },
-    {
-      "epoch": 0.8271728271728271,
-      "grad_norm": 18.642436981201172,
-      "learning_rate": 4.412001605796729e-07,
-      "logits/chosen": -12.188549041748047,
-      "logits/rejected": -12.106082916259766,
-      "logps/chosen": -140.7406463623047,
-      "logps/rejected": -182.560302734375,
-      "loss": 0.2375,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": -6.7588958740234375,
-      "rewards/margins": 3.9190545082092285,
-      "rewards/rejected": -10.677949905395508,
-      "step": 10350
-    },
-    {
-      "epoch": 0.827972027972028,
-      "grad_norm": 19.192689895629883,
-      "learning_rate": 4.3725128347992333e-07,
-      "logits/chosen": -12.201032638549805,
-      "logits/rejected": -12.16457462310791,
-      "logps/chosen": -138.3088836669922,
-      "logps/rejected": -167.8157501220703,
-      "loss": 0.3351,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -6.428294658660889,
-      "rewards/margins": 3.089503526687622,
-      "rewards/rejected": -9.51779842376709,
-      "step": 10360
-    },
-    {
-      "epoch": 0.8287712287712288,
-      "grad_norm": 20.119293212890625,
-      "learning_rate": 4.333184635502799e-07,
-      "logits/chosen": -12.128152847290039,
-      "logits/rejected": -12.17394733428955,
-      "logps/chosen": -136.6043243408203,
-      "logps/rejected": -173.94883728027344,
-      "loss": 0.2907,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": -6.320436000823975,
-      "rewards/margins": 3.464686155319214,
-      "rewards/rejected": -9.78512191772461,
-      "step": 10370
-    },
-    {
-      "epoch": 0.8295704295704296,
-      "grad_norm": 22.33707618713379,
-      "learning_rate": 4.29401731405211e-07,
-      "logits/chosen": -12.098845481872559,
-      "logits/rejected": -12.080906867980957,
-      "logps/chosen": -135.38162231445312,
-      "logps/rejected": -172.11766052246094,
-      "loss": 0.3086,
-      "rewards/accuracies": 0.8375000357627869,
-      "rewards/chosen": -6.632022857666016,
-      "rewards/margins": 3.298658847808838,
-      "rewards/rejected": -9.930682182312012,
-      "step": 10380
-    },
-    {
-      "epoch": 0.8303696303696304,
-      "grad_norm": 30.665090560913086,
-      "learning_rate": 4.255011175339546e-07,
-      "logits/chosen": -12.189809799194336,
-      "logits/rejected": -12.070557594299316,
-      "logps/chosen": -140.25616455078125,
-      "logps/rejected": -171.6234130859375,
-      "loss": 0.3802,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -6.804786682128906,
-      "rewards/margins": 3.030289649963379,
-      "rewards/rejected": -9.835077285766602,
-      "step": 10390
-    },
-    {
-      "epoch": 0.8311688311688312,
-      "grad_norm": 22.98872947692871,
-      "learning_rate": 4.2161665230027653e-07,
-      "logits/chosen": -12.083369255065918,
-      "logits/rejected": -12.15929889678955,
-      "logps/chosen": -142.56761169433594,
-      "logps/rejected": -189.2682342529297,
-      "loss": 0.3522,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -7.187714576721191,
-      "rewards/margins": 4.043059825897217,
-      "rewards/rejected": -11.23077392578125,
-      "step": 10400
-    },
-    {
-      "epoch": 0.8319680319680319,
-      "grad_norm": 15.565129280090332,
-      "learning_rate": 4.1774836594223764e-07,
-      "logits/chosen": -12.198086738586426,
-      "logits/rejected": -12.197803497314453,
-      "logps/chosen": -138.96861267089844,
-      "logps/rejected": -171.10215759277344,
-      "loss": 0.29,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -6.621096134185791,
-      "rewards/margins": 3.101761817932129,
-      "rewards/rejected": -9.722857475280762,
-      "step": 10410
-    },
-    {
-      "epoch": 0.8327672327672327,
-      "grad_norm": 20.284570693969727,
-      "learning_rate": 4.138962885719547e-07,
-      "logits/chosen": -12.14477825164795,
-      "logits/rejected": -12.159278869628906,
-      "logps/chosen": -139.76828002929688,
-      "logps/rejected": -173.5370330810547,
-      "loss": 0.4517,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -6.914497375488281,
-      "rewards/margins": 3.0154054164886475,
-      "rewards/rejected": -9.929903030395508,
-      "step": 10420
-    },
-    {
-      "epoch": 0.8335664335664336,
-      "grad_norm": 2.897404909133911,
-      "learning_rate": 4.100604501753691e-07,
-      "logits/chosen": -12.057295799255371,
-      "logits/rejected": -12.046319961547852,
-      "logps/chosen": -138.5201873779297,
-      "logps/rejected": -174.9243621826172,
-      "loss": 0.2181,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": -6.613814830780029,
-      "rewards/margins": 3.574422836303711,
-      "rewards/rejected": -10.188237190246582,
-      "step": 10430
-    },
-    {
-      "epoch": 0.8343656343656344,
-      "grad_norm": 30.06218910217285,
-      "learning_rate": 4.0624088061201135e-07,
-      "logits/chosen": -12.217287063598633,
-      "logits/rejected": -12.225836753845215,
-      "logps/chosen": -140.658447265625,
-      "logps/rejected": -177.8080291748047,
-      "loss": 0.4982,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -7.158817291259766,
-      "rewards/margins": 3.102315902709961,
-      "rewards/rejected": -10.261134147644043,
-      "step": 10440
-    },
-    {
-      "epoch": 0.8351648351648352,
-      "grad_norm": 7.964390754699707,
-      "learning_rate": 4.0243760961477085e-07,
-      "logits/chosen": -12.152517318725586,
-      "logits/rejected": -12.103405952453613,
-      "logps/chosen": -130.033447265625,
-      "logps/rejected": -167.84007263183594,
-      "loss": 0.2398,
-      "rewards/accuracies": 0.9000000357627869,
-      "rewards/chosen": -5.904440402984619,
-      "rewards/margins": 3.5004100799560547,
-      "rewards/rejected": -9.404850006103516,
-      "step": 10450
-    },
-    {
-      "epoch": 0.835964035964036,
-      "grad_norm": 9.261895179748535,
-      "learning_rate": 3.9865066678966306e-07,
-      "logits/chosen": -12.21778392791748,
-      "logits/rejected": -12.143588066101074,
-      "logps/chosen": -139.89956665039062,
-      "logps/rejected": -175.14895629882812,
-      "loss": 0.271,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": -6.525847911834717,
-      "rewards/margins": 3.342827558517456,
-      "rewards/rejected": -9.868675231933594,
-      "step": 10460
-    },
-    {
-      "epoch": 0.8367632367632367,
-      "grad_norm": 23.332996368408203,
-      "learning_rate": 3.9488008161559843e-07,
-      "logits/chosen": -12.257316589355469,
-      "logits/rejected": -12.18096923828125,
-      "logps/chosen": -135.1853790283203,
-      "logps/rejected": -164.73094177246094,
-      "loss": 0.4373,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -6.175811767578125,
-      "rewards/margins": 3.0222136974334717,
-      "rewards/rejected": -9.198025703430176,
-      "step": 10470
-    },
-    {
-      "epoch": 0.8375624375624375,
-      "grad_norm": 12.654902458190918,
-      "learning_rate": 3.911258834441528e-07,
-      "logits/chosen": -12.211465835571289,
-      "logits/rejected": -12.11547565460205,
-      "logps/chosen": -138.2267303466797,
-      "logps/rejected": -178.36595153808594,
-      "loss": 0.2804,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": -6.743424892425537,
-      "rewards/margins": 3.8180739879608154,
-      "rewards/rejected": -10.56149959564209,
-      "step": 10480
-    },
-    {
-      "epoch": 0.8383616383616384,
-      "grad_norm": 1.3398302793502808,
-      "learning_rate": 3.873881014993422e-07,
-      "logits/chosen": -12.042580604553223,
-      "logits/rejected": -12.07327651977539,
-      "logps/chosen": -141.6290740966797,
-      "logps/rejected": -183.6087646484375,
-      "loss": 0.2692,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": -6.4923224449157715,
-      "rewards/margins": 3.701305389404297,
-      "rewards/rejected": -10.193628311157227,
-      "step": 10490
-    },
-    {
-      "epoch": 0.8391608391608392,
-      "grad_norm": 13.167179107666016,
-      "learning_rate": 3.836667648773906e-07,
-      "logits/chosen": -12.16629695892334,
-      "logits/rejected": -12.098822593688965,
-      "logps/chosen": -134.96987915039062,
-      "logps/rejected": -170.8475341796875,
-      "loss": 0.3101,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -6.37162971496582,
-      "rewards/margins": 3.663099765777588,
-      "rewards/rejected": -10.03472900390625,
-      "step": 10500
-    },
-    {
-      "epoch": 0.83996003996004,
-      "grad_norm": 7.546113014221191,
-      "learning_rate": 3.7996190254650676e-07,
-      "logits/chosen": -12.154616355895996,
-      "logits/rejected": -12.048171997070312,
-      "logps/chosen": -143.42547607421875,
-      "logps/rejected": -179.03738403320312,
-      "loss": 0.2919,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -6.9918341636657715,
-      "rewards/margins": 3.362591505050659,
-      "rewards/rejected": -10.354426383972168,
-      "step": 10510
-    },
-    {
-      "epoch": 0.8407592407592408,
-      "grad_norm": 17.197948455810547,
-      "learning_rate": 3.76273543346658e-07,
-      "logits/chosen": -11.95683479309082,
-      "logits/rejected": -12.121813774108887,
-      "logps/chosen": -134.8517303466797,
-      "logps/rejected": -186.2017364501953,
-      "loss": 0.2273,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": -6.213758945465088,
-      "rewards/margins": 4.007318019866943,
-      "rewards/rejected": -10.221076965332031,
-      "step": 10520
-    },
-    {
-      "epoch": 0.8415584415584415,
-      "grad_norm": 12.129951477050781,
-      "learning_rate": 3.7260171598934387e-07,
-      "logits/chosen": -12.04438304901123,
-      "logits/rejected": -12.055785179138184,
-      "logps/chosen": -140.61270141601562,
-      "logps/rejected": -176.677490234375,
-      "loss": 0.3684,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -6.702260494232178,
-      "rewards/margins": 3.185041904449463,
-      "rewards/rejected": -9.88730239868164,
-      "step": 10530
-    },
-    {
-      "epoch": 0.8423576423576423,
-      "grad_norm": 20.976486206054688,
-      "learning_rate": 3.6894644905737686e-07,
-      "logits/chosen": -12.012260437011719,
-      "logits/rejected": -12.076272964477539,
-      "logps/chosen": -134.97862243652344,
-      "logps/rejected": -179.43833923339844,
-      "loss": 0.2228,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": -6.172288417816162,
-      "rewards/margins": 3.9545044898986816,
-      "rewards/rejected": -10.126792907714844,
-      "step": 10540
-    },
-    {
-      "epoch": 0.8431568431568431,
-      "grad_norm": 17.600021362304688,
-      "learning_rate": 3.653077710046546e-07,
-      "logits/chosen": -11.831256866455078,
-      "logits/rejected": -12.127699851989746,
-      "logps/chosen": -134.6081085205078,
-      "logps/rejected": -182.1595458984375,
-      "loss": 0.2977,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -6.171046257019043,
-      "rewards/margins": 4.103275299072266,
-      "rewards/rejected": -10.274321556091309,
-      "step": 10550
-    },
-    {
-      "epoch": 0.843956043956044,
-      "grad_norm": 18.08307456970215,
-      "learning_rate": 3.616857101559423e-07,
-      "logits/chosen": -12.137450218200684,
-      "logits/rejected": -12.193292617797852,
-      "logps/chosen": -142.39990234375,
-      "logps/rejected": -190.8633575439453,
-      "loss": 0.348,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -6.868846416473389,
-      "rewards/margins": 4.1033935546875,
-      "rewards/rejected": -10.97223949432373,
-      "step": 10560
-    },
-    {
-      "epoch": 0.8447552447552448,
-      "grad_norm": 7.774749755859375,
-      "learning_rate": 3.580802947066497e-07,
-      "logits/chosen": -12.165448188781738,
-      "logits/rejected": -12.102495193481445,
-      "logps/chosen": -138.64378356933594,
-      "logps/rejected": -178.8787384033203,
-      "loss": 0.2215,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": -6.2754106521606445,
-      "rewards/margins": 4.2504096031188965,
-      "rewards/rejected": -10.5258207321167,
-      "step": 10570
-    },
-    {
-      "epoch": 0.8455544455544456,
-      "grad_norm": 18.022235870361328,
-      "learning_rate": 3.544915527226148e-07,
-      "logits/chosen": -12.11390209197998,
-      "logits/rejected": -12.088874816894531,
-      "logps/chosen": -134.8641815185547,
-      "logps/rejected": -182.22903442382812,
-      "loss": 0.3137,
-      "rewards/accuracies": 0.9000000357627869,
-      "rewards/chosen": -6.313209533691406,
-      "rewards/margins": 4.085341930389404,
-      "rewards/rejected": -10.398551940917969,
-      "step": 10580
-    },
-    {
-      "epoch": 0.8463536463536464,
-      "grad_norm": 2.5682432651519775,
-      "learning_rate": 3.5091951213988257e-07,
-      "logits/chosen": -12.168818473815918,
-      "logits/rejected": -12.01273250579834,
-      "logps/chosen": -145.1033172607422,
-      "logps/rejected": -188.1717987060547,
-      "loss": 0.217,
-      "rewards/accuracies": 0.9625000357627869,
-      "rewards/chosen": -7.114346504211426,
-      "rewards/margins": 4.457431793212891,
-      "rewards/rejected": -11.57177734375,
-      "step": 10590
-    },
-    {
-      "epoch": 0.8471528471528471,
-      "grad_norm": 32.71928787231445,
-      "learning_rate": 3.4736420076448784e-07,
-      "logits/chosen": -12.167445182800293,
-      "logits/rejected": -12.143555641174316,
-      "logps/chosen": -141.9297637939453,
-      "logps/rejected": -169.91268920898438,
-      "loss": 0.5423,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -6.946034908294678,
-      "rewards/margins": 2.7336297035217285,
-      "rewards/rejected": -9.679664611816406,
-      "step": 10600
-    },
-    {
-      "epoch": 0.8479520479520479,
-      "grad_norm": 16.17058753967285,
-      "learning_rate": 3.438256462722389e-07,
-      "logits/chosen": -12.157225608825684,
-      "logits/rejected": -12.100897789001465,
-      "logps/chosen": -142.099853515625,
-      "logps/rejected": -173.55535888671875,
-      "loss": 0.4578,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -7.051429748535156,
-      "rewards/margins": 2.980304718017578,
-      "rewards/rejected": -10.03173542022705,
-      "step": 10610
-    },
-    {
-      "epoch": 0.8487512487512487,
-      "grad_norm": 10.812273025512695,
-      "learning_rate": 3.4030387620850423e-07,
-      "logits/chosen": -12.141456604003906,
-      "logits/rejected": -12.102397918701172,
-      "logps/chosen": -134.1421356201172,
-      "logps/rejected": -169.86358642578125,
-      "loss": 0.3399,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -6.196483135223389,
-      "rewards/margins": 3.3775691986083984,
-      "rewards/rejected": -9.574052810668945,
-      "step": 10620
-    },
-    {
-      "epoch": 0.8495504495504496,
-      "grad_norm": 22.22776985168457,
-      "learning_rate": 3.3679891798799436e-07,
-      "logits/chosen": -12.221122741699219,
-      "logits/rejected": -12.129753112792969,
-      "logps/chosen": -138.2547607421875,
-      "logps/rejected": -173.70166015625,
-      "loss": 0.2854,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": -6.654529571533203,
-      "rewards/margins": 3.360424757003784,
-      "rewards/rejected": -10.014955520629883,
-      "step": 10630
-    },
-    {
-      "epoch": 0.8503496503496504,
-      "grad_norm": 3.8180270195007324,
-      "learning_rate": 3.333107988945525e-07,
-      "logits/chosen": -12.003371238708496,
-      "logits/rejected": -11.987526893615723,
-      "logps/chosen": -136.1119842529297,
-      "logps/rejected": -175.46153259277344,
-      "loss": 0.2747,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": -6.134881973266602,
-      "rewards/margins": 3.7061469554901123,
-      "rewards/rejected": -9.841029167175293,
-      "step": 10640
-    },
-    {
-      "epoch": 0.8511488511488512,
-      "grad_norm": 13.84496784210205,
-      "learning_rate": 3.2983954608093865e-07,
-      "logits/chosen": -12.047846794128418,
-      "logits/rejected": -12.076037406921387,
-      "logps/chosen": -129.37425231933594,
-      "logps/rejected": -177.8155059814453,
-      "loss": 0.3415,
-      "rewards/accuracies": 0.8375000357627869,
-      "rewards/chosen": -5.984645366668701,
-      "rewards/margins": 4.121144771575928,
-      "rewards/rejected": -10.105789184570312,
-      "step": 10650
-    },
-    {
-      "epoch": 0.8519480519480519,
-      "grad_norm": 17.24859046936035,
-      "learning_rate": 3.263851865686193e-07,
-      "logits/chosen": -12.130047798156738,
-      "logits/rejected": -12.14078140258789,
-      "logps/chosen": -141.49380493164062,
-      "logps/rejected": -179.06809997558594,
-      "loss": 0.3386,
-      "rewards/accuracies": 0.8375000357627869,
-      "rewards/chosen": -6.58367919921875,
-      "rewards/margins": 3.32883620262146,
-      "rewards/rejected": -9.912515640258789,
-      "step": 10660
-    },
-    {
-      "epoch": 0.8527472527472527,
-      "grad_norm": 8.034584045410156,
-      "learning_rate": 3.229477472475595e-07,
-      "logits/chosen": -12.072859764099121,
-      "logits/rejected": -12.024895668029785,
-      "logps/chosen": -140.22471618652344,
-      "logps/rejected": -185.44012451171875,
-      "loss": 0.256,
-      "rewards/accuracies": 0.9000000357627869,
-      "rewards/chosen": -7.080970764160156,
-      "rewards/margins": 4.0248703956604,
-      "rewards/rejected": -11.105840682983398,
-      "step": 10670
-    },
-    {
-      "epoch": 0.8535464535464535,
-      "grad_norm": 16.01206398010254,
-      "learning_rate": 3.195272548760098e-07,
-      "logits/chosen": -12.225024223327637,
-      "logits/rejected": -12.13732624053955,
-      "logps/chosen": -138.72377014160156,
-      "logps/rejected": -177.1475830078125,
-      "loss": 0.339,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": -6.93843412399292,
-      "rewards/margins": 3.2993996143341064,
-      "rewards/rejected": -10.237833023071289,
-      "step": 10680
-    },
-    {
-      "epoch": 0.8543456543456543,
-      "grad_norm": 21.512990951538086,
-      "learning_rate": 3.161237360802996e-07,
-      "logits/chosen": -12.219748497009277,
-      "logits/rejected": -12.112735748291016,
-      "logps/chosen": -139.71531677246094,
-      "logps/rejected": -168.2742462158203,
-      "loss": 0.2829,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": -6.587482452392578,
-      "rewards/margins": 3.3703789710998535,
-      "rewards/rejected": -9.95786190032959,
-      "step": 10690
-    },
-    {
-      "epoch": 0.8551448551448552,
-      "grad_norm": 23.293672561645508,
-      "learning_rate": 3.1273721735463056e-07,
-      "logits/chosen": -12.109244346618652,
-      "logits/rejected": -12.028656959533691,
-      "logps/chosen": -144.08865356445312,
-      "logps/rejected": -180.2097930908203,
-      "loss": 0.3785,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -7.098779201507568,
-      "rewards/margins": 3.3399174213409424,
-      "rewards/rejected": -10.43869686126709,
-      "step": 10700
-    },
-    {
-      "epoch": 0.855944055944056,
-      "grad_norm": 20.30375099182129,
-      "learning_rate": 3.0936772506086983e-07,
-      "logits/chosen": -12.03030776977539,
-      "logits/rejected": -12.0470609664917,
-      "logps/chosen": -143.78897094726562,
-      "logps/rejected": -186.3387908935547,
-      "loss": 0.399,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -6.951087474822998,
-      "rewards/margins": 3.4350249767303467,
-      "rewards/rejected": -10.386112213134766,
-      "step": 10710
-    },
-    {
-      "epoch": 0.8567432567432568,
-      "grad_norm": 19.17446517944336,
-      "learning_rate": 3.060152854283449e-07,
-      "logits/chosen": -12.038987159729004,
-      "logits/rejected": -12.120071411132812,
-      "logps/chosen": -137.2480010986328,
-      "logps/rejected": -175.432861328125,
-      "loss": 0.2768,
-      "rewards/accuracies": 0.9000000357627869,
-      "rewards/chosen": -6.602427959442139,
-      "rewards/margins": 3.4754281044006348,
-      "rewards/rejected": -10.07785701751709,
-      "step": 10720
-    },
-    {
-      "epoch": 0.8575424575424575,
-      "grad_norm": 1.7568986415863037,
-      "learning_rate": 3.026799245536391e-07,
-      "logits/chosen": -11.961782455444336,
-      "logits/rejected": -12.159167289733887,
-      "logps/chosen": -135.4373016357422,
-      "logps/rejected": -196.30941772460938,
-      "loss": 0.2398,
-      "rewards/accuracies": 0.9000000357627869,
-      "rewards/chosen": -6.538186550140381,
-      "rewards/margins": 4.806034088134766,
-      "rewards/rejected": -11.344220161437988,
-      "step": 10730
-    },
-    {
-      "epoch": 0.8583416583416583,
-      "grad_norm": 28.450851440429688,
-      "learning_rate": 2.993616684003872e-07,
-      "logits/chosen": -11.978656768798828,
-      "logits/rejected": -12.048949241638184,
-      "logps/chosen": -137.84996032714844,
-      "logps/rejected": -192.80581665039062,
-      "loss": 0.2842,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -7.039955139160156,
-      "rewards/margins": 4.183957576751709,
-      "rewards/rejected": -11.22391128540039,
-      "step": 10740
-    },
-    {
-      "epoch": 0.8591408591408591,
-      "grad_norm": 10.396132469177246,
-      "learning_rate": 2.960605427990773e-07,
-      "logits/chosen": -12.08303451538086,
-      "logits/rejected": -12.138566970825195,
-      "logps/chosen": -135.46119689941406,
-      "logps/rejected": -180.00743103027344,
-      "loss": 0.5349,
-      "rewards/accuracies": 0.7750000357627869,
-      "rewards/chosen": -6.640995979309082,
-      "rewards/margins": 3.3721396923065186,
-      "rewards/rejected": -10.01313591003418,
-      "step": 10750
-    },
-    {
-      "epoch": 0.85994005994006,
-      "grad_norm": 9.47109603881836,
-      "learning_rate": 2.927765734468452e-07,
-      "logits/chosen": -12.186187744140625,
-      "logits/rejected": -12.143821716308594,
-      "logps/chosen": -144.10935974121094,
-      "logps/rejected": -177.11013793945312,
-      "loss": 0.4752,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -7.030910015106201,
-      "rewards/margins": 3.3619518280029297,
-      "rewards/rejected": -10.392861366271973,
-      "step": 10760
-    },
-    {
-      "epoch": 0.8607392607392608,
-      "grad_norm": 18.48225975036621,
-      "learning_rate": 2.8950978590727765e-07,
-      "logits/chosen": -12.099197387695312,
-      "logits/rejected": -12.052027702331543,
-      "logps/chosen": -134.29701232910156,
-      "logps/rejected": -168.2634735107422,
-      "loss": 0.2574,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": -6.681613445281982,
-      "rewards/margins": 3.1023693084716797,
-      "rewards/rejected": -9.78398323059082,
-      "step": 10770
-    },
-    {
-      "epoch": 0.8615384615384616,
-      "grad_norm": 12.598806381225586,
-      "learning_rate": 2.862602056102109e-07,
-      "logits/chosen": -11.862014770507812,
-      "logits/rejected": -12.058731079101562,
-      "logps/chosen": -136.7904052734375,
-      "logps/rejected": -194.06690979003906,
-      "loss": 0.2397,
-      "rewards/accuracies": 0.9000000357627869,
-      "rewards/chosen": -6.401876926422119,
-      "rewards/margins": 4.612299919128418,
-      "rewards/rejected": -11.014177322387695,
-      "step": 10780
-    },
-    {
-      "epoch": 0.8623376623376623,
-      "grad_norm": 20.411109924316406,
-      "learning_rate": 2.830278578515344e-07,
-      "logits/chosen": -12.142359733581543,
-      "logits/rejected": -12.092851638793945,
-      "logps/chosen": -144.94229125976562,
-      "logps/rejected": -176.30453491210938,
-      "loss": 0.3706,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -7.375135898590088,
-      "rewards/margins": 2.886025905609131,
-      "rewards/rejected": -10.261161804199219,
-      "step": 10790
-    },
-    {
-      "epoch": 0.8631368631368631,
-      "grad_norm": 23.18751335144043,
-      "learning_rate": 2.7981276779299404e-07,
-      "logits/chosen": -12.196181297302246,
-      "logits/rejected": -12.175948143005371,
-      "logps/chosen": -141.89129638671875,
-      "logps/rejected": -172.25442504882812,
-      "loss": 0.4192,
-      "rewards/accuracies": 0.8375000357627869,
-      "rewards/chosen": -7.126303195953369,
-      "rewards/margins": 2.7036802768707275,
-      "rewards/rejected": -9.82998275756836,
-      "step": 10800
-    },
-    {
-      "epoch": 0.8639360639360639,
-      "grad_norm": 12.476435661315918,
-      "learning_rate": 2.76614960461995e-07,
-      "logits/chosen": -12.119833946228027,
-      "logits/rejected": -12.15268325805664,
-      "logps/chosen": -142.76870727539062,
-      "logps/rejected": -182.99853515625,
-      "loss": 0.2933,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": -6.96331262588501,
-      "rewards/margins": 3.6282379627227783,
-      "rewards/rejected": -10.591551780700684,
-      "step": 10810
-    },
-    {
-      "epoch": 0.8647352647352647,
-      "grad_norm": 15.832107543945312,
-      "learning_rate": 2.734344607514064e-07,
-      "logits/chosen": -12.045928001403809,
-      "logits/rejected": -12.072885513305664,
-      "logps/chosen": -144.3911590576172,
-      "logps/rejected": -178.28700256347656,
-      "loss": 0.4083,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -7.199207305908203,
-      "rewards/margins": 3.2281653881073,
-      "rewards/rejected": -10.427372932434082,
-      "step": 10820
-    },
-    {
-      "epoch": 0.8655344655344656,
-      "grad_norm": 11.510897636413574,
-      "learning_rate": 2.702712934193713e-07,
-      "logits/chosen": -12.160696029663086,
-      "logits/rejected": -12.04643726348877,
-      "logps/chosen": -134.90798950195312,
-      "logps/rejected": -170.30433654785156,
-      "loss": 0.3538,
-      "rewards/accuracies": 0.8375000357627869,
-      "rewards/chosen": -6.516582012176514,
-      "rewards/margins": 3.514622449874878,
-      "rewards/rejected": -10.031204223632812,
-      "step": 10830
-    },
-    {
-      "epoch": 0.8663336663336664,
-      "grad_norm": 17.94367218017578,
-      "learning_rate": 2.671254830891087e-07,
-      "logits/chosen": -11.955897331237793,
-      "logits/rejected": -12.071390151977539,
-      "logps/chosen": -137.3775634765625,
-      "logps/rejected": -187.6778564453125,
-      "loss": 0.3693,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -6.714229106903076,
-      "rewards/margins": 4.067649841308594,
-      "rewards/rejected": -10.781879425048828,
-      "step": 10840
-    },
-    {
-      "epoch": 0.8671328671328671,
-      "grad_norm": 25.66269302368164,
-      "learning_rate": 2.639970542487272e-07,
-      "logits/chosen": -12.123631477355957,
-      "logits/rejected": -12.093123435974121,
-      "logps/chosen": -136.3466339111328,
-      "logps/rejected": -171.80010986328125,
-      "loss": 0.3018,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -6.394751071929932,
-      "rewards/margins": 3.6002824306488037,
-      "rewards/rejected": -9.995034217834473,
-      "step": 10850
-    },
-    {
-      "epoch": 0.8679320679320679,
-      "grad_norm": 6.9430623054504395,
-      "learning_rate": 2.608860312510292e-07,
-      "logits/chosen": -12.12710189819336,
-      "logits/rejected": -12.160303115844727,
-      "logps/chosen": -133.0308074951172,
-      "logps/rejected": -174.6834259033203,
-      "loss": 0.3287,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -6.425089359283447,
-      "rewards/margins": 3.454911947250366,
-      "rewards/rejected": -9.880001068115234,
-      "step": 10860
-    },
-    {
-      "epoch": 0.8687312687312687,
-      "grad_norm": 13.682426452636719,
-      "learning_rate": 2.5779243831332483e-07,
-      "logits/chosen": -12.013086318969727,
-      "logits/rejected": -12.08768367767334,
-      "logps/chosen": -138.04283142089844,
-      "logps/rejected": -178.7208251953125,
-      "loss": 0.2208,
-      "rewards/accuracies": 0.9000000357627869,
-      "rewards/chosen": -6.580657958984375,
-      "rewards/margins": 3.7502384185791016,
-      "rewards/rejected": -10.330897331237793,
-      "step": 10870
-    },
-    {
-      "epoch": 0.8695304695304695,
-      "grad_norm": 8.23641586303711,
-      "learning_rate": 2.5471629951724254e-07,
-      "logits/chosen": -11.983410835266113,
-      "logits/rejected": -12.092520713806152,
-      "logps/chosen": -141.30172729492188,
-      "logps/rejected": -179.2637176513672,
-      "loss": 0.4192,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -7.077406406402588,
-      "rewards/margins": 3.230938673019409,
-      "rewards/rejected": -10.308344841003418,
-      "step": 10880
-    },
-    {
-      "epoch": 0.8703296703296703,
-      "grad_norm": 14.91490650177002,
-      "learning_rate": 2.516576388085423e-07,
-      "logits/chosen": -12.135549545288086,
-      "logits/rejected": -12.170912742614746,
-      "logps/chosen": -140.25498962402344,
-      "logps/rejected": -177.84315490722656,
-      "loss": 0.2938,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -7.026414394378662,
-      "rewards/margins": 3.538450241088867,
-      "rewards/rejected": -10.564865112304688,
-      "step": 10890
-    },
-    {
-      "epoch": 0.8711288711288712,
-      "grad_norm": 5.159121990203857,
-      "learning_rate": 2.4861647999692707e-07,
-      "logits/chosen": -12.14677906036377,
-      "logits/rejected": -12.03211498260498,
-      "logps/chosen": -133.95310974121094,
-      "logps/rejected": -170.25076293945312,
-      "loss": 0.3555,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -6.32609748840332,
-      "rewards/margins": 3.308582067489624,
-      "rewards/rejected": -9.634679794311523,
-      "step": 10900
-    },
-    {
-      "epoch": 0.871928071928072,
-      "grad_norm": 23.8902587890625,
-      "learning_rate": 2.455928467558591e-07,
-      "logits/chosen": -12.208062171936035,
-      "logits/rejected": -12.149476051330566,
-      "logps/chosen": -139.66233825683594,
-      "logps/rejected": -169.35792541503906,
-      "loss": 0.4848,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -6.687965393066406,
-      "rewards/margins": 2.9306533336639404,
-      "rewards/rejected": -9.61861801147461,
-      "step": 10910
-    },
-    {
-      "epoch": 0.8727272727272727,
-      "grad_norm": 14.184701919555664,
-      "learning_rate": 2.425867626223752e-07,
-      "logits/chosen": -12.04153823852539,
-      "logits/rejected": -12.058610916137695,
-      "logps/chosen": -139.36703491210938,
-      "logps/rejected": -182.6197052001953,
-      "loss": 0.3641,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -6.83993673324585,
-      "rewards/margins": 3.871415853500366,
-      "rewards/rejected": -10.711352348327637,
-      "step": 10920
-    },
-    {
-      "epoch": 0.8735264735264735,
-      "grad_norm": 22.574005126953125,
-      "learning_rate": 2.3959825099690527e-07,
-      "logits/chosen": -12.059715270996094,
-      "logits/rejected": -12.120372772216797,
-      "logps/chosen": -144.21421813964844,
-      "logps/rejected": -185.50328063964844,
-      "loss": 0.4553,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -7.327327251434326,
-      "rewards/margins": 3.542572021484375,
-      "rewards/rejected": -10.86989974975586,
-      "step": 10930
-    },
-    {
-      "epoch": 0.8743256743256743,
-      "grad_norm": 15.533249855041504,
-      "learning_rate": 2.3662733514308672e-07,
-      "logits/chosen": -12.048454284667969,
-      "logits/rejected": -12.107256889343262,
-      "logps/chosen": -143.77369689941406,
-      "logps/rejected": -185.3140106201172,
-      "loss": 0.3152,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": -7.142881870269775,
-      "rewards/margins": 3.8074440956115723,
-      "rewards/rejected": -10.950325965881348,
-      "step": 10940
-    },
-    {
-      "epoch": 0.8751248751248751,
-      "grad_norm": 11.031457901000977,
-      "learning_rate": 2.3367403818758517e-07,
-      "logits/chosen": -12.140776634216309,
-      "logits/rejected": -12.098328590393066,
-      "logps/chosen": -137.74928283691406,
-      "logps/rejected": -179.05752563476562,
-      "loss": 0.2245,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": -6.3768534660339355,
-      "rewards/margins": 3.9960744380950928,
-      "rewards/rejected": -10.372928619384766,
-      "step": 10950
-    },
-    {
-      "epoch": 0.875924075924076,
-      "grad_norm": 19.85507583618164,
-      "learning_rate": 2.3073838311991585e-07,
-      "logits/chosen": -12.188044548034668,
-      "logits/rejected": -12.118308067321777,
-      "logps/chosen": -145.9187469482422,
-      "logps/rejected": -185.7565155029297,
-      "loss": 0.2746,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": -7.1380510330200195,
-      "rewards/margins": 3.42366099357605,
-      "rewards/rejected": -10.561711311340332,
-      "step": 10960
-    },
-    {
-      "epoch": 0.8767232767232768,
-      "grad_norm": 8.78612232208252,
-      "learning_rate": 2.2782039279226314e-07,
-      "logits/chosen": -12.240248680114746,
-      "logits/rejected": -12.182053565979004,
-      "logps/chosen": -135.05906677246094,
-      "logps/rejected": -173.6546173095703,
-      "loss": 0.2803,
-      "rewards/accuracies": 0.8375000357627869,
-      "rewards/chosen": -6.302867889404297,
-      "rewards/margins": 3.6004528999328613,
-      "rewards/rejected": -9.9033203125,
-      "step": 10970
-    },
-    {
-      "epoch": 0.8775224775224775,
-      "grad_norm": 24.24924087524414,
-      "learning_rate": 2.249200899193013e-07,
-      "logits/chosen": -12.208819389343262,
-      "logits/rejected": -12.20406436920166,
-      "logps/chosen": -143.41123962402344,
-      "logps/rejected": -176.8545684814453,
-      "loss": 0.344,
-      "rewards/accuracies": 0.8375000357627869,
-      "rewards/chosen": -6.89662504196167,
-      "rewards/margins": 3.219505786895752,
-      "rewards/rejected": -10.116129875183105,
-      "step": 10980
-    },
-    {
-      "epoch": 0.8783216783216783,
-      "grad_norm": 18.083410263061523,
-      "learning_rate": 2.2203749707802065e-07,
-      "logits/chosen": -11.894429206848145,
-      "logits/rejected": -12.043258666992188,
-      "logps/chosen": -138.2560272216797,
-      "logps/rejected": -191.5687255859375,
-      "loss": 0.3481,
-      "rewards/accuracies": 0.8375000357627869,
-      "rewards/chosen": -6.866316318511963,
-      "rewards/margins": 3.753493070602417,
-      "rewards/rejected": -10.619811058044434,
-      "step": 10990
-    },
-    {
-      "epoch": 0.8791208791208791,
-      "grad_norm": 27.339874267578125,
-      "learning_rate": 2.191726367075489e-07,
-      "logits/chosen": -12.166648864746094,
-      "logits/rejected": -12.097114562988281,
-      "logps/chosen": -145.0471649169922,
-      "logps/rejected": -177.5843505859375,
-      "loss": 0.3664,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -7.172407627105713,
-      "rewards/margins": 3.2057037353515625,
-      "rewards/rejected": -10.378110885620117,
-      "step": 11000
-    },
-    {
-      "epoch": 0.8799200799200799,
-      "grad_norm": 14.747498512268066,
-      "learning_rate": 2.1632553110897901e-07,
-      "logits/chosen": -12.114860534667969,
-      "logits/rejected": -12.227449417114258,
-      "logps/chosen": -137.62611389160156,
-      "logps/rejected": -183.92100524902344,
-      "loss": 0.3733,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -6.520303249359131,
-      "rewards/margins": 3.7674014568328857,
-      "rewards/rejected": -10.287704467773438,
-      "step": 11010
-    },
-    {
-      "epoch": 0.8807192807192807,
-      "grad_norm": 13.001787185668945,
-      "learning_rate": 2.1349620244519532e-07,
-      "logits/chosen": -12.220236778259277,
-      "logits/rejected": -12.138198852539062,
-      "logps/chosen": -139.2691192626953,
-      "logps/rejected": -172.18833923339844,
-      "loss": 0.2763,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -6.483604431152344,
-      "rewards/margins": 3.112400531768799,
-      "rewards/rejected": -9.5960054397583,
-      "step": 11020
-    },
-    {
-      "epoch": 0.8815184815184816,
-      "grad_norm": 27.398509979248047,
-      "learning_rate": 2.1068467274069777e-07,
-      "logits/chosen": -12.172769546508789,
-      "logits/rejected": -12.119073867797852,
-      "logps/chosen": -145.311279296875,
-      "logps/rejected": -182.9757537841797,
-      "loss": 0.3326,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -7.312690258026123,
-      "rewards/margins": 3.659343719482422,
-      "rewards/rejected": -10.972034454345703,
-      "step": 11030
-    },
-    {
-      "epoch": 0.8823176823176824,
-      "grad_norm": 7.1155219078063965,
-      "learning_rate": 2.078909638814347e-07,
-      "logits/chosen": -12.199603080749512,
-      "logits/rejected": -12.104254722595215,
-      "logps/chosen": -139.73626708984375,
-      "logps/rejected": -174.13973999023438,
-      "loss": 0.2717,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -6.387345314025879,
-      "rewards/margins": 3.5788333415985107,
-      "rewards/rejected": -9.966178894042969,
-      "step": 11040
-    },
-    {
-      "epoch": 0.8831168831168831,
-      "grad_norm": 11.887413024902344,
-      "learning_rate": 2.051150976146296e-07,
-      "logits/chosen": -12.16929817199707,
-      "logits/rejected": -12.10260009765625,
-      "logps/chosen": -137.5562286376953,
-      "logps/rejected": -171.06033325195312,
-      "loss": 0.2354,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": -6.805556774139404,
-      "rewards/margins": 3.2055814266204834,
-      "rewards/rejected": -10.011137962341309,
-      "step": 11050
-    },
-    {
-      "epoch": 0.8839160839160839,
-      "grad_norm": 30.15117835998535,
-      "learning_rate": 2.0235709554861426e-07,
-      "logits/chosen": -11.892851829528809,
-      "logits/rejected": -12.195289611816406,
-      "logps/chosen": -150.29608154296875,
-      "logps/rejected": -188.00277709960938,
-      "loss": 0.4469,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -7.3255157470703125,
-      "rewards/margins": 3.2896828651428223,
-      "rewards/rejected": -10.615199089050293,
-      "step": 11060
-    },
-    {
-      "epoch": 0.8847152847152847,
-      "grad_norm": 15.347460746765137,
-      "learning_rate": 1.9961697915265726e-07,
-      "logits/chosen": -12.008756637573242,
-      "logits/rejected": -12.093561172485352,
-      "logps/chosen": -146.1922149658203,
-      "logps/rejected": -181.2917938232422,
-      "loss": 0.377,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": -7.283280849456787,
-      "rewards/margins": 3.269043445587158,
-      "rewards/rejected": -10.552325248718262,
-      "step": 11070
-    },
-    {
-      "epoch": 0.8855144855144855,
-      "grad_norm": 15.325662612915039,
-      "learning_rate": 1.9689476975680049e-07,
-      "logits/chosen": -11.950624465942383,
-      "logits/rejected": -12.060503959655762,
-      "logps/chosen": -144.68038940429688,
-      "logps/rejected": -189.8571319580078,
-      "loss": 0.3056,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -7.268769264221191,
-      "rewards/margins": 3.7435100078582764,
-      "rewards/rejected": -11.012279510498047,
-      "step": 11080
-    },
-    {
-      "epoch": 0.8863136863136863,
-      "grad_norm": 2.5712718963623047,
-      "learning_rate": 1.9419048855169042e-07,
-      "logits/chosen": -12.213506698608398,
-      "logits/rejected": -12.121146202087402,
-      "logps/chosen": -136.6543426513672,
-      "logps/rejected": -176.46031188964844,
-      "loss": 0.2304,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": -6.617370128631592,
-      "rewards/margins": 3.8303534984588623,
-      "rewards/rejected": -10.447723388671875,
-      "step": 11090
-    },
-    {
-      "epoch": 0.8871128871128872,
-      "grad_norm": 20.8624210357666,
-      "learning_rate": 1.915041565884146e-07,
-      "logits/chosen": -12.201539993286133,
-      "logits/rejected": -12.194937705993652,
-      "logps/chosen": -139.3544464111328,
-      "logps/rejected": -171.07421875,
-      "loss": 0.2721,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": -6.441978454589844,
-      "rewards/margins": 3.3355090618133545,
-      "rewards/rejected": -9.777487754821777,
-      "step": 11100
-    },
-    {
-      "epoch": 0.8879120879120879,
-      "grad_norm": 16.42822265625,
-      "learning_rate": 1.8883579477833675e-07,
-      "logits/chosen": -12.136874198913574,
-      "logits/rejected": -12.14762020111084,
-      "logps/chosen": -143.7156219482422,
-      "logps/rejected": -177.20179748535156,
-      "loss": 0.3926,
-      "rewards/accuracies": 0.8375000357627869,
-      "rewards/chosen": -7.318294525146484,
-      "rewards/margins": 2.78669810295105,
-      "rewards/rejected": -10.104992866516113,
-      "step": 11110
-    },
-    {
-      "epoch": 0.8887112887112887,
-      "grad_norm": 3.7879602909088135,
-      "learning_rate": 1.8618542389293503e-07,
-      "logits/chosen": -12.310805320739746,
-      "logits/rejected": -12.169222831726074,
-      "logps/chosen": -142.086181640625,
-      "logps/rejected": -171.9422149658203,
-      "loss": 0.3815,
-      "rewards/accuracies": 0.8375000357627869,
-      "rewards/chosen": -6.605237007141113,
-      "rewards/margins": 3.247476577758789,
-      "rewards/rejected": -9.852712631225586,
-      "step": 11120
-    },
-    {
-      "epoch": 0.8895104895104895,
-      "grad_norm": 3.963949680328369,
-      "learning_rate": 1.8355306456363936e-07,
-      "logits/chosen": -12.052377700805664,
-      "logits/rejected": -12.10938549041748,
-      "logps/chosen": -141.03855895996094,
-      "logps/rejected": -188.73849487304688,
-      "loss": 0.2487,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": -6.936820983886719,
-      "rewards/margins": 3.966825246810913,
-      "rewards/rejected": -10.903645515441895,
-      "step": 11130
-    },
-    {
-      "epoch": 0.8903096903096903,
-      "grad_norm": 9.979516983032227,
-      "learning_rate": 1.809387372816715e-07,
-      "logits/chosen": -12.165946006774902,
-      "logits/rejected": -12.168734550476074,
-      "logps/chosen": -138.1473388671875,
-      "logps/rejected": -170.96653747558594,
-      "loss": 0.3368,
-      "rewards/accuracies": 0.8375000357627869,
-      "rewards/chosen": -6.863381385803223,
-      "rewards/margins": 2.888176202774048,
-      "rewards/rejected": -9.751558303833008,
-      "step": 11140
-    },
-    {
-      "epoch": 0.8911088911088911,
-      "grad_norm": 10.558443069458008,
-      "learning_rate": 1.7834246239788606e-07,
-      "logits/chosen": -11.903914451599121,
-      "logits/rejected": -12.112013816833496,
-      "logps/chosen": -143.57591247558594,
-      "logps/rejected": -193.04493713378906,
-      "loss": 0.3453,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -7.3947649002075195,
-      "rewards/margins": 3.492025136947632,
-      "rewards/rejected": -10.88679027557373,
-      "step": 11150
-    },
-    {
-      "epoch": 0.891908091908092,
-      "grad_norm": 15.507352828979492,
-      "learning_rate": 1.7576426012261032e-07,
-      "logits/chosen": -12.030251502990723,
-      "logits/rejected": -12.052300453186035,
-      "logps/chosen": -145.94993591308594,
-      "logps/rejected": -180.9749755859375,
-      "loss": 0.4192,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -7.191389560699463,
-      "rewards/margins": 3.2191715240478516,
-      "rewards/rejected": -10.410560607910156,
-      "step": 11160
-    },
-    {
-      "epoch": 0.8927072927072927,
-      "grad_norm": 8.570359230041504,
-      "learning_rate": 1.7320415052548834e-07,
-      "logits/chosen": -12.181605339050293,
-      "logits/rejected": -12.127579689025879,
-      "logps/chosen": -141.369140625,
-      "logps/rejected": -176.7362518310547,
-      "loss": 0.322,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": -6.974536418914795,
-      "rewards/margins": 3.302635669708252,
-      "rewards/rejected": -10.277172088623047,
-      "step": 11170
-    },
-    {
-      "epoch": 0.8935064935064935,
-      "grad_norm": 9.344218254089355,
-      "learning_rate": 1.7066215353532394e-07,
-      "logits/chosen": -12.135668754577637,
-      "logits/rejected": -12.06782054901123,
-      "logps/chosen": -136.95291137695312,
-      "logps/rejected": -180.69093322753906,
-      "loss": 0.2488,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -6.69739294052124,
-      "rewards/margins": 3.9601311683654785,
-      "rewards/rejected": -10.657524108886719,
-      "step": 11180
-    },
-    {
-      "epoch": 0.8943056943056943,
-      "grad_norm": 6.622384071350098,
-      "learning_rate": 1.681382889399269e-07,
-      "logits/chosen": -12.132062911987305,
-      "logits/rejected": -12.138354301452637,
-      "logps/chosen": -148.08267211914062,
-      "logps/rejected": -191.41111755371094,
-      "loss": 0.2968,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -7.329235076904297,
-      "rewards/margins": 4.1383376121521,
-      "rewards/rejected": -11.467572212219238,
-      "step": 11190
-    },
-    {
-      "epoch": 0.8951048951048951,
-      "grad_norm": 14.760344505310059,
-      "learning_rate": 1.6563257638595703e-07,
-      "logits/chosen": -12.04784870147705,
-      "logits/rejected": -12.109944343566895,
-      "logps/chosen": -141.94107055664062,
-      "logps/rejected": -183.73020935058594,
-      "loss": 0.3737,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -7.185077667236328,
-      "rewards/margins": 3.3049895763397217,
-      "rewards/rejected": -10.490067481994629,
-      "step": 11200
-    },
-    {
-      "epoch": 0.8959040959040959,
-      "grad_norm": 4.125710964202881,
-      "learning_rate": 1.631450353787728e-07,
-      "logits/chosen": -12.147462844848633,
-      "logits/rejected": -12.089739799499512,
-      "logps/chosen": -140.201416015625,
-      "logps/rejected": -170.1114959716797,
-      "loss": 0.3505,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -6.962253570556641,
-      "rewards/margins": 3.0473086833953857,
-      "rewards/rejected": -10.009562492370605,
-      "step": 11210
-    },
-    {
-      "epoch": 0.8967032967032967,
-      "grad_norm": 3.558934211730957,
-      "learning_rate": 1.6067568528227857e-07,
-      "logits/chosen": -12.088164329528809,
-      "logits/rejected": -12.038786888122559,
-      "logps/chosen": -144.1628875732422,
-      "logps/rejected": -183.6881103515625,
-      "loss": 0.2611,
-      "rewards/accuracies": 0.9000000357627869,
-      "rewards/chosen": -6.996456146240234,
-      "rewards/margins": 3.743065595626831,
-      "rewards/rejected": -10.739521980285645,
-      "step": 11220
-    },
-    {
-      "epoch": 0.8975024975024976,
-      "grad_norm": 15.309100151062012,
-      "learning_rate": 1.5822454531877473e-07,
-      "logits/chosen": -11.99691104888916,
-      "logits/rejected": -12.15250301361084,
-      "logps/chosen": -145.2863006591797,
-      "logps/rejected": -181.60943603515625,
-      "loss": 0.4,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -7.161596775054932,
-      "rewards/margins": 3.382127523422241,
-      "rewards/rejected": -10.54372501373291,
-      "step": 11230
-    },
-    {
-      "epoch": 0.8983016983016983,
-      "grad_norm": 14.38172721862793,
-      "learning_rate": 1.557916345688068e-07,
-      "logits/chosen": -12.177797317504883,
-      "logits/rejected": -12.161993980407715,
-      "logps/chosen": -138.96372985839844,
-      "logps/rejected": -184.83738708496094,
-      "loss": 0.2691,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -6.895704746246338,
-      "rewards/margins": 3.689136505126953,
-      "rewards/rejected": -10.58484172821045,
-      "step": 11240
-    },
-    {
-      "epoch": 0.8991008991008991,
-      "grad_norm": 20.583328247070312,
-      "learning_rate": 1.5337697197101764e-07,
-      "logits/chosen": -12.19485092163086,
-      "logits/rejected": -12.1310453414917,
-      "logps/chosen": -137.73843383789062,
-      "logps/rejected": -167.0582275390625,
-      "loss": 0.3208,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": -6.594809055328369,
-      "rewards/margins": 3.060863494873047,
-      "rewards/rejected": -9.655673027038574,
-      "step": 11250
-    },
-    {
-      "epoch": 0.8999000999000999,
-      "grad_norm": 13.13209056854248,
-      "learning_rate": 1.5098057632200015e-07,
-      "logits/chosen": -12.032218933105469,
-      "logits/rejected": -12.093347549438477,
-      "logps/chosen": -142.83226013183594,
-      "logps/rejected": -190.25369262695312,
-      "loss": 0.3146,
-      "rewards/accuracies": 0.8375000357627869,
-      "rewards/chosen": -7.127984523773193,
-      "rewards/margins": 3.8201286792755127,
-      "rewards/rejected": -10.948113441467285,
-      "step": 11260
-    },
-    {
-      "epoch": 0.9006993006993007,
-      "grad_norm": 23.343156814575195,
-      "learning_rate": 1.486024662761512e-07,
-      "logits/chosen": -12.112887382507324,
-      "logits/rejected": -12.075703620910645,
-      "logps/chosen": -145.25071716308594,
-      "logps/rejected": -180.26568603515625,
-      "loss": 0.3571,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -7.184394836425781,
-      "rewards/margins": 3.347008228302002,
-      "rewards/rejected": -10.531403541564941,
-      "step": 11270
-    },
-    {
-      "epoch": 0.9014985014985015,
-      "grad_norm": 4.328427314758301,
-      "learning_rate": 1.4624266034552581e-07,
-      "logits/chosen": -12.15752124786377,
-      "logits/rejected": -12.11303424835205,
-      "logps/chosen": -138.3636016845703,
-      "logps/rejected": -178.57713317871094,
-      "loss": 0.2066,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": -6.618314266204834,
-      "rewards/margins": 3.6482951641082764,
-      "rewards/rejected": -10.266609191894531,
-      "step": 11280
-    },
-    {
-      "epoch": 0.9022977022977023,
-      "grad_norm": 12.545734405517578,
-      "learning_rate": 1.4390117689969313e-07,
-      "logits/chosen": -12.010832786560059,
-      "logits/rejected": -12.075207710266113,
-      "logps/chosen": -149.4794464111328,
-      "logps/rejected": -193.1590118408203,
-      "loss": 0.4009,
-      "rewards/accuracies": 0.7750000357627869,
-      "rewards/chosen": -7.62520170211792,
-      "rewards/margins": 3.6079254150390625,
-      "rewards/rejected": -11.23312759399414,
-      "step": 11290
-    },
-    {
-      "epoch": 0.903096903096903,
-      "grad_norm": 10.450174331665039,
-      "learning_rate": 1.415780341655937e-07,
-      "logits/chosen": -12.12866497039795,
-      "logits/rejected": -12.173235893249512,
-      "logps/chosen": -142.45465087890625,
-      "logps/rejected": -181.1151580810547,
-      "loss": 0.2819,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -6.886268615722656,
-      "rewards/margins": 3.59879994392395,
-      "rewards/rejected": -10.485069274902344,
-      "step": 11300
-    },
-    {
-      "epoch": 0.9038961038961039,
-      "grad_norm": 9.732522964477539,
-      "learning_rate": 1.3927325022739636e-07,
-      "logits/chosen": -12.177970886230469,
-      "logits/rejected": -12.152162551879883,
-      "logps/chosen": -141.92689514160156,
-      "logps/rejected": -180.7978057861328,
-      "loss": 0.4002,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -7.239105224609375,
-      "rewards/margins": 3.294792890548706,
-      "rewards/rejected": -10.53389835357666,
-      "step": 11310
-    },
-    {
-      "epoch": 0.9046953046953047,
-      "grad_norm": 26.04492950439453,
-      "learning_rate": 1.3698684302636072e-07,
-      "logits/chosen": -12.270537376403809,
-      "logits/rejected": -12.146252632141113,
-      "logps/chosen": -145.70913696289062,
-      "logps/rejected": -181.5695037841797,
-      "loss": 0.347,
-      "rewards/accuracies": 0.8375000357627869,
-      "rewards/chosen": -6.99813985824585,
-      "rewards/margins": 3.4410901069641113,
-      "rewards/rejected": -10.439230918884277,
-      "step": 11320
-    },
-    {
-      "epoch": 0.9054945054945055,
-      "grad_norm": 9.686415672302246,
-      "learning_rate": 1.347188303606936e-07,
-      "logits/chosen": -12.160639762878418,
-      "logits/rejected": -12.144186973571777,
-      "logps/chosen": -148.03050231933594,
-      "logps/rejected": -183.37631225585938,
-      "loss": 0.3839,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -7.75714635848999,
-      "rewards/margins": 3.0008792877197266,
-      "rewards/rejected": -10.758026123046875,
-      "step": 11330
-    },
-    {
-      "epoch": 0.9062937062937063,
-      "grad_norm": 4.3069868087768555,
-      "learning_rate": 1.32469229885413e-07,
-      "logits/chosen": -12.03880500793457,
-      "logits/rejected": -12.115633964538574,
-      "logps/chosen": -135.54534912109375,
-      "logps/rejected": -199.5832977294922,
-      "loss": 0.2597,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": -6.464694976806641,
-      "rewards/margins": 4.987657070159912,
-      "rewards/rejected": -11.452353477478027,
-      "step": 11340
-    },
-    {
-      "epoch": 0.9070929070929071,
-      "grad_norm": 7.014191627502441,
-      "learning_rate": 1.3023805911220932e-07,
-      "logits/chosen": -12.130812644958496,
-      "logits/rejected": -12.099139213562012,
-      "logps/chosen": -145.58694458007812,
-      "logps/rejected": -179.8974151611328,
-      "loss": 0.2706,
-      "rewards/accuracies": 0.9000000357627869,
-      "rewards/chosen": -7.570567607879639,
-      "rewards/margins": 3.1895532608032227,
-      "rewards/rejected": -10.760120391845703,
-      "step": 11350
-    },
-    {
-      "epoch": 0.907892107892108,
-      "grad_norm": 14.711112022399902,
-      "learning_rate": 1.2802533540931e-07,
-      "logits/chosen": -12.197344779968262,
-      "logits/rejected": -12.112601280212402,
-      "logps/chosen": -138.8059844970703,
-      "logps/rejected": -177.68499755859375,
-      "loss": 0.3009,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -6.727143287658691,
-      "rewards/margins": 3.7012345790863037,
-      "rewards/rejected": -10.428378105163574,
-      "step": 11360
-    },
-    {
-      "epoch": 0.9086913086913087,
-      "grad_norm": 2.6263272762298584,
-      "learning_rate": 1.2583107600134397e-07,
-      "logits/chosen": -12.192839622497559,
-      "logits/rejected": -12.134058952331543,
-      "logps/chosen": -142.9783935546875,
-      "logps/rejected": -189.3479461669922,
-      "loss": 0.3984,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -7.235992431640625,
-      "rewards/margins": 3.6093201637268066,
-      "rewards/rejected": -10.845314025878906,
-      "step": 11370
-    },
-    {
-      "epoch": 0.9094905094905095,
-      "grad_norm": 15.218456268310547,
-      "learning_rate": 1.2365529796920683e-07,
-      "logits/chosen": -12.21568775177002,
-      "logits/rejected": -12.1705904006958,
-      "logps/chosen": -140.85592651367188,
-      "logps/rejected": -171.6045379638672,
-      "loss": 0.3658,
-      "rewards/accuracies": 0.8375000357627869,
-      "rewards/chosen": -6.7305908203125,
-      "rewards/margins": 3.100639581680298,
-      "rewards/rejected": -9.831230163574219,
-      "step": 11380
-    },
-    {
-      "epoch": 0.9102897102897103,
-      "grad_norm": 28.925613403320312,
-      "learning_rate": 1.2149801824992978e-07,
-      "logits/chosen": -12.130620956420898,
-      "logits/rejected": -12.152144432067871,
-      "logps/chosen": -138.81594848632812,
-      "logps/rejected": -181.22683715820312,
-      "loss": 0.3087,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": -6.973075866699219,
-      "rewards/margins": 3.68393874168396,
-      "rewards/rejected": -10.657012939453125,
-      "step": 11390
-    },
-    {
-      "epoch": 0.9110889110889111,
-      "grad_norm": 10.945894241333008,
-      "learning_rate": 1.193592536365451e-07,
-      "logits/chosen": -12.239226341247559,
-      "logits/rejected": -12.139704704284668,
-      "logps/chosen": -141.37960815429688,
-      "logps/rejected": -172.1630859375,
-      "loss": 0.3308,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -6.880627632141113,
-      "rewards/margins": 3.2108795642852783,
-      "rewards/rejected": -10.091506958007812,
-      "step": 11400
-    },
-    {
-      "epoch": 0.9118881118881119,
-      "grad_norm": 25.067874908447266,
-      "learning_rate": 1.172390207779589e-07,
-      "logits/chosen": -12.043214797973633,
-      "logits/rejected": -12.094691276550293,
-      "logps/chosen": -141.40554809570312,
-      "logps/rejected": -179.3896942138672,
-      "loss": 0.5788,
-      "rewards/accuracies": 0.7750000357627869,
-      "rewards/chosen": -7.221440315246582,
-      "rewards/margins": 3.061936140060425,
-      "rewards/rejected": -10.283376693725586,
-      "step": 11410
-    },
-    {
-      "epoch": 0.9126873126873127,
-      "grad_norm": 20.299720764160156,
-      "learning_rate": 1.1513733617881717e-07,
-      "logits/chosen": -12.118206977844238,
-      "logits/rejected": -12.14696979522705,
-      "logps/chosen": -147.19097900390625,
-      "logps/rejected": -188.11705017089844,
-      "loss": 0.3988,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -7.1025590896606445,
-      "rewards/margins": 3.8628814220428467,
-      "rewards/rejected": -10.96544075012207,
-      "step": 11420
-    },
-    {
-      "epoch": 0.9134865134865134,
-      "grad_norm": 11.613714218139648,
-      "learning_rate": 1.130542161993814e-07,
-      "logits/chosen": -12.026485443115234,
-      "logits/rejected": -12.121634483337402,
-      "logps/chosen": -138.91427612304688,
-      "logps/rejected": -184.34483337402344,
-      "loss": 0.4363,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -6.884099006652832,
-      "rewards/margins": 3.651947021484375,
-      "rewards/rejected": -10.536046981811523,
-      "step": 11430
-    },
-    {
-      "epoch": 0.9142857142857143,
-      "grad_norm": 14.64468765258789,
-      "learning_rate": 1.1098967705539837e-07,
-      "logits/chosen": -12.131985664367676,
-      "logits/rejected": -12.135344505310059,
-      "logps/chosen": -142.1411590576172,
-      "logps/rejected": -178.7441864013672,
-      "loss": 0.3102,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -6.912040710449219,
-      "rewards/margins": 3.3559327125549316,
-      "rewards/rejected": -10.267973899841309,
-      "step": 11440
-    },
-    {
-      "epoch": 0.9150849150849151,
-      "grad_norm": 16.624940872192383,
-      "learning_rate": 1.089437348179756e-07,
-      "logits/chosen": -12.144472122192383,
-      "logits/rejected": -12.148900032043457,
-      "logps/chosen": -138.50674438476562,
-      "logps/rejected": -172.43345642089844,
-      "loss": 0.3167,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -6.557750701904297,
-      "rewards/margins": 3.4968667030334473,
-      "rewards/rejected": -10.054617881774902,
-      "step": 11450
-    },
-    {
-      "epoch": 0.9158841158841159,
-      "grad_norm": 16.663084030151367,
-      "learning_rate": 1.0691640541345611e-07,
-      "logits/chosen": -12.19025707244873,
-      "logits/rejected": -12.166657447814941,
-      "logps/chosen": -144.22909545898438,
-      "logps/rejected": -181.26046752929688,
-      "loss": 0.3901,
-      "rewards/accuracies": 0.8375000357627869,
-      "rewards/chosen": -7.048381328582764,
-      "rewards/margins": 3.3578667640686035,
-      "rewards/rejected": -10.406249046325684,
-      "step": 11460
-    },
-    {
-      "epoch": 0.9166833166833167,
-      "grad_norm": 3.90826153755188,
-      "learning_rate": 1.0490770462329324e-07,
-      "logits/chosen": -12.168716430664062,
-      "logits/rejected": -12.146541595458984,
-      "logps/chosen": -139.70530700683594,
-      "logps/rejected": -179.1303253173828,
-      "loss": 0.2721,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -6.979737758636475,
-      "rewards/margins": 3.451366901397705,
-      "rewards/rejected": -10.431105613708496,
-      "step": 11470
-    },
-    {
-      "epoch": 0.9174825174825175,
-      "grad_norm": 6.73113489151001,
-      "learning_rate": 1.0291764808392834e-07,
-      "logits/chosen": -12.06814956665039,
-      "logits/rejected": -12.093987464904785,
-      "logps/chosen": -135.8857421875,
-      "logps/rejected": -179.59378051757812,
-      "loss": 0.1696,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": -6.513049602508545,
-      "rewards/margins": 3.9879231452941895,
-      "rewards/rejected": -10.500971794128418,
-      "step": 11480
-    },
-    {
-      "epoch": 0.9182817182817182,
-      "grad_norm": 17.070049285888672,
-      "learning_rate": 1.0094625128667046e-07,
-      "logits/chosen": -12.045641899108887,
-      "logits/rejected": -12.19097900390625,
-      "logps/chosen": -145.172119140625,
-      "logps/rejected": -192.4705047607422,
-      "loss": 0.3933,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -7.310245513916016,
-      "rewards/margins": 3.6979501247406006,
-      "rewards/rejected": -11.008195877075195,
-      "step": 11490
-    },
-    {
-      "epoch": 0.919080919080919,
-      "grad_norm": 5.244506359100342,
-      "learning_rate": 9.899352957757347e-08,
-      "logits/chosen": -12.071609497070312,
-      "logits/rejected": -12.066329002380371,
-      "logps/chosen": -152.07449340820312,
-      "logps/rejected": -192.8920135498047,
-      "loss": 0.3755,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -7.8375139236450195,
-      "rewards/margins": 3.7171878814697266,
-      "rewards/rejected": -11.554701805114746,
-      "step": 11500
-    },
-    {
-      "epoch": 0.9198801198801199,
-      "grad_norm": 13.711586952209473,
-      "learning_rate": 9.70594981573178e-08,
-      "logits/chosen": -12.109350204467773,
-      "logits/rejected": -12.08617877960205,
-      "logps/chosen": -141.0697021484375,
-      "logps/rejected": -195.55575561523438,
-      "loss": 0.2736,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": -7.073464393615723,
-      "rewards/margins": 4.658313751220703,
-      "rewards/rejected": -11.73177719116211,
-      "step": 11510
-    },
-    {
-      "epoch": 0.9206793206793207,
-      "grad_norm": 4.770391941070557,
-      "learning_rate": 9.514417208109305e-08,
-      "logits/chosen": -12.223596572875977,
-      "logits/rejected": -12.182744979858398,
-      "logps/chosen": -140.45697021484375,
-      "logps/rejected": -186.9087371826172,
-      "loss": 0.2201,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -6.730265140533447,
-      "rewards/margins": 4.169605255126953,
-      "rewards/rejected": -10.899870872497559,
-      "step": 11520
-    },
-    {
-      "epoch": 0.9214785214785215,
-      "grad_norm": 14.373335838317871,
-      "learning_rate": 9.324756625847809e-08,
-      "logits/chosen": -12.240012168884277,
-      "logits/rejected": -12.127127647399902,
-      "logps/chosen": -140.56094360351562,
-      "logps/rejected": -180.5613250732422,
-      "loss": 0.2432,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": -6.6390581130981445,
-      "rewards/margins": 4.117955684661865,
-      "rewards/rejected": -10.757013320922852,
-      "step": 11530
-    },
-    {
-      "epoch": 0.9222777222777223,
-      "grad_norm": 29.063602447509766,
-      "learning_rate": 9.136969545332857e-08,
-      "logits/chosen": -12.132627487182617,
-      "logits/rejected": -12.115193367004395,
-      "logps/chosen": -135.2618865966797,
-      "logps/rejected": -182.9955291748047,
-      "loss": 0.3113,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -6.292425155639648,
-      "rewards/margins": 4.105528354644775,
-      "rewards/rejected": -10.397953987121582,
-      "step": 11540
-    },
-    {
-      "epoch": 0.9230769230769231,
-      "grad_norm": 13.068032264709473,
-      "learning_rate": 8.951057428365855e-08,
-      "logits/chosen": -12.111024856567383,
-      "logits/rejected": -12.142207145690918,
-      "logps/chosen": -140.51512145996094,
-      "logps/rejected": -180.5317840576172,
-      "loss": 0.2695,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -7.0589599609375,
-      "rewards/margins": 3.596850633621216,
-      "rewards/rejected": -10.655810356140137,
-      "step": 11550
-    },
-    {
-      "epoch": 0.9238761238761238,
-      "grad_norm": 10.938501358032227,
-      "learning_rate": 8.767021722152908e-08,
-      "logits/chosen": -12.198969841003418,
-      "logits/rejected": -12.123109817504883,
-      "logps/chosen": -141.29638671875,
-      "logps/rejected": -175.1110076904297,
-      "loss": 0.3735,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": -6.7761125564575195,
-      "rewards/margins": 3.200077533721924,
-      "rewards/rejected": -9.976190567016602,
-      "step": 11560
-    },
-    {
-      "epoch": 0.9246753246753247,
-      "grad_norm": 15.554697036743164,
-      "learning_rate": 8.584863859293385e-08,
-      "logits/chosen": -12.198037147521973,
-      "logits/rejected": -12.126375198364258,
-      "logps/chosen": -133.32798767089844,
-      "logps/rejected": -167.6688232421875,
-      "loss": 0.2902,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": -6.283897399902344,
-      "rewards/margins": 3.4049949645996094,
-      "rewards/rejected": -9.688892364501953,
-      "step": 11570
-    },
-    {
-      "epoch": 0.9254745254745255,
-      "grad_norm": 25.71439552307129,
-      "learning_rate": 8.404585257769021e-08,
-      "logits/chosen": -12.257412910461426,
-      "logits/rejected": -12.184131622314453,
-      "logps/chosen": -143.7526092529297,
-      "logps/rejected": -173.32212829589844,
-      "loss": 0.4184,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -6.999759197235107,
-      "rewards/margins": 3.095311403274536,
-      "rewards/rejected": -10.095070838928223,
-      "step": 11580
-    },
-    {
-      "epoch": 0.9262737262737263,
-      "grad_norm": 4.686800479888916,
-      "learning_rate": 8.226187320932555e-08,
-      "logits/chosen": -12.089967727661133,
-      "logits/rejected": -12.117823600769043,
-      "logps/chosen": -141.90736389160156,
-      "logps/rejected": -181.68092346191406,
-      "loss": 0.5152,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -7.0700812339782715,
-      "rewards/margins": 3.3681488037109375,
-      "rewards/rejected": -10.438230514526367,
-      "step": 11590
-    },
-    {
-      "epoch": 0.9270729270729271,
-      "grad_norm": 15.116299629211426,
-      "learning_rate": 8.04967143749702e-08,
-      "logits/chosen": -12.272290229797363,
-      "logits/rejected": -12.17949390411377,
-      "logps/chosen": -141.92835998535156,
-      "logps/rejected": -176.8623046875,
-      "loss": 0.3318,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -6.855081081390381,
-      "rewards/margins": 3.5146610736846924,
-      "rewards/rejected": -10.369742393493652,
-      "step": 11600
-    },
-    {
-      "epoch": 0.9278721278721279,
-      "grad_norm": 19.947345733642578,
-      "learning_rate": 7.875038981524863e-08,
-      "logits/chosen": -12.222296714782715,
-      "logits/rejected": -12.234938621520996,
-      "logps/chosen": -147.04368591308594,
-      "logps/rejected": -190.43624877929688,
-      "loss": 0.4146,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -7.438308238983154,
-      "rewards/margins": 3.9465534687042236,
-      "rewards/rejected": -11.38486099243164,
-      "step": 11610
-    },
-    {
-      "epoch": 0.9286713286713286,
-      "grad_norm": 14.012457847595215,
-      "learning_rate": 7.702291312417288e-08,
-      "logits/chosen": -12.1261625289917,
-      "logits/rejected": -12.131247520446777,
-      "logps/chosen": -143.88194274902344,
-      "logps/rejected": -178.21478271484375,
-      "loss": 0.361,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -7.317866802215576,
-      "rewards/margins": 3.1575796604156494,
-      "rewards/rejected": -10.475445747375488,
-      "step": 11620
-    },
-    {
-      "epoch": 0.9294705294705294,
-      "grad_norm": 5.700544357299805,
-      "learning_rate": 7.531429774903592e-08,
-      "logits/chosen": -12.217572212219238,
-      "logits/rejected": -12.122444152832031,
-      "logps/chosen": -141.95225524902344,
-      "logps/rejected": -188.8566436767578,
-      "loss": 0.225,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": -7.1136980056762695,
-      "rewards/margins": 4.214980602264404,
-      "rewards/rejected": -11.3286771774292,
-      "step": 11630
-    },
-    {
-      "epoch": 0.9302697302697303,
-      "grad_norm": 22.29363250732422,
-      "learning_rate": 7.362455699030852e-08,
-      "logits/chosen": -12.111021995544434,
-      "logits/rejected": -12.158370971679688,
-      "logps/chosen": -135.1507110595703,
-      "logps/rejected": -174.01841735839844,
-      "loss": 0.3569,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -6.440254211425781,
-      "rewards/margins": 3.4107701778411865,
-      "rewards/rejected": -9.85102367401123,
-      "step": 11640
-    },
-    {
-      "epoch": 0.9310689310689311,
-      "grad_norm": 14.608246803283691,
-      "learning_rate": 7.195370400153361e-08,
-      "logits/chosen": -12.069113731384277,
-      "logits/rejected": -12.157475471496582,
-      "logps/chosen": -141.6149444580078,
-      "logps/rejected": -195.135986328125,
-      "loss": 0.293,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -6.937130928039551,
-      "rewards/margins": 4.240560054779053,
-      "rewards/rejected": -11.177690505981445,
-      "step": 11650
-    },
-    {
-      "epoch": 0.9318681318681319,
-      "grad_norm": 26.39864158630371,
-      "learning_rate": 7.030175178922544e-08,
-      "logits/chosen": -12.185307502746582,
-      "logits/rejected": -12.0947904586792,
-      "logps/chosen": -147.5071563720703,
-      "logps/rejected": -185.26947021484375,
-      "loss": 0.3305,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -7.545223236083984,
-      "rewards/margins": 3.358900547027588,
-      "rewards/rejected": -10.904123306274414,
-      "step": 11660
-    },
-    {
-      "epoch": 0.9326673326673327,
-      "grad_norm": 8.494548797607422,
-      "learning_rate": 6.866871321276836e-08,
-      "logits/chosen": -12.169219017028809,
-      "logits/rejected": -12.20798110961914,
-      "logps/chosen": -139.478515625,
-      "logps/rejected": -181.73487854003906,
-      "loss": 0.2705,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -7.0764641761779785,
-      "rewards/margins": 3.543333053588867,
-      "rewards/rejected": -10.619797706604004,
-      "step": 11670
-    },
-    {
-      "epoch": 0.9334665334665335,
-      "grad_norm": 7.79030704498291,
-      "learning_rate": 6.705460098431538e-08,
-      "logits/chosen": -12.041778564453125,
-      "logits/rejected": -12.084114074707031,
-      "logps/chosen": -139.99081420898438,
-      "logps/rejected": -181.37826538085938,
-      "loss": 0.4212,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -6.744085788726807,
-      "rewards/margins": 3.7479870319366455,
-      "rewards/rejected": -10.492074012756348,
-      "step": 11680
-    },
-    {
-      "epoch": 0.9342657342657342,
-      "grad_norm": 1.9751633405685425,
-      "learning_rate": 6.545942766869068e-08,
-      "logits/chosen": -12.246904373168945,
-      "logits/rejected": -12.102598190307617,
-      "logps/chosen": -143.68170166015625,
-      "logps/rejected": -175.8052978515625,
-      "loss": 0.3529,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": -6.916490077972412,
-      "rewards/margins": 3.482625722885132,
-      "rewards/rejected": -10.399115562438965,
-      "step": 11690
-    },
-    {
-      "epoch": 0.935064935064935,
-      "grad_norm": 16.443294525146484,
-      "learning_rate": 6.38832056832911e-08,
-      "logits/chosen": -12.140036582946777,
-      "logits/rejected": -12.157798767089844,
-      "logps/chosen": -133.92630004882812,
-      "logps/rejected": -179.73403930664062,
-      "loss": 0.1838,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": -6.433578014373779,
-      "rewards/margins": 4.091385841369629,
-      "rewards/rejected": -10.52496337890625,
-      "step": 11700
-    },
-    {
-      "epoch": 0.9358641358641359,
-      "grad_norm": 25.141427993774414,
-      "learning_rate": 6.232594729798952e-08,
-      "logits/chosen": -12.057530403137207,
-      "logits/rejected": -12.124978065490723,
-      "logps/chosen": -142.56961059570312,
-      "logps/rejected": -181.40882873535156,
-      "loss": 0.4755,
-      "rewards/accuracies": 0.7125000357627869,
-      "rewards/chosen": -7.270411968231201,
-      "rewards/margins": 3.3534324169158936,
-      "rewards/rejected": -10.6238431930542,
-      "step": 11710
-    },
-    {
-      "epoch": 0.9366633366633367,
-      "grad_norm": 19.727022171020508,
-      "learning_rate": 6.078766463503943e-08,
-      "logits/chosen": -12.049223899841309,
-      "logits/rejected": -12.220019340515137,
-      "logps/chosen": -145.16839599609375,
-      "logps/rejected": -197.7736358642578,
-      "loss": 0.2452,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": -7.285755157470703,
-      "rewards/margins": 4.309337615966797,
-      "rewards/rejected": -11.5950927734375,
-      "step": 11720
-    },
-    {
-      "epoch": 0.9374625374625375,
-      "grad_norm": 16.37355613708496,
-      "learning_rate": 5.926836966898053e-08,
-      "logits/chosen": -12.226702690124512,
-      "logits/rejected": -12.159193992614746,
-      "logps/chosen": -146.072021484375,
-      "logps/rejected": -173.3068084716797,
-      "loss": 0.5354,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -7.181363105773926,
-      "rewards/margins": 2.876451253890991,
-      "rewards/rejected": -10.057814598083496,
-      "step": 11730
-    },
-    {
-      "epoch": 0.9382617382617383,
-      "grad_norm": 21.10970687866211,
-      "learning_rate": 5.7768074226545455e-08,
-      "logits/chosen": -12.044598579406738,
-      "logits/rejected": -12.221430778503418,
-      "logps/chosen": -134.897216796875,
-      "logps/rejected": -179.3510284423828,
-      "loss": 0.3047,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -6.374969959259033,
-      "rewards/margins": 3.6452813148498535,
-      "rewards/rejected": -10.020251274108887,
-      "step": 11740
-    },
-    {
-      "epoch": 0.939060939060939,
-      "grad_norm": 23.36241340637207,
-      "learning_rate": 5.628678998656767e-08,
-      "logits/chosen": -11.977896690368652,
-      "logits/rejected": -12.104609489440918,
-      "logps/chosen": -143.4762420654297,
-      "logps/rejected": -191.56216430664062,
-      "loss": 0.3524,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -7.417935371398926,
-      "rewards/margins": 3.7863247394561768,
-      "rewards/rejected": -11.204259872436523,
-      "step": 11750
-    },
-    {
-      "epoch": 0.9398601398601398,
-      "grad_norm": 13.88594913482666,
-      "learning_rate": 5.4824528479891226e-08,
-      "logits/chosen": -12.167976379394531,
-      "logits/rejected": -12.12037181854248,
-      "logps/chosen": -149.12303161621094,
-      "logps/rejected": -181.83834838867188,
-      "loss": 0.4871,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -7.781958103179932,
-      "rewards/margins": 2.880955934524536,
-      "rewards/rejected": -10.662914276123047,
-      "step": 11760
-    },
-    {
-      "epoch": 0.9406593406593406,
-      "grad_norm": 12.120220184326172,
-      "learning_rate": 5.3381301089280026e-08,
-      "logits/chosen": -12.218108177185059,
-      "logits/rejected": -12.156213760375977,
-      "logps/chosen": -137.8723602294922,
-      "logps/rejected": -177.2712860107422,
-      "loss": 0.1844,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": -6.491710186004639,
-      "rewards/margins": 3.6720311641693115,
-      "rewards/rejected": -10.163741111755371,
-      "step": 11770
-    },
-    {
-      "epoch": 0.9414585414585415,
-      "grad_norm": 23.410181045532227,
-      "learning_rate": 5.195711904932954e-08,
-      "logits/chosen": -12.194025993347168,
-      "logits/rejected": -12.071700096130371,
-      "logps/chosen": -137.27830505371094,
-      "logps/rejected": -179.50047302246094,
-      "loss": 0.1684,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": -6.572581768035889,
-      "rewards/margins": 3.964038848876953,
-      "rewards/rejected": -10.536620140075684,
-      "step": 11780
-    },
-    {
-      "epoch": 0.9422577422577423,
-      "grad_norm": 27.969982147216797,
-      "learning_rate": 5.05519934463794e-08,
-      "logits/chosen": -12.201484680175781,
-      "logits/rejected": -12.137736320495605,
-      "logps/chosen": -143.1024627685547,
-      "logps/rejected": -175.3437957763672,
-      "loss": 0.3639,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -6.90608549118042,
-      "rewards/margins": 3.5148932933807373,
-      "rewards/rejected": -10.420979499816895,
-      "step": 11790
-    },
-    {
-      "epoch": 0.9430569430569431,
-      "grad_norm": 17.451602935791016,
-      "learning_rate": 4.916593521842789e-08,
-      "logits/chosen": -12.084578514099121,
-      "logits/rejected": -12.234366416931152,
-      "logps/chosen": -143.42250061035156,
-      "logps/rejected": -192.15847778320312,
-      "loss": 0.3703,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": -7.217949867248535,
-      "rewards/margins": 3.7198922634124756,
-      "rewards/rejected": -10.93784236907959,
-      "step": 11800
-    },
-    {
-      "epoch": 0.9438561438561438,
-      "grad_norm": 17.09610939025879,
-      "learning_rate": 4.779895515504534e-08,
-      "logits/chosen": -12.139369010925293,
-      "logits/rejected": -12.08332347869873,
-      "logps/chosen": -153.97105407714844,
-      "logps/rejected": -182.6417999267578,
-      "loss": 0.5226,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -7.957244396209717,
-      "rewards/margins": 2.9765899181365967,
-      "rewards/rejected": -10.933833122253418,
-      "step": 11810
-    },
-    {
-      "epoch": 0.9446553446553446,
-      "grad_norm": 16.509763717651367,
-      "learning_rate": 4.6451063897290926e-08,
-      "logits/chosen": -12.187828063964844,
-      "logits/rejected": -12.128993034362793,
-      "logps/chosen": -148.00172424316406,
-      "logps/rejected": -181.6998748779297,
-      "loss": 0.3024,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -7.520776271820068,
-      "rewards/margins": 3.63492751121521,
-      "rewards/rejected": -11.1557035446167,
-      "step": 11820
-    },
-    {
-      "epoch": 0.9454545454545454,
-      "grad_norm": 28.419601440429688,
-      "learning_rate": 4.512227193762958e-08,
-      "logits/chosen": -12.218345642089844,
-      "logits/rejected": -12.184550285339355,
-      "logps/chosen": -142.48135375976562,
-      "logps/rejected": -175.76931762695312,
-      "loss": 0.4688,
-      "rewards/accuracies": 0.7750000357627869,
-      "rewards/chosen": -6.714509010314941,
-      "rewards/margins": 3.4109256267547607,
-      "rewards/rejected": -10.125434875488281,
-      "step": 11830
-    },
-    {
-      "epoch": 0.9462537462537463,
-      "grad_norm": 19.852542877197266,
-      "learning_rate": 4.381258961985158e-08,
-      "logits/chosen": -12.247458457946777,
-      "logits/rejected": -12.195066452026367,
-      "logps/chosen": -135.31651306152344,
-      "logps/rejected": -172.49363708496094,
-      "loss": 0.2524,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -6.439352512359619,
-      "rewards/margins": 3.663742780685425,
-      "rewards/rejected": -10.103096008300781,
-      "step": 11840
-    },
-    {
-      "epoch": 0.9470529470529471,
-      "grad_norm": 12.137730598449707,
-      "learning_rate": 4.252202713898978e-08,
-      "logits/chosen": -12.157485008239746,
-      "logits/rejected": -12.161755561828613,
-      "logps/chosen": -138.1735382080078,
-      "logps/rejected": -176.99217224121094,
-      "loss": 0.2025,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": -6.236845970153809,
-      "rewards/margins": 4.082284450531006,
-      "rewards/rejected": -10.319129943847656,
-      "step": 11850
-    },
-    {
-      "epoch": 0.9478521478521479,
-      "grad_norm": 22.578289031982422,
-      "learning_rate": 4.12505945412417e-08,
-      "logits/chosen": -12.10041332244873,
-      "logits/rejected": -12.087085723876953,
-      "logps/chosen": -141.77891540527344,
-      "logps/rejected": -180.81619262695312,
-      "loss": 0.2362,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": -7.202554225921631,
-      "rewards/margins": 3.46720814704895,
-      "rewards/rejected": -10.669761657714844,
-      "step": 11860
-    },
-    {
-      "epoch": 0.9486513486513487,
-      "grad_norm": 10.379511833190918,
-      "learning_rate": 3.999830172389113e-08,
-      "logits/chosen": -12.189763069152832,
-      "logits/rejected": -12.137022972106934,
-      "logps/chosen": -140.01731872558594,
-      "logps/rejected": -175.5948486328125,
-      "loss": 0.3291,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -6.823362827301025,
-      "rewards/margins": 3.4296226501464844,
-      "rewards/rejected": -10.252985000610352,
-      "step": 11870
-    },
-    {
-      "epoch": 0.9494505494505494,
-      "grad_norm": 24.13884162902832,
-      "learning_rate": 3.876515843523138e-08,
-      "logits/chosen": -12.224791526794434,
-      "logits/rejected": -12.062413215637207,
-      "logps/chosen": -142.55905151367188,
-      "logps/rejected": -178.15188598632812,
-      "loss": 0.3209,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -6.978152751922607,
-      "rewards/margins": 3.278456926345825,
-      "rewards/rejected": -10.256609916687012,
-      "step": 11880
-    },
-    {
-      "epoch": 0.9502497502497502,
-      "grad_norm": 19.529830932617188,
-      "learning_rate": 3.7551174274488014e-08,
-      "logits/chosen": -12.198746681213379,
-      "logits/rejected": -12.138190269470215,
-      "logps/chosen": -140.47999572753906,
-      "logps/rejected": -174.5394287109375,
-      "loss": 0.4892,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -7.092100620269775,
-      "rewards/margins": 3.004765033721924,
-      "rewards/rejected": -10.0968656539917,
-      "step": 11890
-    },
-    {
-      "epoch": 0.951048951048951,
-      "grad_norm": 11.378500938415527,
-      "learning_rate": 3.6356358691746754e-08,
-      "logits/chosen": -12.013643264770508,
-      "logits/rejected": -12.170820236206055,
-      "logps/chosen": -141.43528747558594,
-      "logps/rejected": -189.858642578125,
-      "loss": 0.3254,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -6.99343729019165,
-      "rewards/margins": 4.094195365905762,
-      "rewards/rejected": -11.087632179260254,
-      "step": 11900
-    },
-    {
-      "epoch": 0.9518481518481519,
-      "grad_norm": 10.397502899169922,
-      "learning_rate": 3.518072098787656e-08,
-      "logits/chosen": -12.045208930969238,
-      "logits/rejected": -12.142157554626465,
-      "logps/chosen": -150.8633270263672,
-      "logps/rejected": -187.67041015625,
-      "loss": 0.343,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -7.344248294830322,
-      "rewards/margins": 3.4809136390686035,
-      "rewards/rejected": -10.825161933898926,
-      "step": 11910
-    },
-    {
-      "epoch": 0.9526473526473527,
-      "grad_norm": 17.897357940673828,
-      "learning_rate": 3.4024270314459705e-08,
-      "logits/chosen": -12.21469783782959,
-      "logits/rejected": -12.232344627380371,
-      "logps/chosen": -138.81558227539062,
-      "logps/rejected": -173.15835571289062,
-      "loss": 0.4063,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -7.040181636810303,
-      "rewards/margins": 2.9336256980895996,
-      "rewards/rejected": -9.973807334899902,
-      "step": 11920
-    },
-    {
-      "epoch": 0.9534465534465535,
-      "grad_norm": 18.923961639404297,
-      "learning_rate": 3.2887015673720156e-08,
-      "logits/chosen": -12.216924667358398,
-      "logits/rejected": -12.130465507507324,
-      "logps/chosen": -143.12115478515625,
-      "logps/rejected": -181.20130920410156,
-      "loss": 0.393,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -7.127814769744873,
-      "rewards/margins": 3.443132162094116,
-      "rewards/rejected": -10.570947647094727,
-      "step": 11930
-    },
-    {
-      "epoch": 0.9542457542457542,
-      "grad_norm": 12.417950630187988,
-      "learning_rate": 3.17689659184528e-08,
-      "logits/chosen": -12.234987258911133,
-      "logits/rejected": -12.094840049743652,
-      "logps/chosen": -140.2808380126953,
-      "logps/rejected": -177.64578247070312,
-      "loss": 0.2897,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -6.798310279846191,
-      "rewards/margins": 3.5095481872558594,
-      "rewards/rejected": -10.30785846710205,
-      "step": 11940
-    },
-    {
-      "epoch": 0.955044955044955,
-      "grad_norm": 15.310285568237305,
-      "learning_rate": 3.067012975195433e-08,
-      "logits/chosen": -12.158228874206543,
-      "logits/rejected": -12.127902030944824,
-      "logps/chosen": -139.69119262695312,
-      "logps/rejected": -178.06370544433594,
-      "loss": 0.2706,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -6.7987799644470215,
-      "rewards/margins": 3.4488632678985596,
-      "rewards/rejected": -10.24764347076416,
-      "step": 11950
-    },
-    {
-      "epoch": 0.9558441558441558,
-      "grad_norm": 25.624116897583008,
-      "learning_rate": 2.9590515727956904e-08,
-      "logits/chosen": -12.10793399810791,
-      "logits/rejected": -12.165340423583984,
-      "logps/chosen": -136.0132293701172,
-      "logps/rejected": -178.293701171875,
-      "loss": 0.3376,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -6.499722957611084,
-      "rewards/margins": 3.3552091121673584,
-      "rewards/rejected": -9.854931831359863,
-      "step": 11960
-    },
-    {
-      "epoch": 0.9566433566433566,
-      "grad_norm": 17.97190284729004,
-      "learning_rate": 2.853013225056045e-08,
-      "logits/chosen": -12.210049629211426,
-      "logits/rejected": -12.131799697875977,
-      "logps/chosen": -135.19961547851562,
-      "logps/rejected": -178.74815368652344,
-      "loss": 0.2255,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -6.623639106750488,
-      "rewards/margins": 3.8452746868133545,
-      "rewards/rejected": -10.468913078308105,
-      "step": 11970
-    },
-    {
-      "epoch": 0.9574425574425575,
-      "grad_norm": 13.427388191223145,
-      "learning_rate": 2.7488987574166847e-08,
-      "logits/chosen": -12.26730728149414,
-      "logits/rejected": -12.16114330291748,
-      "logps/chosen": -142.74407958984375,
-      "logps/rejected": -178.0996856689453,
-      "loss": 0.2564,
-      "rewards/accuracies": 0.9000000357627869,
-      "rewards/chosen": -6.955391883850098,
-      "rewards/margins": 3.7375335693359375,
-      "rewards/rejected": -10.692926406860352,
-      "step": 11980
-    },
-    {
-      "epoch": 0.9582417582417583,
-      "grad_norm": 21.4708251953125,
-      "learning_rate": 2.6467089803417222e-08,
-      "logits/chosen": -12.042070388793945,
-      "logits/rejected": -12.067957878112793,
-      "logps/chosen": -148.7872772216797,
-      "logps/rejected": -185.75885009765625,
-      "loss": 0.3433,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -7.67165994644165,
-      "rewards/margins": 3.261652708053589,
-      "rewards/rejected": -10.933313369750977,
-      "step": 11990
-    },
-    {
-      "epoch": 0.9590409590409591,
-      "grad_norm": 18.269641876220703,
-      "learning_rate": 2.546444689312727e-08,
-      "logits/chosen": -12.02833080291748,
-      "logits/rejected": -12.082822799682617,
-      "logps/chosen": -143.31980895996094,
-      "logps/rejected": -184.5691375732422,
-      "loss": 0.2801,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -6.849358558654785,
-      "rewards/margins": 4.077477931976318,
-      "rewards/rejected": -10.926836967468262,
-      "step": 12000
-    },
-    {
-      "epoch": 0.9598401598401598,
-      "grad_norm": 16.704710006713867,
-      "learning_rate": 2.4481066648225914e-08,
-      "logits/chosen": -12.133151054382324,
-      "logits/rejected": -12.095044136047363,
-      "logps/chosen": -143.1634063720703,
-      "logps/rejected": -174.17373657226562,
-      "loss": 0.4509,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -7.353865146636963,
-      "rewards/margins": 3.3502871990203857,
-      "rewards/rejected": -10.70415210723877,
-      "step": 12010
-    },
-    {
-      "epoch": 0.9606393606393606,
-      "grad_norm": 24.82450294494629,
-      "learning_rate": 2.3516956723695915e-08,
-      "logits/chosen": -12.156756401062012,
-      "logits/rejected": -12.152548789978027,
-      "logps/chosen": -145.46524047851562,
-      "logps/rejected": -185.9225311279297,
-      "loss": 0.4166,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -7.228430271148682,
-      "rewards/margins": 3.535753011703491,
-      "rewards/rejected": -10.764182090759277,
-      "step": 12020
-    },
-    {
-      "epoch": 0.9614385614385614,
-      "grad_norm": 13.609478950500488,
-      "learning_rate": 2.2572124624511137e-08,
-      "logits/chosen": -12.013975143432617,
-      "logits/rejected": -12.181368827819824,
-      "logps/chosen": -139.7754669189453,
-      "logps/rejected": -192.8511199951172,
-      "loss": 0.2167,
-      "rewards/accuracies": 0.9000000357627869,
-      "rewards/chosen": -6.662395000457764,
-      "rewards/margins": 4.419895648956299,
-      "rewards/rejected": -11.082290649414062,
-      "step": 12030
-    },
-    {
-      "epoch": 0.9622377622377623,
-      "grad_norm": 4.9325995445251465,
-      "learning_rate": 2.164657770558104e-08,
-      "logits/chosen": -12.163172721862793,
-      "logits/rejected": -12.170525550842285,
-      "logps/chosen": -139.52618408203125,
-      "logps/rejected": -189.97119140625,
-      "loss": 0.3521,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": -6.936926364898682,
-      "rewards/margins": 3.94622802734375,
-      "rewards/rejected": -10.88315486907959,
-      "step": 12040
-    },
-    {
-      "epoch": 0.9630369630369631,
-      "grad_norm": 23.423389434814453,
-      "learning_rate": 2.074032317169239e-08,
-      "logits/chosen": -11.975293159484863,
-      "logits/rejected": -12.141617774963379,
-      "logps/chosen": -141.0369415283203,
-      "logps/rejected": -195.87942504882812,
-      "loss": 0.3177,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -7.191354274749756,
-      "rewards/margins": 4.318080425262451,
-      "rewards/rejected": -11.50943374633789,
-      "step": 12050
-    },
-    {
-      "epoch": 0.9638361638361639,
-      "grad_norm": 9.3110933303833,
-      "learning_rate": 1.9853368077453193e-08,
-      "logits/chosen": -12.086065292358398,
-      "logits/rejected": -12.122870445251465,
-      "logps/chosen": -145.0602569580078,
-      "logps/rejected": -174.85476684570312,
-      "loss": 0.5119,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -7.432656764984131,
-      "rewards/margins": 2.796722173690796,
-      "rewards/rejected": -10.229378700256348,
-      "step": 12060
-    },
-    {
-      "epoch": 0.9646353646353646,
-      "grad_norm": 45.97776794433594,
-      "learning_rate": 1.8985719327237195e-08,
-      "logits/chosen": -12.041017532348633,
-      "logits/rejected": -12.20025634765625,
-      "logps/chosen": -138.621337890625,
-      "logps/rejected": -186.0052032470703,
-      "loss": 0.306,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": -6.827723979949951,
-      "rewards/margins": 3.751528263092041,
-      "rewards/rejected": -10.579251289367676,
-      "step": 12070
-    },
-    {
-      "epoch": 0.9654345654345654,
-      "grad_norm": 21.881406784057617,
-      "learning_rate": 1.8137383675131126e-08,
-      "logits/chosen": -12.05231761932373,
-      "logits/rejected": -12.107765197753906,
-      "logps/chosen": -138.9774627685547,
-      "logps/rejected": -186.87173461914062,
-      "loss": 0.2931,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": -6.664384365081787,
-      "rewards/margins": 4.059634685516357,
-      "rewards/rejected": -10.724019050598145,
-      "step": 12080
-    },
-    {
-      "epoch": 0.9662337662337662,
-      "grad_norm": 25.772367477416992,
-      "learning_rate": 1.7308367724881704e-08,
-      "logits/chosen": -12.13994026184082,
-      "logits/rejected": -12.127341270446777,
-      "logps/chosen": -145.2510528564453,
-      "logps/rejected": -188.2342529296875,
-      "loss": 0.2576,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -7.391953468322754,
-      "rewards/margins": 3.7807624340057373,
-      "rewards/rejected": -11.17271614074707,
-      "step": 12090
-    },
-    {
-      "epoch": 0.967032967032967,
-      "grad_norm": 23.114334106445312,
-      "learning_rate": 1.6498677929844275e-08,
-      "logits/chosen": -11.955042839050293,
-      "logits/rejected": -12.07641887664795,
-      "logps/chosen": -143.2186737060547,
-      "logps/rejected": -190.68206787109375,
-      "loss": 0.3346,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -7.024468898773193,
-      "rewards/margins": 3.688183546066284,
-      "rewards/rejected": -10.712653160095215,
-      "step": 12100
-    },
-    {
-      "epoch": 0.9678321678321679,
-      "grad_norm": 19.457019805908203,
-      "learning_rate": 1.5708320592932037e-08,
-      "logits/chosen": -12.273749351501465,
-      "logits/rejected": -12.252847671508789,
-      "logps/chosen": -143.63401794433594,
-      "logps/rejected": -185.97496032714844,
-      "loss": 0.2693,
-      "rewards/accuracies": 0.9000000357627869,
-      "rewards/chosen": -7.345334053039551,
-      "rewards/margins": 3.5560805797576904,
-      "rewards/rejected": -10.901413917541504,
-      "step": 12110
-    },
-    {
-      "epoch": 0.9686313686313687,
-      "grad_norm": 28.185293197631836,
-      "learning_rate": 1.4937301866568e-08,
-      "logits/chosen": -12.17640209197998,
-      "logits/rejected": -12.077238082885742,
-      "logps/chosen": -144.3380889892578,
-      "logps/rejected": -176.7383575439453,
-      "loss": 0.3904,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -7.284399509429932,
-      "rewards/margins": 3.317735433578491,
-      "rewards/rejected": -10.602134704589844,
-      "step": 12120
-    },
-    {
-      "epoch": 0.9694305694305694,
-      "grad_norm": 22.689584732055664,
-      "learning_rate": 1.4185627752635877e-08,
-      "logits/chosen": -12.205166816711426,
-      "logits/rejected": -12.222441673278809,
-      "logps/chosen": -142.81065368652344,
-      "logps/rejected": -185.1542510986328,
-      "loss": 0.3771,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -7.152032375335693,
-      "rewards/margins": 3.928156614303589,
-      "rewards/rejected": -11.08018970489502,
-      "step": 12130
-    },
-    {
-      "epoch": 0.9702297702297702,
-      "grad_norm": 13.144206047058105,
-      "learning_rate": 1.3453304102435116e-08,
-      "logits/chosen": -12.267791748046875,
-      "logits/rejected": -12.163886070251465,
-      "logps/chosen": -145.26620483398438,
-      "logps/rejected": -176.0245361328125,
-      "loss": 0.4643,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": -7.458654880523682,
-      "rewards/margins": 2.8742153644561768,
-      "rewards/rejected": -10.332869529724121,
-      "step": 12140
-    },
-    {
-      "epoch": 0.971028971028971,
-      "grad_norm": 7.357172966003418,
-      "learning_rate": 1.2740336616633153e-08,
-      "logits/chosen": -12.163433074951172,
-      "logits/rejected": -12.215336799621582,
-      "logps/chosen": -134.6800994873047,
-      "logps/rejected": -175.87059020996094,
-      "loss": 0.3146,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -7.0116496086120605,
-      "rewards/margins": 3.160313367843628,
-      "rewards/rejected": -10.171961784362793,
-      "step": 12150
-    },
-    {
-      "epoch": 0.9718281718281718,
-      "grad_norm": 11.066679000854492,
-      "learning_rate": 1.2046730845222954e-08,
-      "logits/chosen": -12.165721893310547,
-      "logits/rejected": -12.247424125671387,
-      "logps/chosen": -142.2310028076172,
-      "logps/rejected": -181.80545043945312,
-      "loss": 0.2556,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -6.753901958465576,
-      "rewards/margins": 3.2718417644500732,
-      "rewards/rejected": -10.02574348449707,
-      "step": 12160
-    },
-    {
-      "epoch": 0.9726273726273726,
-      "grad_norm": 19.638484954833984,
-      "learning_rate": 1.13724921874786e-08,
-      "logits/chosen": -12.116413116455078,
-      "logits/rejected": -12.17465877532959,
-      "logps/chosen": -148.0548095703125,
-      "logps/rejected": -183.07615661621094,
-      "loss": 0.2891,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -7.8129425048828125,
-      "rewards/margins": 2.8797948360443115,
-      "rewards/rejected": -10.692736625671387,
-      "step": 12170
-    },
-    {
-      "epoch": 0.9734265734265735,
-      "grad_norm": 11.567111015319824,
-      "learning_rate": 1.0717625891913663e-08,
-      "logits/chosen": -12.213236808776855,
-      "logits/rejected": -12.198195457458496,
-      "logps/chosen": -143.07872009277344,
-      "logps/rejected": -182.51890563964844,
-      "loss": 0.2656,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -6.833310604095459,
-      "rewards/margins": 3.6042211055755615,
-      "rewards/rejected": -10.437532424926758,
-      "step": 12180
-    },
-    {
-      "epoch": 0.9742257742257743,
-      "grad_norm": 25.231388092041016,
-      "learning_rate": 1.0082137056240393e-08,
-      "logits/chosen": -12.150675773620605,
-      "logits/rejected": -12.152809143066406,
-      "logps/chosen": -146.83596801757812,
-      "logps/rejected": -188.74839782714844,
-      "loss": 0.3471,
-      "rewards/accuracies": 0.8375000357627869,
-      "rewards/chosen": -7.528872013092041,
-      "rewards/margins": 3.5252857208251953,
-      "rewards/rejected": -11.054158210754395,
-      "step": 12190
-    },
-    {
-      "epoch": 0.975024975024975,
-      "grad_norm": 9.929890632629395,
-      "learning_rate": 9.466030627330314e-09,
-      "logits/chosen": -12.160919189453125,
-      "logits/rejected": -12.197257995605469,
-      "logps/chosen": -150.54312133789062,
-      "logps/rejected": -191.14772033691406,
-      "loss": 0.4265,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": -7.746136665344238,
-      "rewards/margins": 3.6992580890655518,
-      "rewards/rejected": -11.445394515991211,
-      "step": 12200
-    },
-    {
-      "epoch": 0.9758241758241758,
-      "grad_norm": 19.653770446777344,
-      "learning_rate": 8.869311401174806e-09,
-      "logits/chosen": -12.134451866149902,
-      "logits/rejected": -12.128806114196777,
-      "logps/chosen": -145.11990356445312,
-      "logps/rejected": -182.68809509277344,
-      "loss": 0.3674,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": -7.359257698059082,
-      "rewards/margins": 3.1145904064178467,
-      "rewards/rejected": -10.473848342895508,
-      "step": 12210
-    },
-    {
-      "epoch": 0.9766233766233766,
-      "grad_norm": 24.320100784301758,
-      "learning_rate": 8.291984022848754e-09,
-      "logits/chosen": -12.176373481750488,
-      "logits/rejected": -12.032011985778809,
-      "logps/chosen": -145.0596923828125,
-      "logps/rejected": -180.66949462890625,
-      "loss": 0.4139,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -7.294722080230713,
-      "rewards/margins": 3.5558063983917236,
-      "rewards/rejected": -10.850528717041016,
-      "step": 12220
-    },
-    {
-      "epoch": 0.9774225774225774,
-      "grad_norm": 13.253820419311523,
-      "learning_rate": 7.734052986474171e-09,
-      "logits/chosen": -12.166831016540527,
-      "logits/rejected": -12.15704345703125,
-      "logps/chosen": -144.79574584960938,
-      "logps/rejected": -176.02711486816406,
-      "loss": 0.3436,
-      "rewards/accuracies": 0.8375000357627869,
-      "rewards/chosen": -7.37438440322876,
-      "rewards/margins": 2.6703507900238037,
-      "rewards/rejected": -10.044734954833984,
-      "step": 12230
-    },
-    {
-      "epoch": 0.9782217782217782,
-      "grad_norm": 15.81540298461914,
-      "learning_rate": 7.195522635184693e-09,
-      "logits/chosen": -12.170473098754883,
-      "logits/rejected": -12.074197769165039,
-      "logps/chosen": -139.63426208496094,
-      "logps/rejected": -175.04075622558594,
-      "loss": 0.2802,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -7.002773284912109,
-      "rewards/margins": 3.437737226486206,
-      "rewards/rejected": -10.440510749816895,
-      "step": 12240
-    },
-    {
-      "epoch": 0.9790209790209791,
-      "grad_norm": 11.409400939941406,
-      "learning_rate": 6.676397161092252e-09,
-      "logits/chosen": -12.261028289794922,
-      "logits/rejected": -12.206972122192383,
-      "logps/chosen": -139.2923126220703,
-      "logps/rejected": -179.90411376953125,
-      "loss": 0.2721,
-      "rewards/accuracies": 0.8375000357627869,
-      "rewards/chosen": -7.036710262298584,
-      "rewards/margins": 3.2437286376953125,
-      "rewards/rejected": -10.280438423156738,
-      "step": 12250
-    },
-    {
-      "epoch": 0.9798201798201798,
-      "grad_norm": 9.474007606506348,
-      "learning_rate": 6.17668060525406e-09,
-      "logits/chosen": -12.108057975769043,
-      "logits/rejected": -12.095232963562012,
-      "logps/chosen": -139.15774536132812,
-      "logps/rejected": -186.77780151367188,
-      "loss": 0.2556,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": -6.9744439125061035,
-      "rewards/margins": 4.00940465927124,
-      "rewards/rejected": -10.983848571777344,
-      "step": 12260
-    },
-    {
-      "epoch": 0.9806193806193806,
-      "grad_norm": 10.860443115234375,
-      "learning_rate": 5.696376857641794e-09,
-      "logits/chosen": -12.198197364807129,
-      "logits/rejected": -12.270419120788574,
-      "logps/chosen": -143.40794372558594,
-      "logps/rejected": -184.77920532226562,
-      "loss": 0.2598,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": -6.8439154624938965,
-      "rewards/margins": 3.8158111572265625,
-      "rewards/rejected": -10.659727096557617,
-      "step": 12270
-    },
-    {
-      "epoch": 0.9814185814185814,
-      "grad_norm": 22.790069580078125,
-      "learning_rate": 5.235489657110515e-09,
-      "logits/chosen": -12.247511863708496,
-      "logits/rejected": -12.111291885375977,
-      "logps/chosen": -146.5253448486328,
-      "logps/rejected": -185.86276245117188,
-      "loss": 0.3728,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -7.388991832733154,
-      "rewards/margins": 3.568990468978882,
-      "rewards/rejected": -10.957980155944824,
-      "step": 12280
-    },
-    {
-      "epoch": 0.9822177822177822,
-      "grad_norm": 7.317874908447266,
-      "learning_rate": 4.794022591370906e-09,
-      "logits/chosen": -11.982527732849121,
-      "logits/rejected": -12.150198936462402,
-      "logps/chosen": -140.60586547851562,
-      "logps/rejected": -193.7384796142578,
-      "loss": 0.2632,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": -6.705893039703369,
-      "rewards/margins": 4.157721996307373,
-      "rewards/rejected": -10.863615989685059,
-      "step": 12290
-    },
-    {
-      "epoch": 0.983016983016983,
-      "grad_norm": 21.70923614501953,
-      "learning_rate": 4.371979096958745e-09,
-      "logits/chosen": -12.167257308959961,
-      "logits/rejected": -12.088598251342773,
-      "logps/chosen": -147.13880920410156,
-      "logps/rejected": -173.38792419433594,
-      "loss": 0.4656,
-      "rewards/accuracies": 0.7750000357627869,
-      "rewards/chosen": -7.5166335105896,
-      "rewards/margins": 2.445549726486206,
-      "rewards/rejected": -9.962183952331543,
-      "step": 12300
-    },
-    {
-      "epoch": 0.9838161838161839,
-      "grad_norm": 21.887248992919922,
-      "learning_rate": 3.9693624592115896e-09,
-      "logits/chosen": -12.167427062988281,
-      "logits/rejected": -12.156765937805176,
-      "logps/chosen": -139.9525604248047,
-      "logps/rejected": -185.19403076171875,
-      "loss": 0.2888,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -6.6415019035339355,
-      "rewards/margins": 4.074194431304932,
-      "rewards/rejected": -10.71569538116455,
-      "step": 12310
-    },
-    {
-      "epoch": 0.9846153846153847,
-      "grad_norm": 15.561605453491211,
-      "learning_rate": 3.5861758122404665e-09,
-      "logits/chosen": -12.219579696655273,
-      "logits/rejected": -12.174565315246582,
-      "logps/chosen": -143.99986267089844,
-      "logps/rejected": -180.14071655273438,
-      "loss": 0.3068,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -7.140140056610107,
-      "rewards/margins": 3.5144131183624268,
-      "rewards/rejected": -10.654552459716797,
-      "step": 12320
-    },
-    {
-      "epoch": 0.9854145854145854,
-      "grad_norm": 18.580284118652344,
-      "learning_rate": 3.2224221389062784e-09,
-      "logits/chosen": -12.085851669311523,
-      "logits/rejected": -12.113747596740723,
-      "logps/chosen": -136.55284118652344,
-      "logps/rejected": -184.46495056152344,
-      "loss": 0.135,
-      "rewards/accuracies": 0.9625000357627869,
-      "rewards/chosen": -6.572741985321045,
-      "rewards/margins": 4.362630844116211,
-      "rewards/rejected": -10.935373306274414,
-      "step": 12330
-    },
-    {
-      "epoch": 0.9862137862137862,
-      "grad_norm": 20.072694778442383,
-      "learning_rate": 2.878104270797322e-09,
-      "logits/chosen": -12.191258430480957,
-      "logits/rejected": -12.136225700378418,
-      "logps/chosen": -147.42079162597656,
-      "logps/rejected": -182.0079345703125,
-      "loss": 0.3718,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -7.390552043914795,
-      "rewards/margins": 3.108107328414917,
-      "rewards/rejected": -10.498659133911133,
-      "step": 12340
-    },
-    {
-      "epoch": 0.987012987012987,
-      "grad_norm": 9.940278053283691,
-      "learning_rate": 2.5532248882062515e-09,
-      "logits/chosen": -12.278543472290039,
-      "logits/rejected": -12.130844116210938,
-      "logps/chosen": -143.10015869140625,
-      "logps/rejected": -180.1986083984375,
-      "loss": 0.1933,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": -6.962545871734619,
-      "rewards/margins": 3.7922286987304688,
-      "rewards/rejected": -10.754775047302246,
-      "step": 12350
-    },
-    {
-      "epoch": 0.9878121878121878,
-      "grad_norm": 4.076949596405029,
-      "learning_rate": 2.2477865201100956e-09,
-      "logits/chosen": -12.23047924041748,
-      "logits/rejected": -12.12234878540039,
-      "logps/chosen": -138.96522521972656,
-      "logps/rejected": -182.3941650390625,
-      "loss": 0.2535,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": -6.828181743621826,
-      "rewards/margins": 3.9510345458984375,
-      "rewards/rejected": -10.779216766357422,
-      "step": 12360
-    },
-    {
-      "epoch": 0.9886113886113886,
-      "grad_norm": 17.571306228637695,
-      "learning_rate": 1.961791544149716e-09,
-      "logits/chosen": -12.111479759216309,
-      "logits/rejected": -12.101297378540039,
-      "logps/chosen": -147.5512237548828,
-      "logps/rejected": -196.501953125,
-      "loss": 0.2575,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -7.914129734039307,
-      "rewards/margins": 3.8419995307922363,
-      "rewards/rejected": -11.756129264831543,
-      "step": 12370
-    },
-    {
-      "epoch": 0.9894105894105895,
-      "grad_norm": 8.976057052612305,
-      "learning_rate": 1.6952421866114898e-09,
-      "logits/chosen": -12.038252830505371,
-      "logits/rejected": -12.203242301940918,
-      "logps/chosen": -144.5151824951172,
-      "logps/rejected": -192.02085876464844,
-      "loss": 0.368,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -7.109652996063232,
-      "rewards/margins": 3.932513952255249,
-      "rewards/rejected": -11.042167663574219,
-      "step": 12380
-    },
-    {
-      "epoch": 0.9902097902097902,
-      "grad_norm": 15.478114128112793,
-      "learning_rate": 1.448140522410657e-09,
-      "logits/chosen": -12.333587646484375,
-      "logits/rejected": -12.295121192932129,
-      "logps/chosen": -139.56268310546875,
-      "logps/rejected": -167.14178466796875,
-      "loss": 0.3403,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -6.812671184539795,
-      "rewards/margins": 2.884084463119507,
-      "rewards/rejected": -9.696754455566406,
-      "step": 12390
-    },
-    {
-      "epoch": 0.991008991008991,
-      "grad_norm": 14.942720413208008,
-      "learning_rate": 1.220488475074666e-09,
-      "logits/chosen": -12.238165855407715,
-      "logits/rejected": -12.107606887817383,
-      "logps/chosen": -140.7101593017578,
-      "logps/rejected": -180.8552703857422,
-      "loss": 0.2679,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -7.05299711227417,
-      "rewards/margins": 3.5227653980255127,
-      "rewards/rejected": -10.575762748718262,
-      "step": 12400
-    },
-    {
-      "epoch": 0.9918081918081918,
-      "grad_norm": 24.497522354125977,
-      "learning_rate": 1.0122878167279082e-09,
-      "logits/chosen": -12.12418270111084,
-      "logits/rejected": -12.164368629455566,
-      "logps/chosen": -145.1948699951172,
-      "logps/rejected": -190.72512817382812,
-      "loss": 0.307,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": -7.261709690093994,
-      "rewards/margins": 3.86576509475708,
-      "rewards/rejected": -11.127473831176758,
-      "step": 12410
-    },
-    {
-      "epoch": 0.9926073926073926,
-      "grad_norm": 24.537384033203125,
-      "learning_rate": 8.235401680786736e-10,
-      "logits/chosen": -12.23515510559082,
-      "logits/rejected": -12.160743713378906,
-      "logps/chosen": -139.62376403808594,
-      "logps/rejected": -177.37010192871094,
-      "loss": 0.4138,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -6.999446392059326,
-      "rewards/margins": 3.274168014526367,
-      "rewards/rejected": -10.273613929748535,
-      "step": 12420
-    },
-    {
-      "epoch": 0.9934065934065934,
-      "grad_norm": 15.764270782470703,
-      "learning_rate": 6.542469984052724e-10,
-      "logits/chosen": -12.062739372253418,
-      "logits/rejected": -12.18981647491455,
-      "logps/chosen": -139.14796447753906,
-      "logps/rejected": -185.07676696777344,
-      "loss": 0.3351,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -6.872987270355225,
-      "rewards/margins": 3.3031094074249268,
-      "rewards/rejected": -10.17609691619873,
-      "step": 12430
-    },
-    {
-      "epoch": 0.9942057942057942,
-      "grad_norm": 16.77101707458496,
-      "learning_rate": 5.044096255463204e-10,
-      "logits/chosen": -12.103509902954102,
-      "logits/rejected": -12.229147911071777,
-      "logps/chosen": -151.34185791015625,
-      "logps/rejected": -189.61412048339844,
-      "loss": 0.5306,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -7.577481269836426,
-      "rewards/margins": 3.126288414001465,
-      "rewards/rejected": -10.703770637512207,
-      "step": 12440
-    },
-    {
-      "epoch": 0.995004995004995,
-      "grad_norm": 12.72045612335205,
-      "learning_rate": 3.740292158893599e-10,
-      "logits/chosen": -12.184672355651855,
-      "logits/rejected": -12.101056098937988,
-      "logps/chosen": -142.92640686035156,
-      "logps/rejected": -188.22525024414062,
-      "loss": 0.2478,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": -6.854015350341797,
-      "rewards/margins": 4.303215026855469,
-      "rewards/rejected": -11.157230377197266,
-      "step": 12450
-    },
-    {
-      "epoch": 0.9958041958041958,
-      "grad_norm": 2.6530280113220215,
-      "learning_rate": 2.631067843619772e-10,
-      "logits/chosen": -12.239202499389648,
-      "logits/rejected": -12.179596900939941,
-      "logps/chosen": -141.46714782714844,
-      "logps/rejected": -174.36326599121094,
-      "loss": 0.2825,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -6.674633026123047,
-      "rewards/margins": 3.6979637145996094,
-      "rewards/rejected": -10.372596740722656,
-      "step": 12460
-    },
-    {
-      "epoch": 0.9966033966033966,
-      "grad_norm": 5.942234992980957,
-      "learning_rate": 1.7164319442347644e-10,
-      "logits/chosen": -12.241859436035156,
-      "logits/rejected": -12.090044975280762,
-      "logps/chosen": -142.4546661376953,
-      "logps/rejected": -177.8150634765625,
-      "loss": 0.3027,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": -7.139862060546875,
-      "rewards/margins": 3.6752471923828125,
-      "rewards/rejected": -10.815109252929688,
-      "step": 12470
-    },
-    {
-      "epoch": 0.9974025974025974,
-      "grad_norm": 29.329647064208984,
-      "learning_rate": 9.963915805905056e-11,
-      "logits/chosen": -12.23520565032959,
-      "logits/rejected": -12.168898582458496,
-      "logps/chosen": -138.92698669433594,
-      "logps/rejected": -185.13182067871094,
-      "loss": 0.33,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -6.8220415115356445,
-      "rewards/margins": 4.1144585609436035,
-      "rewards/rejected": -10.936500549316406,
-      "step": 12480
-    },
-    {
-      "epoch": 0.9982017982017982,
-      "grad_norm": 23.34520149230957,
-      "learning_rate": 4.7095235774230474e-11,
-      "logits/chosen": -12.166956901550293,
-      "logits/rejected": -12.068496704101562,
-      "logps/chosen": -139.09324645996094,
-      "logps/rejected": -172.1549072265625,
-      "loss": 0.366,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -6.676534175872803,
-      "rewards/margins": 2.9019792079925537,
-      "rewards/rejected": -9.578513145446777,
-      "step": 12490
-    },
-    {
-      "epoch": 0.999000999000999,
-      "grad_norm": 21.092517852783203,
-      "learning_rate": 1.4011836588778693e-11,
-      "logits/chosen": -12.220911979675293,
-      "logits/rejected": -12.099786758422852,
-      "logps/chosen": -148.6799774169922,
-      "logps/rejected": -177.0653076171875,
-      "loss": 0.4121,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -7.52353048324585,
-      "rewards/margins": 2.9356906414031982,
-      "rewards/rejected": -10.459221839904785,
-      "step": 12500
-    },
-    {
-      "epoch": 0.9998001998001999,
-      "grad_norm": 9.327104568481445,
-      "learning_rate": 3.89218036411787e-13,
-      "logits/chosen": -12.23544979095459,
-      "logits/rejected": -12.17452335357666,
-      "logps/chosen": -141.15048217773438,
-      "logps/rejected": -180.34933471679688,
-      "loss": 0.3821,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -7.203601360321045,
-      "rewards/margins": 3.3228776454925537,
-      "rewards/rejected": -10.52647876739502,
-      "step": 12510
-    },
-    {
-      "epoch": 0.99996003996004,
-      "step": 12512,
-      "total_flos": 6.814329228177064e+18,
-      "train_loss": 0.41961536931869625,
-      "train_runtime": 114854.9374,
-      "train_samples_per_second": 0.872,
-      "train_steps_per_second": 0.109
-    }
-  ],
-  "logging_steps": 10,
-  "max_steps": 12512,
-  "num_input_tokens_seen": 0,
-  "num_train_epochs": 1,
-  "save_steps": 500,
-  "stateful_callbacks": {
-    "TrainerControl": {
-      "args": {
-        "should_epoch_stop": false,
-        "should_evaluate": false,
-        "should_log": false,
-        "should_save": true,
-        "should_training_stop": true
-      },
-      "attributes": {}
-    }
-  },
-  "total_flos": 6.814329228177064e+18,
-  "train_batch_size": 1,
-  "trial_name": null,
-  "trial_params": null
-}