{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.0,
  "eval_steps": 500,
  "global_step": 1803,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0005546311702717693,
      "grad_norm": 2.562516450881958,
      "learning_rate": 0.0,
      "logits/chosen": 0.5525987148284912,
      "logits/rejected": 0.8582919239997864,
      "logps/chosen": -280.66448974609375,
      "logps/rejected": -198.26028442382812,
      "loss": 0.2528,
      "loss/chosen-sft": 1.1612701416015625,
      "loss/dpo": 0.2528122663497925,
      "rewards/accuracies": 0.28125,
      "rewards/chosen": -0.00369430985301733,
      "rewards/margins": -0.003086067270487547,
      "rewards/rejected": -0.0006082424661144614,
      "step": 1
    },
    {
      "epoch": 0.0027731558513588465,
      "grad_norm": 1.8309712409973145,
      "learning_rate": 1.1049723756906076e-08,
      "logits/chosen": 0.6242256164550781,
      "logits/rejected": 0.7471870183944702,
      "logps/chosen": -365.79705810546875,
      "logps/rejected": -280.31201171875,
      "loss": 0.2317,
      "loss/chosen-sft": 1.3244354724884033,
      "loss/dpo": 0.23169046640396118,
      "rewards/accuracies": 0.421875,
      "rewards/chosen": -0.00040742673445492983,
      "rewards/margins": -0.0013081450015306473,
      "rewards/rejected": 0.0009007179760374129,
      "step": 5
    },
    {
      "epoch": 0.005546311702717693,
      "grad_norm": 2.4382247924804688,
      "learning_rate": 2.486187845303867e-08,
      "logits/chosen": 0.649553656578064,
      "logits/rejected": 0.8831444978713989,
      "logps/chosen": -284.4432067871094,
      "logps/rejected": -248.49337768554688,
      "loss": 0.2305,
      "loss/chosen-sft": 1.1865930557250977,
      "loss/dpo": 0.23052072525024414,
      "rewards/accuracies": 0.4625000059604645,
      "rewards/chosen": -0.0007243253057822585,
      "rewards/margins": -5.907495506107807e-06,
      "rewards/rejected": -0.0007184178684838116,
      "step": 10
    },
    {
      "epoch": 0.008319467554076539,
      "grad_norm": 2.216749668121338,
      "learning_rate": 3.867403314917127e-08,
      "logits/chosen": 0.7230191826820374,
      "logits/rejected": 0.9627832174301147,
      "logps/chosen": -302.41400146484375,
      "logps/rejected": -264.18963623046875,
      "loss": 0.2317,
      "loss/chosen-sft": 1.27865731716156,
      "loss/dpo": 0.23168079555034637,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": 0.000250987388426438,
      "rewards/margins": 0.0003366722376085818,
      "rewards/rejected": -8.568489283788949e-05,
      "step": 15
    },
    {
      "epoch": 0.011092623405435386,
      "grad_norm": 2.429425001144409,
      "learning_rate": 5.248618784530386e-08,
      "logits/chosen": 0.7443245649337769,
      "logits/rejected": 0.9213382601737976,
      "logps/chosen": -296.67926025390625,
      "logps/rejected": -233.2224578857422,
      "loss": 0.2435,
      "loss/chosen-sft": 1.2470526695251465,
      "loss/dpo": 0.24353936314582825,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": 0.000928783614654094,
      "rewards/margins": 0.0006695252959616482,
      "rewards/rejected": 0.00025925817317329347,
      "step": 20
    },
    {
      "epoch": 0.013865779256794232,
      "grad_norm": 2.4279961585998535,
      "learning_rate": 6.629834254143646e-08,
      "logits/chosen": 0.8054523468017578,
      "logits/rejected": 1.0713064670562744,
      "logps/chosen": -301.8001403808594,
      "logps/rejected": -265.5172424316406,
      "loss": 0.2257,
      "loss/chosen-sft": 1.2987916469573975,
      "loss/dpo": 0.22568467259407043,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0008613772806711495,
      "rewards/margins": -5.407678145274986e-06,
      "rewards/rejected": -0.0008559696143493056,
      "step": 25
    },
    {
      "epoch": 0.016638935108153077,
      "grad_norm": 1.9755202531814575,
      "learning_rate": 8.011049723756906e-08,
      "logits/chosen": 0.8093868494033813,
      "logits/rejected": 0.8907510042190552,
      "logps/chosen": -317.31488037109375,
      "logps/rejected": -238.03701782226562,
      "loss": 0.2252,
      "loss/chosen-sft": 1.2658107280731201,
      "loss/dpo": 0.225176140666008,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": -0.0001813205162761733,
      "rewards/margins": -0.0007404539501294494,
      "rewards/rejected": 0.0005591334775090218,
      "step": 30
    },
    {
      "epoch": 0.019412090959511925,
      "grad_norm": 2.0302202701568604,
      "learning_rate": 9.392265193370165e-08,
      "logits/chosen": 0.7306667566299438,
      "logits/rejected": 0.9124480485916138,
      "logps/chosen": -326.53631591796875,
      "logps/rejected": -278.6661682128906,
      "loss": 0.2324,
      "loss/chosen-sft": 1.2703967094421387,
      "loss/dpo": 0.23235292732715607,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -9.17384386411868e-05,
      "rewards/margins": 0.00045947995386086404,
      "rewards/rejected": -0.0005512182251550257,
      "step": 35
    },
    {
      "epoch": 0.022185246810870772,
      "grad_norm": 2.4935853481292725,
      "learning_rate": 1.0773480662983425e-07,
      "logits/chosen": 0.8364327549934387,
      "logits/rejected": 0.9892117381095886,
      "logps/chosen": -299.20513916015625,
      "logps/rejected": -260.8147888183594,
      "loss": 0.237,
      "loss/chosen-sft": 1.221801519393921,
      "loss/dpo": 0.23703160881996155,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.00010150570597033948,
      "rewards/margins": 0.00015612409333698452,
      "rewards/rejected": -5.461848195409402e-05,
      "step": 40
    },
    {
      "epoch": 0.024958402662229616,
      "grad_norm": 2.5195465087890625,
      "learning_rate": 1.2154696132596685e-07,
      "logits/chosen": 0.7738581895828247,
      "logits/rejected": 1.0265463590621948,
      "logps/chosen": -318.85101318359375,
      "logps/rejected": -237.57666015625,
      "loss": 0.2307,
      "loss/chosen-sft": 1.2462958097457886,
      "loss/dpo": 0.23071476817131042,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -8.556898683309555e-05,
      "rewards/margins": 0.0009564283536747098,
      "rewards/rejected": -0.0010419972240924835,
      "step": 45
    },
    {
      "epoch": 0.027731558513588463,
      "grad_norm": 2.201871871948242,
      "learning_rate": 1.3535911602209942e-07,
      "logits/chosen": 0.7524826526641846,
      "logits/rejected": 0.8071184158325195,
      "logps/chosen": -343.429443359375,
      "logps/rejected": -256.6566162109375,
      "loss": 0.2343,
      "loss/chosen-sft": 1.2330760955810547,
      "loss/dpo": 0.2342543601989746,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.00010758457938209176,
      "rewards/margins": 0.0004232854989822954,
      "rewards/rejected": -0.00031570097780786455,
      "step": 50
    },
    {
      "epoch": 0.03050471436494731,
      "grad_norm": 2.1549618244171143,
      "learning_rate": 1.4917127071823204e-07,
      "logits/chosen": 0.6793915033340454,
      "logits/rejected": 0.9533787965774536,
      "logps/chosen": -286.3639221191406,
      "logps/rejected": -248.7386932373047,
      "loss": 0.2127,
      "loss/chosen-sft": 1.2288844585418701,
      "loss/dpo": 0.21267767250537872,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": 0.0002672958071343601,
      "rewards/margins": 0.0003136277082376182,
      "rewards/rejected": -4.6331912017194554e-05,
      "step": 55
    },
    {
      "epoch": 0.033277870216306155,
      "grad_norm": 2.341963291168213,
      "learning_rate": 1.6298342541436463e-07,
      "logits/chosen": 0.7014708518981934,
      "logits/rejected": 1.0665433406829834,
      "logps/chosen": -310.38134765625,
      "logps/rejected": -242.14944458007812,
      "loss": 0.2263,
      "loss/chosen-sft": 1.2344342470169067,
      "loss/dpo": 0.2262984812259674,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": 0.0001726890041027218,
      "rewards/margins": 0.000270530057605356,
      "rewards/rejected": -9.784109715837985e-05,
      "step": 60
    },
    {
      "epoch": 0.036051026067665005,
      "grad_norm": 2.35201096534729,
      "learning_rate": 1.7679558011049722e-07,
      "logits/chosen": 0.7712376117706299,
      "logits/rejected": 0.9752995371818542,
      "logps/chosen": -323.18377685546875,
      "logps/rejected": -256.13909912109375,
      "loss": 0.2288,
      "loss/chosen-sft": 1.301064133644104,
      "loss/dpo": 0.228795126080513,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.0009808921022340655,
      "rewards/margins": 0.000625107903033495,
      "rewards/rejected": 0.0003557841700967401,
      "step": 65
    },
    {
      "epoch": 0.03882418191902385,
      "grad_norm": 2.6178271770477295,
      "learning_rate": 1.9060773480662984e-07,
      "logits/chosen": 0.7605900168418884,
      "logits/rejected": 0.8606400489807129,
      "logps/chosen": -351.55023193359375,
      "logps/rejected": -263.1789855957031,
      "loss": 0.2374,
      "loss/chosen-sft": 1.300065040588379,
      "loss/dpo": 0.23744645714759827,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": 0.0018242821097373962,
      "rewards/margins": 0.0014699746388942003,
      "rewards/rejected": 0.0003543071507010609,
      "step": 70
    },
    {
      "epoch": 0.04159733777038269,
      "grad_norm": 1.9824270009994507,
      "learning_rate": 2.0441988950276244e-07,
      "logits/chosen": 0.8183063268661499,
      "logits/rejected": 1.056434988975525,
      "logps/chosen": -304.6941833496094,
      "logps/rejected": -246.3820343017578,
      "loss": 0.2305,
      "loss/chosen-sft": 1.2456873655319214,
      "loss/dpo": 0.23054738342761993,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0013592742616310716,
      "rewards/margins": 0.0002732311259023845,
      "rewards/rejected": 0.0010860430775210261,
      "step": 75
    },
    {
      "epoch": 0.044370493621741544,
      "grad_norm": 2.122025489807129,
      "learning_rate": 2.1823204419889503e-07,
      "logits/chosen": 0.7255276441574097,
      "logits/rejected": 0.9549322128295898,
      "logps/chosen": -319.54302978515625,
      "logps/rejected": -265.33941650390625,
      "loss": 0.2322,
      "loss/chosen-sft": 1.2196282148361206,
      "loss/dpo": 0.23221342265605927,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": 0.002971996320411563,
      "rewards/margins": 0.002218194305896759,
      "rewards/rejected": 0.0007538016652688384,
      "step": 80
    },
    {
      "epoch": 0.04714364947310039,
      "grad_norm": 2.24088454246521,
      "learning_rate": 2.320441988950276e-07,
      "logits/chosen": 0.9074276685714722,
      "logits/rejected": 0.8777278661727905,
      "logps/chosen": -311.9112243652344,
      "logps/rejected": -235.16073608398438,
      "loss": 0.2216,
      "loss/chosen-sft": 1.3307913541793823,
      "loss/dpo": 0.22161617875099182,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": 0.0005854293704032898,
      "rewards/margins": 0.00022579837241210043,
      "rewards/rejected": 0.0003596309688873589,
      "step": 85
    },
    {
      "epoch": 0.04991680532445923,
      "grad_norm": 2.139037609100342,
      "learning_rate": 2.4585635359116024e-07,
      "logits/chosen": 0.5329002141952515,
      "logits/rejected": 0.8354529142379761,
      "logps/chosen": -297.3415832519531,
      "logps/rejected": -252.3900604248047,
      "loss": 0.2337,
      "loss/chosen-sft": 1.1595966815948486,
      "loss/dpo": 0.23370489478111267,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": 0.002842614660039544,
      "rewards/margins": 0.003834578674286604,
      "rewards/rejected": -0.0009919643634930253,
      "step": 90
    },
    {
      "epoch": 0.05268996117581808,
      "grad_norm": 2.091688632965088,
      "learning_rate": 2.596685082872928e-07,
      "logits/chosen": 0.743569016456604,
      "logits/rejected": 0.9253543615341187,
      "logps/chosen": -319.1153564453125,
      "logps/rejected": -271.44854736328125,
      "loss": 0.2319,
      "loss/chosen-sft": 1.252418041229248,
      "loss/dpo": 0.2319067418575287,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.0031713570933789015,
      "rewards/margins": 0.002358622383326292,
      "rewards/rejected": 0.0008127348264679313,
      "step": 95
    },
    {
      "epoch": 0.05546311702717693,
      "grad_norm": 2.084585189819336,
      "learning_rate": 2.734806629834254e-07,
      "logits/chosen": 0.7346712350845337,
      "logits/rejected": 0.8320924639701843,
      "logps/chosen": -297.8607482910156,
      "logps/rejected": -247.6679229736328,
      "loss": 0.2305,
      "loss/chosen-sft": 1.1974223852157593,
      "loss/dpo": 0.23046079277992249,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": 0.0038714460097253323,
      "rewards/margins": 0.004991450812667608,
      "rewards/rejected": -0.001120004802942276,
      "step": 100
    },
    {
      "epoch": 0.05823627287853577,
      "grad_norm": 2.2226288318634033,
      "learning_rate": 2.87292817679558e-07,
      "logits/chosen": 0.7988497614860535,
      "logits/rejected": 1.012565016746521,
      "logps/chosen": -322.70849609375,
      "logps/rejected": -248.58609008789062,
      "loss": 0.2288,
      "loss/chosen-sft": 1.2408934831619263,
      "loss/dpo": 0.228750079870224,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": 0.006251711398363113,
      "rewards/margins": 0.007590452674776316,
      "rewards/rejected": -0.001338740810751915,
      "step": 105
    },
    {
      "epoch": 0.06100942872989462,
      "grad_norm": 2.297593593597412,
      "learning_rate": 3.011049723756906e-07,
      "logits/chosen": 0.726061224937439,
      "logits/rejected": 1.0324876308441162,
      "logps/chosen": -281.13897705078125,
      "logps/rejected": -251.3507537841797,
      "loss": 0.2497,
      "loss/chosen-sft": 1.165907859802246,
      "loss/dpo": 0.24974966049194336,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": 0.0033826460130512714,
      "rewards/margins": 0.0045662978664040565,
      "rewards/rejected": -0.0011836517369374633,
      "step": 110
    },
    {
      "epoch": 0.06378258458125347,
      "grad_norm": 1.887861728668213,
      "learning_rate": 3.149171270718232e-07,
      "logits/chosen": 0.7413018345832825,
      "logits/rejected": 1.1185897588729858,
      "logps/chosen": -276.33709716796875,
      "logps/rejected": -222.5193328857422,
      "loss": 0.2343,
      "loss/chosen-sft": 1.2043964862823486,
      "loss/dpo": 0.23427948355674744,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": 0.006615237798541784,
      "rewards/margins": 0.009055422618985176,
      "rewards/rejected": -0.002440184820443392,
      "step": 115
    },
    {
      "epoch": 0.06655574043261231,
      "grad_norm": 1.9467929601669312,
      "learning_rate": 3.287292817679558e-07,
      "logits/chosen": 0.779278039932251,
      "logits/rejected": 1.0071234703063965,
      "logps/chosen": -309.67266845703125,
      "logps/rejected": -277.6393127441406,
      "loss": 0.2353,
      "loss/chosen-sft": 1.250942587852478,
      "loss/dpo": 0.23532333970069885,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.005773581098765135,
      "rewards/margins": 0.008276325650513172,
      "rewards/rejected": -0.0025027443189173937,
      "step": 120
    },
    {
      "epoch": 0.06932889628397115,
      "grad_norm": 1.7528764009475708,
      "learning_rate": 3.425414364640884e-07,
      "logits/chosen": 0.7105622291564941,
      "logits/rejected": 0.9455775022506714,
      "logps/chosen": -247.330810546875,
      "logps/rejected": -234.4990692138672,
      "loss": 0.2187,
      "loss/chosen-sft": 1.204803466796875,
      "loss/dpo": 0.21866345405578613,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": 0.005182682536542416,
      "rewards/margins": 0.00711778411641717,
      "rewards/rejected": -0.0019351018127053976,
      "step": 125
    },
    {
      "epoch": 0.07210205213533001,
      "grad_norm": 2.430612802505493,
      "learning_rate": 3.56353591160221e-07,
      "logits/chosen": 0.8638172149658203,
      "logits/rejected": 1.0668808221817017,
      "logps/chosen": -282.2446594238281,
      "logps/rejected": -227.82461547851562,
      "loss": 0.2437,
      "loss/chosen-sft": 1.202839970588684,
      "loss/dpo": 0.243655726313591,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.0073145912028849125,
      "rewards/margins": 0.013259527273476124,
      "rewards/rejected": -0.005944933742284775,
      "step": 130
    },
    {
      "epoch": 0.07487520798668885,
      "grad_norm": 2.054070472717285,
      "learning_rate": 3.7016574585635355e-07,
      "logits/chosen": 0.6850544214248657,
      "logits/rejected": 0.9055711627006531,
      "logps/chosen": -329.90472412109375,
      "logps/rejected": -257.50933837890625,
      "loss": 0.232,
      "loss/chosen-sft": 1.2098791599273682,
      "loss/dpo": 0.2320254147052765,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.008226691745221615,
      "rewards/margins": 0.014872360043227673,
      "rewards/rejected": -0.006645667366683483,
      "step": 135
    },
    {
      "epoch": 0.0776483638380477,
      "grad_norm": 2.211216688156128,
      "learning_rate": 3.8397790055248617e-07,
      "logits/chosen": 0.7939187288284302,
      "logits/rejected": 0.9478033185005188,
      "logps/chosen": -318.32879638671875,
      "logps/rejected": -252.41796875,
      "loss": 0.2326,
      "loss/chosen-sft": 1.2694523334503174,
      "loss/dpo": 0.2325749397277832,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": 0.010530084371566772,
      "rewards/margins": 0.015220420435070992,
      "rewards/rejected": -0.004690336063504219,
      "step": 140
    },
    {
      "epoch": 0.08042151968940654,
      "grad_norm": 2.2831854820251465,
      "learning_rate": 3.9779005524861873e-07,
      "logits/chosen": 0.7261658906936646,
      "logits/rejected": 0.7962777614593506,
      "logps/chosen": -322.5554504394531,
      "logps/rejected": -283.42431640625,
      "loss": 0.2576,
      "loss/chosen-sft": 1.2340948581695557,
      "loss/dpo": 0.25764793157577515,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": 0.0055245086550712585,
      "rewards/margins": 0.013468381948769093,
      "rewards/rejected": -0.00794387236237526,
      "step": 145
    },
    {
      "epoch": 0.08319467554076539,
      "grad_norm": 2.4128849506378174,
      "learning_rate": 4.1160220994475135e-07,
      "logits/chosen": 0.715286374092102,
      "logits/rejected": 0.9870149493217468,
      "logps/chosen": -293.37115478515625,
      "logps/rejected": -232.06796264648438,
      "loss": 0.2465,
      "loss/chosen-sft": 1.1868611574172974,
      "loss/dpo": 0.2465299814939499,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.008405817672610283,
      "rewards/margins": 0.021539855748414993,
      "rewards/rejected": -0.013134037144482136,
      "step": 150
    },
    {
      "epoch": 0.08596783139212424,
      "grad_norm": 2.3715765476226807,
      "learning_rate": 4.2541436464088397e-07,
      "logits/chosen": 0.7736817598342896,
      "logits/rejected": 0.842314600944519,
      "logps/chosen": -321.31622314453125,
      "logps/rejected": -244.69161987304688,
      "loss": 0.2485,
      "loss/chosen-sft": 1.2318105697631836,
      "loss/dpo": 0.24850551784038544,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.011977704241871834,
      "rewards/margins": 0.02396266534924507,
      "rewards/rejected": -0.011984961107373238,
      "step": 155
    },
    {
      "epoch": 0.08874098724348309,
      "grad_norm": 2.3416831493377686,
      "learning_rate": 4.3922651933701654e-07,
      "logits/chosen": 0.7620590925216675,
      "logits/rejected": 1.0964124202728271,
      "logps/chosen": -287.9859313964844,
      "logps/rejected": -227.1605682373047,
      "loss": 0.2379,
      "loss/chosen-sft": 1.2391362190246582,
      "loss/dpo": 0.23787717521190643,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.008434845134615898,
      "rewards/margins": 0.027770137414336205,
      "rewards/rejected": -0.019335290417075157,
      "step": 160
    },
    {
      "epoch": 0.09151414309484193,
      "grad_norm": 2.5888402462005615,
      "learning_rate": 4.5303867403314916e-07,
      "logits/chosen": 0.7178218960762024,
      "logits/rejected": 1.0630967617034912,
      "logps/chosen": -300.853759765625,
      "logps/rejected": -226.2397918701172,
      "loss": 0.2489,
      "loss/chosen-sft": 1.2291874885559082,
      "loss/dpo": 0.2488756626844406,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": 0.0051612951792776585,
      "rewards/margins": 0.02443910576403141,
      "rewards/rejected": -0.01927780732512474,
      "step": 165
    },
    {
      "epoch": 0.09428729894620078,
      "grad_norm": 2.380986213684082,
      "learning_rate": 4.668508287292817e-07,
      "logits/chosen": 0.769342303276062,
      "logits/rejected": 1.0527831315994263,
      "logps/chosen": -301.633056640625,
      "logps/rejected": -271.94549560546875,
      "loss": 0.2273,
      "loss/chosen-sft": 1.2724863290786743,
      "loss/dpo": 0.22729595005512238,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.005723125766962767,
      "rewards/margins": 0.024905353784561157,
      "rewards/rejected": -0.019182229414582253,
      "step": 170
    },
    {
      "epoch": 0.09706045479755962,
      "grad_norm": 2.346064329147339,
      "learning_rate": 4.806629834254143e-07,
      "logits/chosen": 0.7390652894973755,
      "logits/rejected": 1.0698118209838867,
      "logps/chosen": -336.51434326171875,
      "logps/rejected": -264.5447692871094,
      "loss": 0.2329,
      "loss/chosen-sft": 1.2428892850875854,
      "loss/dpo": 0.2328546941280365,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": 0.013324853964149952,
      "rewards/margins": 0.037267185747623444,
      "rewards/rejected": -0.023942332714796066,
      "step": 175
    },
    {
      "epoch": 0.09983361064891846,
      "grad_norm": 2.174849271774292,
      "learning_rate": 4.944751381215469e-07,
      "logits/chosen": 0.6287063360214233,
      "logits/rejected": 0.839179515838623,
      "logps/chosen": -281.9019470214844,
      "logps/rejected": -246.0726318359375,
      "loss": 0.2247,
      "loss/chosen-sft": 1.2559473514556885,
      "loss/dpo": 0.22472822666168213,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.0028144109528511763,
      "rewards/margins": 0.03012235462665558,
      "rewards/rejected": -0.03293676674365997,
      "step": 180
    },
    {
      "epoch": 0.10260676650027732,
      "grad_norm": 2.126122236251831,
      "learning_rate": 4.999957796414774e-07,
      "logits/chosen": 0.7778046727180481,
      "logits/rejected": 0.8593734502792358,
      "logps/chosen": -305.8876037597656,
      "logps/rejected": -231.57009887695312,
      "loss": 0.2087,
      "loss/chosen-sft": 1.2814704179763794,
      "loss/dpo": 0.20866894721984863,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.0015260865911841393,
      "rewards/margins": 0.032867975533008575,
      "rewards/rejected": -0.03439406305551529,
      "step": 185
    },
    {
      "epoch": 0.10537992235163617,
      "grad_norm": 2.394864082336426,
      "learning_rate": 4.999699890776339e-07,
      "logits/chosen": 0.7574223279953003,
      "logits/rejected": 0.981839656829834,
      "logps/chosen": -307.5397033691406,
      "logps/rejected": -260.81689453125,
      "loss": 0.2244,
      "loss/chosen-sft": 1.2601521015167236,
      "loss/dpo": 0.22438263893127441,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.001994546502828598,
      "rewards/margins": 0.03860156610608101,
      "rewards/rejected": -0.040596116334199905,
      "step": 190
    },
    {
      "epoch": 0.10815307820299501,
      "grad_norm": 2.5312979221343994,
      "learning_rate": 4.999207550094137e-07,
      "logits/chosen": 0.5832860469818115,
      "logits/rejected": 0.9633687138557434,
      "logps/chosen": -320.2137145996094,
      "logps/rejected": -255.04135131835938,
      "loss": 0.2567,
      "loss/chosen-sft": 1.1643339395523071,
      "loss/dpo": 0.25674429535865784,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.009322636760771275,
      "rewards/margins": 0.04288625717163086,
      "rewards/rejected": -0.05220889300107956,
      "step": 195
    },
    {
      "epoch": 0.11092623405435385,
      "grad_norm": 2.321089506149292,
      "learning_rate": 4.998480820542476e-07,
      "logits/chosen": 0.5820528268814087,
      "logits/rejected": 0.7807351350784302,
      "logps/chosen": -270.94195556640625,
      "logps/rejected": -220.6343231201172,
      "loss": 0.2267,
      "loss/chosen-sft": 1.1440722942352295,
      "loss/dpo": 0.22673571109771729,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.0010490523418411613,
      "rewards/margins": 0.05040453001856804,
      "rewards/rejected": -0.05145358294248581,
      "step": 200
    },
    {
      "epoch": 0.1136993899057127,
      "grad_norm": 2.525728940963745,
      "learning_rate": 4.997519770277884e-07,
      "logits/chosen": 0.44963234663009644,
      "logits/rejected": 0.9135919809341431,
      "logps/chosen": -272.36224365234375,
      "logps/rejected": -245.6114044189453,
      "loss": 0.2211,
      "loss/chosen-sft": 1.1812119483947754,
      "loss/dpo": 0.22114984691143036,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.022441323846578598,
      "rewards/margins": 0.04137767106294632,
      "rewards/rejected": -0.06381900608539581,
      "step": 205
    },
    {
      "epoch": 0.11647254575707154,
      "grad_norm": 2.5504720211029053,
      "learning_rate": 4.99632448943273e-07,
      "logits/chosen": 0.7174805998802185,
      "logits/rejected": 1.01616632938385,
      "logps/chosen": -292.91656494140625,
      "logps/rejected": -247.65097045898438,
      "loss": 0.2198,
      "loss/chosen-sft": 1.2257840633392334,
      "loss/dpo": 0.2198173552751541,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.025929760187864304,
      "rewards/margins": 0.05272764712572098,
      "rewards/rejected": -0.07865741103887558,
      "step": 210
    },
    {
      "epoch": 0.1192457016084304,
      "grad_norm": 2.452855110168457,
      "learning_rate": 4.994895090106754e-07,
      "logits/chosen": 0.6810121536254883,
      "logits/rejected": 1.0060815811157227,
      "logps/chosen": -282.73846435546875,
      "logps/rejected": -267.6331481933594,
      "loss": 0.222,
      "loss/chosen-sft": 1.192631483078003,
      "loss/dpo": 0.2220323085784912,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.02360682748258114,
      "rewards/margins": 0.048061199486255646,
      "rewards/rejected": -0.07166802138090134,
      "step": 215
    },
    {
      "epoch": 0.12201885745978924,
      "grad_norm": 2.1740057468414307,
      "learning_rate": 4.993231706356567e-07,
      "logits/chosen": 0.6147283911705017,
      "logits/rejected": 0.8315596580505371,
      "logps/chosen": -336.437255859375,
      "logps/rejected": -253.25320434570312,
      "loss": 0.2107,
      "loss/chosen-sft": 1.323418378829956,
      "loss/dpo": 0.21066264808177948,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.03315647691488266,
      "rewards/margins": 0.057157598435878754,
      "rewards/rejected": -0.09031407535076141,
      "step": 220
    },
    {
      "epoch": 0.12479201331114809,
      "grad_norm": 2.743816375732422,
      "learning_rate": 4.991334494183074e-07,
      "logits/chosen": 0.6287509202957153,
      "logits/rejected": 0.9196675419807434,
      "logps/chosen": -317.734375,
      "logps/rejected": -269.40240478515625,
      "loss": 0.2176,
      "loss/chosen-sft": 1.2351751327514648,
      "loss/dpo": 0.21760547161102295,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.026932349428534508,
      "rewards/margins": 0.06921146810054779,
      "rewards/rejected": -0.09614382684230804,
      "step": 225
    },
    {
      "epoch": 0.12756516916250693,
      "grad_norm": 2.0664100646972656,
      "learning_rate": 4.989203631516842e-07,
      "logits/chosen": 0.5628304481506348,
      "logits/rejected": 0.9230527877807617,
      "logps/chosen": -290.1402282714844,
      "logps/rejected": -230.798828125,
      "loss": 0.2193,
      "loss/chosen-sft": 1.1874725818634033,
      "loss/dpo": 0.21933284401893616,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.04623774066567421,
      "rewards/margins": 0.04349591210484505,
      "rewards/rejected": -0.08973364531993866,
      "step": 230
    },
    {
      "epoch": 0.13033832501386577,
      "grad_norm": 2.657771587371826,
      "learning_rate": 4.986839318201412e-07,
      "logits/chosen": 0.5794273018836975,
      "logits/rejected": 0.8498503565788269,
      "logps/chosen": -283.7383117675781,
      "logps/rejected": -270.8831787109375,
      "loss": 0.2113,
      "loss/chosen-sft": 1.2316051721572876,
      "loss/dpo": 0.2112795114517212,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.04904647916555405,
      "rewards/margins": 0.06235337257385254,
      "rewards/rejected": -0.11139985173940659,
      "step": 235
    },
    {
      "epoch": 0.13311148086522462,
      "grad_norm": 2.395413875579834,
      "learning_rate": 4.984241775974562e-07,
      "logits/chosen": 0.6039844751358032,
      "logits/rejected": 0.9748314619064331,
      "logps/chosen": -343.57330322265625,
      "logps/rejected": -282.4112548828125,
      "loss": 0.2046,
      "loss/chosen-sft": 1.2928663492202759,
      "loss/dpo": 0.20455893874168396,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.05615786463022232,
      "rewards/margins": 0.057152897119522095,
      "rewards/rejected": -0.11331076920032501,
      "step": 240
    },
    {
      "epoch": 0.13588463671658346,
      "grad_norm": 2.410879373550415,
      "learning_rate": 4.981411248447506e-07,
      "logits/chosen": 0.5458131432533264,
      "logits/rejected": 0.7987755537033081,
      "logps/chosen": -320.3564453125,
      "logps/rejected": -274.52349853515625,
      "loss": 0.2127,
      "loss/chosen-sft": 1.2482545375823975,
      "loss/dpo": 0.21267366409301758,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.05543135479092598,
      "rewards/margins": 0.07835234701633453,
      "rewards/rejected": -0.13378369808197021,
      "step": 245
    },
    {
      "epoch": 0.1386577925679423,
      "grad_norm": 2.126710891723633,
      "learning_rate": 4.978348001082048e-07,
      "logits/chosen": 0.5291799306869507,
      "logits/rejected": 0.7740551829338074,
      "logps/chosen": -305.97369384765625,
      "logps/rejected": -258.32623291015625,
      "loss": 0.1986,
      "loss/chosen-sft": 1.317022442817688,
      "loss/dpo": 0.19859442114830017,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.056786395609378815,
      "rewards/margins": 0.06887609511613846,
      "rewards/rejected": -0.12566249072551727,
      "step": 250
    },
    {
      "epoch": 0.14143094841930118,
      "grad_norm": 2.286869525909424,
      "learning_rate": 4.975052321165688e-07,
      "logits/chosen": 0.4996607303619385,
      "logits/rejected": 0.8591309785842896,
      "logps/chosen": -315.6490783691406,
      "logps/rejected": -246.1661834716797,
      "loss": 0.2036,
      "loss/chosen-sft": 1.2581641674041748,
      "loss/dpo": 0.2035764753818512,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.0983358696103096,
      "rewards/margins": 0.06135006994009018,
      "rewards/rejected": -0.15968593955039978,
      "step": 255
    },
    {
      "epoch": 0.14420410427066002,
      "grad_norm": 2.3020002841949463,
      "learning_rate": 4.971524517784676e-07,
      "logits/chosen": 0.6534574627876282,
      "logits/rejected": 0.9783307313919067,
      "logps/chosen": -315.9989318847656,
      "logps/rejected": -300.7061767578125,
      "loss": 0.1887,
      "loss/chosen-sft": 1.3003921508789062,
      "loss/dpo": 0.18872812390327454,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.0870845764875412,
      "rewards/margins": 0.09078534692525864,
      "rewards/rejected": -0.17786994576454163,
      "step": 260
    },
    {
      "epoch": 0.14697726012201887,
      "grad_norm": 2.2956702709198,
      "learning_rate": 4.967764921795026e-07,
      "logits/chosen": 0.4762963354587555,
      "logits/rejected": 0.7974573373794556,
      "logps/chosen": -320.36944580078125,
      "logps/rejected": -253.922119140625,
      "loss": 0.1965,
      "loss/chosen-sft": 1.2646160125732422,
      "loss/dpo": 0.19646450877189636,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.10069359838962555,
      "rewards/margins": 0.07882802188396454,
      "rewards/rejected": -0.1795216202735901,
      "step": 265
    },
    {
      "epoch": 0.1497504159733777,
      "grad_norm": 2.5840351581573486,
      "learning_rate": 4.963773885791484e-07,
      "logits/chosen": 0.46905121207237244,
      "logits/rejected": 0.6983851194381714,
      "logps/chosen": -319.40997314453125,
      "logps/rejected": -264.07574462890625,
      "loss": 0.2039,
      "loss/chosen-sft": 1.2846953868865967,
      "loss/dpo": 0.2038877308368683,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.14658531546592712,
      "rewards/margins": 0.03527539223432541,
      "rewards/rejected": -0.18186071515083313,
      "step": 270
    },
    {
      "epoch": 0.15252357182473655,
      "grad_norm": 2.3462185859680176,
      "learning_rate": 4.959551784074461e-07,
      "logits/chosen": 0.5223512053489685,
      "logits/rejected": 0.673526406288147,
      "logps/chosen": -353.185546875,
      "logps/rejected": -287.7161865234375,
      "loss": 0.1814,
      "loss/chosen-sft": 1.3422802686691284,
      "loss/dpo": 0.18135803937911987,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.11449305713176727,
      "rewards/margins": 0.09264171123504639,
      "rewards/rejected": -0.20713476836681366,
      "step": 275
    },
    {
      "epoch": 0.1552967276760954,
      "grad_norm": 2.2738516330718994,
      "learning_rate": 4.955099012614933e-07,
      "logits/chosen": 0.589896023273468,
      "logits/rejected": 0.8485240936279297,
      "logps/chosen": -340.37811279296875,
      "logps/rejected": -278.1029968261719,
      "loss": 0.1715,
      "loss/chosen-sft": 1.3321316242218018,
      "loss/dpo": 0.17148999869823456,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.12517675757408142,
      "rewards/margins": 0.09786481410264969,
      "rewards/rejected": -0.2230415791273117,
      "step": 280
    },
    {
      "epoch": 0.15806988352745424,
      "grad_norm": 2.3262434005737305,
      "learning_rate": 4.9504159890173e-07,
      "logits/chosen": 0.5216140747070312,
      "logits/rejected": 0.760550320148468,
      "logps/chosen": -334.56842041015625,
      "logps/rejected": -273.66644287109375,
      "loss": 0.1817,
      "loss/chosen-sft": 1.3172911405563354,
      "loss/dpo": 0.18172022700309753,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.15752620995044708,
      "rewards/margins": 0.09383035451173782,
      "rewards/rejected": -0.2513565421104431,
      "step": 285
    },
    {
      "epoch": 0.16084303937881309,
      "grad_norm": 2.1715376377105713,
      "learning_rate": 4.945503152480221e-07,
      "logits/chosen": 0.4839824140071869,
      "logits/rejected": 0.7215232849121094,
      "logps/chosen": -348.7749938964844,
      "logps/rejected": -296.0406188964844,
      "loss": 0.1779,
      "loss/chosen-sft": 1.3408844470977783,
      "loss/dpo": 0.17787811160087585,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.13581573963165283,
      "rewards/margins": 0.10350732505321503,
      "rewards/rejected": -0.23932304978370667,
      "step": 290
    },
    {
      "epoch": 0.16361619523017193,
      "grad_norm": 1.9679666757583618,
      "learning_rate": 4.940360963755426e-07,
      "logits/chosen": 0.31268057227134705,
      "logits/rejected": 0.45721864700317383,
      "logps/chosen": -306.3217468261719,
      "logps/rejected": -259.17010498046875,
      "loss": 0.168,
      "loss/chosen-sft": 1.3279306888580322,
      "loss/dpo": 0.16799867153167725,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.14948979020118713,
      "rewards/margins": 0.11183127015829086,
      "rewards/rejected": -0.2613210082054138,
      "step": 295
    },
    {
      "epoch": 0.16638935108153077,
      "grad_norm": 2.079015016555786,
      "learning_rate": 4.934989905104502e-07,
      "logits/chosen": 0.38664117455482483,
      "logits/rejected": 0.5652385354042053,
      "logps/chosen": -327.77191162109375,
      "logps/rejected": -252.3622589111328,
      "loss": 0.1615,
      "loss/chosen-sft": 1.3207409381866455,
      "loss/dpo": 0.1614522486925125,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.15085890889167786,
      "rewards/margins": 0.1116364598274231,
      "rewards/rejected": -0.26249536871910095,
      "step": 300
    },
    {
      "epoch": 0.16916250693288962,
      "grad_norm": 1.7469545602798462,
      "learning_rate": 4.929390480253667e-07,
      "logits/chosen": 0.4214795231819153,
      "logits/rejected": 0.5547307729721069,
      "logps/chosen": -345.3471984863281,
      "logps/rejected": -296.81610107421875,
      "loss": 0.1597,
      "loss/chosen-sft": 1.3892717361450195,
      "loss/dpo": 0.15971598029136658,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.1656077653169632,
      "rewards/margins": 0.13407020270824432,
      "rewards/rejected": -0.2996779680252075,
      "step": 305
    },
    {
      "epoch": 0.1719356627842485,
      "grad_norm": 2.5472302436828613,
      "learning_rate": 4.923563214346525e-07,
      "logits/chosen": 0.2179243117570877,
      "logits/rejected": 0.5859929323196411,
      "logps/chosen": -359.1659240722656,
      "logps/rejected": -323.3959045410156,
      "loss": 0.1693,
      "loss/chosen-sft": 1.3262033462524414,
      "loss/dpo": 0.16931983828544617,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1735382229089737,
      "rewards/margins": 0.17630478739738464,
      "rewards/rejected": -0.34984302520751953,
      "step": 310
    },
    {
      "epoch": 0.17470881863560733,
      "grad_norm": 1.927839756011963,
      "learning_rate": 4.917508653894817e-07,
      "logits/chosen": 0.2809959352016449,
      "logits/rejected": 0.6429028511047363,
      "logps/chosen": -306.2704162597656,
      "logps/rejected": -285.570556640625,
      "loss": 0.1554,
      "loss/chosen-sft": 1.3090779781341553,
      "loss/dpo": 0.15539118647575378,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.19202136993408203,
      "rewards/margins": 0.13031750917434692,
      "rewards/rejected": -0.32233884930610657,
      "step": 315
    },
    {
      "epoch": 0.17748197448696618,
      "grad_norm": 1.4578968286514282,
      "learning_rate": 4.911227366727166e-07,
      "logits/chosen": 0.3806017339229584,
      "logits/rejected": 0.646203339099884,
      "logps/chosen": -343.3798828125,
      "logps/rejected": -279.1783752441406,
      "loss": 0.1326,
      "loss/chosen-sft": 1.4282009601593018,
      "loss/dpo": 0.1326414793729782,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.20244988799095154,
      "rewards/margins": 0.11332936584949493,
      "rewards/rejected": -0.31577926874160767,
      "step": 320
    },
    {
      "epoch": 0.18025513033832502,
      "grad_norm": 2.2041287422180176,
      "learning_rate": 4.904719941935818e-07,
      "logits/chosen": 0.3938923478126526,
      "logits/rejected": 0.5521587133407593,
      "logps/chosen": -330.3848876953125,
      "logps/rejected": -284.7240905761719,
      "loss": 0.1679,
      "loss/chosen-sft": 1.335784673690796,
      "loss/dpo": 0.16794539988040924,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.2126191109418869,
      "rewards/margins": 0.12985429167747498,
      "rewards/rejected": -0.34247341752052307,
      "step": 325
    },
    {
      "epoch": 0.18302828618968386,
      "grad_norm": 2.1968367099761963,
      "learning_rate": 4.897986989821405e-07,
      "logits/chosen": 0.27726611495018005,
      "logits/rejected": 0.42112284898757935,
      "logps/chosen": -304.1949462890625,
      "logps/rejected": -284.40496826171875,
      "loss": 0.1461,
      "loss/chosen-sft": 1.3867931365966797,
      "loss/dpo": 0.1461399793624878,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.24662432074546814,
      "rewards/margins": 0.13204023241996765,
      "rewards/rejected": -0.3786645829677582,
      "step": 330
    },
    {
      "epoch": 0.1858014420410427,
      "grad_norm": 1.8825585842132568,
      "learning_rate": 4.891029141835697e-07,
      "logits/chosen": 0.320846825838089,
      "logits/rejected": 0.5570532083511353,
      "logps/chosen": -327.5889587402344,
      "logps/rejected": -277.48101806640625,
      "loss": 0.1485,
      "loss/chosen-sft": 1.314846396446228,
      "loss/dpo": 0.1484725922346115,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.23770102858543396,
      "rewards/margins": 0.13969966769218445,
      "rewards/rejected": -0.3774007260799408,
      "step": 335
    },
    {
      "epoch": 0.18857459789240155,
      "grad_norm": 2.134202718734741,
      "learning_rate": 4.883847050522388e-07,
      "logits/chosen": 0.32603126764297485,
      "logits/rejected": 0.5855879783630371,
      "logps/chosen": -306.82940673828125,
      "logps/rejected": -306.5520324707031,
      "loss": 0.1386,
      "loss/chosen-sft": 1.3495581150054932,
      "loss/dpo": 0.138578861951828,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.25742700695991516,
      "rewards/margins": 0.12370799481868744,
      "rewards/rejected": -0.3811350464820862,
      "step": 340
    },
    {
      "epoch": 0.1913477537437604,
      "grad_norm": 1.7875416278839111,
      "learning_rate": 4.876441389455892e-07,
      "logits/chosen": 0.24733588099479675,
      "logits/rejected": 0.45660385489463806,
      "logps/chosen": -326.81988525390625,
      "logps/rejected": -290.23175048828125,
      "loss": 0.1307,
      "loss/chosen-sft": 1.372218132019043,
      "loss/dpo": 0.130675807595253,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.24057714641094208,
      "rewards/margins": 0.166746586561203,
      "rewards/rejected": -0.4073237478733063,
      "step": 345
    },
    {
      "epoch": 0.19412090959511924,
      "grad_norm": 1.9340115785598755,
      "learning_rate": 4.868812853178174e-07,
      "logits/chosen": 0.2431308925151825,
      "logits/rejected": 0.5894955992698669,
      "logps/chosen": -318.0989990234375,
      "logps/rejected": -283.68896484375,
      "loss": 0.1256,
      "loss/chosen-sft": 1.354346513748169,
      "loss/dpo": 0.12558932602405548,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.2501407861709595,
      "rewards/margins": 0.15478388965129852,
      "rewards/rejected": -0.4049246907234192,
      "step": 350
    },
    {
      "epoch": 0.19689406544647808,
      "grad_norm": 2.164641857147217,
      "learning_rate": 4.860962157133614e-07,
      "logits/chosen": 0.042165856808423996,
      "logits/rejected": 0.2751336395740509,
      "logps/chosen": -312.33636474609375,
      "logps/rejected": -293.5239562988281,
      "loss": 0.1378,
      "loss/chosen-sft": 1.307612657546997,
      "loss/dpo": 0.1378055065870285,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.2775038182735443,
      "rewards/margins": 0.1561960130929947,
      "rewards/rejected": -0.4336997866630554,
      "step": 355
    },
    {
      "epoch": 0.19966722129783693,
      "grad_norm": 1.9318678379058838,
      "learning_rate": 4.852890037601906e-07,
      "logits/chosen": 0.16851834952831268,
      "logits/rejected": 0.38687795400619507,
      "logps/chosen": -339.43377685546875,
      "logps/rejected": -299.70611572265625,
      "loss": 0.1384,
      "loss/chosen-sft": 1.4045716524124146,
      "loss/dpo": 0.1384141445159912,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.2960866689682007,
      "rewards/margins": 0.16027973592281342,
      "rewards/rejected": -0.4563663899898529,
      "step": 360
    },
    {
      "epoch": 0.20244037714919577,
      "grad_norm": 1.874963402748108,
      "learning_rate": 4.844597251629008e-07,
      "logits/chosen": 0.21792784333229065,
      "logits/rejected": 0.3441501259803772,
      "logps/chosen": -325.2286682128906,
      "logps/rejected": -281.23638916015625,
      "loss": 0.1352,
      "loss/chosen-sft": 1.3615696430206299,
      "loss/dpo": 0.13523094356060028,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.3180539608001709,
      "rewards/margins": 0.1318327933549881,
      "rewards/rejected": -0.4498867392539978,
      "step": 365
    },
    {
      "epoch": 0.20521353300055464,
      "grad_norm": 2.0710604190826416,
      "learning_rate": 4.836084576956137e-07,
      "logits/chosen": 0.3437557816505432,
      "logits/rejected": 0.5105153322219849,
      "logps/chosen": -339.3221435546875,
      "logps/rejected": -306.34954833984375,
      "loss": 0.1202,
      "loss/chosen-sft": 1.4008045196533203,
      "loss/dpo": 0.12022168934345245,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.295442670583725,
      "rewards/margins": 0.2085741013288498,
      "rewards/rejected": -0.5040167570114136,
      "step": 370
    },
    {
      "epoch": 0.2079866888519135,
      "grad_norm": 2.0409181118011475,
      "learning_rate": 4.827352811946839e-07,
      "logits/chosen": 0.10601860284805298,
      "logits/rejected": 0.19340696930885315,
      "logps/chosen": -363.87677001953125,
      "logps/rejected": -313.70306396484375,
      "loss": 0.1444,
      "loss/chosen-sft": 1.4021615982055664,
      "loss/dpo": 0.14439384639263153,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.3728375732898712,
      "rewards/margins": 0.1328699290752411,
      "rewards/rejected": -0.5057075023651123,
      "step": 375
    },
    {
      "epoch": 0.21075984470327233,
      "grad_norm": 1.8490561246871948,
      "learning_rate": 4.818402775512101e-07,
      "logits/chosen": 0.12518136203289032,
      "logits/rejected": 0.1990843415260315,
      "logps/chosen": -330.71282958984375,
      "logps/rejected": -289.8016052246094,
      "loss": 0.1272,
      "loss/chosen-sft": 1.3432165384292603,
      "loss/dpo": 0.1271766871213913,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.35159942507743835,
      "rewards/margins": 0.19611182808876038,
      "rewards/rejected": -0.5477112531661987,
      "step": 380
    },
    {
      "epoch": 0.21353300055463117,
      "grad_norm": 2.0964889526367188,
      "learning_rate": 4.80923530703356e-07,
      "logits/chosen": 0.10008511692285538,
      "logits/rejected": 0.3669296205043793,
      "logps/chosen": -345.01715087890625,
      "logps/rejected": -335.6979064941406,
      "loss": 0.1178,
      "loss/chosen-sft": 1.375633955001831,
      "loss/dpo": 0.1178009957075119,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.33166831731796265,
      "rewards/margins": 0.257099449634552,
      "rewards/rejected": -0.5887677073478699,
      "step": 385
    },
    {
      "epoch": 0.21630615640599002,
      "grad_norm": 1.3537511825561523,
      "learning_rate": 4.799851266284776e-07,
      "logits/chosen": 0.029593368992209435,
      "logits/rejected": 0.2654074728488922,
      "logps/chosen": -345.6829528808594,
      "logps/rejected": -295.1934814453125,
      "loss": 0.1097,
      "loss/chosen-sft": 1.4121724367141724,
      "loss/dpo": 0.10970698297023773,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.36835533380508423,
      "rewards/margins": 0.22947156429290771,
      "rewards/rejected": -0.5978268980979919,
      "step": 390
    },
    {
      "epoch": 0.21907931225734886,
      "grad_norm": 2.3625447750091553,
      "learning_rate": 4.790251533350597e-07,
      "logits/chosen": 0.10083159059286118,
      "logits/rejected": 0.31363213062286377,
      "logps/chosen": -332.98846435546875,
      "logps/rejected": -296.0115051269531,
      "loss": 0.1227,
      "loss/chosen-sft": 1.3798365592956543,
      "loss/dpo": 0.12265870720148087,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.4115685820579529,
      "rewards/margins": 0.17796972393989563,
      "rewards/rejected": -0.5895382761955261,
      "step": 395
    },
    {
      "epoch": 0.2218524681087077,
      "grad_norm": 1.8092533349990845,
      "learning_rate": 4.780437008544628e-07,
      "logits/chosen": 0.0437210276722908,
      "logits/rejected": 0.28447720408439636,
      "logps/chosen": -328.55364990234375,
      "logps/rejected": -299.6891174316406,
      "loss": 0.1123,
      "loss/chosen-sft": 1.4356980323791504,
      "loss/dpo": 0.11227130889892578,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3866044580936432,
      "rewards/margins": 0.21935506165027618,
      "rewards/rejected": -0.6059595346450806,
      "step": 400
    },
    {
      "epoch": 0.22462562396006655,
      "grad_norm": 1.755149245262146,
      "learning_rate": 4.770408612324783e-07,
      "logits/chosen": 0.06074788048863411,
      "logits/rejected": 0.29286664724349976,
      "logps/chosen": -342.78314208984375,
      "logps/rejected": -324.9090270996094,
      "loss": 0.1229,
      "loss/chosen-sft": 1.3950598239898682,
      "loss/dpo": 0.12285208702087402,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.40826615691185,
      "rewards/margins": 0.23281535506248474,
      "rewards/rejected": -0.6410815119743347,
      "step": 405
    },
    {
      "epoch": 0.2273987798114254,
      "grad_norm": 1.7682974338531494,
      "learning_rate": 4.760167285206968e-07,
      "logits/chosen": 0.03603815287351608,
      "logits/rejected": 0.14143748581409454,
      "logps/chosen": -357.19000244140625,
      "logps/rejected": -312.27984619140625,
      "loss": 0.1099,
      "loss/chosen-sft": 1.4361236095428467,
      "loss/dpo": 0.10990427434444427,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.4452105462551117,
      "rewards/margins": 0.2284567803144455,
      "rewards/rejected": -0.673667311668396,
      "step": 410
    },
    {
      "epoch": 0.23017193566278424,
      "grad_norm": 1.7545676231384277,
      "learning_rate": 4.749713987676871e-07,
      "logits/chosen": 0.060189586132764816,
      "logits/rejected": 0.1871364414691925,
      "logps/chosen": -355.28924560546875,
      "logps/rejected": -317.2358093261719,
      "loss": 0.1093,
      "loss/chosen-sft": 1.3852781057357788,
      "loss/dpo": 0.10928022861480713,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.43528875708580017,
      "rewards/margins": 0.2679973840713501,
      "rewards/rejected": -0.7032861709594727,
      "step": 415
    },
    {
      "epoch": 0.23294509151414308,
      "grad_norm": 1.7819030284881592,
      "learning_rate": 4.7390497000998853e-07,
      "logits/chosen": -0.06483317911624908,
      "logits/rejected": 0.0744047462940216,
      "logps/chosen": -327.4217834472656,
      "logps/rejected": -300.5835876464844,
      "loss": 0.1031,
      "loss/chosen-sft": 1.4440373182296753,
      "loss/dpo": 0.10313411056995392,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.420682817697525,
      "rewards/margins": 0.2728646397590637,
      "rewards/rejected": -0.6935475468635559,
      "step": 420
    },
    {
      "epoch": 0.23571824736550195,
      "grad_norm": 1.5001357793807983,
      "learning_rate": 4.7281754226291627e-07,
      "logits/chosen": -0.01330060325562954,
      "logits/rejected": 0.23702244460582733,
      "logps/chosen": -346.0107116699219,
      "logps/rejected": -335.9864501953125,
      "loss": 0.0908,
      "loss/chosen-sft": 1.3927044868469238,
      "loss/dpo": 0.09075666218996048,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.4789350628852844,
      "rewards/margins": 0.313282310962677,
      "rewards/rejected": -0.7922172546386719,
      "step": 425
    },
    {
      "epoch": 0.2384914032168608,
      "grad_norm": 1.6260863542556763,
      "learning_rate": 4.717092175111814e-07,
      "logits/chosen": 0.0023514986969530582,
      "logits/rejected": 0.2060910165309906,
      "logps/chosen": -372.6448059082031,
      "logps/rejected": -335.29534912109375,
      "loss": 0.101,
      "loss/chosen-sft": 1.4546012878417969,
      "loss/dpo": 0.10103006660938263,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.5173962712287903,
      "rewards/margins": 0.24449630081653595,
      "rewards/rejected": -0.7618924975395203,
      "step": 430
    },
    {
      "epoch": 0.24126455906821964,
      "grad_norm": 1.8368723392486572,
      "learning_rate": 4.7058009969932666e-07,
      "logits/chosen": -0.010248428210616112,
      "logits/rejected": 0.10792305320501328,
      "logps/chosen": -364.9273376464844,
      "logps/rejected": -310.1815490722656,
      "loss": 0.104,
      "loss/chosen-sft": 1.4473296403884888,
      "loss/dpo": 0.10400193929672241,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.5129069089889526,
      "rewards/margins": 0.27995672821998596,
      "rewards/rejected": -0.792863667011261,
      "step": 435
    },
    {
      "epoch": 0.24403771491957849,
      "grad_norm": 1.4362239837646484,
      "learning_rate": 4.694302947219775e-07,
      "logits/chosen": -0.04999478533864021,
      "logits/rejected": 0.15826420485973358,
      "logps/chosen": -371.3370361328125,
      "logps/rejected": -349.93511962890625,
      "loss": 0.0862,
      "loss/chosen-sft": 1.5128648281097412,
      "loss/dpo": 0.08617158234119415,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.5207723379135132,
      "rewards/margins": 0.3013128638267517,
      "rewards/rejected": -0.8220852017402649,
      "step": 440
    },
    {
      "epoch": 0.24681087077093733,
      "grad_norm": 1.7123464345932007,
      "learning_rate": 4.6825991041391067e-07,
      "logits/chosen": -0.131384015083313,
      "logits/rejected": 0.14633068442344666,
      "logps/chosen": -360.64166259765625,
      "logps/rejected": -330.9405822753906,
      "loss": 0.0844,
      "loss/chosen-sft": 1.433370590209961,
      "loss/dpo": 0.08443091064691544,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.49853143095970154,
      "rewards/margins": 0.3446124196052551,
      "rewards/rejected": -0.8431438207626343,
      "step": 445
    },
    {
      "epoch": 0.24958402662229617,
      "grad_norm": 1.4186034202575684,
      "learning_rate": 4.670690565399415e-07,
      "logits/chosen": -0.17499125003814697,
      "logits/rejected": 0.10050486028194427,
      "logps/chosen": -378.93157958984375,
      "logps/rejected": -318.13885498046875,
      "loss": 0.0908,
      "loss/chosen-sft": 1.4939839839935303,
      "loss/dpo": 0.09077504277229309,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.5793511271476746,
      "rewards/margins": 0.26338380575180054,
      "rewards/rejected": -0.8427349328994751,
      "step": 450
    },
    {
      "epoch": 0.252357182473655,
      "grad_norm": 1.2995809316635132,
      "learning_rate": 4.65857844784629e-07,
      "logits/chosen": -0.1015838161110878,
      "logits/rejected": 0.014798527583479881,
      "logps/chosen": -328.8960266113281,
      "logps/rejected": -330.8209228515625,
      "loss": 0.0818,
      "loss/chosen-sft": 1.502540946006775,
      "loss/dpo": 0.08181241899728775,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.55858314037323,
      "rewards/margins": 0.30958643555641174,
      "rewards/rejected": -0.8681696057319641,
      "step": 455
    },
    {
      "epoch": 0.25513033832501386,
      "grad_norm": 1.888208270072937,
      "learning_rate": 4.6462638874180173e-07,
      "logits/chosen": -0.0846543088555336,
      "logits/rejected": -0.07818257808685303,
      "logps/chosen": -361.853271484375,
      "logps/rejected": -330.44915771484375,
      "loss": 0.0959,
      "loss/chosen-sft": 1.524152398109436,
      "loss/dpo": 0.0959150493144989,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.630092978477478,
      "rewards/margins": 0.2400503158569336,
      "rewards/rejected": -0.8701432943344116,
      "step": 460
    },
    {
      "epoch": 0.2579034941763727,
      "grad_norm": 1.4479496479034424,
      "learning_rate": 4.633748039039044e-07,
      "logits/chosen": -0.17475584149360657,
      "logits/rejected": -0.1427185982465744,
      "logps/chosen": -379.28741455078125,
      "logps/rejected": -354.30877685546875,
      "loss": 0.0791,
      "loss/chosen-sft": 1.561486005783081,
      "loss/dpo": 0.0790865495800972,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.5844415426254272,
      "rewards/margins": 0.31326019763946533,
      "rewards/rejected": -0.8977017402648926,
      "step": 465
    },
    {
      "epoch": 0.26067665002773155,
      "grad_norm": 2.0286715030670166,
      "learning_rate": 4.621032076511662e-07,
      "logits/chosen": -0.17609013617038727,
      "logits/rejected": -0.021662216633558273,
      "logps/chosen": -377.0018005371094,
      "logps/rejected": -337.9693603515625,
      "loss": 0.0829,
      "loss/chosen-sft": 1.5360634326934814,
      "loss/dpo": 0.08291391283273697,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.652755856513977,
      "rewards/margins": 0.29427844285964966,
      "rewards/rejected": -0.9470342397689819,
      "step": 470
    },
    {
      "epoch": 0.2634498058790904,
      "grad_norm": 1.7011218070983887,
      "learning_rate": 4.6081171924059245e-07,
      "logits/chosen": -0.164344921708107,
      "logits/rejected": 0.05261586979031563,
      "logps/chosen": -335.7785339355469,
      "logps/rejected": -326.3609313964844,
      "loss": 0.0754,
      "loss/chosen-sft": 1.5235470533370972,
      "loss/dpo": 0.07544606924057007,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.571322500705719,
      "rewards/margins": 0.26280707120895386,
      "rewards/rejected": -0.8341296315193176,
      "step": 475
    },
    {
      "epoch": 0.26622296173044924,
      "grad_norm": 2.2942631244659424,
      "learning_rate": 4.5950045979478004e-07,
      "logits/chosen": -0.2594403624534607,
      "logits/rejected": -0.0034357428085058928,
      "logps/chosen": -354.83197021484375,
      "logps/rejected": -359.304443359375,
      "loss": 0.0882,
      "loss/chosen-sft": 1.4360209703445435,
      "loss/dpo": 0.08824630081653595,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.6372642517089844,
      "rewards/margins": 0.3590429425239563,
      "rewards/rejected": -0.9963071942329407,
      "step": 480
    },
    {
      "epoch": 0.2689961175818081,
      "grad_norm": 2.1030640602111816,
      "learning_rate": 4.5816955229055776e-07,
      "logits/chosen": -0.3849255442619324,
      "logits/rejected": -0.1723778247833252,
      "logps/chosen": -331.4612731933594,
      "logps/rejected": -328.07037353515625,
      "loss": 0.0885,
      "loss/chosen-sft": 1.4904568195343018,
      "loss/dpo": 0.08851548284292221,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.6340426206588745,
      "rewards/margins": 0.27626457810401917,
      "rewards/rejected": -0.9103072881698608,
      "step": 485
    },
    {
      "epoch": 0.2717692734331669,
      "grad_norm": 1.9212195873260498,
      "learning_rate": 4.56819121547453e-07,
      "logits/chosen": -0.13772639632225037,
      "logits/rejected": -0.04014406353235245,
      "logps/chosen": -368.6828918457031,
      "logps/rejected": -321.24365234375,
      "loss": 0.0897,
      "loss/chosen-sft": 1.4933507442474365,
      "loss/dpo": 0.08970335870981216,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.7113620638847351,
      "rewards/margins": 0.1993238627910614,
      "rewards/rejected": -0.9106858968734741,
      "step": 490
    },
    {
      "epoch": 0.27454242928452577,
      "grad_norm": 1.695456624031067,
      "learning_rate": 4.554492942159855e-07,
      "logits/chosen": -0.42002072930336,
      "logits/rejected": -0.18559008836746216,
      "logps/chosen": -350.0926818847656,
      "logps/rejected": -358.0768127441406,
      "loss": 0.0807,
      "loss/chosen-sft": 1.4628541469573975,
      "loss/dpo": 0.08066307753324509,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.6113547086715698,
      "rewards/margins": 0.36685535311698914,
      "rewards/rejected": -0.9782101511955261,
      "step": 495
    },
    {
      "epoch": 0.2773155851358846,
      "grad_norm": 2.2545158863067627,
      "learning_rate": 4.540601987657893e-07,
      "logits/chosen": -0.27345049381256104,
      "logits/rejected": -0.0689864382147789,
      "logps/chosen": -370.7053527832031,
      "logps/rejected": -339.49029541015625,
      "loss": 0.0875,
      "loss/chosen-sft": 1.5028795003890991,
      "loss/dpo": 0.08748678863048553,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.6775597333908081,
      "rewards/margins": 0.23481640219688416,
      "rewards/rejected": -0.9123761057853699,
      "step": 500
    },
    {
      "epoch": 0.28008874098724346,
      "grad_norm": 1.761813759803772,
      "learning_rate": 4.5265196547356453e-07,
      "logits/chosen": -0.4009808599948883,
      "logits/rejected": -0.2716490626335144,
      "logps/chosen": -375.43145751953125,
      "logps/rejected": -348.8159484863281,
      "loss": 0.0836,
      "loss/chosen-sft": 1.5295841693878174,
      "loss/dpo": 0.08356954157352448,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.6790491342544556,
      "rewards/margins": 0.30093619227409363,
      "rewards/rejected": -0.979985237121582,
      "step": 505
    },
    {
      "epoch": 0.28286189683860236,
      "grad_norm": 1.7265639305114746,
      "learning_rate": 4.5122472641085887e-07,
      "logits/chosen": -0.2951423227787018,
      "logits/rejected": -0.06516732275485992,
      "logps/chosen": -327.62579345703125,
      "logps/rejected": -322.10516357421875,
      "loss": 0.0822,
      "loss/chosen-sft": 1.4988086223602295,
      "loss/dpo": 0.08221860229969025,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.6168748140335083,
      "rewards/margins": 0.27371373772621155,
      "rewards/rejected": -0.8905885815620422,
      "step": 510
    },
    {
      "epoch": 0.2856350526899612,
      "grad_norm": 2.8561835289001465,
      "learning_rate": 4.497786154316815e-07,
      "logits/chosen": -0.3033020496368408,
      "logits/rejected": -0.1509987711906433,
      "logps/chosen": -405.4481506347656,
      "logps/rejected": -363.7564392089844,
      "loss": 0.0839,
      "loss/chosen-sft": 1.5500370264053345,
      "loss/dpo": 0.08386242389678955,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.7174540758132935,
      "rewards/margins": 0.30318009853363037,
      "rewards/rejected": -1.0206342935562134,
      "step": 515
    },
    {
      "epoch": 0.28840820854132004,
      "grad_norm": 1.3990095853805542,
      "learning_rate": 4.483137681599495e-07,
      "logits/chosen": -0.4407324194908142,
      "logits/rejected": -0.3505704998970032,
      "logps/chosen": -393.1653747558594,
      "logps/rejected": -341.4064636230469,
      "loss": 0.0787,
      "loss/chosen-sft": 1.5403501987457275,
      "loss/dpo": 0.07872845977544785,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.6447398066520691,
      "rewards/margins": 0.4145263731479645,
      "rewards/rejected": -1.0592660903930664,
      "step": 520
    },
    {
      "epoch": 0.2911813643926789,
      "grad_norm": 1.8205024003982544,
      "learning_rate": 4.468303219767683e-07,
      "logits/chosen": -0.3818379044532776,
      "logits/rejected": -0.13233591616153717,
      "logps/chosen": -356.06488037109375,
      "logps/rejected": -356.4136047363281,
      "loss": 0.0827,
      "loss/chosen-sft": 1.497135043144226,
      "loss/dpo": 0.08271769434213638,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.6697732210159302,
      "rewards/margins": 0.2621932625770569,
      "rewards/rejected": -0.9319664835929871,
      "step": 525
    },
    {
      "epoch": 0.29395452024403773,
      "grad_norm": 1.3097429275512695,
      "learning_rate": 4.453284160075473e-07,
      "logits/chosen": -0.4805290699005127,
      "logits/rejected": -0.3880520164966583,
      "logps/chosen": -391.4358825683594,
      "logps/rejected": -344.5740661621094,
      "loss": 0.0787,
      "loss/chosen-sft": 1.4888331890106201,
      "loss/dpo": 0.07873310893774033,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.7074838280677795,
      "rewards/margins": 0.3189099431037903,
      "rewards/rejected": -1.0263937711715698,
      "step": 530
    },
    {
      "epoch": 0.2967276760953966,
      "grad_norm": 1.8334152698516846,
      "learning_rate": 4.438081911089522e-07,
      "logits/chosen": -0.1407683938741684,
      "logits/rejected": -0.08939669281244278,
      "logps/chosen": -368.7952880859375,
      "logps/rejected": -324.58660888671875,
      "loss": 0.0787,
      "loss/chosen-sft": 1.536380410194397,
      "loss/dpo": 0.07865273952484131,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7058097124099731,
      "rewards/margins": 0.27695003151893616,
      "rewards/rejected": -0.9827596545219421,
      "step": 535
    },
    {
      "epoch": 0.2995008319467554,
      "grad_norm": 1.0951838493347168,
      "learning_rate": 4.422697898556945e-07,
      "logits/chosen": -0.1452847272157669,
      "logits/rejected": -0.12602165341377258,
      "logps/chosen": -388.03533935546875,
      "logps/rejected": -322.50787353515625,
      "loss": 0.0711,
      "loss/chosen-sft": 1.6297862529754639,
      "loss/dpo": 0.07111659646034241,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.6191781759262085,
      "rewards/margins": 0.31716564297676086,
      "rewards/rejected": -0.936343789100647,
      "step": 540
    },
    {
      "epoch": 0.30227398779811426,
      "grad_norm": 1.171819806098938,
      "learning_rate": 4.4071335652716004e-07,
      "logits/chosen": -0.28522688150405884,
      "logits/rejected": 0.05751846358180046,
      "logps/chosen": -330.20245361328125,
      "logps/rejected": -353.6244812011719,
      "loss": 0.0786,
      "loss/chosen-sft": 1.4777370691299438,
      "loss/dpo": 0.07859226316213608,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.649788498878479,
      "rewards/margins": 0.34457841515541077,
      "rewards/rejected": -0.9943668246269226,
      "step": 545
    },
    {
      "epoch": 0.3050471436494731,
      "grad_norm": 2.0230906009674072,
      "learning_rate": 4.391390370938777e-07,
      "logits/chosen": -0.2850199043750763,
      "logits/rejected": -0.1109178215265274,
      "logps/chosen": -364.53826904296875,
      "logps/rejected": -338.34283447265625,
      "loss": 0.0867,
      "loss/chosen-sft": 1.5412209033966064,
      "loss/dpo": 0.08672328293323517,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.6998955607414246,
      "rewards/margins": 0.25897401571273804,
      "rewards/rejected": -0.9588696360588074,
      "step": 550
    },
    {
      "epoch": 0.30782029950083195,
      "grad_norm": 1.2038522958755493,
      "learning_rate": 4.3754697920383006e-07,
      "logits/chosen": -0.34611284732818604,
      "logits/rejected": -0.049505867063999176,
      "logps/chosen": -355.5877990722656,
      "logps/rejected": -355.5694274902344,
      "loss": 0.0705,
      "loss/chosen-sft": 1.5247033834457397,
      "loss/dpo": 0.0704929307103157,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.6597936749458313,
      "rewards/margins": 0.4111254811286926,
      "rewards/rejected": -1.070919156074524,
      "step": 555
    },
    {
      "epoch": 0.3105934553521908,
      "grad_norm": 1.383131980895996,
      "learning_rate": 4.359373321686053e-07,
      "logits/chosen": -0.47573018074035645,
      "logits/rejected": -0.38598376512527466,
      "logps/chosen": -388.10736083984375,
      "logps/rejected": -381.851318359375,
      "loss": 0.0792,
      "loss/chosen-sft": 1.4690505266189575,
      "loss/dpo": 0.07924602925777435,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.6938058137893677,
      "rewards/margins": 0.39269718527793884,
      "rewards/rejected": -1.086503028869629,
      "step": 560
    },
    {
      "epoch": 0.31336661120354964,
      "grad_norm": 1.9160140752792358,
      "learning_rate": 4.343102469493947e-07,
      "logits/chosen": -0.3587764501571655,
      "logits/rejected": -0.2721293866634369,
      "logps/chosen": -403.26788330078125,
      "logps/rejected": -358.91046142578125,
      "loss": 0.068,
      "loss/chosen-sft": 1.6094681024551392,
      "loss/dpo": 0.06795226037502289,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7775420546531677,
      "rewards/margins": 0.31379085779190063,
      "rewards/rejected": -1.091333031654358,
      "step": 565
    },
    {
      "epoch": 0.3161397670549085,
      "grad_norm": 1.0593386888504028,
      "learning_rate": 4.326658761428342e-07,
      "logits/chosen": -0.4315316677093506,
      "logits/rejected": -0.2712632417678833,
      "logps/chosen": -381.10260009765625,
      "logps/rejected": -344.9892883300781,
      "loss": 0.0726,
      "loss/chosen-sft": 1.552372694015503,
      "loss/dpo": 0.07259351015090942,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.800487220287323,
      "rewards/margins": 0.31337645649909973,
      "rewards/rejected": -1.1138637065887451,
      "step": 570
    },
    {
      "epoch": 0.3189129229062673,
      "grad_norm": 1.4632201194763184,
      "learning_rate": 4.310043739666937e-07,
      "logits/chosen": -0.49550876021385193,
      "logits/rejected": -0.3280448317527771,
      "logps/chosen": -401.83856201171875,
      "logps/rejected": -399.66729736328125,
      "loss": 0.0516,
      "loss/chosen-sft": 1.647662878036499,
      "loss/dpo": 0.05160484462976456,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.8304476737976074,
      "rewards/margins": 0.4837685227394104,
      "rewards/rejected": -1.314216136932373,
      "step": 575
    },
    {
      "epoch": 0.32168607875762617,
      "grad_norm": 1.8497745990753174,
      "learning_rate": 4.2932589624541296e-07,
      "logits/chosen": -0.3687261641025543,
      "logits/rejected": -0.27867016196250916,
      "logps/chosen": -376.83660888671875,
      "logps/rejected": -353.30670166015625,
      "loss": 0.0618,
      "loss/chosen-sft": 1.6445457935333252,
      "loss/dpo": 0.06179194524884224,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.8887649774551392,
      "rewards/margins": 0.33651480078697205,
      "rewards/rejected": -1.2252798080444336,
      "step": 580
    },
    {
      "epoch": 0.324459234608985,
      "grad_norm": 1.1594619750976562,
      "learning_rate": 4.276306003954881e-07,
      "logits/chosen": -0.46756500005722046,
      "logits/rejected": -0.4428383708000183,
      "logps/chosen": -405.39593505859375,
      "logps/rejected": -375.8490905761719,
      "loss": 0.0648,
      "loss/chosen-sft": 1.6012241840362549,
      "loss/dpo": 0.06479503959417343,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.0120012760162354,
      "rewards/margins": 0.2799423336982727,
      "rewards/rejected": -1.2919435501098633,
      "step": 585
    },
    {
      "epoch": 0.32723239046034386,
      "grad_norm": 1.4882887601852417,
      "learning_rate": 4.25918645410708e-07,
      "logits/chosen": -0.5149391889572144,
      "logits/rejected": -0.43163880705833435,
      "logps/chosen": -395.2956237792969,
      "logps/rejected": -388.7841491699219,
      "loss": 0.0565,
      "loss/chosen-sft": 1.619074821472168,
      "loss/dpo": 0.056492336094379425,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.8736220598220825,
      "rewards/margins": 0.49877220392227173,
      "rewards/rejected": -1.372394323348999,
      "step": 590
    },
    {
      "epoch": 0.3300055463117027,
      "grad_norm": 1.6804783344268799,
      "learning_rate": 4.2419019184724316e-07,
      "logits/chosen": -0.4885106086730957,
      "logits/rejected": -0.2503131926059723,
      "logps/chosen": -417.79327392578125,
      "logps/rejected": -404.0755920410156,
      "loss": 0.0634,
      "loss/chosen-sft": 1.5858757495880127,
      "loss/dpo": 0.06336269527673721,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.9350301623344421,
      "rewards/margins": 0.4474121630191803,
      "rewards/rejected": -1.3824422359466553,
      "step": 595
    },
    {
      "epoch": 0.33277870216306155,
      "grad_norm": 1.5922985076904297,
      "learning_rate": 4.224454018085878e-07,
      "logits/chosen": -0.3938080966472626,
      "logits/rejected": -0.20645618438720703,
      "logps/chosen": -417.41644287109375,
      "logps/rejected": -377.573486328125,
      "loss": 0.0521,
      "loss/chosen-sft": 1.6786199808120728,
      "loss/dpo": 0.05210758373141289,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.9713231325149536,
      "rewards/margins": 0.3552365303039551,
      "rewards/rejected": -1.3265597820281982,
      "step": 600
    },
    {
      "epoch": 0.3355518580144204,
      "grad_norm": 1.135326862335205,
      "learning_rate": 4.206844389303569e-07,
      "logits/chosen": -0.5038084387779236,
      "logits/rejected": -0.1152668371796608,
      "logps/chosen": -368.07244873046875,
      "logps/rejected": -371.786865234375,
      "loss": 0.059,
      "loss/chosen-sft": 1.6389780044555664,
      "loss/dpo": 0.058953166007995605,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.9387050867080688,
      "rewards/margins": 0.34334003925323486,
      "rewards/rejected": -1.2820451259613037,
      "step": 605
    },
    {
      "epoch": 0.33832501386577923,
      "grad_norm": 1.6886770725250244,
      "learning_rate": 4.1890746836493987e-07,
      "logits/chosen": -0.5036560893058777,
      "logits/rejected": -0.3065970242023468,
      "logps/chosen": -398.92559814453125,
      "logps/rejected": -394.6751403808594,
      "loss": 0.063,
      "loss/chosen-sft": 1.580437421798706,
      "loss/dpo": 0.0630171000957489,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.9837394952774048,
      "rewards/margins": 0.3374081552028656,
      "rewards/rejected": -1.3211476802825928,
      "step": 610
    },
    {
      "epoch": 0.3410981697171381,
      "grad_norm": 1.2646410465240479,
      "learning_rate": 4.171146567660112e-07,
      "logits/chosen": -0.34558913111686707,
      "logits/rejected": -0.22349652647972107,
      "logps/chosen": -397.49908447265625,
      "logps/rejected": -370.159912109375,
      "loss": 0.0511,
      "loss/chosen-sft": 1.6435205936431885,
      "loss/dpo": 0.05106610804796219,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.8979086875915527,
      "rewards/margins": 0.5196878910064697,
      "rewards/rejected": -1.4175965785980225,
      "step": 615
    },
    {
      "epoch": 0.343871325568497,
      "grad_norm": 1.721474528312683,
      "learning_rate": 4.153061722729013e-07,
      "logits/chosen": -0.4159115254878998,
      "logits/rejected": -0.3842083811759949,
      "logps/chosen": -425.87567138671875,
      "logps/rejected": -361.23883056640625,
      "loss": 0.0537,
      "loss/chosen-sft": 1.750331163406372,
      "loss/dpo": 0.05366608500480652,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.9684473276138306,
      "rewards/margins": 0.3472323715686798,
      "rewards/rejected": -1.3156797885894775,
      "step": 620
    },
    {
      "epoch": 0.3466444814198558,
      "grad_norm": 0.8142698407173157,
      "learning_rate": 4.1348218449482723e-07,
      "logits/chosen": -0.5395044088363647,
      "logits/rejected": -0.4873916208744049,
      "logps/chosen": -400.04217529296875,
      "logps/rejected": -377.87615966796875,
      "loss": 0.052,
      "loss/chosen-sft": 1.6478407382965088,
      "loss/dpo": 0.05204028636217117,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.9235207438468933,
      "rewards/margins": 0.4557490348815918,
      "rewards/rejected": -1.3792698383331299,
      "step": 625
    },
    {
      "epoch": 0.34941763727121466,
      "grad_norm": 1.6725468635559082,
      "learning_rate": 4.1164286449498584e-07,
      "logits/chosen": -0.5160374641418457,
      "logits/rejected": -0.267566978931427,
      "logps/chosen": -390.87774658203125,
      "logps/rejected": -379.4105224609375,
      "loss": 0.0671,
      "loss/chosen-sft": 1.6147594451904297,
      "loss/dpo": 0.06710793823003769,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -1.0025107860565186,
      "rewards/margins": 0.2850556969642639,
      "rewards/rejected": -1.2875664234161377,
      "step": 630
    },
    {
      "epoch": 0.3521907931225735,
      "grad_norm": 1.5668613910675049,
      "learning_rate": 4.0978838477451065e-07,
      "logits/chosen": -0.36808034777641296,
      "logits/rejected": -0.3304065763950348,
      "logps/chosen": -414.526123046875,
      "logps/rejected": -388.3052062988281,
      "loss": 0.0505,
      "loss/chosen-sft": 1.8385422229766846,
      "loss/dpo": 0.05049455910921097,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.9938969612121582,
      "rewards/margins": 0.41056522727012634,
      "rewards/rejected": -1.404462218284607,
      "step": 635
    },
    {
      "epoch": 0.35496394897393235,
      "grad_norm": 1.7163115739822388,
      "learning_rate": 4.079189192562938e-07,
      "logits/chosen": -0.597520649433136,
      "logits/rejected": -0.5070825815200806,
      "logps/chosen": -420.65447998046875,
      "logps/rejected": -367.829345703125,
      "loss": 0.0641,
      "loss/chosen-sft": 1.6417961120605469,
      "loss/dpo": 0.0640825405716896,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.9867199659347534,
      "rewards/margins": 0.3000028729438782,
      "rewards/rejected": -1.2867228984832764,
      "step": 640
    },
    {
      "epoch": 0.3577371048252912,
      "grad_norm": 1.513505220413208,
      "learning_rate": 4.0603464326867456e-07,
      "logits/chosen": -0.3763376474380493,
      "logits/rejected": -0.37528449296951294,
      "logps/chosen": -425.38043212890625,
      "logps/rejected": -415.4266662597656,
      "loss": 0.0531,
      "loss/chosen-sft": 1.692238211631775,
      "loss/dpo": 0.053089797496795654,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.8854948878288269,
      "rewards/margins": 0.5308740735054016,
      "rewards/rejected": -1.416368842124939,
      "step": 645
    },
    {
      "epoch": 0.36051026067665004,
      "grad_norm": 2.2651264667510986,
      "learning_rate": 4.041357335289962e-07,
      "logits/chosen": -0.6403440833091736,
      "logits/rejected": -0.5450460314750671,
      "logps/chosen": -418.7191467285156,
      "logps/rejected": -380.0513610839844,
      "loss": 0.0605,
      "loss/chosen-sft": 1.6550674438476562,
      "loss/dpo": 0.060495972633361816,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -1.0474960803985596,
      "rewards/margins": 0.3858080506324768,
      "rewards/rejected": -1.4333041906356812,
      "step": 650
    },
    {
      "epoch": 0.3632834165280089,
      "grad_norm": 1.6215060949325562,
      "learning_rate": 4.0222236812703247e-07,
      "logits/chosen": -0.43776410818099976,
      "logits/rejected": -0.33489981293678284,
      "logps/chosen": -383.8864440917969,
      "logps/rejected": -378.69873046875,
      "loss": 0.0604,
      "loss/chosen-sft": 1.619616150856018,
      "loss/dpo": 0.060428131371736526,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.8916348218917847,
      "rewards/margins": 0.45284873247146606,
      "rewards/rejected": -1.3444836139678955,
      "step": 655
    },
    {
      "epoch": 0.36605657237936773,
      "grad_norm": 1.3622971773147583,
      "learning_rate": 4.002947265082854e-07,
      "logits/chosen": -0.5182867646217346,
      "logits/rejected": -0.41873541474342346,
      "logps/chosen": -382.6905212402344,
      "logps/rejected": -354.53240966796875,
      "loss": 0.0451,
      "loss/chosen-sft": 1.6769845485687256,
      "loss/dpo": 0.04514864459633827,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.910293698310852,
      "rewards/margins": 0.40142306685447693,
      "rewards/rejected": -1.3117166757583618,
      "step": 660
    },
    {
      "epoch": 0.3688297282307266,
      "grad_norm": 1.7427003383636475,
      "learning_rate": 3.983529894571558e-07,
      "logits/chosen": -0.5258729457855225,
      "logits/rejected": -0.35827913880348206,
      "logps/chosen": -410.27655029296875,
      "logps/rejected": -398.0840759277344,
      "loss": 0.0515,
      "loss/chosen-sft": 1.6551071405410767,
      "loss/dpo": 0.051458604633808136,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.9875022172927856,
      "rewards/margins": 0.4931188225746155,
      "rewards/rejected": -1.4806209802627563,
      "step": 665
    },
    {
      "epoch": 0.3716028840820854,
      "grad_norm": 1.5637753009796143,
      "learning_rate": 3.963973390799887e-07,
      "logits/chosen": -0.3731691241264343,
      "logits/rejected": -0.3140907287597656,
      "logps/chosen": -387.2796936035156,
      "logps/rejected": -389.09967041015625,
      "loss": 0.0573,
      "loss/chosen-sft": 1.664841890335083,
      "loss/dpo": 0.057298194617033005,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.9336613416671753,
      "rewards/margins": 0.4270879626274109,
      "rewards/rejected": -1.360749363899231,
      "step": 670
    },
    {
      "epoch": 0.37437603993344426,
      "grad_norm": 1.713516116142273,
      "learning_rate": 3.944279587879942e-07,
      "logits/chosen": -0.6181343197822571,
      "logits/rejected": -0.4861913323402405,
      "logps/chosen": -399.0451354980469,
      "logps/rejected": -375.4527282714844,
      "loss": 0.0681,
      "loss/chosen-sft": 1.6289466619491577,
      "loss/dpo": 0.06814940273761749,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.020996332168579,
      "rewards/margins": 0.3115329444408417,
      "rewards/rejected": -1.3325293064117432,
      "step": 675
    },
    {
      "epoch": 0.3771491957848031,
      "grad_norm": 1.3407127857208252,
      "learning_rate": 3.9244503328004606e-07,
      "logits/chosen": -0.7082573175430298,
      "logits/rejected": -0.5655766725540161,
      "logps/chosen": -412.4944763183594,
      "logps/rejected": -384.94476318359375,
      "loss": 0.0535,
      "loss/chosen-sft": 1.6391160488128662,
      "loss/dpo": 0.05352095887064934,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -1.062304139137268,
      "rewards/margins": 0.3755071461200714,
      "rewards/rejected": -1.4378111362457275,
      "step": 680
    },
    {
      "epoch": 0.37992235163616195,
      "grad_norm": 0.8834872841835022,
      "learning_rate": 3.9044874852536013e-07,
      "logits/chosen": -0.5957868099212646,
      "logits/rejected": -0.3501487076282501,
      "logps/chosen": -390.2853088378906,
      "logps/rejected": -378.35296630859375,
      "loss": 0.0609,
      "loss/chosen-sft": 1.706976294517517,
      "loss/dpo": 0.06085295230150223,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -1.0856157541275024,
      "rewards/margins": 0.3497942090034485,
      "rewards/rejected": -1.4354099035263062,
      "step": 685
    },
    {
      "epoch": 0.3826955074875208,
      "grad_norm": 1.8584004640579224,
      "learning_rate": 3.8843929174605283e-07,
      "logits/chosen": -0.5268598794937134,
      "logits/rejected": -0.5594683885574341,
      "logps/chosen": -399.52593994140625,
      "logps/rejected": -352.721923828125,
      "loss": 0.0588,
      "loss/chosen-sft": 1.7252442836761475,
      "loss/dpo": 0.058761436492204666,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.9334037899971008,
      "rewards/margins": 0.39858493208885193,
      "rewards/rejected": -1.3319886922836304,
      "step": 690
    },
    {
      "epoch": 0.38546866333887964,
      "grad_norm": 1.2674587965011597,
      "learning_rate": 3.8641685139958234e-07,
      "logits/chosen": -0.5243244171142578,
      "logits/rejected": -0.3986015319824219,
      "logps/chosen": -428.97662353515625,
      "logps/rejected": -402.95074462890625,
      "loss": 0.0568,
      "loss/chosen-sft": 1.6090924739837646,
      "loss/dpo": 0.056836675852537155,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.9936995506286621,
      "rewards/margins": 0.45986804366111755,
      "rewards/rejected": -1.4535675048828125,
      "step": 695
    },
    {
      "epoch": 0.3882418191902385,
      "grad_norm": 1.1967881917953491,
      "learning_rate": 3.8438161716107453e-07,
      "logits/chosen": -0.35420164465904236,
      "logits/rejected": -0.16235880553722382,
      "logps/chosen": -388.86260986328125,
      "logps/rejected": -373.76336669921875,
      "loss": 0.0531,
      "loss/chosen-sft": 1.7332239151000977,
      "loss/dpo": 0.05314627289772034,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.9368470907211304,
      "rewards/margins": 0.39190053939819336,
      "rewards/rejected": -1.3287477493286133,
      "step": 700
    },
    {
      "epoch": 0.3910149750415973,
      "grad_norm": 1.0293500423431396,
      "learning_rate": 3.8233377990553376e-07,
      "logits/chosen": -0.5038026571273804,
      "logits/rejected": -0.45793700218200684,
      "logps/chosen": -392.74078369140625,
      "logps/rejected": -383.5316467285156,
      "loss": 0.0469,
      "loss/chosen-sft": 1.6530288457870483,
      "loss/dpo": 0.04691758006811142,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.9149090051651001,
      "rewards/margins": 0.5396536588668823,
      "rewards/rejected": -1.4545625448226929,
      "step": 705
    },
    {
      "epoch": 0.39378813089295617,
      "grad_norm": 1.4593671560287476,
      "learning_rate": 3.80273531689942e-07,
      "logits/chosen": -0.5035001039505005,
      "logits/rejected": -0.39359813928604126,
      "logps/chosen": -410.6475524902344,
      "logps/rejected": -410.08355712890625,
      "loss": 0.0516,
      "loss/chosen-sft": 1.5783100128173828,
      "loss/dpo": 0.05163818597793579,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.9400428533554077,
      "rewards/margins": 0.5864478945732117,
      "rewards/rejected": -1.5264908075332642,
      "step": 710
    },
    {
      "epoch": 0.396561286744315,
      "grad_norm": 1.9627400636672974,
      "learning_rate": 3.7820106573524645e-07,
      "logits/chosen": -0.3761715888977051,
      "logits/rejected": -0.33153462409973145,
      "logps/chosen": -387.8844909667969,
      "logps/rejected": -367.91168212890625,
      "loss": 0.0642,
      "loss/chosen-sft": 1.6243873834609985,
      "loss/dpo": 0.0642186850309372,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.0104358196258545,
      "rewards/margins": 0.39246731996536255,
      "rewards/rejected": -1.4029031991958618,
      "step": 715
    },
    {
      "epoch": 0.39933444259567386,
      "grad_norm": 1.1319611072540283,
      "learning_rate": 3.7611657640823825e-07,
      "logits/chosen": -0.5168325901031494,
      "logits/rejected": -0.41257476806640625,
      "logps/chosen": -418.1272888183594,
      "logps/rejected": -379.2960510253906,
      "loss": 0.0503,
      "loss/chosen-sft": 1.699724793434143,
      "loss/dpo": 0.050268955528736115,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -1.030914545059204,
      "rewards/margins": 0.43790435791015625,
      "rewards/rejected": -1.46881902217865,
      "step": 720
    },
    {
      "epoch": 0.4021075984470327,
      "grad_norm": 1.7412174940109253,
      "learning_rate": 3.74020259203324e-07,
      "logits/chosen": -0.6365585327148438,
      "logits/rejected": -0.4046885371208191,
      "logps/chosen": -392.642822265625,
      "logps/rejected": -416.04986572265625,
      "loss": 0.0541,
      "loss/chosen-sft": 1.6220734119415283,
      "loss/dpo": 0.05405275151133537,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.0092906951904297,
      "rewards/margins": 0.5087220072746277,
      "rewards/rejected": -1.5180127620697021,
      "step": 725
    },
    {
      "epoch": 0.40488075429839154,
      "grad_norm": 1.3539364337921143,
      "learning_rate": 3.7191231072419095e-07,
      "logits/chosen": -0.41355252265930176,
      "logits/rejected": -0.23317034542560577,
      "logps/chosen": -427.7068786621094,
      "logps/rejected": -423.05108642578125,
      "loss": 0.045,
      "loss/chosen-sft": 1.7072865962982178,
      "loss/dpo": 0.04499746486544609,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -1.0050982236862183,
      "rewards/margins": 0.5913174748420715,
      "rewards/rejected": -1.5964157581329346,
      "step": 730
    },
    {
      "epoch": 0.40765391014975044,
      "grad_norm": 2.4430134296417236,
      "learning_rate": 3.6979292866536864e-07,
      "logits/chosen": -0.5284135937690735,
      "logits/rejected": -0.39231494069099426,
      "logps/chosen": -444.33062744140625,
      "logps/rejected": -440.982177734375,
      "loss": 0.0645,
      "loss/chosen-sft": 1.6972286701202393,
      "loss/dpo": 0.06453467905521393,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -1.0674129724502563,
      "rewards/margins": 0.47073474526405334,
      "rewards/rejected": -1.5381478071212769,
      "step": 735
    },
    {
      "epoch": 0.4104270660011093,
      "grad_norm": 1.5764257907867432,
      "learning_rate": 3.6766231179368815e-07,
      "logits/chosen": -0.6887901425361633,
      "logits/rejected": -0.5167983174324036,
      "logps/chosen": -443.1559143066406,
      "logps/rejected": -440.83489990234375,
      "loss": 0.0519,
      "loss/chosen-sft": 1.635066270828247,
      "loss/dpo": 0.05188404396176338,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -1.104542851448059,
      "rewards/margins": 0.5738195180892944,
      "rewards/rejected": -1.678362250328064,
      "step": 740
    },
    {
      "epoch": 0.41320022185246813,
      "grad_norm": 1.0604594945907593,
      "learning_rate": 3.6552065992964043e-07,
      "logits/chosen": -0.6154565215110779,
      "logits/rejected": -0.3693044185638428,
      "logps/chosen": -406.3134460449219,
      "logps/rejected": -377.28228759765625,
      "loss": 0.0516,
      "loss/chosen-sft": 1.6899700164794922,
      "loss/dpo": 0.05156542733311653,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -1.0907108783721924,
      "rewards/margins": 0.35733407735824585,
      "rewards/rejected": -1.448045015335083,
      "step": 745
    },
    {
      "epoch": 0.415973377703827,
      "grad_norm": 1.4795838594436646,
      "learning_rate": 3.6336817392863625e-07,
      "logits/chosen": -0.4953575134277344,
      "logits/rejected": -0.3335438668727875,
      "logps/chosen": -399.979736328125,
      "logps/rejected": -392.7442321777344,
      "loss": 0.0494,
      "loss/chosen-sft": 1.6881647109985352,
      "loss/dpo": 0.04941638186573982,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -1.1530383825302124,
      "rewards/margins": 0.4033992886543274,
      "rewards/rejected": -1.5564377307891846,
      "step": 750
    },
    {
      "epoch": 0.4187465335551858,
      "grad_norm": 1.2639776468276978,
      "learning_rate": 3.6120505566216906e-07,
      "logits/chosen": -0.669465184211731,
      "logits/rejected": -0.5370808839797974,
      "logps/chosen": -412.94366455078125,
      "logps/rejected": -406.2000427246094,
      "loss": 0.0412,
      "loss/chosen-sft": 1.7509901523590088,
      "loss/dpo": 0.04121355339884758,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -1.085228681564331,
      "rewards/margins": 0.5851390361785889,
      "rewards/rejected": -1.6703678369522095,
      "step": 755
    },
    {
      "epoch": 0.42151968940654466,
      "grad_norm": 1.776004672050476,
      "learning_rate": 3.5903150799888215e-07,
      "logits/chosen": -0.562536358833313,
      "logits/rejected": -0.49020713567733765,
      "logps/chosen": -412.8077087402344,
      "logps/rejected": -399.482421875,
      "loss": 0.0529,
      "loss/chosen-sft": 1.6743816137313843,
      "loss/dpo": 0.052908383309841156,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -1.026395320892334,
      "rewards/margins": 0.47851258516311646,
      "rewards/rejected": -1.5049078464508057,
      "step": 760
    },
    {
      "epoch": 0.4242928452579035,
      "grad_norm": 1.3225823640823364,
      "learning_rate": 3.5684773478554255e-07,
      "logits/chosen": -0.5152966976165771,
      "logits/rejected": -0.5033225417137146,
      "logps/chosen": -405.97271728515625,
      "logps/rejected": -377.90875244140625,
      "loss": 0.0571,
      "loss/chosen-sft": 1.7368265390396118,
      "loss/dpo": 0.05706434324383736,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -1.0174944400787354,
      "rewards/margins": 0.47621211409568787,
      "rewards/rejected": -1.493706464767456,
      "step": 765
    },
    {
      "epoch": 0.42706600110926235,
      "grad_norm": 1.3815367221832275,
      "learning_rate": 3.546539408279235e-07,
      "logits/chosen": -0.49167943000793457,
      "logits/rejected": -0.2837482988834381,
      "logps/chosen": -405.2371520996094,
      "logps/rejected": -391.68670654296875,
      "loss": 0.0601,
      "loss/chosen-sft": 1.7111915349960327,
      "loss/dpo": 0.060050565749406815,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -1.0395854711532593,
      "rewards/margins": 0.37245672941207886,
      "rewards/rejected": -1.412042260169983,
      "step": 770
    },
    {
      "epoch": 0.4298391569606212,
      "grad_norm": 1.1186020374298096,
      "learning_rate": 3.5245033187159647e-07,
      "logits/chosen": -0.5566674470901489,
      "logits/rejected": -0.4645940661430359,
      "logps/chosen": -405.1397399902344,
      "logps/rejected": -374.271728515625,
      "loss": 0.0514,
      "loss/chosen-sft": 1.6212804317474365,
      "loss/dpo": 0.05135621875524521,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.9940582513809204,
      "rewards/margins": 0.45282116532325745,
      "rewards/rejected": -1.4468793869018555,
      "step": 775
    },
    {
      "epoch": 0.43261231281198004,
      "grad_norm": 1.5393182039260864,
      "learning_rate": 3.502371145826352e-07,
      "logits/chosen": -0.5995725989341736,
      "logits/rejected": -0.5144038200378418,
      "logps/chosen": -400.0671081542969,
      "logps/rejected": -385.246337890625,
      "loss": 0.0464,
      "loss/chosen-sft": 1.6027672290802002,
      "loss/dpo": 0.04639893397688866,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.9513114094734192,
      "rewards/margins": 0.5679217576980591,
      "rewards/rejected": -1.5192331075668335,
      "step": 780
    },
    {
      "epoch": 0.4353854686633389,
      "grad_norm": 1.7037338018417358,
      "learning_rate": 3.4801449652823374e-07,
      "logits/chosen": -0.8535438776016235,
      "logits/rejected": -0.6764947175979614,
      "logps/chosen": -405.5221252441406,
      "logps/rejected": -374.32440185546875,
      "loss": 0.0488,
      "loss/chosen-sft": 1.583449363708496,
      "loss/dpo": 0.048822566866874695,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.958821177482605,
      "rewards/margins": 0.5157214999198914,
      "rewards/rejected": -1.4745426177978516,
      "step": 785
    },
    {
      "epoch": 0.4381586245146977,
      "grad_norm": 1.703331708908081,
      "learning_rate": 3.4578268615723924e-07,
      "logits/chosen": -0.561161458492279,
      "logits/rejected": -0.5696666836738586,
      "logps/chosen": -420.22381591796875,
      "logps/rejected": -396.88299560546875,
      "loss": 0.0437,
      "loss/chosen-sft": 1.7269833087921143,
      "loss/dpo": 0.04372150078415871,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -1.0612952709197998,
      "rewards/margins": 0.5599007606506348,
      "rewards/rejected": -1.6211960315704346,
      "step": 790
    },
    {
      "epoch": 0.44093178036605657,
      "grad_norm": 1.3786600828170776,
      "learning_rate": 3.4354189278060317e-07,
      "logits/chosen": -0.7774502635002136,
      "logits/rejected": -0.6075506806373596,
      "logps/chosen": -412.4271545410156,
      "logps/rejected": -393.829345703125,
      "loss": 0.0453,
      "loss/chosen-sft": 1.675945520401001,
      "loss/dpo": 0.04528175666928291,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -1.1789729595184326,
      "rewards/margins": 0.48232507705688477,
      "rewards/rejected": -1.6612980365753174,
      "step": 795
    },
    {
      "epoch": 0.4437049362174154,
      "grad_norm": 1.5878552198410034,
      "learning_rate": 3.412923265517503e-07,
      "logits/chosen": -0.7400108575820923,
      "logits/rejected": -0.7512958645820618,
      "logps/chosen": -412.6996154785156,
      "logps/rejected": -379.6897277832031,
      "loss": 0.0488,
      "loss/chosen-sft": 1.7094510793685913,
      "loss/dpo": 0.048782043159008026,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -1.1738940477371216,
      "rewards/margins": 0.429801881313324,
      "rewards/rejected": -1.6036958694458008,
      "step": 800
    },
    {
      "epoch": 0.44647809206877426,
      "grad_norm": 1.7441812753677368,
      "learning_rate": 3.390341984468699e-07,
      "logits/chosen": -0.7113388776779175,
      "logits/rejected": -0.6850872039794922,
      "logps/chosen": -428.8340759277344,
      "logps/rejected": -400.9315490722656,
      "loss": 0.0473,
      "loss/chosen-sft": 1.7859251499176025,
      "loss/dpo": 0.047270990908145905,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.1826239824295044,
      "rewards/margins": 0.4750004708766937,
      "rewards/rejected": -1.6576244831085205,
      "step": 805
    },
    {
      "epoch": 0.4492512479201331,
      "grad_norm": 1.530585765838623,
      "learning_rate": 3.367677202451292e-07,
      "logits/chosen": -0.6908737421035767,
      "logits/rejected": -0.5597144365310669,
      "logps/chosen": -455.83758544921875,
      "logps/rejected": -452.01385498046875,
      "loss": 0.0584,
      "loss/chosen-sft": 1.6678813695907593,
      "loss/dpo": 0.05841095373034477,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.1806871891021729,
      "rewards/margins": 0.6453709602355957,
      "rewards/rejected": -1.8260581493377686,
      "step": 810
    },
    {
      "epoch": 0.45202440377149194,
      "grad_norm": 0.9433002471923828,
      "learning_rate": 3.3449310450881164e-07,
      "logits/chosen": -0.7155448198318481,
      "logits/rejected": -0.6245291829109192,
      "logps/chosen": -421.4974060058594,
      "logps/rejected": -427.9629821777344,
      "loss": 0.0364,
      "loss/chosen-sft": 1.7820746898651123,
      "loss/dpo": 0.03638064116239548,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -1.212754249572754,
      "rewards/margins": 0.6936885714530945,
      "rewards/rejected": -1.9064428806304932,
      "step": 815
    },
    {
      "epoch": 0.4547975596228508,
      "grad_norm": 1.6761059761047363,
      "learning_rate": 3.322105645633813e-07,
      "logits/chosen": -0.6909006834030151,
      "logits/rejected": -0.5668486952781677,
      "logps/chosen": -440.548095703125,
      "logps/rejected": -421.02392578125,
      "loss": 0.0613,
      "loss/chosen-sft": 1.691504716873169,
      "loss/dpo": 0.06133972853422165,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -1.250173807144165,
      "rewards/margins": 0.3592910170555115,
      "rewards/rejected": -1.6094646453857422,
      "step": 820
    },
    {
      "epoch": 0.45757071547420963,
      "grad_norm": 1.7704241275787354,
      "learning_rate": 3.299203144774767e-07,
      "logits/chosen": -0.717354953289032,
      "logits/rejected": -0.6114916801452637,
      "logps/chosen": -436.1394958496094,
      "logps/rejected": -417.92218017578125,
      "loss": 0.0443,
      "loss/chosen-sft": 1.7568438053131104,
      "loss/dpo": 0.044312089681625366,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -1.234349250793457,
      "rewards/margins": 0.4998023509979248,
      "rewards/rejected": -1.7341516017913818,
      "step": 825
    },
    {
      "epoch": 0.4603438713255685,
      "grad_norm": 1.0728057622909546,
      "learning_rate": 3.276225690428338e-07,
      "logits/chosen": -0.7584112882614136,
      "logits/rejected": -0.7765355110168457,
      "logps/chosen": -411.444580078125,
      "logps/rejected": -426.4891052246094,
      "loss": 0.0411,
      "loss/chosen-sft": 1.7121307849884033,
      "loss/dpo": 0.04111065715551376,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -1.1426109075546265,
      "rewards/margins": 0.6041598916053772,
      "rewards/rejected": -1.7467708587646484,
      "step": 830
    },
    {
      "epoch": 0.4631170271769273,
      "grad_norm": 1.3771647214889526,
      "learning_rate": 3.2531754375414206e-07,
      "logits/chosen": -0.7121502161026001,
      "logits/rejected": -0.732271671295166,
      "logps/chosen": -458.24688720703125,
      "logps/rejected": -436.6095275878906,
      "loss": 0.0516,
      "loss/chosen-sft": 1.7345997095108032,
      "loss/dpo": 0.051555633544921875,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -1.2151943445205688,
      "rewards/margins": 0.511760413646698,
      "rewards/rejected": -1.7269548177719116,
      "step": 835
    },
    {
      "epoch": 0.46589018302828616,
      "grad_norm": 1.6047500371932983,
      "learning_rate": 3.230054547888339e-07,
      "logits/chosen": -0.763964831829071,
      "logits/rejected": -0.7086871862411499,
      "logps/chosen": -406.05316162109375,
      "logps/rejected": -391.0845642089844,
      "loss": 0.0415,
      "loss/chosen-sft": 1.747057318687439,
      "loss/dpo": 0.04153294861316681,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -1.1723990440368652,
      "rewards/margins": 0.46168699860572815,
      "rewards/rejected": -1.634086012840271,
      "step": 840
    },
    {
      "epoch": 0.468663338879645,
      "grad_norm": 1.445021152496338,
      "learning_rate": 3.2068651898681076e-07,
      "logits/chosen": -0.742651104927063,
      "logits/rejected": -0.7188843488693237,
      "logps/chosen": -458.99652099609375,
      "logps/rejected": -454.03533935546875,
      "loss": 0.0491,
      "loss/chosen-sft": 1.6705642938613892,
      "loss/dpo": 0.0490889772772789,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -1.282627820968628,
      "rewards/margins": 0.5290722846984863,
      "rewards/rejected": -1.8117001056671143,
      "step": 845
    },
    {
      "epoch": 0.4714364947310039,
      "grad_norm": 2.1770212650299072,
      "learning_rate": 3.183609538301065e-07,
      "logits/chosen": -0.8837400674819946,
      "logits/rejected": -0.7734067440032959,
      "logps/chosen": -432.4523010253906,
      "logps/rejected": -402.35064697265625,
      "loss": 0.041,
      "loss/chosen-sft": 1.7752116918563843,
      "loss/dpo": 0.040977347642183304,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -1.2817370891571045,
      "rewards/margins": 0.4654463827610016,
      "rewards/rejected": -1.7471835613250732,
      "step": 850
    },
    {
      "epoch": 0.47420965058236275,
      "grad_norm": 1.3028136491775513,
      "learning_rate": 3.1602897742249077e-07,
      "logits/chosen": -0.9581116437911987,
      "logits/rejected": -0.7644520998001099,
      "logps/chosen": -424.92352294921875,
      "logps/rejected": -427.76806640625,
      "loss": 0.0447,
      "loss/chosen-sft": 1.6508781909942627,
      "loss/dpo": 0.04467002674937248,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.2735633850097656,
      "rewards/margins": 0.5769675970077515,
      "rewards/rejected": -1.850530982017517,
      "step": 855
    },
    {
      "epoch": 0.4769828064337216,
      "grad_norm": 1.5738519430160522,
      "learning_rate": 3.136908084690142e-07,
      "logits/chosen": -0.7455793619155884,
      "logits/rejected": -0.6809624433517456,
      "logps/chosen": -430.9115295410156,
      "logps/rejected": -438.87164306640625,
      "loss": 0.0372,
      "loss/chosen-sft": 1.8043369054794312,
      "loss/dpo": 0.037248264998197556,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -1.165959119796753,
      "rewards/margins": 0.6415104866027832,
      "rewards/rejected": -1.8074697256088257,
      "step": 860
    },
    {
      "epoch": 0.47975596228508044,
      "grad_norm": 0.9466845989227295,
      "learning_rate": 3.113466662554971e-07,
      "logits/chosen": -0.8632648587226868,
      "logits/rejected": -0.7621570825576782,
      "logps/chosen": -416.5755310058594,
      "logps/rejected": -404.36383056640625,
      "loss": 0.0405,
      "loss/chosen-sft": 1.7127773761749268,
      "loss/dpo": 0.04047512635588646,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.1657226085662842,
      "rewards/margins": 0.5985251665115356,
      "rewards/rejected": -1.7642476558685303,
      "step": 865
    },
    {
      "epoch": 0.4825291181364393,
      "grad_norm": 1.4616957902908325,
      "learning_rate": 3.0899677062796356e-07,
      "logits/chosen": -0.9384894371032715,
      "logits/rejected": -0.7627253532409668,
      "logps/chosen": -454.59613037109375,
      "logps/rejected": -447.65667724609375,
      "loss": 0.0443,
      "loss/chosen-sft": 1.7974565029144287,
      "loss/dpo": 0.04427001625299454,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -1.3299897909164429,
      "rewards/margins": 0.5200726389884949,
      "rewards/rejected": -1.850062370300293,
      "step": 870
    },
    {
      "epoch": 0.4853022739877981,
      "grad_norm": 1.1953011751174927,
      "learning_rate": 3.066413419720231e-07,
      "logits/chosen": -1.0708543062210083,
      "logits/rejected": -0.9470682144165039,
      "logps/chosen": -411.50323486328125,
      "logps/rejected": -438.43023681640625,
      "loss": 0.0373,
      "loss/chosen-sft": 1.778357744216919,
      "loss/dpo": 0.03726748377084732,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -1.4992625713348389,
      "rewards/margins": 0.5397129058837891,
      "rewards/rejected": -2.038975477218628,
      "step": 875
    },
    {
      "epoch": 0.48807542983915697,
      "grad_norm": 1.2079335451126099,
      "learning_rate": 3.042806011922021e-07,
      "logits/chosen": -0.9137675166130066,
      "logits/rejected": -0.8694272041320801,
      "logps/chosen": -465.96954345703125,
      "logps/rejected": -473.0062561035156,
      "loss": 0.0423,
      "loss/chosen-sft": 1.8091964721679688,
      "loss/dpo": 0.04231221228837967,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -1.3828151226043701,
      "rewards/margins": 0.6629751920700073,
      "rewards/rejected": -2.045790195465088,
      "step": 880
    },
    {
      "epoch": 0.4908485856905158,
      "grad_norm": 2.0538318157196045,
      "learning_rate": 3.019147696912256e-07,
      "logits/chosen": -0.8203264474868774,
      "logits/rejected": -0.8118319511413574,
      "logps/chosen": -457.15557861328125,
      "logps/rejected": -443.97412109375,
      "loss": 0.0359,
      "loss/chosen-sft": 1.819947600364685,
      "loss/dpo": 0.03590567782521248,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -1.4309660196304321,
      "rewards/margins": 0.6293816566467285,
      "rewards/rejected": -2.060347557067871,
      "step": 885
    },
    {
      "epoch": 0.49362174154187466,
      "grad_norm": 1.3179012537002563,
      "learning_rate": 2.9954406934925353e-07,
      "logits/chosen": -1.0511844158172607,
      "logits/rejected": -0.8907807469367981,
      "logps/chosen": -441.5658264160156,
      "logps/rejected": -442.07025146484375,
      "loss": 0.0387,
      "loss/chosen-sft": 1.8559125661849976,
      "loss/dpo": 0.0386546328663826,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -1.4241468906402588,
      "rewards/margins": 0.45265036821365356,
      "rewards/rejected": -1.8767973184585571,
      "step": 890
    },
    {
      "epoch": 0.4963948973932335,
      "grad_norm": 1.4792488813400269,
      "learning_rate": 2.9716872250307153e-07,
      "logits/chosen": -0.9463046789169312,
      "logits/rejected": -0.8543429374694824,
      "logps/chosen": -422.39288330078125,
      "logps/rejected": -453.1312561035156,
      "loss": 0.0381,
      "loss/chosen-sft": 1.7985944747924805,
      "loss/dpo": 0.0381343699991703,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.2584816217422485,
      "rewards/margins": 0.6484702825546265,
      "rewards/rejected": -1.906951904296875,
      "step": 895
    },
    {
      "epoch": 0.49916805324459235,
      "grad_norm": 1.479712724685669,
      "learning_rate": 2.9478895192523867e-07,
      "logits/chosen": -0.9726383090019226,
      "logits/rejected": -0.8202565312385559,
      "logps/chosen": -414.5870056152344,
      "logps/rejected": -435.7919006347656,
      "loss": 0.0542,
      "loss/chosen-sft": 1.651619553565979,
      "loss/dpo": 0.05416691303253174,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -1.2731552124023438,
      "rewards/margins": 0.5164347290992737,
      "rewards/rejected": -1.7895901203155518,
      "step": 900
    },
    {
      "epoch": 0.5019412090959512,
      "grad_norm": 1.1789251565933228,
      "learning_rate": 2.9240498080319503e-07,
      "logits/chosen": -0.8077837228775024,
      "logits/rejected": -0.7600533366203308,
      "logps/chosen": -395.1336975097656,
      "logps/rejected": -399.80316162109375,
      "loss": 0.0351,
      "loss/chosen-sft": 1.7661956548690796,
      "loss/dpo": 0.03513758257031441,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.110447883605957,
      "rewards/margins": 0.6378698348999023,
      "rewards/rejected": -1.7483177185058594,
      "step": 905
    },
    {
      "epoch": 0.50471436494731,
      "grad_norm": 1.0820693969726562,
      "learning_rate": 2.9001703271832987e-07,
      "logits/chosen": -0.839047908782959,
      "logits/rejected": -0.7069706916809082,
      "logps/chosen": -418.8855895996094,
      "logps/rejected": -439.6836853027344,
      "loss": 0.0325,
      "loss/chosen-sft": 1.841740369796753,
      "loss/dpo": 0.03246615082025528,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -1.2718727588653564,
      "rewards/margins": 0.6148180365562439,
      "rewards/rejected": -1.8866908550262451,
      "step": 910
    },
    {
      "epoch": 0.5074875207986689,
      "grad_norm": 1.1299471855163574,
      "learning_rate": 2.8762533162501306e-07,
      "logits/chosen": -0.8962306976318359,
      "logits/rejected": -0.7364431619644165,
      "logps/chosen": -440.30419921875,
      "logps/rejected": -406.437744140625,
      "loss": 0.048,
      "loss/chosen-sft": 1.8074274063110352,
      "loss/dpo": 0.04800194129347801,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -1.4012686014175415,
      "rewards/margins": 0.2889956831932068,
      "rewards/rejected": -1.690264344215393,
      "step": 915
    },
    {
      "epoch": 0.5102606766500277,
      "grad_norm": 1.7517938613891602,
      "learning_rate": 2.852301018295914e-07,
      "logits/chosen": -1.0834687948226929,
      "logits/rejected": -0.7870742082595825,
      "logps/chosen": -392.78662109375,
      "logps/rejected": -423.45294189453125,
      "loss": 0.0458,
      "loss/chosen-sft": 1.6456248760223389,
      "loss/dpo": 0.045752983540296555,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -1.2369292974472046,
      "rewards/margins": 0.5108424425125122,
      "rewards/rejected": -1.7477716207504272,
      "step": 920
    },
    {
      "epoch": 0.5130338325013866,
      "grad_norm": 1.3080066442489624,
      "learning_rate": 2.828315679693518e-07,
      "logits/chosen": -0.7610379457473755,
      "logits/rejected": -0.6710953712463379,
      "logps/chosen": -434.26953125,
      "logps/rejected": -448.9422912597656,
      "loss": 0.0385,
      "loss/chosen-sft": 1.773113489151001,
      "loss/dpo": 0.03848852962255478,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -1.2653634548187256,
      "rewards/margins": 0.6251915097236633,
      "rewards/rejected": -1.8905551433563232,
      "step": 925
    },
    {
      "epoch": 0.5158069883527454,
      "grad_norm": 1.5302928686141968,
      "learning_rate": 2.80429954991454e-07,
      "logits/chosen": -0.8365219831466675,
      "logits/rejected": -0.787796139717102,
      "logps/chosen": -418.86383056640625,
      "logps/rejected": -427.78790283203125,
      "loss": 0.0488,
      "loss/chosen-sft": 1.701751470565796,
      "loss/dpo": 0.048776544630527496,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -1.2631399631500244,
      "rewards/margins": 0.5425049066543579,
      "rewards/rejected": -1.8056447505950928,
      "step": 930
    },
    {
      "epoch": 0.5185801442041043,
      "grad_norm": 1.1883572340011597,
      "learning_rate": 2.7802548813183364e-07,
      "logits/chosen": -0.8014055490493774,
      "logits/rejected": -0.8269286155700684,
      "logps/chosen": -445.94378662109375,
      "logps/rejected": -428.35107421875,
      "loss": 0.0446,
      "loss/chosen-sft": 1.728753685951233,
      "loss/dpo": 0.04457592964172363,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1154412031173706,
      "rewards/margins": 0.6285417675971985,
      "rewards/rejected": -1.7439830303192139,
      "step": 935
    },
    {
      "epoch": 0.5213533000554631,
      "grad_norm": 1.7934128046035767,
      "learning_rate": 2.756183928940784e-07,
      "logits/chosen": -0.7623527646064758,
      "logits/rejected": -0.7455809712409973,
      "logps/chosen": -425.8876037597656,
      "logps/rejected": -401.5950927734375,
      "loss": 0.0398,
      "loss/chosen-sft": 1.7588945627212524,
      "loss/dpo": 0.03979960083961487,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -1.2177762985229492,
      "rewards/margins": 0.4867513179779053,
      "rewards/rejected": -1.704527497291565,
      "step": 940
    },
    {
      "epoch": 0.5241264559068219,
      "grad_norm": 1.373268723487854,
      "learning_rate": 2.7320889502827905e-07,
      "logits/chosen": -0.8963130712509155,
      "logits/rejected": -0.8474391102790833,
      "logps/chosen": -429.3999938964844,
      "logps/rejected": -427.1219177246094,
      "loss": 0.0446,
      "loss/chosen-sft": 1.7064225673675537,
      "loss/dpo": 0.04461617022752762,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -1.2002841234207153,
      "rewards/margins": 0.6481701731681824,
      "rewards/rejected": -1.848454236984253,
      "step": 945
    },
    {
      "epoch": 0.5268996117581808,
      "grad_norm": 1.2301034927368164,
      "learning_rate": 2.707972205098576e-07,
      "logits/chosen": -0.9842405319213867,
      "logits/rejected": -1.0120937824249268,
      "logps/chosen": -438.3604431152344,
      "logps/rejected": -427.14764404296875,
      "loss": 0.0411,
      "loss/chosen-sft": 1.813296914100647,
      "loss/dpo": 0.041075654327869415,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -1.2374821901321411,
      "rewards/margins": 0.6461740732192993,
      "rewards/rejected": -1.8836562633514404,
      "step": 950
    },
    {
      "epoch": 0.5296727676095396,
      "grad_norm": 1.549978256225586,
      "learning_rate": 2.68383595518374e-07,
      "logits/chosen": -0.8700895309448242,
      "logits/rejected": -0.8361631631851196,
      "logps/chosen": -442.2728576660156,
      "logps/rejected": -430.29986572265625,
      "loss": 0.0374,
      "loss/chosen-sft": 1.7138087749481201,
      "loss/dpo": 0.03742986172437668,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.300048589706421,
      "rewards/margins": 0.5438514947891235,
      "rewards/rejected": -1.8438999652862549,
      "step": 955
    },
    {
      "epoch": 0.5324459234608985,
      "grad_norm": 1.7318216562271118,
      "learning_rate": 2.659682464163138e-07,
      "logits/chosen": -1.0281853675842285,
      "logits/rejected": -1.0226585865020752,
      "logps/chosen": -469.50238037109375,
      "logps/rejected": -455.428955078125,
      "loss": 0.0578,
      "loss/chosen-sft": 1.7266900539398193,
      "loss/dpo": 0.057835765182971954,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -1.2760839462280273,
      "rewards/margins": 0.5891796946525574,
      "rewards/rejected": -1.8652637004852295,
      "step": 960
    },
    {
      "epoch": 0.5352190793122573,
      "grad_norm": 1.1333941221237183,
      "learning_rate": 2.6355139972785885e-07,
      "logits/chosen": -0.9396898150444031,
      "logits/rejected": -0.849763035774231,
      "logps/chosen": -423.746337890625,
      "logps/rejected": -441.26837158203125,
      "loss": 0.0365,
      "loss/chosen-sft": 1.7641470432281494,
      "loss/dpo": 0.03645756468176842,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -1.1291122436523438,
      "rewards/margins": 0.7151139378547668,
      "rewards/rejected": -1.8442262411117554,
      "step": 965
    },
    {
      "epoch": 0.5379922351636162,
      "grad_norm": 1.0045897960662842,
      "learning_rate": 2.6113328211764235e-07,
      "logits/chosen": -1.0510257482528687,
      "logits/rejected": -1.082379698753357,
      "logps/chosen": -449.94561767578125,
      "logps/rejected": -432.62933349609375,
      "loss": 0.0391,
      "loss/chosen-sft": 1.7299797534942627,
      "loss/dpo": 0.039098747074604034,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -1.1797751188278198,
      "rewards/margins": 0.6614251136779785,
      "rewards/rejected": -1.8412002325057983,
      "step": 970
    },
    {
      "epoch": 0.540765391014975,
      "grad_norm": 1.4864047765731812,
      "learning_rate": 2.5871412036949153e-07,
      "logits/chosen": -0.956852912902832,
      "logits/rejected": -0.8994203805923462,
      "logps/chosen": -428.5096130371094,
      "logps/rejected": -419.07330322265625,
      "loss": 0.0463,
      "loss/chosen-sft": 1.774070143699646,
      "loss/dpo": 0.04627335071563721,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -1.2709102630615234,
      "rewards/margins": 0.5661468505859375,
      "rewards/rejected": -1.837057113647461,
      "step": 975
    },
    {
      "epoch": 0.5435385468663338,
      "grad_norm": 1.1833720207214355,
      "learning_rate": 2.5629414136515825e-07,
      "logits/chosen": -0.7828256487846375,
      "logits/rejected": -0.6822995543479919,
      "logps/chosen": -444.83697509765625,
      "logps/rejected": -446.899658203125,
      "loss": 0.0362,
      "loss/chosen-sft": 1.8193594217300415,
      "loss/dpo": 0.03620842099189758,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.2841075658798218,
      "rewards/margins": 0.670619547367096,
      "rewards/rejected": -1.9547271728515625,
      "step": 980
    },
    {
      "epoch": 0.5463117027176927,
      "grad_norm": 1.4741407632827759,
      "learning_rate": 2.5387357206304077e-07,
      "logits/chosen": -0.9925743341445923,
      "logits/rejected": -0.7799035310745239,
      "logps/chosen": -441.3125,
      "logps/rejected": -427.43115234375,
      "loss": 0.0341,
      "loss/chosen-sft": 1.7645461559295654,
      "loss/dpo": 0.034115031361579895,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -1.332214117050171,
      "rewards/margins": 0.5084002017974854,
      "rewards/rejected": -1.8406140804290771,
      "step": 985
    },
    {
      "epoch": 0.5490848585690515,
      "grad_norm": 2.5760786533355713,
      "learning_rate": 2.514526394768989e-07,
      "logits/chosen": -0.8460060358047485,
      "logits/rejected": -0.8547107577323914,
      "logps/chosen": -451.8817443847656,
      "logps/rejected": -449.7991638183594,
      "loss": 0.0474,
      "loss/chosen-sft": 1.7884471416473389,
      "loss/dpo": 0.04735780879855156,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.3389358520507812,
      "rewards/margins": 0.6204281449317932,
      "rewards/rejected": -1.9593639373779297,
      "step": 990
    },
    {
      "epoch": 0.5518580144204104,
      "grad_norm": 1.9366549253463745,
      "learning_rate": 2.490315706545631e-07,
      "logits/chosen": -0.8773177862167358,
      "logits/rejected": -0.9315664172172546,
      "logps/chosen": -423.4974670410156,
      "logps/rejected": -398.5281066894531,
      "loss": 0.0402,
      "loss/chosen-sft": 1.7923309803009033,
      "loss/dpo": 0.04023148491978645,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -1.3467055559158325,
      "rewards/margins": 0.4263473451137543,
      "rewards/rejected": -1.7730529308319092,
      "step": 995
    },
    {
      "epoch": 0.5546311702717692,
      "grad_norm": 1.0191140174865723,
      "learning_rate": 2.466105926566405e-07,
      "logits/chosen": -0.8884904980659485,
      "logits/rejected": -0.8517130017280579,
      "logps/chosen": -412.35595703125,
      "logps/rejected": -398.6607360839844,
      "loss": 0.0425,
      "loss/chosen-sft": 1.7367550134658813,
      "loss/dpo": 0.0425165630877018,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -1.207376480102539,
      "rewards/margins": 0.44255828857421875,
      "rewards/rejected": -1.6499347686767578,
      "step": 1000
    },
    {
      "epoch": 0.5574043261231281,
      "grad_norm": 1.5985798835754395,
      "learning_rate": 2.441899325352205e-07,
      "logits/chosen": -0.8503344655036926,
      "logits/rejected": -0.7258619070053101,
      "logps/chosen": -405.3533630371094,
      "logps/rejected": -423.4427795410156,
      "loss": 0.038,
      "loss/chosen-sft": 1.7873739004135132,
      "loss/dpo": 0.03796621412038803,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -1.2806193828582764,
      "rewards/margins": 0.5564595460891724,
      "rewards/rejected": -1.8370788097381592,
      "step": 1005
    },
    {
      "epoch": 0.5601774819744869,
      "grad_norm": 2.0438764095306396,
      "learning_rate": 2.417698173125804e-07,
      "logits/chosen": -0.8835655450820923,
      "logits/rejected": -0.8549100160598755,
      "logps/chosen": -443.4007263183594,
      "logps/rejected": -412.9955139160156,
      "loss": 0.0476,
      "loss/chosen-sft": 1.781747817993164,
      "loss/dpo": 0.04762732982635498,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.2374616861343384,
      "rewards/margins": 0.514583945274353,
      "rewards/rejected": -1.7520456314086914,
      "step": 1010
    },
    {
      "epoch": 0.5629506378258459,
      "grad_norm": 2.029576539993286,
      "learning_rate": 2.393504739598938e-07,
      "logits/chosen": -0.8470139503479004,
      "logits/rejected": -0.8256238102912903,
      "logps/chosen": -434.9107971191406,
      "logps/rejected": -436.6976623535156,
      "loss": 0.047,
      "loss/chosen-sft": 1.7188364267349243,
      "loss/dpo": 0.04702724516391754,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.2836484909057617,
      "rewards/margins": 0.6734278202056885,
      "rewards/rejected": -1.9570764303207397,
      "step": 1015
    },
    {
      "epoch": 0.5657237936772047,
      "grad_norm": 1.5274903774261475,
      "learning_rate": 2.3693212937594436e-07,
      "logits/chosen": -0.6356366872787476,
      "logits/rejected": -0.5753142237663269,
      "logps/chosen": -411.853271484375,
      "logps/rejected": -436.2920837402344,
      "loss": 0.0407,
      "loss/chosen-sft": 1.8935306072235107,
      "loss/dpo": 0.040696293115615845,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.258502721786499,
      "rewards/margins": 0.4864071011543274,
      "rewards/rejected": -1.7449098825454712,
      "step": 1020
    },
    {
      "epoch": 0.5684969495285636,
      "grad_norm": 1.1039131879806519,
      "learning_rate": 2.3451501036584604e-07,
      "logits/chosen": -0.7690117359161377,
      "logits/rejected": -0.7704351544380188,
      "logps/chosen": -429.619140625,
      "logps/rejected": -413.0948791503906,
      "loss": 0.0304,
      "loss/chosen-sft": 1.7215017080307007,
      "loss/dpo": 0.030385727062821388,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -1.2474838495254517,
      "rewards/margins": 0.6526226997375488,
      "rewards/rejected": -1.900106430053711,
      "step": 1025
    },
    {
      "epoch": 0.5712701053799224,
      "grad_norm": 2.5751590728759766,
      "learning_rate": 2.3209934361977194e-07,
      "logits/chosen": -0.9362251162528992,
      "logits/rejected": -0.8153011202812195,
      "logps/chosen": -431.95391845703125,
      "logps/rejected": -430.9054260253906,
      "loss": 0.0599,
      "loss/chosen-sft": 1.7196757793426514,
      "loss/dpo": 0.05986684560775757,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -1.2645565271377563,
      "rewards/margins": 0.46636122465133667,
      "rewards/rejected": -1.7309175729751587,
      "step": 1030
    },
    {
      "epoch": 0.5740432612312812,
      "grad_norm": 1.936566710472107,
      "learning_rate": 2.296853556916941e-07,
      "logits/chosen": -0.6564992070198059,
      "logits/rejected": -0.532072126865387,
      "logps/chosen": -406.83099365234375,
      "logps/rejected": -410.1024475097656,
      "loss": 0.0359,
      "loss/chosen-sft": 1.8352649211883545,
      "loss/dpo": 0.03590982407331467,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -1.2317606210708618,
      "rewards/margins": 0.5972455739974976,
      "rewards/rejected": -1.8290061950683594,
      "step": 1035
    },
    {
      "epoch": 0.5768164170826401,
      "grad_norm": 1.4029645919799805,
      "learning_rate": 2.2727327297813613e-07,
      "logits/chosen": -0.9582229852676392,
      "logits/rejected": -0.9992238283157349,
      "logps/chosen": -444.89752197265625,
      "logps/rejected": -424.8453674316406,
      "loss": 0.0355,
      "loss/chosen-sft": 1.804992914199829,
      "loss/dpo": 0.03546611964702606,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -1.2539708614349365,
      "rewards/margins": 0.5192979574203491,
      "rewards/rejected": -1.7732689380645752,
      "step": 1040
    },
    {
      "epoch": 0.5795895729339989,
      "grad_norm": 1.8364492654800415,
      "learning_rate": 2.2486332169694095e-07,
      "logits/chosen": -0.7399333715438843,
      "logits/rejected": -0.7711300253868103,
      "logps/chosen": -425.58721923828125,
      "logps/rejected": -407.39349365234375,
      "loss": 0.0461,
      "loss/chosen-sft": 1.7470451593399048,
      "loss/dpo": 0.046109430491924286,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -1.3421345949172974,
      "rewards/margins": 0.5037655830383301,
      "rewards/rejected": -1.8459001779556274,
      "step": 1045
    },
    {
      "epoch": 0.5823627287853578,
      "grad_norm": 1.4886951446533203,
      "learning_rate": 2.224557278660539e-07,
      "logits/chosen": -0.7063448429107666,
      "logits/rejected": -0.7334424257278442,
      "logps/chosen": -429.91693115234375,
      "logps/rejected": -450.59893798828125,
      "loss": 0.0326,
      "loss/chosen-sft": 1.7843490839004517,
      "loss/dpo": 0.03259655088186264,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -1.2502191066741943,
      "rewards/margins": 0.8139023780822754,
      "rewards/rejected": -2.064121723175049,
      "step": 1050
    },
    {
      "epoch": 0.5851358846367166,
      "grad_norm": 1.7785381078720093,
      "learning_rate": 2.200507172823268e-07,
      "logits/chosen": -0.822382926940918,
      "logits/rejected": -0.698542058467865,
      "logps/chosen": -440.42431640625,
      "logps/rejected": -438.05169677734375,
      "loss": 0.0392,
      "loss/chosen-sft": 1.7739006280899048,
      "loss/dpo": 0.0392305888235569,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -1.3763277530670166,
      "rewards/margins": 0.5221567153930664,
      "rewards/rejected": -1.8984845876693726,
      "step": 1055
    },
    {
      "epoch": 0.5879090404880755,
      "grad_norm": 1.6011799573898315,
      "learning_rate": 2.176485155003405e-07,
      "logits/chosen": -0.7036235928535461,
      "logits/rejected": -0.6139532327651978,
      "logps/chosen": -427.38519287109375,
      "logps/rejected": -418.2265625,
      "loss": 0.0372,
      "loss/chosen-sft": 1.8186620473861694,
      "loss/dpo": 0.03721202537417412,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -1.3542938232421875,
      "rewards/margins": 0.4948766827583313,
      "rewards/rejected": -1.8491706848144531,
      "step": 1060
    },
    {
      "epoch": 0.5906821963394343,
      "grad_norm": 1.0207229852676392,
      "learning_rate": 2.1524934781125164e-07,
      "logits/chosen": -0.977032482624054,
      "logits/rejected": -0.9711772799491882,
      "logps/chosen": -445.39263916015625,
      "logps/rejected": -431.14007568359375,
      "loss": 0.0468,
      "loss/chosen-sft": 1.7722899913787842,
      "loss/dpo": 0.04676477983593941,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.318066120147705,
      "rewards/margins": 0.6112589836120605,
      "rewards/rejected": -1.9293251037597656,
      "step": 1065
    },
    {
      "epoch": 0.5934553521907932,
      "grad_norm": 1.426311731338501,
      "learning_rate": 2.1285343922166393e-07,
      "logits/chosen": -0.8198713064193726,
      "logits/rejected": -0.7067451477050781,
      "logps/chosen": -472.1852111816406,
      "logps/rejected": -444.33935546875,
      "loss": 0.0446,
      "loss/chosen-sft": 1.7742096185684204,
      "loss/dpo": 0.04458652064204216,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -1.3653810024261475,
      "rewards/margins": 0.4912968575954437,
      "rewards/rejected": -1.8566780090332031,
      "step": 1070
    },
    {
      "epoch": 0.596228508042152,
      "grad_norm": 1.9216793775558472,
      "learning_rate": 2.104610144325252e-07,
      "logits/chosen": -0.9030396342277527,
      "logits/rejected": -0.9881137609481812,
      "logps/chosen": -442.3429260253906,
      "logps/rejected": -423.65838623046875,
      "loss": 0.0469,
      "loss/chosen-sft": 1.7527765035629272,
      "loss/dpo": 0.0468582846224308,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.2682268619537354,
      "rewards/margins": 0.5394538640975952,
      "rewards/rejected": -1.8076804876327515,
      "step": 1075
    },
    {
      "epoch": 0.5990016638935108,
      "grad_norm": 1.1101206541061401,
      "learning_rate": 2.0807229781805415e-07,
      "logits/chosen": -0.9745734930038452,
      "logits/rejected": -0.7166577577590942,
      "logps/chosen": -407.92266845703125,
      "logps/rejected": -429.6647033691406,
      "loss": 0.0472,
      "loss/chosen-sft": 1.707130789756775,
      "loss/dpo": 0.04721622169017792,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -1.232958197593689,
      "rewards/margins": 0.6351754069328308,
      "rewards/rejected": -1.8681339025497437,
      "step": 1080
    },
    {
      "epoch": 0.6017748197448697,
      "grad_norm": 0.9766126871109009,
      "learning_rate": 2.056875134046976e-07,
      "logits/chosen": -0.772416889667511,
      "logits/rejected": -0.6425715684890747,
      "logps/chosen": -451.41204833984375,
      "logps/rejected": -415.15338134765625,
      "loss": 0.0364,
      "loss/chosen-sft": 1.8030576705932617,
      "loss/dpo": 0.036396466195583344,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -1.2050635814666748,
      "rewards/margins": 0.5893860459327698,
      "rewards/rejected": -1.7944495677947998,
      "step": 1085
    },
    {
      "epoch": 0.6045479755962285,
      "grad_norm": 1.6555626392364502,
      "learning_rate": 2.0330688485011926e-07,
      "logits/chosen": -0.8580873608589172,
      "logits/rejected": -0.8450067639350891,
      "logps/chosen": -426.8406677246094,
      "logps/rejected": -436.485595703125,
      "loss": 0.0461,
      "loss/chosen-sft": 1.7769439220428467,
      "loss/dpo": 0.04613155499100685,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.2692389488220215,
      "rewards/margins": 0.5166956186294556,
      "rewards/rejected": -1.7859344482421875,
      "step": 1090
    },
    {
      "epoch": 0.6073211314475874,
      "grad_norm": 1.8412526845932007,
      "learning_rate": 2.0093063542222508e-07,
      "logits/chosen": -0.8168280720710754,
      "logits/rejected": -0.662460446357727,
      "logps/chosen": -419.52154541015625,
      "logps/rejected": -474.30926513671875,
      "loss": 0.037,
      "loss/chosen-sft": 1.7386878728866577,
      "loss/dpo": 0.03695227950811386,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.2162078619003296,
      "rewards/margins": 0.7796363830566406,
      "rewards/rejected": -1.9958442449569702,
      "step": 1095
    },
    {
      "epoch": 0.6100942872989462,
      "grad_norm": 0.9794536232948303,
      "learning_rate": 1.9855898797822295e-07,
      "logits/chosen": -0.6690505743026733,
      "logits/rejected": -0.6739727258682251,
      "logps/chosen": -416.6888732910156,
      "logps/rejected": -418.7158203125,
      "loss": 0.0298,
      "loss/chosen-sft": 1.8950073719024658,
      "loss/dpo": 0.029840370640158653,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.3056199550628662,
      "rewards/margins": 0.519034743309021,
      "rewards/rejected": -1.8246548175811768,
      "step": 1100
    },
    {
      "epoch": 0.6128674431503051,
      "grad_norm": 1.5625451803207397,
      "learning_rate": 1.9619216494372258e-07,
      "logits/chosen": -0.8960970044136047,
      "logits/rejected": -0.851621150970459,
      "logps/chosen": -416.8877868652344,
      "logps/rejected": -431.9337463378906,
      "loss": 0.0397,
      "loss/chosen-sft": 1.755854606628418,
      "loss/dpo": 0.039700526744127274,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -1.3231550455093384,
      "rewards/margins": 0.486478716135025,
      "rewards/rejected": -1.8096338510513306,
      "step": 1105
    },
    {
      "epoch": 0.6156405990016639,
      "grad_norm": 0.9809625744819641,
      "learning_rate": 1.9383038829187523e-07,
      "logits/chosen": -0.8659313321113586,
      "logits/rejected": -0.7378997802734375,
      "logps/chosen": -466.25006103515625,
      "logps/rejected": -437.8448181152344,
      "loss": 0.0338,
      "loss/chosen-sft": 1.7783132791519165,
      "loss/dpo": 0.03376453369855881,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.262998342514038,
      "rewards/margins": 0.7884011268615723,
      "rewards/rejected": -2.0513997077941895,
      "step": 1110
    },
    {
      "epoch": 0.6184137548530227,
      "grad_norm": 1.4257228374481201,
      "learning_rate": 1.914738795225556e-07,
      "logits/chosen": -0.8100174069404602,
      "logits/rejected": -0.9278467893600464,
      "logps/chosen": -423.9007263183594,
      "logps/rejected": -412.9117126464844,
      "loss": 0.0436,
      "loss/chosen-sft": 1.7985398769378662,
      "loss/dpo": 0.04363011568784714,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -1.3259937763214111,
      "rewards/margins": 0.5338584184646606,
      "rewards/rejected": -1.8598521947860718,
      "step": 1115
    },
    {
      "epoch": 0.6211869107043816,
      "grad_norm": 1.4828506708145142,
      "learning_rate": 1.8912285964158856e-07,
      "logits/chosen": -0.8464866876602173,
      "logits/rejected": -0.8284673690795898,
      "logps/chosen": -445.93475341796875,
      "logps/rejected": -457.19384765625,
      "loss": 0.0344,
      "loss/chosen-sft": 1.7983877658843994,
      "loss/dpo": 0.03439199924468994,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -1.3726943731307983,
      "rewards/margins": 0.6521696448326111,
      "rewards/rejected": -2.0248641967773438,
      "step": 1120
    },
    {
      "epoch": 0.6239600665557404,
      "grad_norm": 1.2299528121948242,
      "learning_rate": 1.8677754914002231e-07,
      "logits/chosen": -0.9932361841201782,
      "logits/rejected": -1.02671480178833,
      "logps/chosen": -434.72509765625,
      "logps/rejected": -444.66766357421875,
      "loss": 0.0389,
      "loss/chosen-sft": 1.8104565143585205,
      "loss/dpo": 0.038856539875268936,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -1.2957760095596313,
      "rewards/margins": 0.6685920357704163,
      "rewards/rejected": -1.9643681049346924,
      "step": 1125
    },
    {
      "epoch": 0.6267332224070993,
      "grad_norm": 1.7034136056900024,
      "learning_rate": 1.8443816797344896e-07,
      "logits/chosen": -0.8144285082817078,
      "logits/rejected": -0.6793054342269897,
      "logps/chosen": -461.71258544921875,
      "logps/rejected": -455.9671325683594,
      "loss": 0.0474,
      "loss/chosen-sft": 1.8361161947250366,
      "loss/dpo": 0.047358639538288116,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.4464315176010132,
      "rewards/margins": 0.4728009104728699,
      "rewards/rejected": -1.9192323684692383,
      "step": 1130
    },
    {
      "epoch": 0.6295063782584581,
      "grad_norm": 1.4374454021453857,
      "learning_rate": 1.821049355413767e-07,
      "logits/chosen": -0.8650090098381042,
      "logits/rejected": -0.6468401551246643,
      "logps/chosen": -443.62274169921875,
      "logps/rejected": -443.31427001953125,
      "loss": 0.0332,
      "loss/chosen-sft": 1.8839133977890015,
      "loss/dpo": 0.03318975493311882,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -1.5003621578216553,
      "rewards/margins": 0.500784158706665,
      "rewards/rejected": -2.0011465549468994,
      "step": 1135
    },
    {
      "epoch": 0.632279534109817,
      "grad_norm": 1.7502909898757935,
      "learning_rate": 1.7977807066665267e-07,
      "logits/chosen": -0.9573124647140503,
      "logits/rejected": -0.8072667121887207,
      "logps/chosen": -418.649658203125,
      "logps/rejected": -418.47589111328125,
      "loss": 0.0452,
      "loss/chosen-sft": 1.7528860569000244,
      "loss/dpo": 0.045168764889240265,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -1.3320482969284058,
      "rewards/margins": 0.5464748740196228,
      "rewards/rejected": -1.8785232305526733,
      "step": 1140
    },
    {
      "epoch": 0.6350526899611758,
      "grad_norm": 1.0295249223709106,
      "learning_rate": 1.7745779157494096e-07,
      "logits/chosen": -0.8213936686515808,
      "logits/rejected": -0.7543411254882812,
      "logps/chosen": -437.8804626464844,
      "logps/rejected": -436.56982421875,
      "loss": 0.0303,
      "loss/chosen-sft": 1.7841157913208008,
      "loss/dpo": 0.030281806364655495,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -1.393178939819336,
      "rewards/margins": 0.679972231388092,
      "rewards/rejected": -2.073151111602783,
      "step": 1145
    },
    {
      "epoch": 0.6378258458125347,
      "grad_norm": 1.2215420007705688,
      "learning_rate": 1.7514431587425622e-07,
      "logits/chosen": -0.7637904286384583,
      "logits/rejected": -0.8227758407592773,
      "logps/chosen": -429.562255859375,
      "logps/rejected": -432.89971923828125,
      "loss": 0.0361,
      "loss/chosen-sft": 1.8592332601547241,
      "loss/dpo": 0.03613410145044327,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -1.4023336172103882,
      "rewards/margins": 0.5858246684074402,
      "rewards/rejected": -1.9881582260131836,
      "step": 1150
    },
    {
      "epoch": 0.6405990016638935,
      "grad_norm": 1.6888597011566162,
      "learning_rate": 1.728378605345553e-07,
      "logits/chosen": -0.7143627405166626,
      "logits/rejected": -0.7377561330795288,
      "logps/chosen": -465.07403564453125,
      "logps/rejected": -435.6494140625,
      "loss": 0.0425,
      "loss/chosen-sft": 1.8083655834197998,
      "loss/dpo": 0.04253358393907547,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.460288166999817,
      "rewards/margins": 0.5263352990150452,
      "rewards/rejected": -1.9866234064102173,
      "step": 1155
    },
    {
      "epoch": 0.6433721575152523,
      "grad_norm": 1.5432595014572144,
      "learning_rate": 1.705386418673882e-07,
      "logits/chosen": -0.8244765996932983,
      "logits/rejected": -0.7667987942695618,
      "logps/chosen": -437.60687255859375,
      "logps/rejected": -436.830078125,
      "loss": 0.0467,
      "loss/chosen-sft": 1.7204382419586182,
      "loss/dpo": 0.04673684015870094,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -1.34583580493927,
      "rewards/margins": 0.5071083903312683,
      "rewards/rejected": -1.8529441356658936,
      "step": 1160
    },
    {
      "epoch": 0.6461453133666112,
      "grad_norm": 1.2096738815307617,
      "learning_rate": 1.6824687550561208e-07,
      "logits/chosen": -0.7485244274139404,
      "logits/rejected": -0.7693239450454712,
      "logps/chosen": -464.8460388183594,
      "logps/rejected": -450.24713134765625,
      "loss": 0.0358,
      "loss/chosen-sft": 1.8007644414901733,
      "loss/dpo": 0.03579232841730118,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -1.335431694984436,
      "rewards/margins": 0.6632755994796753,
      "rewards/rejected": -1.9987071752548218,
      "step": 1165
    },
    {
      "epoch": 0.64891846921797,
      "grad_norm": 1.1814310550689697,
      "learning_rate": 1.659627763831671e-07,
      "logits/chosen": -0.7120985984802246,
      "logits/rejected": -0.6295477747917175,
      "logps/chosen": -436.17083740234375,
      "logps/rejected": -424.6572265625,
      "loss": 0.0357,
      "loss/chosen-sft": 1.8579403162002563,
      "loss/dpo": 0.035725630819797516,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -1.285359263420105,
      "rewards/margins": 0.5941758155822754,
      "rewards/rejected": -1.8795350790023804,
      "step": 1170
    },
    {
      "epoch": 0.6516916250693289,
      "grad_norm": 1.1809978485107422,
      "learning_rate": 1.6368655871491975e-07,
      "logits/chosen": -0.7668045163154602,
      "logits/rejected": -0.7772132754325867,
      "logps/chosen": -462.0221252441406,
      "logps/rejected": -435.96954345703125,
      "loss": 0.0424,
      "loss/chosen-sft": 1.831925630569458,
      "loss/dpo": 0.04238344356417656,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.3251338005065918,
      "rewards/margins": 0.5784658193588257,
      "rewards/rejected": -1.903599500656128,
      "step": 1175
    },
    {
      "epoch": 0.6544647809206877,
      "grad_norm": 1.225167155265808,
      "learning_rate": 1.6141843597657172e-07,
      "logits/chosen": -0.8141440153121948,
      "logits/rejected": -0.7542480230331421,
      "logps/chosen": -421.45635986328125,
      "logps/rejected": -441.95025634765625,
      "loss": 0.0411,
      "loss/chosen-sft": 1.7617318630218506,
      "loss/dpo": 0.04108492285013199,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -1.3422725200653076,
      "rewards/margins": 0.5261791944503784,
      "rewards/rejected": -1.8684518337249756,
      "step": 1180
    },
    {
      "epoch": 0.6572379367720466,
      "grad_norm": 1.3029569387435913,
      "learning_rate": 1.5915862088463968e-07,
      "logits/chosen": -0.996296226978302,
      "logits/rejected": -0.9346402287483215,
      "logps/chosen": -446.56622314453125,
      "logps/rejected": -449.5144958496094,
      "loss": 0.0419,
      "loss/chosen-sft": 1.6928361654281616,
      "loss/dpo": 0.041938044130802155,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -1.3604621887207031,
      "rewards/margins": 0.7254354357719421,
      "rewards/rejected": -2.085897922515869,
      "step": 1185
    },
    {
      "epoch": 0.6600110926234054,
      "grad_norm": 1.26889967918396,
      "learning_rate": 1.5690732537650546e-07,
      "logits/chosen": -0.7706629037857056,
      "logits/rejected": -0.7561334371566772,
      "logps/chosen": -435.36602783203125,
      "logps/rejected": -470.29901123046875,
      "loss": 0.0294,
      "loss/chosen-sft": 1.8877112865447998,
      "loss/dpo": 0.029442256316542625,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -1.4551079273223877,
      "rewards/margins": 0.6810146570205688,
      "rewards/rejected": -2.136122703552246,
      "step": 1190
    },
    {
      "epoch": 0.6627842484747642,
      "grad_norm": 1.2422409057617188,
      "learning_rate": 1.546647605905393e-07,
      "logits/chosen": -0.8117157816886902,
      "logits/rejected": -0.7619314193725586,
      "logps/chosen": -432.6480407714844,
      "logps/rejected": -441.0057678222656,
      "loss": 0.0352,
      "loss/chosen-sft": 1.8744285106658936,
      "loss/dpo": 0.035161614418029785,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -1.305821180343628,
      "rewards/margins": 0.5970739722251892,
      "rewards/rejected": -1.9028953313827515,
      "step": 1195
    },
    {
      "epoch": 0.6655574043261231,
      "grad_norm": 0.8602780699729919,
      "learning_rate": 1.52431136846298e-07,
      "logits/chosen": -0.9312012791633606,
      "logits/rejected": -0.7855014204978943,
      "logps/chosen": -424.40911865234375,
      "logps/rejected": -425.8111877441406,
      "loss": 0.0254,
      "loss/chosen-sft": 1.8612302541732788,
      "loss/dpo": 0.025435030460357666,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -1.3491990566253662,
      "rewards/margins": 0.6118738055229187,
      "rewards/rejected": -1.9610726833343506,
      "step": 1200
    },
    {
      "epoch": 0.6683305601774819,
      "grad_norm": 1.010581135749817,
      "learning_rate": 1.5020666362480084e-07,
      "logits/chosen": -0.8483907580375671,
      "logits/rejected": -0.7495467066764832,
      "logps/chosen": -495.5743103027344,
      "logps/rejected": -482.33209228515625,
      "loss": 0.0387,
      "loss/chosen-sft": 1.8878498077392578,
      "loss/dpo": 0.03866366669535637,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -1.4326591491699219,
      "rewards/margins": 0.7254186868667603,
      "rewards/rejected": -2.1580777168273926,
      "step": 1205
    },
    {
      "epoch": 0.6711037160288408,
      "grad_norm": 1.6181249618530273,
      "learning_rate": 1.4799154954888222e-07,
      "logits/chosen": -1.0427324771881104,
      "logits/rejected": -0.809046745300293,
      "logps/chosen": -415.38909912109375,
      "logps/rejected": -420.14453125,
      "loss": 0.0316,
      "loss/chosen-sft": 1.8586212396621704,
      "loss/dpo": 0.031649235635995865,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -1.452163577079773,
      "rewards/margins": 0.537935197353363,
      "rewards/rejected": -1.9900987148284912,
      "step": 1210
    },
    {
      "epoch": 0.6738768718801996,
      "grad_norm": 0.7974721789360046,
      "learning_rate": 1.4578600236362697e-07,
      "logits/chosen": -0.8521813154220581,
      "logits/rejected": -0.6828973889350891,
      "logps/chosen": -456.5556640625,
      "logps/rejected": -470.51971435546875,
      "loss": 0.0311,
      "loss/chosen-sft": 1.945521593093872,
      "loss/dpo": 0.03109545074403286,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -1.4207074642181396,
      "rewards/margins": 0.7211933732032776,
      "rewards/rejected": -2.1419005393981934,
      "step": 1215
    },
    {
      "epoch": 0.6766500277315585,
      "grad_norm": 1.4059048891067505,
      "learning_rate": 1.435902289168861e-07,
      "logits/chosen": -0.9025594592094421,
      "logits/rejected": -0.835253894329071,
      "logps/chosen": -480.26593017578125,
      "logps/rejected": -467.8526306152344,
      "loss": 0.0337,
      "loss/chosen-sft": 1.9281394481658936,
      "loss/dpo": 0.033659420907497406,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -1.5400809049606323,
      "rewards/margins": 0.5880559086799622,
      "rewards/rejected": -2.1281368732452393,
      "step": 1220
    },
    {
      "epoch": 0.6794231835829173,
      "grad_norm": 1.8963831663131714,
      "learning_rate": 1.4140443513987807e-07,
      "logits/chosen": -1.0080327987670898,
      "logits/rejected": -0.9464845657348633,
      "logps/chosen": -455.85968017578125,
      "logps/rejected": -438.92840576171875,
      "loss": 0.0337,
      "loss/chosen-sft": 1.840531349182129,
      "loss/dpo": 0.033664338290691376,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.5052850246429443,
      "rewards/margins": 0.6105053424835205,
      "rewards/rejected": -2.115790843963623,
      "step": 1225
    },
    {
      "epoch": 0.6821963394342762,
      "grad_norm": 1.1899809837341309,
      "learning_rate": 1.3922882602787523e-07,
      "logits/chosen": -0.7441499829292297,
      "logits/rejected": -0.7358977198600769,
      "logps/chosen": -479.97064208984375,
      "logps/rejected": -476.0712890625,
      "loss": 0.0317,
      "loss/chosen-sft": 1.9372377395629883,
      "loss/dpo": 0.03166855126619339,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -1.5703338384628296,
      "rewards/margins": 0.6448060870170593,
      "rewards/rejected": -2.2151401042938232,
      "step": 1230
    },
    {
      "epoch": 0.684969495285635,
      "grad_norm": 1.3064395189285278,
      "learning_rate": 1.3706360562097797e-07,
      "logits/chosen": -0.9031554460525513,
      "logits/rejected": -0.8757888674736023,
      "logps/chosen": -471.3316345214844,
      "logps/rejected": -446.17120361328125,
      "loss": 0.0316,
      "loss/chosen-sft": 1.8330347537994385,
      "loss/dpo": 0.031595904380083084,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -1.457507848739624,
      "rewards/margins": 0.67474764585495,
      "rewards/rejected": -2.1322555541992188,
      "step": 1235
    },
    {
      "epoch": 0.687742651136994,
      "grad_norm": 1.1494730710983276,
      "learning_rate": 1.3490897698497983e-07,
      "logits/chosen": -0.9943927526473999,
      "logits/rejected": -0.9441978335380554,
      "logps/chosen": -467.54266357421875,
      "logps/rejected": -473.6890563964844,
      "loss": 0.0293,
      "loss/chosen-sft": 1.9200853109359741,
      "loss/dpo": 0.02934952639043331,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -1.563489556312561,
      "rewards/margins": 0.7292519211769104,
      "rewards/rejected": -2.292741298675537,
      "step": 1240
    },
    {
      "epoch": 0.6905158069883528,
      "grad_norm": 1.2559505701065063,
      "learning_rate": 1.3276514219232142e-07,
      "logits/chosen": -0.6624242067337036,
      "logits/rejected": -0.8091727495193481,
      "logps/chosen": -471.83941650390625,
      "logps/rejected": -457.29510498046875,
      "loss": 0.0342,
      "loss/chosen-sft": 1.9265739917755127,
      "loss/dpo": 0.034207794815301895,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -1.4902698993682861,
      "rewards/margins": 0.6235011219978333,
      "rewards/rejected": -2.1137709617614746,
      "step": 1245
    },
    {
      "epoch": 0.6932889628397116,
      "grad_norm": 1.1430951356887817,
      "learning_rate": 1.3063230230314027e-07,
      "logits/chosen": -0.9455582499504089,
      "logits/rejected": -0.8169571757316589,
      "logps/chosen": -444.04193115234375,
      "logps/rejected": -469.19354248046875,
      "loss": 0.0347,
      "loss/chosen-sft": 1.893571138381958,
      "loss/dpo": 0.034717872738838196,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -1.5202562808990479,
      "rewards/margins": 0.6696333289146423,
      "rewards/rejected": -2.189889669418335,
      "step": 1250
    },
    {
      "epoch": 0.6960621186910705,
      "grad_norm": 1.2610682249069214,
      "learning_rate": 1.2851065734641364e-07,
      "logits/chosen": -0.9905312657356262,
      "logits/rejected": -0.8740830421447754,
      "logps/chosen": -422.3661193847656,
      "logps/rejected": -433.2808532714844,
      "loss": 0.0288,
      "loss/chosen-sft": 1.7975590229034424,
      "loss/dpo": 0.028823431581258774,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -1.442055583000183,
      "rewards/margins": 0.5924633741378784,
      "rewards/rejected": -2.0345191955566406,
      "step": 1255
    },
    {
      "epoch": 0.6988352745424293,
      "grad_norm": 0.9623591303825378,
      "learning_rate": 1.2640040630119916e-07,
      "logits/chosen": -0.7741963267326355,
      "logits/rejected": -0.8283836245536804,
      "logps/chosen": -455.494140625,
      "logps/rejected": -467.2837829589844,
      "loss": 0.0324,
      "loss/chosen-sft": 2.0400707721710205,
      "loss/dpo": 0.0323946438729763,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -1.560781717300415,
      "rewards/margins": 0.5312785506248474,
      "rewards/rejected": -2.0920603275299072,
      "step": 1260
    },
    {
      "epoch": 0.7016084303937882,
      "grad_norm": 1.337944746017456,
      "learning_rate": 1.243017470779729e-07,
      "logits/chosen": -1.0406231880187988,
      "logits/rejected": -0.8966327905654907,
      "logps/chosen": -426.70361328125,
      "logps/rejected": -453.76031494140625,
      "loss": 0.0359,
      "loss/chosen-sft": 1.8434947729110718,
      "loss/dpo": 0.03585369139909744,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -1.4888029098510742,
      "rewards/margins": 0.7223333120346069,
      "rewards/rejected": -2.2111363410949707,
      "step": 1265
    },
    {
      "epoch": 0.704381586245147,
      "grad_norm": 0.8494482040405273,
      "learning_rate": 1.222148765000694e-07,
      "logits/chosen": -0.8672950863838196,
      "logits/rejected": -0.8264138102531433,
      "logps/chosen": -444.71246337890625,
      "logps/rejected": -443.8768615722656,
      "loss": 0.0316,
      "loss/chosen-sft": 1.826744794845581,
      "loss/dpo": 0.03159435838460922,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.518831491470337,
      "rewards/margins": 0.6539059281349182,
      "rewards/rejected": -2.1727373600006104,
      "step": 1270
    },
    {
      "epoch": 0.7071547420965059,
      "grad_norm": 1.5342941284179688,
      "learning_rate": 1.2013999028522104e-07,
      "logits/chosen": -0.9066953659057617,
      "logits/rejected": -0.9141793251037598,
      "logps/chosen": -475.5211486816406,
      "logps/rejected": -463.80810546875,
      "loss": 0.0517,
      "loss/chosen-sft": 1.7181308269500732,
      "loss/dpo": 0.05173926800489426,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.5922820568084717,
      "rewards/margins": 0.48251956701278687,
      "rewards/rejected": -2.0748016834259033,
      "step": 1275
    },
    {
      "epoch": 0.7099278979478647,
      "grad_norm": 1.1075633764266968,
      "learning_rate": 1.1807728302720418e-07,
      "logits/chosen": -0.8835026025772095,
      "logits/rejected": -0.8582647442817688,
      "logps/chosen": -450.48870849609375,
      "logps/rejected": -440.99871826171875,
      "loss": 0.0319,
      "loss/chosen-sft": 1.9053627252578735,
      "loss/dpo": 0.03186877816915512,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -1.4612672328948975,
      "rewards/margins": 0.6721949577331543,
      "rewards/rejected": -2.133462429046631,
      "step": 1280
    },
    {
      "epoch": 0.7127010537992235,
      "grad_norm": 2.3948493003845215,
      "learning_rate": 1.1602694817758773e-07,
      "logits/chosen": -0.9616036415100098,
      "logits/rejected": -0.9405485987663269,
      "logps/chosen": -455.7767639160156,
      "logps/rejected": -471.544677734375,
      "loss": 0.0506,
      "loss/chosen-sft": 1.7842986583709717,
      "loss/dpo": 0.0505877360701561,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -1.4995403289794922,
      "rewards/margins": 0.608048141002655,
      "rewards/rejected": -2.107588291168213,
      "step": 1285
    },
    {
      "epoch": 0.7154742096505824,
      "grad_norm": 1.8153194189071655,
      "learning_rate": 1.139891780275912e-07,
      "logits/chosen": -1.0241485834121704,
      "logits/rejected": -0.897274374961853,
      "logps/chosen": -420.4510803222656,
      "logps/rejected": -436.85003662109375,
      "loss": 0.0345,
      "loss/chosen-sft": 1.7239220142364502,
      "loss/dpo": 0.034504033625125885,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -1.3688867092132568,
      "rewards/margins": 0.6113638877868652,
      "rewards/rejected": -1.980250597000122,
      "step": 1290
    },
    {
      "epoch": 0.7182473655019412,
      "grad_norm": 1.2911590337753296,
      "learning_rate": 1.119641636900502e-07,
      "logits/chosen": -0.9305152893066406,
      "logits/rejected": -0.9541549682617188,
      "logps/chosen": -439.82513427734375,
      "logps/rejected": -432.65167236328125,
      "loss": 0.0283,
      "loss/chosen-sft": 1.806671142578125,
      "loss/dpo": 0.02832832559943199,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -1.3666812181472778,
      "rewards/margins": 0.6614538431167603,
      "rewards/rejected": -2.028134822845459,
      "step": 1295
    },
    {
      "epoch": 0.7210205213533001,
      "grad_norm": 1.451220989227295,
      "learning_rate": 1.0995209508149306e-07,
      "logits/chosen": -0.9968698620796204,
      "logits/rejected": -0.9646986722946167,
      "logps/chosen": -460.3720703125,
      "logps/rejected": -457.362548828125,
      "loss": 0.0421,
      "loss/chosen-sft": 1.8072360754013062,
      "loss/dpo": 0.04210533946752548,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -1.4575574398040771,
      "rewards/margins": 0.5370498895645142,
      "rewards/rejected": -1.9946073293685913,
      "step": 1300
    },
    {
      "epoch": 0.7237936772046589,
      "grad_norm": 1.028705358505249,
      "learning_rate": 1.0795316090432893e-07,
      "logits/chosen": -0.7744545936584473,
      "logits/rejected": -0.6203697323799133,
      "logps/chosen": -474.93524169921875,
      "logps/rejected": -458.9364318847656,
      "loss": 0.0387,
      "loss/chosen-sft": 1.8132940530776978,
      "loss/dpo": 0.03869001194834709,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -1.3626388311386108,
      "rewards/margins": 0.6128473281860352,
      "rewards/rejected": -1.975486397743225,
      "step": 1305
    },
    {
      "epoch": 0.7265668330560178,
      "grad_norm": 1.3705142736434937,
      "learning_rate": 1.0596754862915136e-07,
      "logits/chosen": -1.0295013189315796,
      "logits/rejected": -0.9204280972480774,
      "logps/chosen": -434.82366943359375,
      "logps/rejected": -449.77423095703125,
      "loss": 0.0357,
      "loss/chosen-sft": 1.7468935251235962,
      "loss/dpo": 0.035688553005456924,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.2829020023345947,
      "rewards/margins": 0.707922101020813,
      "rewards/rejected": -1.9908241033554077,
      "step": 1310
    },
    {
      "epoch": 0.7293399889073766,
      "grad_norm": 0.8645944595336914,
      "learning_rate": 1.0399544447715494e-07,
      "logits/chosen": -0.797434389591217,
      "logits/rejected": -0.6160916090011597,
      "logps/chosen": -464.4483947753906,
      "logps/rejected": -458.0086364746094,
      "loss": 0.0418,
      "loss/chosen-sft": 1.9083738327026367,
      "loss/dpo": 0.04177068918943405,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.5665171146392822,
      "rewards/margins": 0.468144029378891,
      "rewards/rejected": -2.034661054611206,
      "step": 1315
    },
    {
      "epoch": 0.7321131447587355,
      "grad_norm": 1.3339563608169556,
      "learning_rate": 1.0203703340267192e-07,
      "logits/chosen": -0.8840950727462769,
      "logits/rejected": -0.8916382789611816,
      "logps/chosen": -446.3101501464844,
      "logps/rejected": -463.23077392578125,
      "loss": 0.0404,
      "loss/chosen-sft": 1.7909902334213257,
      "loss/dpo": 0.0403718575835228,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -1.371553659439087,
      "rewards/margins": 0.7114600539207458,
      "rewards/rejected": -2.0830137729644775,
      "step": 1320
    },
    {
      "epoch": 0.7348863006100943,
      "grad_norm": 1.4580516815185547,
      "learning_rate": 1.0009249907582485e-07,
      "logits/chosen": -1.012226939201355,
      "logits/rejected": -0.7340652346611023,
      "logps/chosen": -443.7181701660156,
      "logps/rejected": -456.22320556640625,
      "loss": 0.0308,
      "loss/chosen-sft": 1.804347276687622,
      "loss/dpo": 0.030787784606218338,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -1.5312182903289795,
      "rewards/margins": 0.6057604551315308,
      "rewards/rejected": -2.1369788646698,
      "step": 1325
    },
    {
      "epoch": 0.7376594564614531,
      "grad_norm": 0.9017294645309448,
      "learning_rate": 9.816202386530199e-08,
      "logits/chosen": -0.8607552647590637,
      "logits/rejected": -0.8176922798156738,
      "logps/chosen": -476.98260498046875,
      "logps/rejected": -490.8758239746094,
      "loss": 0.0357,
      "loss/chosen-sft": 1.7674789428710938,
      "loss/dpo": 0.035682059824466705,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -1.4640976190567017,
      "rewards/margins": 0.7083105444908142,
      "rewards/rejected": -2.172408103942871,
      "step": 1330
    },
    {
      "epoch": 0.740432612312812,
      "grad_norm": 1.749300241470337,
      "learning_rate": 9.62457888212535e-08,
      "logits/chosen": -0.9406915903091431,
      "logits/rejected": -0.8014345169067383,
      "logps/chosen": -438.89849853515625,
      "logps/rejected": -451.3016662597656,
      "loss": 0.0358,
      "loss/chosen-sft": 1.861494779586792,
      "loss/dpo": 0.03579792380332947,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.349241018295288,
      "rewards/margins": 0.7064675688743591,
      "rewards/rejected": -2.055708408355713,
      "step": 1335
    },
    {
      "epoch": 0.7432057681641708,
      "grad_norm": 1.16977059841156,
      "learning_rate": 9.434397365831162e-08,
      "logits/chosen": -0.8669630289077759,
      "logits/rejected": -0.8108338117599487,
      "logps/chosen": -437.13848876953125,
      "logps/rejected": -468.7396545410156,
      "loss": 0.0317,
      "loss/chosen-sft": 1.831624984741211,
      "loss/dpo": 0.03170974552631378,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -1.3475593328475952,
      "rewards/margins": 0.8466382026672363,
      "rewards/rejected": -2.194197654724121,
      "step": 1340
    },
    {
      "epoch": 0.7459789240155297,
      "grad_norm": 2.1936497688293457,
      "learning_rate": 9.245675673873577e-08,
      "logits/chosen": -0.9459794163703918,
      "logits/rejected": -0.9377508163452148,
      "logps/chosen": -516.6649780273438,
      "logps/rejected": -491.7989807128906,
      "loss": 0.0465,
      "loss/chosen-sft": 1.8350709676742554,
      "loss/dpo": 0.04651065915822983,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -1.5745521783828735,
      "rewards/margins": 0.6551668047904968,
      "rewards/rejected": -2.2297191619873047,
      "step": 1345
    },
    {
      "epoch": 0.7487520798668885,
      "grad_norm": 1.4931093454360962,
      "learning_rate": 9.058431505568562e-08,
      "logits/chosen": -0.7636462450027466,
      "logits/rejected": -0.8169956207275391,
      "logps/chosen": -464.9522399902344,
      "logps/rejected": -456.91650390625,
      "loss": 0.0312,
      "loss/chosen-sft": 1.9652206897735596,
      "loss/dpo": 0.031192084774374962,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.4719616174697876,
      "rewards/margins": 0.6275067329406738,
      "rewards/rejected": -2.099468469619751,
      "step": 1350
    },
    {
      "epoch": 0.7515252357182474,
      "grad_norm": 1.4099466800689697,
      "learning_rate": 8.872682421662068e-08,
      "logits/chosen": -0.787927508354187,
      "logits/rejected": -0.7053574323654175,
      "logps/chosen": -463.33087158203125,
      "logps/rejected": -486.23675537109375,
      "loss": 0.0261,
      "loss/chosen-sft": 1.898769736289978,
      "loss/dpo": 0.026135969907045364,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -1.4200894832611084,
      "rewards/margins": 0.8304470181465149,
      "rewards/rejected": -2.2505364418029785,
      "step": 1355
    },
    {
      "epoch": 0.7542983915696062,
      "grad_norm": 1.183593511581421,
      "learning_rate": 8.688445842683173e-08,
      "logits/chosen": -0.9655888676643372,
      "logits/rejected": -1.0269839763641357,
      "logps/chosen": -465.0248107910156,
      "logps/rejected": -462.23638916015625,
      "loss": 0.037,
      "loss/chosen-sft": 1.8918052911758423,
      "loss/dpo": 0.036974623799324036,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -1.49183988571167,
      "rewards/margins": 0.5933989882469177,
      "rewards/rejected": -2.0852386951446533,
      "step": 1360
    },
    {
      "epoch": 0.757071547420965,
      "grad_norm": 1.654320240020752,
      "learning_rate": 8.505739047310257e-08,
      "logits/chosen": -0.9955303072929382,
      "logits/rejected": -0.8278782963752747,
      "logps/chosen": -464.4917907714844,
      "logps/rejected": -458.9212951660156,
      "loss": 0.0295,
      "loss/chosen-sft": 1.8392751216888428,
      "loss/dpo": 0.02950271964073181,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -1.4409606456756592,
      "rewards/margins": 0.6720392107963562,
      "rewards/rejected": -2.11299991607666,
      "step": 1365
    },
    {
      "epoch": 0.7598447032723239,
      "grad_norm": 2.021111249923706,
      "learning_rate": 8.324579170750518e-08,
      "logits/chosen": -1.111428141593933,
      "logits/rejected": -1.0972769260406494,
      "logps/chosen": -464.07928466796875,
      "logps/rejected": -461.49462890625,
      "loss": 0.0334,
      "loss/chosen-sft": 1.842139482498169,
      "loss/dpo": 0.03337743133306503,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -1.577466368675232,
      "rewards/margins": 0.6026979684829712,
      "rewards/rejected": -2.1801645755767822,
      "step": 1370
    },
    {
      "epoch": 0.7626178591236827,
      "grad_norm": 2.152736186981201,
      "learning_rate": 8.14498320313296e-08,
      "logits/chosen": -0.8720847964286804,
      "logits/rejected": -0.8584582209587097,
      "logps/chosen": -433.6831970214844,
      "logps/rejected": -444.64703369140625,
      "loss": 0.0365,
      "loss/chosen-sft": 1.9097425937652588,
      "loss/dpo": 0.03647807240486145,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -1.5121186971664429,
      "rewards/margins": 0.6217208504676819,
      "rewards/rejected": -2.1338393688201904,
      "step": 1375
    },
    {
      "epoch": 0.7653910149750416,
      "grad_norm": 1.0486515760421753,
      "learning_rate": 7.966967987914932e-08,
      "logits/chosen": -0.8114673495292664,
      "logits/rejected": -0.8360360264778137,
      "logps/chosen": -451.40081787109375,
      "logps/rejected": -462.27850341796875,
      "loss": 0.0316,
      "loss/chosen-sft": 1.915459394454956,
      "loss/dpo": 0.03155887499451637,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -1.4397701025009155,
      "rewards/margins": 0.7389817237854004,
      "rewards/rejected": -2.1787517070770264,
      "step": 1380
    },
    {
      "epoch": 0.7681641708264004,
      "grad_norm": 1.6471340656280518,
      "learning_rate": 7.7905502203025e-08,
      "logits/chosen": -0.8942529559135437,
      "logits/rejected": -0.7852484583854675,
      "logps/chosen": -430.1280822753906,
      "logps/rejected": -426.33160400390625,
      "loss": 0.0318,
      "loss/chosen-sft": 1.8815257549285889,
      "loss/dpo": 0.03184288740158081,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -1.3281621932983398,
      "rewards/margins": 0.6897465586662292,
      "rewards/rejected": -2.0179085731506348,
      "step": 1385
    },
    {
      "epoch": 0.7709373266777593,
      "grad_norm": 1.7929314374923706,
      "learning_rate": 7.615746445684665e-08,
      "logits/chosen": -1.137683391571045,
      "logits/rejected": -1.0650242567062378,
      "logps/chosen": -483.86932373046875,
      "logps/rejected": -496.49847412109375,
      "loss": 0.0351,
      "loss/chosen-sft": 1.70965576171875,
      "loss/dpo": 0.03509928658604622,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -1.5360954999923706,
      "rewards/margins": 0.8399691581726074,
      "rewards/rejected": -2.3760645389556885,
      "step": 1390
    },
    {
      "epoch": 0.7737104825291181,
      "grad_norm": 1.5007047653198242,
      "learning_rate": 7.442573058081644e-08,
      "logits/chosen": -1.022068738937378,
      "logits/rejected": -0.8677648305892944,
      "logps/chosen": -446.630859375,
      "logps/rejected": -453.5487365722656,
      "loss": 0.0413,
      "loss/chosen-sft": 1.7940679788589478,
      "loss/dpo": 0.04133762791752815,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -1.49905264377594,
      "rewards/margins": 0.5288316011428833,
      "rewards/rejected": -2.0278842449188232,
      "step": 1395
    },
    {
      "epoch": 0.776483638380477,
      "grad_norm": 1.9248064756393433,
      "learning_rate": 7.271046298607365e-08,
      "logits/chosen": -0.8683999180793762,
      "logits/rejected": -0.9180633425712585,
      "logps/chosen": -461.62811279296875,
      "logps/rejected": -475.0498046875,
      "loss": 0.0317,
      "loss/chosen-sft": 1.8975083827972412,
      "loss/dpo": 0.03167451545596123,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -1.472059726715088,
      "rewards/margins": 0.7187052965164185,
      "rewards/rejected": -2.190764904022217,
      "step": 1400
    },
    {
      "epoch": 0.7792567942318358,
      "grad_norm": 1.3342360258102417,
      "learning_rate": 7.101182253946281e-08,
      "logits/chosen": -0.8948928117752075,
      "logits/rejected": -0.6546001434326172,
      "logps/chosen": -442.4454650878906,
      "logps/rejected": -462.285888671875,
      "loss": 0.0349,
      "loss/chosen-sft": 1.9095041751861572,
      "loss/dpo": 0.034947969019412994,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -1.5290088653564453,
      "rewards/margins": 0.6571269631385803,
      "rewards/rejected": -2.186135768890381,
      "step": 1405
    },
    {
      "epoch": 0.7820299500831946,
      "grad_norm": 1.7256712913513184,
      "learning_rate": 6.932996854844658e-08,
      "logits/chosen": -1.0015538930892944,
      "logits/rejected": -0.9933465719223022,
      "logps/chosen": -425.962646484375,
      "logps/rejected": -465.14178466796875,
      "loss": 0.0328,
      "loss/chosen-sft": 1.6965240240097046,
      "loss/dpo": 0.03280189260840416,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -1.3612712621688843,
      "rewards/margins": 0.8677095174789429,
      "rewards/rejected": -2.228980779647827,
      "step": 1410
    },
    {
      "epoch": 0.7848031059345535,
      "grad_norm": 1.550096035003662,
      "learning_rate": 6.766505874616571e-08,
      "logits/chosen": -0.9519731402397156,
      "logits/rejected": -0.9299766421318054,
      "logps/chosen": -441.6609802246094,
      "logps/rejected": -446.9225158691406,
      "loss": 0.0292,
      "loss/chosen-sft": 1.765045166015625,
      "loss/dpo": 0.029176611453294754,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -1.4031174182891846,
      "rewards/margins": 0.6740747094154358,
      "rewards/rejected": -2.0771920680999756,
      "step": 1415
    },
    {
      "epoch": 0.7875762617859123,
      "grad_norm": 1.3679091930389404,
      "learning_rate": 6.601724927664492e-08,
      "logits/chosen": -0.7503564953804016,
      "logits/rejected": -0.7818638682365417,
      "logps/chosen": -428.4849548339844,
      "logps/rejected": -410.2527770996094,
      "loss": 0.0353,
      "loss/chosen-sft": 1.8490569591522217,
      "loss/dpo": 0.03528434783220291,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -1.3704160451889038,
      "rewards/margins": 0.5347402691841125,
      "rewards/rejected": -1.9051563739776611,
      "step": 1420
    },
    {
      "epoch": 0.7903494176372712,
      "grad_norm": 1.4390736818313599,
      "learning_rate": 6.438669468015018e-08,
      "logits/chosen": -1.0825473070144653,
      "logits/rejected": -1.1610548496246338,
      "logps/chosen": -407.60589599609375,
      "logps/rejected": -412.09747314453125,
      "loss": 0.04,
      "loss/chosen-sft": 1.8394687175750732,
      "loss/dpo": 0.04001585766673088,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -1.425935983657837,
      "rewards/margins": 0.44705772399902344,
      "rewards/rejected": -1.8729937076568604,
      "step": 1425
    },
    {
      "epoch": 0.79312257348863,
      "grad_norm": 1.4306334257125854,
      "learning_rate": 6.277354787869385e-08,
      "logits/chosen": -1.0177534818649292,
      "logits/rejected": -0.9789898991584778,
      "logps/chosen": -461.71197509765625,
      "logps/rejected": -433.2428283691406,
      "loss": 0.0298,
      "loss/chosen-sft": 1.7876182794570923,
      "loss/dpo": 0.029793113470077515,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -1.4716134071350098,
      "rewards/margins": 0.5654765367507935,
      "rewards/rejected": -2.0370900630950928,
      "step": 1430
    },
    {
      "epoch": 0.7958957293399889,
      "grad_norm": 1.1099703311920166,
      "learning_rate": 6.117796016169374e-08,
      "logits/chosen": -1.059214472770691,
      "logits/rejected": -1.051519751548767,
      "logps/chosen": -492.03424072265625,
      "logps/rejected": -475.771484375,
      "loss": 0.0285,
      "loss/chosen-sft": 1.937116265296936,
      "loss/dpo": 0.02849132940173149,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -1.6160300970077515,
      "rewards/margins": 0.6453025937080383,
      "rewards/rejected": -2.2613329887390137,
      "step": 1435
    },
    {
      "epoch": 0.7986688851913477,
      "grad_norm": 1.9658957719802856,
      "learning_rate": 5.9600081171784e-08,
      "logits/chosen": -0.9570498466491699,
      "logits/rejected": -0.9356196522712708,
      "logps/chosen": -460.81982421875,
      "logps/rejected": -476.683837890625,
      "loss": 0.0404,
      "loss/chosen-sft": 1.7493022680282593,
      "loss/dpo": 0.04036666080355644,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.3861143589019775,
      "rewards/margins": 0.7235819697380066,
      "rewards/rejected": -2.109696388244629,
      "step": 1440
    },
    {
      "epoch": 0.8014420410427066,
      "grad_norm": 1.9839755296707153,
      "learning_rate": 5.8040058890781035e-08,
      "logits/chosen": -0.8933590054512024,
      "logits/rejected": -0.9177519083023071,
      "logps/chosen": -447.789794921875,
      "logps/rejected": -467.09344482421875,
      "loss": 0.0316,
      "loss/chosen-sft": 1.9308321475982666,
      "loss/dpo": 0.03159358352422714,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -1.4023634195327759,
      "rewards/margins": 0.8167144656181335,
      "rewards/rejected": -2.2190778255462646,
      "step": 1445
    },
    {
      "epoch": 0.8042151968940654,
      "grad_norm": 1.4874529838562012,
      "learning_rate": 5.6498039625804574e-08,
      "logits/chosen": -0.9050949215888977,
      "logits/rejected": -0.8587920069694519,
      "logps/chosen": -458.5428161621094,
      "logps/rejected": -460.24542236328125,
      "loss": 0.0277,
      "loss/chosen-sft": 1.8650929927825928,
      "loss/dpo": 0.027697661891579628,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.4219458103179932,
      "rewards/margins": 0.7902048826217651,
      "rewards/rejected": -2.2121505737304688,
      "step": 1450
    },
    {
      "epoch": 0.8069883527454242,
      "grad_norm": 1.2237356901168823,
      "learning_rate": 5.4974167995556955e-08,
      "logits/chosen": -1.0649144649505615,
      "logits/rejected": -0.894222617149353,
      "logps/chosen": -449.2327575683594,
      "logps/rejected": -476.51300048828125,
      "loss": 0.0296,
      "loss/chosen-sft": 1.8358997106552124,
      "loss/dpo": 0.029581155627965927,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -1.4774913787841797,
      "rewards/margins": 0.7366743087768555,
      "rewards/rejected": -2.2141659259796143,
      "step": 1455
    },
    {
      "epoch": 0.8097615085967831,
      "grad_norm": 1.6183357238769531,
      "learning_rate": 5.346858691675915e-08,
      "logits/chosen": -0.8394588232040405,
      "logits/rejected": -0.8869924545288086,
      "logps/chosen": -450.36273193359375,
      "logps/rejected": -425.9990234375,
      "loss": 0.0354,
      "loss/chosen-sft": 1.9527273178100586,
      "loss/dpo": 0.035379212349653244,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.555987000465393,
      "rewards/margins": 0.40505796670913696,
      "rewards/rejected": -1.9610449075698853,
      "step": 1460
    },
    {
      "epoch": 0.812534664448142,
      "grad_norm": 1.2106329202651978,
      "learning_rate": 5.198143759074813e-08,
      "logits/chosen": -0.8181321024894714,
      "logits/rejected": -0.8268686532974243,
      "logps/chosen": -422.52069091796875,
      "logps/rejected": -439.79437255859375,
      "loss": 0.0218,
      "loss/chosen-sft": 1.9324089288711548,
      "loss/dpo": 0.02178550697863102,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.4317119121551514,
      "rewards/margins": 0.7136107087135315,
      "rewards/rejected": -2.145322322845459,
      "step": 1465
    },
    {
      "epoch": 0.8153078202995009,
      "grad_norm": 2.4801158905029297,
      "learning_rate": 5.051285949023354e-08,
      "logits/chosen": -0.9226022958755493,
      "logits/rejected": -0.8704797625541687,
      "logps/chosen": -494.6982421875,
      "logps/rejected": -470.52734375,
      "loss": 0.0366,
      "loss/chosen-sft": 1.9025272130966187,
      "loss/dpo": 0.036645907908678055,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -1.6271445751190186,
      "rewards/margins": 0.5947997570037842,
      "rewards/rejected": -2.2219443321228027,
      "step": 1470
    },
    {
      "epoch": 0.8180809761508597,
      "grad_norm": 1.554540753364563,
      "learning_rate": 4.906299034621761e-08,
      "logits/chosen": -0.918652355670929,
      "logits/rejected": -0.7777332067489624,
      "logps/chosen": -438.8077697753906,
      "logps/rejected": -469.95928955078125,
      "loss": 0.0358,
      "loss/chosen-sft": 1.8588802814483643,
      "loss/dpo": 0.035846300423145294,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -1.3037010431289673,
      "rewards/margins": 0.817149817943573,
      "rewards/rejected": -2.1208510398864746,
      "step": 1475
    },
    {
      "epoch": 0.8208541320022186,
      "grad_norm": 1.5162110328674316,
      "learning_rate": 4.7631966135077974e-08,
      "logits/chosen": -0.9379565119743347,
      "logits/rejected": -0.8222628831863403,
      "logps/chosen": -460.7021484375,
      "logps/rejected": -467.64739990234375,
      "loss": 0.027,
      "loss/chosen-sft": 1.943677306175232,
      "loss/dpo": 0.026978474110364914,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -1.556178331375122,
      "rewards/margins": 0.6195975542068481,
      "rewards/rejected": -2.1757760047912598,
      "step": 1480
    },
    {
      "epoch": 0.8236272878535774,
      "grad_norm": 1.4927583932876587,
      "learning_rate": 4.621992106581504e-08,
      "logits/chosen": -1.0209442377090454,
      "logits/rejected": -0.7870509028434753,
      "logps/chosen": -411.0484924316406,
      "logps/rejected": -458.4707946777344,
      "loss": 0.0281,
      "loss/chosen-sft": 1.8953702449798584,
      "loss/dpo": 0.02814427576959133,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.4694321155548096,
      "rewards/margins": 0.6849014163017273,
      "rewards/rejected": -2.1543333530426025,
      "step": 1485
    },
    {
      "epoch": 0.8264004437049363,
      "grad_norm": 1.1416542530059814,
      "learning_rate": 4.482698756746506e-08,
      "logits/chosen": -0.8766797184944153,
      "logits/rejected": -0.6899703145027161,
      "logps/chosen": -426.66851806640625,
      "logps/rejected": -439.741455078125,
      "loss": 0.0395,
      "loss/chosen-sft": 1.8772623538970947,
      "loss/dpo": 0.039542876183986664,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -1.4856750965118408,
      "rewards/margins": 0.5127314329147339,
      "rewards/rejected": -1.9984066486358643,
      "step": 1490
    },
    {
      "epoch": 0.8291735995562951,
      "grad_norm": 1.0494132041931152,
      "learning_rate": 4.3453296276680375e-08,
      "logits/chosen": -0.7310534715652466,
      "logits/rejected": -0.7772519588470459,
      "logps/chosen": -459.5819396972656,
      "logps/rejected": -443.21087646484375,
      "loss": 0.0361,
      "loss/chosen-sft": 1.916404366493225,
      "loss/dpo": 0.03613205999135971,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -1.4593141078948975,
      "rewards/margins": 0.5708898305892944,
      "rewards/rejected": -2.0302040576934814,
      "step": 1495
    },
    {
      "epoch": 0.831946755407654,
      "grad_norm": 1.4319831132888794,
      "learning_rate": 4.209897602547768e-08,
      "logits/chosen": -0.8863110542297363,
      "logits/rejected": -0.8555914759635925,
      "logps/chosen": -444.46142578125,
      "logps/rejected": -446.90155029296875,
      "loss": 0.0231,
      "loss/chosen-sft": 1.974479079246521,
      "loss/dpo": 0.023065898567438126,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -1.5314444303512573,
      "rewards/margins": 0.6271489858627319,
      "rewards/rejected": -2.1585934162139893,
      "step": 1500
    },
    {
      "epoch": 0.8347199112590128,
      "grad_norm": 1.5065139532089233,
      "learning_rate": 4.076415382915527e-08,
      "logits/chosen": -0.9296368360519409,
      "logits/rejected": -0.8830710649490356,
      "logps/chosen": -458.8507385253906,
      "logps/rejected": -464.94677734375,
      "loss": 0.0392,
      "loss/chosen-sft": 1.758873701095581,
      "loss/dpo": 0.039175163954496384,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -1.4754732847213745,
      "rewards/margins": 0.8373439908027649,
      "rewards/rejected": -2.312817335128784,
      "step": 1505
    },
    {
      "epoch": 0.8374930671103716,
      "grad_norm": 2.0096213817596436,
      "learning_rate": 3.944895487438102e-08,
      "logits/chosen": -0.8368681073188782,
      "logits/rejected": -0.8462071418762207,
      "logps/chosen": -440.0353088378906,
      "logps/rejected": -455.104736328125,
      "loss": 0.0277,
      "loss/chosen-sft": 1.8118178844451904,
      "loss/dpo": 0.02766679786145687,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -1.4319641590118408,
      "rewards/margins": 0.7568520307540894,
      "rewards/rejected": -2.1888160705566406,
      "step": 1510
    },
    {
      "epoch": 0.8402662229617305,
      "grad_norm": 1.322721242904663,
      "learning_rate": 3.8153502507451727e-08,
      "logits/chosen": -0.8090510368347168,
      "logits/rejected": -0.7667346596717834,
      "logps/chosen": -448.12725830078125,
      "logps/rejected": -465.86297607421875,
      "loss": 0.0352,
      "loss/chosen-sft": 1.8181946277618408,
      "loss/dpo": 0.03516048565506935,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -1.5605225563049316,
      "rewards/margins": 0.6109769940376282,
      "rewards/rejected": -2.171499729156494,
      "step": 1515
    },
    {
      "epoch": 0.8430393788130893,
      "grad_norm": 0.6515668034553528,
      "learning_rate": 3.687791822272493e-08,
      "logits/chosen": -0.8708721995353699,
      "logits/rejected": -0.8009916543960571,
      "logps/chosen": -413.84649658203125,
      "logps/rejected": -422.46844482421875,
      "loss": 0.0289,
      "loss/chosen-sft": 1.8942821025848389,
      "loss/dpo": 0.028922390192747116,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -1.4152357578277588,
      "rewards/margins": 0.6280101537704468,
      "rewards/rejected": -2.043246030807495,
      "step": 1520
    },
    {
      "epoch": 0.8458125346644482,
      "grad_norm": 1.2580747604370117,
      "learning_rate": 3.562232165122461e-08,
      "logits/chosen": -0.9807391166687012,
      "logits/rejected": -0.8759809732437134,
      "logps/chosen": -468.5948791503906,
      "logps/rejected": -469.4264221191406,
      "loss": 0.0312,
      "loss/chosen-sft": 1.8805710077285767,
      "loss/dpo": 0.03115806356072426,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.6106830835342407,
      "rewards/margins": 0.6084949374198914,
      "rewards/rejected": -2.2191781997680664,
      "step": 1525
    },
    {
      "epoch": 0.848585690515807,
      "grad_norm": 1.6682928800582886,
      "learning_rate": 3.438683054942154e-08,
      "logits/chosen": -0.9435014724731445,
      "logits/rejected": -0.9378792643547058,
      "logps/chosen": -497.65216064453125,
      "logps/rejected": -514.5195922851562,
      "loss": 0.0276,
      "loss/chosen-sft": 1.8878179788589478,
      "loss/dpo": 0.02764343097805977,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -1.5859915018081665,
      "rewards/margins": 0.8114764094352722,
      "rewards/rejected": -2.397467851638794,
      "step": 1530
    },
    {
      "epoch": 0.8513588463671659,
      "grad_norm": 1.870586633682251,
      "learning_rate": 3.3171560788189555e-08,
      "logits/chosen": -0.7878917455673218,
      "logits/rejected": -0.8808261752128601,
      "logps/chosen": -448.91168212890625,
      "logps/rejected": -445.90191650390625,
      "loss": 0.03,
      "loss/chosen-sft": 1.9762178659439087,
      "loss/dpo": 0.029997188597917557,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.4799820184707642,
      "rewards/margins": 0.5908954739570618,
      "rewards/rejected": -2.0708775520324707,
      "step": 1535
    },
    {
      "epoch": 0.8541320022185247,
      "grad_norm": 1.2771514654159546,
      "learning_rate": 3.19766263419384e-08,
      "logits/chosen": -0.9804418683052063,
      "logits/rejected": -0.9339153170585632,
      "logps/chosen": -451.52972412109375,
      "logps/rejected": -487.1279296875,
      "loss": 0.0356,
      "loss/chosen-sft": 1.733094573020935,
      "loss/dpo": 0.03555992618203163,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.3957600593566895,
      "rewards/margins": 0.8448952436447144,
      "rewards/rejected": -2.2406554222106934,
      "step": 1540
    },
    {
      "epoch": 0.8569051580698835,
      "grad_norm": 1.825315237045288,
      "learning_rate": 3.080213927792471e-08,
      "logits/chosen": -0.9238882064819336,
      "logits/rejected": -0.9216313362121582,
      "logps/chosen": -449.4173278808594,
      "logps/rejected": -422.06097412109375,
      "loss": 0.0379,
      "loss/chosen-sft": 1.8662887811660767,
      "loss/dpo": 0.037861116230487823,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -1.5062415599822998,
      "rewards/margins": 0.5276464223861694,
      "rewards/rejected": -2.0338878631591797,
      "step": 1545
    },
    {
      "epoch": 0.8596783139212424,
      "grad_norm": 1.1899453401565552,
      "learning_rate": 2.9648209745741838e-08,
      "logits/chosen": -0.9589411020278931,
      "logits/rejected": -0.8292596936225891,
      "logps/chosen": -456.76092529296875,
      "logps/rejected": -475.18206787109375,
      "loss": 0.0399,
      "loss/chosen-sft": 1.9160964488983154,
      "loss/dpo": 0.03985082358121872,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.461181879043579,
      "rewards/margins": 0.6755915880203247,
      "rewards/rejected": -2.1367735862731934,
      "step": 1550
    },
    {
      "epoch": 0.8624514697726012,
      "grad_norm": 0.9635495543479919,
      "learning_rate": 2.8514945966989085e-08,
      "logits/chosen": -1.0600335597991943,
      "logits/rejected": -0.9049699902534485,
      "logps/chosen": -442.63555908203125,
      "logps/rejected": -459.53570556640625,
      "loss": 0.0311,
      "loss/chosen-sft": 1.8001188039779663,
      "loss/dpo": 0.03108590468764305,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -1.4419844150543213,
      "rewards/margins": 0.7059625387191772,
      "rewards/rejected": -2.147946834564209,
      "step": 1555
    },
    {
      "epoch": 0.8652246256239601,
      "grad_norm": 1.15142822265625,
      "learning_rate": 2.7402454225122744e-08,
      "logits/chosen": -1.0275518894195557,
      "logits/rejected": -0.8722109794616699,
      "logps/chosen": -416.63787841796875,
      "logps/rejected": -473.12542724609375,
      "loss": 0.0299,
      "loss/chosen-sft": 1.8142344951629639,
      "loss/dpo": 0.029919322580099106,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -1.4069955348968506,
      "rewards/margins": 0.8083630800247192,
      "rewards/rejected": -2.2153587341308594,
      "step": 1560
    },
    {
      "epoch": 0.8679977814753189,
      "grad_norm": 1.1056207418441772,
      "learning_rate": 2.631083885548749e-08,
      "logits/chosen": -0.9130992889404297,
      "logits/rejected": -0.7697763442993164,
      "logps/chosen": -439.4908752441406,
      "logps/rejected": -480.8926696777344,
      "loss": 0.0255,
      "loss/chosen-sft": 1.9055465459823608,
      "loss/dpo": 0.025480791926383972,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -1.5075581073760986,
      "rewards/margins": 0.8769866824150085,
      "rewards/rejected": -2.384544849395752,
      "step": 1565
    },
    {
      "epoch": 0.8707709373266778,
      "grad_norm": 1.4552932977676392,
      "learning_rate": 2.524020223553208e-08,
      "logits/chosen": -0.8160026669502258,
      "logits/rejected": -0.8994795083999634,
      "logps/chosen": -484.19598388671875,
      "logps/rejected": -469.77947998046875,
      "loss": 0.0308,
      "loss/chosen-sft": 1.9661228656768799,
      "loss/dpo": 0.03077336587011814,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -1.574336051940918,
      "rewards/margins": 0.6738288998603821,
      "rewards/rejected": -2.2481648921966553,
      "step": 1570
    },
    {
      "epoch": 0.8735440931780366,
      "grad_norm": 1.7502833604812622,
      "learning_rate": 2.4190644775207075e-08,
      "logits/chosen": -0.9263548851013184,
      "logits/rejected": -0.8413556814193726,
      "logps/chosen": -437.6075744628906,
      "logps/rejected": -467.7953186035156,
      "loss": 0.0372,
      "loss/chosen-sft": 1.796007513999939,
      "loss/dpo": 0.03721100836992264,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -1.3773536682128906,
      "rewards/margins": 0.7799967527389526,
      "rewards/rejected": -2.157350540161133,
      "step": 1575
    },
    {
      "epoch": 0.8763172490293955,
      "grad_norm": 0.8901669383049011,
      "learning_rate": 2.316226490754844e-08,
      "logits/chosen": -0.9022026062011719,
      "logits/rejected": -0.9546858072280884,
      "logps/chosen": -455.98046875,
      "logps/rejected": -470.28582763671875,
      "loss": 0.0353,
      "loss/chosen-sft": 1.8491671085357666,
      "loss/dpo": 0.035286836326122284,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.4982842206954956,
      "rewards/margins": 0.7641604542732239,
      "rewards/rejected": -2.2624447345733643,
      "step": 1580
    },
    {
      "epoch": 0.8790904048807543,
      "grad_norm": 0.9218027591705322,
      "learning_rate": 2.215515907944576e-08,
      "logits/chosen": -0.9104015231132507,
      "logits/rejected": -0.8896605372428894,
      "logps/chosen": -470.4886169433594,
      "logps/rejected": -467.1653747558594,
      "loss": 0.0378,
      "loss/chosen-sft": 1.9249181747436523,
      "loss/dpo": 0.037777096033096313,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.5565952062606812,
      "rewards/margins": 0.5057957768440247,
      "rewards/rejected": -2.0623910427093506,
      "step": 1585
    },
    {
      "epoch": 0.8818635607321131,
      "grad_norm": 1.4500895738601685,
      "learning_rate": 2.1169421742596923e-08,
      "logits/chosen": -0.8355404734611511,
      "logits/rejected": -0.8188692331314087,
      "logps/chosen": -468.78759765625,
      "logps/rejected": -436.91619873046875,
      "loss": 0.0279,
      "loss/chosen-sft": 1.9327714443206787,
      "loss/dpo": 0.02788035199046135,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -1.53583824634552,
      "rewards/margins": 0.596767783164978,
      "rewards/rejected": -2.132605791091919,
      "step": 1590
    },
    {
      "epoch": 0.884636716583472,
      "grad_norm": 1.5247975587844849,
      "learning_rate": 2.02051453446499e-08,
      "logits/chosen": -0.9076545834541321,
      "logits/rejected": -0.8217021822929382,
      "logps/chosen": -462.9772033691406,
      "logps/rejected": -458.1619567871094,
      "loss": 0.0357,
      "loss/chosen-sft": 1.872582197189331,
      "loss/dpo": 0.035654447972774506,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.4506585597991943,
      "rewards/margins": 0.6560076475143433,
      "rewards/rejected": -2.106666088104248,
      "step": 1595
    },
    {
      "epoch": 0.8874098724348308,
      "grad_norm": 2.1406707763671875,
      "learning_rate": 1.9262420320532768e-08,
      "logits/chosen": -0.9324251413345337,
      "logits/rejected": -0.7913103103637695,
      "logps/chosen": -460.52789306640625,
      "logps/rejected": -489.23614501953125,
      "loss": 0.0312,
      "loss/chosen-sft": 1.8192570209503174,
      "loss/dpo": 0.03123651072382927,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -1.4744288921356201,
      "rewards/margins": 0.7909864187240601,
      "rewards/rejected": -2.2654154300689697,
      "step": 1600
    },
    {
      "epoch": 0.8901830282861897,
      "grad_norm": 2.196462631225586,
      "learning_rate": 1.8341335083971815e-08,
      "logits/chosen": -0.7642068862915039,
      "logits/rejected": -0.7473465800285339,
      "logps/chosen": -469.9163513183594,
      "logps/rejected": -457.0773010253906,
      "loss": 0.0376,
      "loss/chosen-sft": 1.9092369079589844,
      "loss/dpo": 0.037584852427244186,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -1.5675512552261353,
      "rewards/margins": 0.54124915599823,
      "rewards/rejected": -2.1088004112243652,
      "step": 1605
    },
    {
      "epoch": 0.8929561841375485,
      "grad_norm": 1.6157978773117065,
      "learning_rate": 1.7441976019200166e-08,
      "logits/chosen": -1.053679347038269,
      "logits/rejected": -0.9105228185653687,
      "logps/chosen": -452.0389709472656,
      "logps/rejected": -464.1913146972656,
      "loss": 0.0342,
      "loss/chosen-sft": 1.905279517173767,
      "loss/dpo": 0.034248754382133484,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.5297168493270874,
      "rewards/margins": 0.5347718596458435,
      "rewards/rejected": -2.064488649368286,
      "step": 1610
    },
    {
      "epoch": 0.8957293399889074,
      "grad_norm": 1.4368631839752197,
      "learning_rate": 1.6564427472855662e-08,
      "logits/chosen": -0.928158164024353,
      "logits/rejected": -0.781406819820404,
      "logps/chosen": -456.40380859375,
      "logps/rejected": -482.193603515625,
      "loss": 0.0359,
      "loss/chosen-sft": 1.7659343481063843,
      "loss/dpo": 0.03594128414988518,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -1.445878267288208,
      "rewards/margins": 0.774651825428009,
      "rewards/rejected": -2.2205300331115723,
      "step": 1615
    },
    {
      "epoch": 0.8985024958402662,
      "grad_norm": 1.0532777309417725,
      "learning_rate": 1.570877174607088e-08,
      "logits/chosen": -0.9026380777359009,
      "logits/rejected": -0.7336758375167847,
      "logps/chosen": -474.20599365234375,
      "logps/rejected": -458.268798828125,
      "loss": 0.039,
      "loss/chosen-sft": 1.8853695392608643,
      "loss/dpo": 0.03895549476146698,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.5534743070602417,
      "rewards/margins": 0.538535475730896,
      "rewards/rejected": -2.0920097827911377,
      "step": 1620
    },
    {
      "epoch": 0.901275651691625,
      "grad_norm": 1.640979290008545,
      "learning_rate": 1.4875089086754111e-08,
      "logits/chosen": -0.9097537994384766,
      "logits/rejected": -0.68541419506073,
      "logps/chosen": -440.2255859375,
      "logps/rejected": -470.017333984375,
      "loss": 0.0278,
      "loss/chosen-sft": 1.8272202014923096,
      "loss/dpo": 0.027846310287714005,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -1.4420444965362549,
      "rewards/margins": 0.7163407206535339,
      "rewards/rejected": -2.1583850383758545,
      "step": 1625
    },
    {
      "epoch": 0.9040488075429839,
      "grad_norm": 1.3717536926269531,
      "learning_rate": 1.4063457682063573e-08,
      "logits/chosen": -1.025506854057312,
      "logits/rejected": -0.8326213955879211,
      "logps/chosen": -396.4437561035156,
      "logps/rejected": -455.72265625,
      "loss": 0.0323,
      "loss/chosen-sft": 1.8843291997909546,
      "loss/dpo": 0.032339178025722504,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -1.4136439561843872,
      "rewards/margins": 0.7241330146789551,
      "rewards/rejected": -2.137777090072632,
      "step": 1630
    },
    {
      "epoch": 0.9068219633943427,
      "grad_norm": 1.1654001474380493,
      "learning_rate": 1.3273953651074393e-08,
      "logits/chosen": -1.0155668258666992,
      "logits/rejected": -0.751710057258606,
      "logps/chosen": -461.4085998535156,
      "logps/rejected": -451.8213806152344,
      "loss": 0.0251,
      "loss/chosen-sft": 1.8744451999664307,
      "loss/dpo": 0.02511768974363804,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -1.5510212182998657,
      "rewards/margins": 0.5998275876045227,
      "rewards/rejected": -2.150848865509033,
      "step": 1635
    },
    {
      "epoch": 0.9095951192457016,
      "grad_norm": 1.6467472314834595,
      "learning_rate": 1.250665103763987e-08,
      "logits/chosen": -0.8909111022949219,
      "logits/rejected": -0.8106695413589478,
      "logps/chosen": -450.2090759277344,
      "logps/rejected": -483.452880859375,
      "loss": 0.0419,
      "loss/chosen-sft": 1.795325517654419,
      "loss/dpo": 0.04194999486207962,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.475182056427002,
      "rewards/margins": 0.7644132971763611,
      "rewards/rejected": -2.239595413208008,
      "step": 1640
    },
    {
      "epoch": 0.9123682750970604,
      "grad_norm": 1.179222583770752,
      "learning_rate": 1.1761621803447336e-08,
      "logits/chosen": -0.8358624577522278,
      "logits/rejected": -0.7992157936096191,
      "logps/chosen": -449.79736328125,
      "logps/rejected": -441.9453125,
      "loss": 0.0362,
      "loss/chosen-sft": 1.9050792455673218,
      "loss/dpo": 0.03616093099117279,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -1.381279468536377,
      "rewards/margins": 0.6649306416511536,
      "rewards/rejected": -2.0462100505828857,
      "step": 1645
    },
    {
      "epoch": 0.9151414309484193,
      "grad_norm": 1.244621753692627,
      "learning_rate": 1.1038935821268941e-08,
      "logits/chosen": -0.833348274230957,
      "logits/rejected": -0.6609139442443848,
      "logps/chosen": -409.50445556640625,
      "logps/rejected": -461.4767150878906,
      "loss": 0.0329,
      "loss/chosen-sft": 1.8607597351074219,
      "loss/dpo": 0.032938309013843536,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -1.4654487371444702,
      "rewards/margins": 0.6329339146614075,
      "rewards/rejected": -2.0983824729919434,
      "step": 1650
    },
    {
      "epoch": 0.9179145867997781,
      "grad_norm": 1.552565336227417,
      "learning_rate": 1.0338660868408927e-08,
      "logits/chosen": -0.7576335668563843,
      "logits/rejected": -0.7207523584365845,
      "logps/chosen": -497.5868225097656,
      "logps/rejected": -482.0480041503906,
      "loss": 0.0367,
      "loss/chosen-sft": 1.8623535633087158,
      "loss/dpo": 0.036711666733026505,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -1.4475281238555908,
      "rewards/margins": 0.6721404790878296,
      "rewards/rejected": -2.119668483734131,
      "step": 1655
    },
    {
      "epoch": 0.920687742651137,
      "grad_norm": 2.2845067977905273,
      "learning_rate": 9.660862620346877e-09,
      "logits/chosen": -0.7926570177078247,
      "logits/rejected": -0.7714422345161438,
      "logps/chosen": -473.17706298828125,
      "logps/rejected": -469.4546813964844,
      "loss": 0.0379,
      "loss/chosen-sft": 1.8994096517562866,
      "loss/dpo": 0.0379011332988739,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.5939931869506836,
      "rewards/margins": 0.5131052136421204,
      "rewards/rejected": -2.107098340988159,
      "step": 1660
    },
    {
      "epoch": 0.9234608985024958,
      "grad_norm": 1.7006827592849731,
      "learning_rate": 9.005604644578473e-09,
      "logits/chosen": -0.8427762985229492,
      "logits/rejected": -0.8619502186775208,
      "logps/chosen": -480.4871520996094,
      "logps/rejected": -467.28363037109375,
      "loss": 0.0539,
      "loss/chosen-sft": 1.8251368999481201,
      "loss/dpo": 0.05387115478515625,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -1.6016905307769775,
      "rewards/margins": 0.5135762691497803,
      "rewards/rejected": -2.115266799926758,
      "step": 1665
    },
    {
      "epoch": 0.9262340543538546,
      "grad_norm": 1.814226508140564,
      "learning_rate": 8.372948394653717e-09,
      "logits/chosen": -0.8938384056091309,
      "logits/rejected": -0.8742619752883911,
      "logps/chosen": -495.4317321777344,
      "logps/rejected": -477.78497314453125,
      "loss": 0.0345,
      "loss/chosen-sft": 1.8645175695419312,
      "loss/dpo": 0.03449907898902893,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.5643129348754883,
      "rewards/margins": 0.605742871761322,
      "rewards/rejected": -2.170055866241455,
      "step": 1670
    },
    {
      "epoch": 0.9290072102052135,
      "grad_norm": 2.352726697921753,
      "learning_rate": 7.762953204413475e-09,
      "logits/chosen": -1.0139451026916504,
      "logits/rejected": -1.1322184801101685,
      "logps/chosen": -484.33172607421875,
      "logps/rejected": -444.97344970703125,
      "loss": 0.044,
      "loss/chosen-sft": 1.884155511856079,
      "loss/dpo": 0.04397551342844963,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -1.5331767797470093,
      "rewards/margins": 0.5316352844238281,
      "rewards/rejected": -2.064812183380127,
      "step": 1675
    },
    {
      "epoch": 0.9317803660565723,
      "grad_norm": 1.378034234046936,
      "learning_rate": 7.175676282424964e-09,
      "logits/chosen": -0.9042679667472839,
      "logits/rejected": -0.8096022605895996,
      "logps/chosen": -460.42950439453125,
      "logps/rejected": -475.254638671875,
      "loss": 0.0342,
      "loss/chosen-sft": 1.8950878381729126,
      "loss/dpo": 0.03422309830784798,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -1.513983130455017,
      "rewards/margins": 0.7315508127212524,
      "rewards/rejected": -2.2455339431762695,
      "step": 1680
    },
    {
      "epoch": 0.9345535219079312,
      "grad_norm": 1.6129722595214844,
      "learning_rate": 6.611172706616291e-09,
      "logits/chosen": -0.9313820004463196,
      "logits/rejected": -0.8965193033218384,
      "logps/chosen": -446.47283935546875,
      "logps/rejected": -424.822509765625,
      "loss": 0.0305,
      "loss/chosen-sft": 1.8703186511993408,
      "loss/dpo": 0.030467282980680466,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -1.5669549703598022,
      "rewards/margins": 0.5169414281845093,
      "rewards/rejected": -2.0838963985443115,
      "step": 1685
    },
    {
      "epoch": 0.93732667775929,
      "grad_norm": 1.3904266357421875,
      "learning_rate": 6.069495419111003e-09,
      "logits/chosen": -0.8602801561355591,
      "logits/rejected": -0.8798410296440125,
      "logps/chosen": -514.514892578125,
      "logps/rejected": -513.6451416015625,
      "loss": 0.0346,
      "loss/chosen-sft": 1.8502308130264282,
      "loss/dpo": 0.03457609936594963,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -1.4935173988342285,
      "rewards/margins": 0.9913633465766907,
      "rewards/rejected": -2.4848809242248535,
      "step": 1690
    },
    {
      "epoch": 0.940099833610649,
      "grad_norm": 1.7256861925125122,
      "learning_rate": 5.550695221263002e-09,
      "logits/chosen": -1.1261186599731445,
      "logits/rejected": -1.107710599899292,
      "logps/chosen": -490.0545959472656,
      "logps/rejected": -447.8335876464844,
      "loss": 0.0385,
      "loss/chosen-sft": 1.7753324508666992,
      "loss/dpo": 0.038456808775663376,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -1.554856538772583,
      "rewards/margins": 0.5833019018173218,
      "rewards/rejected": -2.1381585597991943,
      "step": 1695
    },
    {
      "epoch": 0.9428729894620078,
      "grad_norm": 1.7229971885681152,
      "learning_rate": 5.054820768891854e-09,
      "logits/chosen": -0.7668136954307556,
      "logits/rejected": -0.7551401257514954,
      "logps/chosen": -470.37451171875,
      "logps/rejected": -467.57177734375,
      "loss": 0.0284,
      "loss/chosen-sft": 1.8557796478271484,
      "loss/dpo": 0.028406251221895218,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -1.5322498083114624,
      "rewards/margins": 0.6786109805107117,
      "rewards/rejected": -2.2108609676361084,
      "step": 1700
    },
    {
      "epoch": 0.9456461453133667,
      "grad_norm": 2.3088462352752686,
      "learning_rate": 4.581918567719917e-09,
      "logits/chosen": -0.9519344568252563,
      "logits/rejected": -0.9092620015144348,
      "logps/chosen": -490.27813720703125,
      "logps/rejected": -480.19171142578125,
      "loss": 0.0364,
      "loss/chosen-sft": 1.8951151371002197,
      "loss/dpo": 0.03644304722547531,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -1.4337555170059204,
      "rewards/margins": 0.7906380891799927,
      "rewards/rejected": -2.224393367767334,
      "step": 1705
    },
    {
      "epoch": 0.9484193011647255,
      "grad_norm": 1.4663399457931519,
      "learning_rate": 4.132032969010546e-09,
      "logits/chosen": -0.9776910543441772,
      "logits/rejected": -0.8744028210639954,
      "logps/chosen": -475.00274658203125,
      "logps/rejected": -476.331298828125,
      "loss": 0.0408,
      "loss/chosen-sft": 1.855271339416504,
      "loss/dpo": 0.040752846747636795,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -1.570431113243103,
      "rewards/margins": 0.5336312055587769,
      "rewards/rejected": -2.10406231880188,
      "step": 1710
    },
    {
      "epoch": 0.9511924570160843,
      "grad_norm": 1.3314710855484009,
      "learning_rate": 3.705206165408703e-09,
      "logits/chosen": -0.9017612338066101,
      "logits/rejected": -0.9057533144950867,
      "logps/chosen": -448.4554138183594,
      "logps/rejected": -430.2477111816406,
      "loss": 0.0334,
      "loss/chosen-sft": 1.8261409997940063,
      "loss/dpo": 0.03343508765101433,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -1.4459483623504639,
      "rewards/margins": 0.5900785326957703,
      "rewards/rejected": -2.0360267162323,
      "step": 1715
    },
    {
      "epoch": 0.9539656128674432,
      "grad_norm": 1.0403763055801392,
      "learning_rate": 3.301478186983897e-09,
      "logits/chosen": -0.9940805435180664,
      "logits/rejected": -1.1318776607513428,
      "logps/chosen": -436.7303161621094,
      "logps/rejected": -440.62969970703125,
      "loss": 0.0342,
      "loss/chosen-sft": 1.7668132781982422,
      "loss/dpo": 0.03422313928604126,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -1.4205049276351929,
      "rewards/margins": 0.8040353655815125,
      "rewards/rejected": -2.2245402336120605,
      "step": 1720
    },
    {
      "epoch": 0.956738768718802,
      "grad_norm": 1.7417303323745728,
      "learning_rate": 2.9208868974759937e-09,
      "logits/chosen": -0.8438510894775391,
      "logits/rejected": -0.8072006106376648,
      "logps/chosen": -452.6871643066406,
      "logps/rejected": -420.52227783203125,
      "loss": 0.0271,
      "loss/chosen-sft": 1.9397671222686768,
      "loss/dpo": 0.027111122384667397,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -1.4771382808685303,
      "rewards/margins": 0.5044761896133423,
      "rewards/rejected": -1.981614351272583,
      "step": 1725
    },
    {
      "epoch": 0.9595119245701609,
      "grad_norm": 1.5373972654342651,
      "learning_rate": 2.5634679907440006e-09,
      "logits/chosen": -0.9225784540176392,
      "logits/rejected": -0.8528642654418945,
      "logps/chosen": -441.6475524902344,
      "logps/rejected": -410.14324951171875,
      "loss": 0.0367,
      "loss/chosen-sft": 1.814764380455017,
      "loss/dpo": 0.03674568608403206,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -1.4582865238189697,
      "rewards/margins": 0.40831345319747925,
      "rewards/rejected": -1.8665997982025146,
      "step": 1730
    },
    {
      "epoch": 0.9622850804215197,
      "grad_norm": 1.2935632467269897,
      "learning_rate": 2.229254987418744e-09,
      "logits/chosen": -0.8527067303657532,
      "logits/rejected": -0.7993227243423462,
      "logps/chosen": -499.4395446777344,
      "logps/rejected": -482.21197509765625,
      "loss": 0.0354,
      "loss/chosen-sft": 1.8621113300323486,
      "loss/dpo": 0.03540179878473282,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -1.4600284099578857,
      "rewards/margins": 0.7969890832901001,
      "rewards/rejected": -2.2570176124572754,
      "step": 1735
    },
    {
      "epoch": 0.9650582362728786,
      "grad_norm": 1.218867540359497,
      "learning_rate": 1.9182792317588294e-09,
      "logits/chosen": -0.9154025912284851,
      "logits/rejected": -0.8462129831314087,
      "logps/chosen": -430.40594482421875,
      "logps/rejected": -414.01397705078125,
      "loss": 0.0362,
      "loss/chosen-sft": 1.7988450527191162,
      "loss/dpo": 0.03615463525056839,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.418278694152832,
      "rewards/margins": 0.5125333666801453,
      "rewards/rejected": -1.930812120437622,
      "step": 1740
    },
    {
      "epoch": 0.9678313921242374,
      "grad_norm": 1.9518085718154907,
      "learning_rate": 1.6305698887113806e-09,
      "logits/chosen": -0.9541261792182922,
      "logits/rejected": -0.8191477656364441,
      "logps/chosen": -407.2529602050781,
      "logps/rejected": -422.18865966796875,
      "loss": 0.0323,
      "loss/chosen-sft": 1.8834625482559204,
      "loss/dpo": 0.0322595052421093,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -1.3437700271606445,
      "rewards/margins": 0.6855727434158325,
      "rewards/rejected": -2.0293426513671875,
      "step": 1745
    },
    {
      "epoch": 0.9706045479755963,
      "grad_norm": 1.7286250591278076,
      "learning_rate": 1.366153941176451e-09,
      "logits/chosen": -1.052943468093872,
      "logits/rejected": -0.9355376362800598,
      "logps/chosen": -460.63153076171875,
      "logps/rejected": -460.62994384765625,
      "loss": 0.0427,
      "loss/chosen-sft": 1.877771019935608,
      "loss/dpo": 0.042682547122240067,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -1.5018032789230347,
      "rewards/margins": 0.6279391050338745,
      "rewards/rejected": -2.129742383956909,
      "step": 1750
    },
    {
      "epoch": 0.9733777038269551,
      "grad_norm": 0.8861921429634094,
      "learning_rate": 1.1250561874766029e-09,
      "logits/chosen": -1.0122451782226562,
      "logits/rejected": -0.9112497568130493,
      "logps/chosen": -472.03948974609375,
      "logps/rejected": -466.75006103515625,
      "loss": 0.0358,
      "loss/chosen-sft": 1.8839130401611328,
      "loss/dpo": 0.03582266345620155,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -1.5027332305908203,
      "rewards/margins": 0.59858238697052,
      "rewards/rejected": -2.101315498352051,
      "step": 1755
    },
    {
      "epoch": 0.9761508596783139,
      "grad_norm": 1.1699074506759644,
      "learning_rate": 9.072992390312117e-10,
      "logits/chosen": -0.8916594386100769,
      "logits/rejected": -0.7321555018424988,
      "logps/chosen": -414.146728515625,
      "logps/rejected": -429.60125732421875,
      "loss": 0.0415,
      "loss/chosen-sft": 1.8041099309921265,
      "loss/dpo": 0.04147591441869736,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.4231741428375244,
      "rewards/margins": 0.5474594235420227,
      "rewards/rejected": -1.9706335067749023,
      "step": 1760
    },
    {
      "epoch": 0.9789240155296728,
      "grad_norm": 2.7589733600616455,
      "learning_rate": 7.12903518235719e-10,
      "logits/chosen": -1.0372343063354492,
      "logits/rejected": -0.8465463519096375,
      "logps/chosen": -406.49755859375,
      "logps/rejected": -411.850341796875,
      "loss": 0.0382,
      "loss/chosen-sft": 1.7656471729278564,
      "loss/dpo": 0.03815629705786705,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -1.410011649131775,
      "rewards/margins": 0.5866178274154663,
      "rewards/rejected": -1.9966297149658203,
      "step": 1765
    },
    {
      "epoch": 0.9816971713810316,
      "grad_norm": 1.103261947631836,
      "learning_rate": 5.418872565464139e-10,
      "logits/chosen": -0.8118406534194946,
      "logits/rejected": -0.6884601712226868,
      "logps/chosen": -441.5873107910156,
      "logps/rejected": -420.6312561035156,
      "loss": 0.0308,
      "loss/chosen-sft": 1.925108551979065,
      "loss/dpo": 0.03077712655067444,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -1.486276388168335,
      "rewards/margins": 0.5229099988937378,
      "rewards/rejected": -2.0091867446899414,
      "step": 1770
    },
    {
      "epoch": 0.9844703272323905,
      "grad_norm": 1.2194581031799316,
      "learning_rate": 3.942664927706063e-10,
      "logits/chosen": -1.0195016860961914,
      "logits/rejected": -0.9845107793807983,
      "logps/chosen": -432.424560546875,
      "logps/rejected": -431.7875061035156,
      "loss": 0.0297,
      "loss/chosen-sft": 1.9019267559051514,
      "loss/dpo": 0.02968726120889187,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -1.5260287523269653,
      "rewards/margins": 0.49342551827430725,
      "rewards/rejected": -2.01945424079895,
      "step": 1775
    },
    {
      "epoch": 0.9872434830837493,
      "grad_norm": 1.3297661542892456,
      "learning_rate": 2.700550715623029e-10,
      "logits/chosen": -0.9309781193733215,
      "logits/rejected": -0.765326201915741,
      "logps/chosen": -445.7706604003906,
      "logps/rejected": -450.1620178222656,
      "loss": 0.0314,
      "loss/chosen-sft": 1.8302927017211914,
      "loss/dpo": 0.03142596781253815,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -1.3721158504486084,
      "rewards/margins": 0.6550136804580688,
      "rewards/rejected": -2.0271294116973877,
      "step": 1780
    },
    {
      "epoch": 0.9900166389351082,
      "grad_norm": 1.6393502950668335,
      "learning_rate": 1.692646421239674e-10,
      "logits/chosen": -0.9880639910697937,
      "logits/rejected": -0.8702710270881653,
      "logps/chosen": -454.1924743652344,
      "logps/rejected": -474.16741943359375,
      "loss": 0.0314,
      "loss/chosen-sft": 1.8532609939575195,
      "loss/dpo": 0.031443167477846146,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.4502158164978027,
      "rewards/margins": 0.7252721786499023,
      "rewards/rejected": -2.175487995147705,
      "step": 1785
    },
    {
      "epoch": 0.992789794786467,
      "grad_norm": 1.782798409461975,
      "learning_rate": 9.190465711375606e-11,
      "logits/chosen": -0.8094294667243958,
      "logits/rejected": -0.825986385345459,
      "logps/chosen": -438.8473205566406,
      "logps/rejected": -475.4677734375,
      "loss": 0.0401,
      "loss/chosen-sft": 1.7468044757843018,
      "loss/dpo": 0.04005669802427292,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -1.3847625255584717,
      "rewards/margins": 0.7623748779296875,
      "rewards/rejected": -2.147137403488159,
      "step": 1790
    },
    {
      "epoch": 0.9955629506378258,
      "grad_norm": 1.64493989944458,
      "learning_rate": 3.798237175925423e-11,
      "logits/chosen": -0.9432379007339478,
      "logits/rejected": -0.890951931476593,
      "logps/chosen": -427.1351623535156,
      "logps/rejected": -463.54705810546875,
      "loss": 0.0352,
      "loss/chosen-sft": 1.7833278179168701,
      "loss/dpo": 0.03518053516745567,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -1.4109861850738525,
      "rewards/margins": 0.7486652135848999,
      "rewards/rejected": -2.159651279449463,
      "step": 1795
    },
    {
      "epoch": 0.9983361064891847,
      "grad_norm": 1.9120142459869385,
      "learning_rate": 7.502843176826478e-12,
      "logits/chosen": -0.9462081789970398,
      "logits/rejected": -0.8944603204727173,
      "logps/chosen": -472.4647521972656,
      "logps/rejected": -491.3990173339844,
      "loss": 0.0391,
      "loss/chosen-sft": 1.819483757019043,
      "loss/dpo": 0.03906460851430893,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.514047384262085,
      "rewards/margins": 0.6475565433502197,
      "rewards/rejected": -2.161604166030884,
      "step": 1800
    },
    {
      "epoch": 1.0,
      "step": 1803,
      "total_flos": 0.0,
      "train_loss": 0.08027586419418396,
      "train_runtime": 2186.6137,
      "train_samples_per_second": 26.375,
      "train_steps_per_second": 0.825
    }
  ],
  "logging_steps": 5,
  "max_steps": 1803,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 500,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 32,
  "trial_name": null,
  "trial_params": null
}