{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.9990726429675425,
  "eval_steps": 100,
  "global_step": 808,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.012364760432766615,
      "grad_norm": 20.75,
      "learning_rate": 1.9512195121951218e-06,
      "log_odds_chosen": 5.523005962371826,
      "log_odds_ratio": -0.27857011556625366,
      "logits/chosen": -0.47717565298080444,
      "logits/rejected": -0.5149000883102417,
      "logps/chosen": -0.09525187313556671,
      "logps/rejected": -3.94183349609375,
      "loss": 0.443,
      "nll_loss": 0.10230422019958496,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.09525187313556671,
      "rewards/margins": 3.846581220626831,
      "rewards/rejected": -3.94183349609375,
      "step": 10
    },
    {
      "epoch": 0.02472952086553323,
      "grad_norm": 14.3125,
      "learning_rate": 3.9024390243902435e-06,
      "log_odds_chosen": 5.220697402954102,
      "log_odds_ratio": -0.32944533228874207,
      "logits/chosen": -0.45673075318336487,
      "logits/rejected": -0.48329925537109375,
      "logps/chosen": -0.11893807351589203,
      "logps/rejected": -3.8670706748962402,
      "loss": 0.4606,
      "nll_loss": 0.1369919627904892,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.11893807351589203,
      "rewards/margins": 3.7481327056884766,
      "rewards/rejected": -3.8670706748962402,
      "step": 20
    },
    {
      "epoch": 0.03709428129829984,
      "grad_norm": 43.25,
      "learning_rate": 5.853658536585366e-06,
      "log_odds_chosen": 5.081484794616699,
      "log_odds_ratio": -0.31304678320884705,
      "logits/chosen": -0.4474870562553406,
      "logits/rejected": -0.4531799256801605,
      "logps/chosen": -0.11334141343832016,
      "logps/rejected": -3.6699836254119873,
      "loss": 0.4347,
      "nll_loss": 0.1262090504169464,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.11334141343832016,
      "rewards/margins": 3.556642532348633,
      "rewards/rejected": -3.6699836254119873,
      "step": 30
    },
    {
      "epoch": 0.04945904173106646,
      "grad_norm": 26.375,
      "learning_rate": 7.804878048780487e-06,
      "log_odds_chosen": 5.873383045196533,
      "log_odds_ratio": -0.2693401575088501,
      "logits/chosen": -0.4661811888217926,
      "logits/rejected": -0.4860765039920807,
      "logps/chosen": -0.12170048803091049,
      "logps/rejected": -4.414368152618408,
      "loss": 0.4025,
      "nll_loss": 0.12971656024456024,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.12170048803091049,
      "rewards/margins": 4.292667388916016,
      "rewards/rejected": -4.414368152618408,
      "step": 40
    },
    {
      "epoch": 0.061823802163833076,
      "grad_norm": 7.71875,
      "learning_rate": 7.997282466850342e-06,
      "log_odds_chosen": 5.318218231201172,
      "log_odds_ratio": -0.2838350236415863,
      "logits/chosen": -0.43940672278404236,
      "logits/rejected": -0.44333672523498535,
      "logps/chosen": -0.13381657004356384,
      "logps/rejected": -3.832505702972412,
      "loss": 0.411,
      "nll_loss": 0.1340525895357132,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.13381657004356384,
      "rewards/margins": 3.6986892223358154,
      "rewards/rejected": -3.832505702972412,
      "step": 50
    },
    {
      "epoch": 0.07418856259659969,
      "grad_norm": 20.125,
      "learning_rate": 7.987893265604088e-06,
      "log_odds_chosen": 5.140921115875244,
      "log_odds_ratio": -0.3448868691921234,
      "logits/chosen": -0.3982415497303009,
      "logits/rejected": -0.4345609247684479,
      "logps/chosen": -0.10162041336297989,
      "logps/rejected": -3.67462420463562,
      "loss": 0.4488,
      "nll_loss": 0.10289975255727768,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.10162041336297989,
      "rewards/margins": 3.5730037689208984,
      "rewards/rejected": -3.67462420463562,
      "step": 60
    },
    {
      "epoch": 0.0865533230293663,
      "grad_norm": 7.15625,
      "learning_rate": 7.971814592585675e-06,
      "log_odds_chosen": 5.889096260070801,
      "log_odds_ratio": -0.2466737926006317,
      "logits/chosen": -0.35493898391723633,
      "logits/rejected": -0.384539932012558,
      "logps/chosen": -0.10166393220424652,
      "logps/rejected": -4.258695602416992,
      "loss": 0.3634,
      "nll_loss": 0.1078365296125412,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10166393220424652,
      "rewards/margins": 4.157031059265137,
      "rewards/rejected": -4.258695602416992,
      "step": 70
    },
    {
      "epoch": 0.09891808346213292,
      "grad_norm": 9.0625,
      "learning_rate": 7.949073418885378e-06,
      "log_odds_chosen": 5.626579284667969,
      "log_odds_ratio": -0.3173326551914215,
      "logits/chosen": -0.37646952271461487,
      "logits/rejected": -0.41776227951049805,
      "logps/chosen": -0.14894399046897888,
      "logps/rejected": -4.165127277374268,
      "loss": 0.4266,
      "nll_loss": 0.1764051616191864,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.14894399046897888,
      "rewards/margins": 4.016183376312256,
      "rewards/rejected": -4.165127277374268,
      "step": 80
    },
    {
      "epoch": 0.11128284389489954,
      "grad_norm": 10.0625,
      "learning_rate": 7.919707891572192e-06,
      "log_odds_chosen": 4.933900833129883,
      "log_odds_ratio": -0.4360424876213074,
      "logits/chosen": -0.38308653235435486,
      "logits/rejected": -0.41619744896888733,
      "logps/chosen": -0.21633660793304443,
      "logps/rejected": -3.6978626251220703,
      "loss": 0.5427,
      "nll_loss": 0.2736307680606842,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.21633660793304443,
      "rewards/margins": 3.4815258979797363,
      "rewards/rejected": -3.6978626251220703,
      "step": 90
    },
    {
      "epoch": 0.12364760432766615,
      "grad_norm": 6.125,
      "learning_rate": 7.883767269704208e-06,
      "log_odds_chosen": 5.787186145782471,
      "log_odds_ratio": -0.24177666008472443,
      "logits/chosen": -0.36189815402030945,
      "logits/rejected": -0.3885316252708435,
      "logps/chosen": -0.10456478595733643,
      "logps/rejected": -4.127496242523193,
      "loss": 0.4223,
      "nll_loss": 0.10984029620885849,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.10456478595733643,
      "rewards/margins": 4.022932052612305,
      "rewards/rejected": -4.127496242523193,
      "step": 100
    },
    {
      "epoch": 0.13601236476043277,
      "grad_norm": 9.75,
      "learning_rate": 7.841311841699256e-06,
      "log_odds_chosen": 5.171591758728027,
      "log_odds_ratio": -0.34959590435028076,
      "logits/chosen": -0.3628004491329193,
      "logits/rejected": -0.41781893372535706,
      "logps/chosen": -0.13123615086078644,
      "logps/rejected": -3.952786922454834,
      "loss": 0.5216,
      "nll_loss": 0.14560718834400177,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.13123615086078644,
      "rewards/margins": 3.821551561355591,
      "rewards/rejected": -3.952786922454834,
      "step": 110
    },
    {
      "epoch": 0.14837712519319937,
      "grad_norm": 8.375,
      "learning_rate": 7.7924128242044e-06,
      "log_odds_chosen": 5.987541198730469,
      "log_odds_ratio": -0.24941344559192657,
      "logits/chosen": -0.41023507714271545,
      "logits/rejected": -0.42857393622398376,
      "logps/chosen": -0.1003207415342331,
      "logps/rejected": -4.482211112976074,
      "loss": 0.3645,
      "nll_loss": 0.10704884678125381,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.1003207415342331,
      "rewards/margins": 4.381890296936035,
      "rewards/rejected": -4.482211112976074,
      "step": 120
    },
    {
      "epoch": 0.160741885625966,
      "grad_norm": 12.75,
      "learning_rate": 7.737152242633916e-06,
      "log_odds_chosen": 6.005825042724609,
      "log_odds_ratio": -0.2806817889213562,
      "logits/chosen": -0.45315614342689514,
      "logits/rejected": -0.5087369680404663,
      "logps/chosen": -0.09657936543226242,
      "logps/rejected": -4.489271640777588,
      "loss": 0.4169,
      "nll_loss": 0.10316102206707001,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.09657936543226242,
      "rewards/margins": 4.392692565917969,
      "rewards/rejected": -4.489271640777588,
      "step": 130
    },
    {
      "epoch": 0.1731066460587326,
      "grad_norm": 9.25,
      "learning_rate": 7.675622793576181e-06,
      "log_odds_chosen": 6.356817722320557,
      "log_odds_ratio": -0.2625463902950287,
      "logits/chosen": -0.42411336302757263,
      "logits/rejected": -0.4512180685997009,
      "logps/chosen": -0.11383982747793198,
      "logps/rejected": -4.8227434158325195,
      "loss": 0.3895,
      "nll_loss": 0.12313707917928696,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.11383982747793198,
      "rewards/margins": 4.708903789520264,
      "rewards/rejected": -4.8227434158325195,
      "step": 140
    },
    {
      "epoch": 0.18547140649149924,
      "grad_norm": 11.8125,
      "learning_rate": 7.607927689300238e-06,
      "log_odds_chosen": 5.872275352478027,
      "log_odds_ratio": -0.27027657628059387,
      "logits/chosen": -0.4047353267669678,
      "logits/rejected": -0.4540864825248718,
      "logps/chosen": -0.10377119481563568,
      "logps/rejected": -4.4931488037109375,
      "loss": 0.3557,
      "nll_loss": 0.11161776632070541,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.10377119481563568,
      "rewards/margins": 4.389377117156982,
      "rewards/rejected": -4.4931488037109375,
      "step": 150
    },
    {
      "epoch": 0.19783616692426584,
      "grad_norm": 5.9375,
      "learning_rate": 7.534180484622877e-06,
      "log_odds_chosen": 6.647129058837891,
      "log_odds_ratio": -0.2904432415962219,
      "logits/chosen": -0.38757234811782837,
      "logits/rejected": -0.44936904311180115,
      "logps/chosen": -0.14150920510292053,
      "logps/rejected": -5.145331382751465,
      "loss": 0.4073,
      "nll_loss": 0.16469481587409973,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.14150920510292053,
      "rewards/margins": 5.003822326660156,
      "rewards/rejected": -5.145331382751465,
      "step": 160
    },
    {
      "epoch": 0.21020092735703247,
      "grad_norm": 11.6875,
      "learning_rate": 7.454504886426683e-06,
      "log_odds_chosen": 7.098265171051025,
      "log_odds_ratio": -0.21199099719524384,
      "logits/chosen": -0.4072357714176178,
      "logits/rejected": -0.46719616651535034,
      "logps/chosen": -0.09355773776769638,
      "logps/rejected": -5.363073348999023,
      "loss": 0.3763,
      "nll_loss": 0.09803523123264313,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.09355773776769638,
      "rewards/margins": 5.2695159912109375,
      "rewards/rejected": -5.363073348999023,
      "step": 170
    },
    {
      "epoch": 0.22256568778979907,
      "grad_norm": 9.0625,
      "learning_rate": 7.369034546148533e-06,
      "log_odds_chosen": 8.294835090637207,
      "log_odds_ratio": -0.1507936716079712,
      "logits/chosen": -0.3593350052833557,
      "logits/rejected": -0.44668951630592346,
      "logps/chosen": -0.10564480721950531,
      "logps/rejected": -6.4017744064331055,
      "loss": 0.3443,
      "nll_loss": 0.1119481548666954,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.10564480721950531,
      "rewards/margins": 6.296130180358887,
      "rewards/rejected": -6.4017744064331055,
      "step": 180
    },
    {
      "epoch": 0.23493044822256567,
      "grad_norm": 10.5625,
      "learning_rate": 7.2779128355866596e-06,
      "log_odds_chosen": 6.6952314376831055,
      "log_odds_ratio": -0.271857351064682,
      "logits/chosen": -0.3374063968658447,
      "logits/rejected": -0.3751365542411804,
      "logps/chosen": -0.1900143027305603,
      "logps/rejected": -5.095300197601318,
      "loss": 0.4874,
      "nll_loss": 0.20837704837322235,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.1900143027305603,
      "rewards/margins": 4.9052863121032715,
      "rewards/rejected": -5.095300197601318,
      "step": 190
    },
    {
      "epoch": 0.2472952086553323,
      "grad_norm": 11.0,
      "learning_rate": 7.181292606402339e-06,
      "log_odds_chosen": 6.087054252624512,
      "log_odds_ratio": -0.24622377753257751,
      "logits/chosen": -0.34358811378479004,
      "logits/rejected": -0.35175061225891113,
      "logps/chosen": -0.10159099102020264,
      "logps/rejected": -4.460976600646973,
      "loss": 0.3813,
      "nll_loss": 0.10912550985813141,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.10159099102020264,
      "rewards/margins": 4.359385013580322,
      "rewards/rejected": -4.460976600646973,
      "step": 200
    },
    {
      "epoch": 0.2596599690880989,
      "grad_norm": 9.625,
      "learning_rate": 7.079335933719625e-06,
      "log_odds_chosen": 5.750031471252441,
      "log_odds_ratio": -0.32498854398727417,
      "logits/chosen": -0.34495121240615845,
      "logits/rejected": -0.42697566747665405,
      "logps/chosen": -0.13423889875411987,
      "logps/rejected": -4.52581787109375,
      "loss": 0.3999,
      "nll_loss": 0.14618520438671112,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.13423889875411987,
      "rewards/margins": 4.391578674316406,
      "rewards/rejected": -4.52581787109375,
      "step": 210
    },
    {
      "epoch": 0.27202472952086554,
      "grad_norm": 7.25,
      "learning_rate": 6.972213844253246e-06,
      "log_odds_chosen": 4.923450469970703,
      "log_odds_ratio": -0.39486002922058105,
      "logits/chosen": -0.3468414545059204,
      "logits/rejected": -0.393020898103714,
      "logps/chosen": -0.12390674650669098,
      "logps/rejected": -3.825371265411377,
      "loss": 0.4458,
      "nll_loss": 0.13455723226070404,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.12390674650669098,
      "rewards/margins": 3.7014641761779785,
      "rewards/rejected": -3.825371265411377,
      "step": 220
    },
    {
      "epoch": 0.28438948995363217,
      "grad_norm": 5.90625,
      "learning_rate": 6.860106029420683e-06,
      "log_odds_chosen": 7.671366214752197,
      "log_odds_ratio": -0.16312380135059357,
      "logits/chosen": -0.3415583074092865,
      "logits/rejected": -0.4085741937160492,
      "logps/chosen": -0.09625446796417236,
      "logps/rejected": -5.770381927490234,
      "loss": 0.2866,
      "nll_loss": 0.10523615032434464,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.09625446796417236,
      "rewards/margins": 5.67412805557251,
      "rewards/rejected": -5.770381927490234,
      "step": 230
    },
    {
      "epoch": 0.29675425038639874,
      "grad_norm": 10.5625,
      "learning_rate": 6.743200543919695e-06,
      "log_odds_chosen": 6.7600297927856445,
      "log_odds_ratio": -0.2927699089050293,
      "logits/chosen": -0.37835583090782166,
      "logits/rejected": -0.422700971364975,
      "logps/chosen": -0.11542928218841553,
      "logps/rejected": -5.151785850524902,
      "loss": 0.3609,
      "nll_loss": 0.1274506151676178,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.11542928218841553,
      "rewards/margins": 5.036356449127197,
      "rewards/rejected": -5.151785850524902,
      "step": 240
    },
    {
      "epoch": 0.3091190108191654,
      "grad_norm": 13.0,
      "learning_rate": 6.6216934902769005e-06,
      "log_odds_chosen": 5.663918972015381,
      "log_odds_ratio": -0.3453647494316101,
      "logits/chosen": -0.34125855565071106,
      "logits/rejected": -0.3729458749294281,
      "logps/chosen": -0.13385739922523499,
      "logps/rejected": -4.3902740478515625,
      "loss": 0.4553,
      "nll_loss": 0.15540793538093567,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.13385739922523499,
      "rewards/margins": 4.256417274475098,
      "rewards/rejected": -4.3902740478515625,
      "step": 250
    },
    {
      "epoch": 0.321483771251932,
      "grad_norm": 11.8125,
      "learning_rate": 6.495788689896567e-06,
      "log_odds_chosen": 5.2633957862854,
      "log_odds_ratio": -0.2864142954349518,
      "logits/chosen": -0.34622472524642944,
      "logits/rejected": -0.34518593549728394,
      "logps/chosen": -0.11769046634435654,
      "logps/rejected": -4.009757041931152,
      "loss": 0.42,
      "nll_loss": 0.12802495062351227,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.11769046634435654,
      "rewards/margins": 3.8920669555664062,
      "rewards/rejected": -4.009757041931152,
      "step": 260
    },
    {
      "epoch": 0.33384853168469864,
      "grad_norm": 9.6875,
      "learning_rate": 6.3656973411614075e-06,
      "log_odds_chosen": 7.293965816497803,
      "log_odds_ratio": -0.216775581240654,
      "logits/chosen": -0.3193885385990143,
      "logits/rejected": -0.40312114357948303,
      "logps/chosen": -0.10884882509708405,
      "logps/rejected": -5.596304893493652,
      "loss": 0.3533,
      "nll_loss": 0.11900696903467178,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.10884882509708405,
      "rewards/margins": 5.487456321716309,
      "rewards/rejected": -5.596304893493652,
      "step": 270
    },
    {
      "epoch": 0.3462132921174652,
      "grad_norm": 10.125,
      "learning_rate": 6.231637665158897e-06,
      "log_odds_chosen": 6.230224609375,
      "log_odds_ratio": -0.2035815268754959,
      "logits/chosen": -0.36509817838668823,
      "logits/rejected": -0.38093799352645874,
      "logps/chosen": -0.0948435515165329,
      "logps/rejected": -4.630681037902832,
      "loss": 0.4417,
      "nll_loss": 0.0979100838303566,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.0948435515165329,
      "rewards/margins": 4.535837650299072,
      "rewards/rejected": -4.630681037902832,
      "step": 280
    },
    {
      "epoch": 0.35857805255023184,
      "grad_norm": 12.0,
      "learning_rate": 6.0938345396274016e-06,
      "log_odds_chosen": 5.3339362144470215,
      "log_odds_ratio": -0.33579060435295105,
      "logits/chosen": -0.29368194937705994,
      "logits/rejected": -0.31918543577194214,
      "logps/chosen": -0.1240597516298294,
      "logps/rejected": -4.045788288116455,
      "loss": 0.3713,
      "nll_loss": 0.13200339674949646,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.1240597516298294,
      "rewards/margins": 3.9217286109924316,
      "rewards/rejected": -4.045788288116455,
      "step": 290
    },
    {
      "epoch": 0.37094281298299847,
      "grad_norm": 10.125,
      "learning_rate": 5.952519121736122e-06,
      "log_odds_chosen": 5.762480735778809,
      "log_odds_ratio": -0.2765099108219147,
      "logits/chosen": -0.33325260877609253,
      "logits/rejected": -0.3311522901058197,
      "logps/chosen": -0.10073937475681305,
      "logps/rejected": -4.253504753112793,
      "loss": 0.3689,
      "nll_loss": 0.10901761054992676,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.10073937475681305,
      "rewards/margins": 4.152765274047852,
      "rewards/rejected": -4.253504753112793,
      "step": 300
    },
    {
      "epoch": 0.38330757341576505,
      "grad_norm": 10.0625,
      "learning_rate": 5.807928460331655e-06,
      "log_odds_chosen": 6.584504127502441,
      "log_odds_ratio": -0.2417345494031906,
      "logits/chosen": -0.33266735076904297,
      "logits/rejected": -0.3785056173801422,
      "logps/chosen": -0.11963678896427155,
      "logps/rejected": -5.059963226318359,
      "loss": 0.3759,
      "nll_loss": 0.13275153934955597,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.11963678896427155,
      "rewards/margins": 4.940326690673828,
      "rewards/rejected": -5.059963226318359,
      "step": 310
    },
    {
      "epoch": 0.3956723338485317,
      "grad_norm": 7.9375,
      "learning_rate": 5.660305098301574e-06,
      "log_odds_chosen": 6.815100193023682,
      "log_odds_ratio": -0.2694624066352844,
      "logits/chosen": -0.343056857585907,
      "logits/rejected": -0.38409820199012756,
      "logps/chosen": -0.13277564942836761,
      "logps/rejected": -5.175203800201416,
      "loss": 0.3839,
      "nll_loss": 0.15725703537464142,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -0.13277564942836761,
      "rewards/margins": 5.042428493499756,
      "rewards/rejected": -5.175203800201416,
      "step": 320
    },
    {
      "epoch": 0.4080370942812983,
      "grad_norm": 9.75,
      "learning_rate": 5.509896665722073e-06,
      "log_odds_chosen": 6.840758323669434,
      "log_odds_ratio": -0.22664757072925568,
      "logits/chosen": -0.32007095217704773,
      "logits/rejected": -0.36283355951309204,
      "logps/chosen": -0.09083802253007889,
      "logps/rejected": -5.145449638366699,
      "loss": 0.3342,
      "nll_loss": 0.09650816023349762,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.09083802253007889,
      "rewards/margins": 5.054611682891846,
      "rewards/rejected": -5.145449638366699,
      "step": 330
    },
    {
      "epoch": 0.42040185471406494,
      "grad_norm": 13.6875,
      "learning_rate": 5.35695546447212e-06,
      "log_odds_chosen": 6.528438568115234,
      "log_odds_ratio": -0.27346697449684143,
      "logits/chosen": -0.3096829950809479,
      "logits/rejected": -0.365972101688385,
      "logps/chosen": -0.1788136065006256,
      "logps/rejected": -5.211609840393066,
      "loss": 0.6287,
      "nll_loss": 0.23896226286888123,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.1788136065006256,
      "rewards/margins": 5.0327959060668945,
      "rewards/rejected": -5.211609840393066,
      "step": 340
    },
    {
      "epoch": 0.4327666151468315,
      "grad_norm": 5.125,
      "learning_rate": 5.2017380450109245e-06,
      "log_odds_chosen": 5.903274059295654,
      "log_odds_ratio": -0.3314878046512604,
      "logits/chosen": -0.2893844246864319,
      "logits/rejected": -0.31566327810287476,
      "logps/chosen": -0.13658109307289124,
      "logps/rejected": -4.494335651397705,
      "loss": 0.5195,
      "nll_loss": 0.16627302765846252,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.13658109307289124,
      "rewards/margins": 4.357754707336426,
      "rewards/rejected": -4.494335651397705,
      "step": 350
    },
    {
      "epoch": 0.44513137557959814,
      "grad_norm": 9.5,
      "learning_rate": 5.044504776028637e-06,
      "log_odds_chosen": 7.735085964202881,
      "log_odds_ratio": -0.24246785044670105,
      "logits/chosen": -0.3523925542831421,
      "logits/rejected": -0.3978312611579895,
      "logps/chosen": -0.10198203474283218,
      "logps/rejected": -6.024114608764648,
      "loss": 0.3647,
      "nll_loss": 0.11343041807413101,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.10198203474283218,
      "rewards/margins": 5.92213249206543,
      "rewards/rejected": -6.024114608764648,
      "step": 360
    },
    {
      "epoch": 0.4574961360123648,
      "grad_norm": 7.28125,
      "learning_rate": 4.8855194076921875e-06,
      "log_odds_chosen": 7.6855010986328125,
      "log_odds_ratio": -0.2234397679567337,
      "logits/chosen": -0.3215215802192688,
      "logits/rejected": -0.3523593544960022,
      "logps/chosen": -0.12617747485637665,
      "logps/rejected": -5.9721503257751465,
      "loss": 0.3307,
      "nll_loss": 0.13751187920570374,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.12617747485637665,
      "rewards/margins": 5.845973014831543,
      "rewards/rejected": -5.9721503257751465,
      "step": 370
    },
    {
      "epoch": 0.46986089644513135,
      "grad_norm": 10.0625,
      "learning_rate": 4.725048629218874e-06,
      "log_odds_chosen": 6.840598106384277,
      "log_odds_ratio": -0.2805066704750061,
      "logits/chosen": -0.3227631449699402,
      "logits/rejected": -0.3682071268558502,
      "logps/chosen": -0.11587204039096832,
      "logps/rejected": -5.246945381164551,
      "loss": 0.4208,
      "nll_loss": 0.13161909580230713,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.11587204039096832,
      "rewards/margins": 5.131073474884033,
      "rewards/rejected": -5.246945381164551,
      "step": 380
    },
    {
      "epoch": 0.482225656877898,
      "grad_norm": 15.9375,
      "learning_rate": 4.56336162151986e-06,
      "log_odds_chosen": 7.6165056228637695,
      "log_odds_ratio": -0.20993241667747498,
      "logits/chosen": -0.3179897367954254,
      "logits/rejected": -0.35824161767959595,
      "logps/chosen": -0.10418138653039932,
      "logps/rejected": -5.913028717041016,
      "loss": 0.3296,
      "nll_loss": 0.1109020933508873,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.10418138653039932,
      "rewards/margins": 5.808846950531006,
      "rewards/rejected": -5.913028717041016,
      "step": 390
    },
    {
      "epoch": 0.4945904173106646,
      "grad_norm": 11.8125,
      "learning_rate": 4.400729605663989e-06,
      "log_odds_chosen": 6.626736640930176,
      "log_odds_ratio": -0.2823312282562256,
      "logits/chosen": -0.3265175521373749,
      "logits/rejected": -0.385267972946167,
      "logps/chosen": -0.12494019418954849,
      "logps/rejected": -5.27275276184082,
      "loss": 0.4187,
      "nll_loss": 0.1347981095314026,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.12494019418954849,
      "rewards/margins": 5.147812843322754,
      "rewards/rejected": -5.27275276184082,
      "step": 400
    },
    {
      "epoch": 0.5069551777434312,
      "grad_norm": 7.71875,
      "learning_rate": 4.237425387919345e-06,
      "log_odds_chosen": 7.125645637512207,
      "log_odds_ratio": -0.24858930706977844,
      "logits/chosen": -0.33324047923088074,
      "logits/rejected": -0.37242159247398376,
      "logps/chosen": -0.16267578303813934,
      "logps/rejected": -5.598741054534912,
      "loss": 0.4448,
      "nll_loss": 0.19792306423187256,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.16267578303813934,
      "rewards/margins": 5.436065673828125,
      "rewards/rejected": -5.598741054534912,
      "step": 410
    },
    {
      "epoch": 0.5193199381761978,
      "grad_norm": 9.5,
      "learning_rate": 4.0737229021357395e-06,
      "log_odds_chosen": 5.5140156745910645,
      "log_odds_ratio": -0.4161587357521057,
      "logits/chosen": -0.3467995524406433,
      "logits/rejected": -0.3643236756324768,
      "logps/chosen": -0.1758127063512802,
      "logps/rejected": -4.3748979568481445,
      "loss": 0.5047,
      "nll_loss": 0.21004962921142578,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.1758127063512802,
      "rewards/margins": 4.199085235595703,
      "rewards/rejected": -4.3748979568481445,
      "step": 420
    },
    {
      "epoch": 0.5316846986089645,
      "grad_norm": 11.5625,
      "learning_rate": 3.909896750235728e-06,
      "log_odds_chosen": 7.526612758636475,
      "log_odds_ratio": -0.21826815605163574,
      "logits/chosen": -0.35854607820510864,
      "logits/rejected": -0.381328284740448,
      "logps/chosen": -0.1035633236169815,
      "logps/rejected": -5.802954196929932,
      "loss": 0.3089,
      "nll_loss": 0.11211247742176056,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.1035633236169815,
      "rewards/margins": 5.6993913650512695,
      "rewards/rejected": -5.802954196929932,
      "step": 430
    },
    {
      "epoch": 0.5440494590417311,
      "grad_norm": 12.0,
      "learning_rate": 3.7462217415849794e-06,
      "log_odds_chosen": 6.769415855407715,
      "log_odds_ratio": -0.2991565763950348,
      "logits/chosen": -0.3402034640312195,
      "logits/rejected": -0.3760274052619934,
      "logps/chosen": -0.10911045968532562,
      "logps/rejected": -5.297713756561279,
      "loss": 0.4848,
      "nll_loss": 0.11359377950429916,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.10911045968532562,
      "rewards/margins": 5.188603401184082,
      "rewards/rejected": -5.297713756561279,
      "step": 440
    },
    {
      "epoch": 0.5564142194744977,
      "grad_norm": 12.6875,
      "learning_rate": 3.5829724320146712e-06,
      "log_odds_chosen": 7.027789115905762,
      "log_odds_ratio": -0.2503162920475006,
      "logits/chosen": -0.367917001247406,
      "logits/rejected": -0.39431366324424744,
      "logps/chosen": -0.16502001881599426,
      "logps/rejected": -5.450506210327148,
      "loss": 0.425,
      "nll_loss": 0.21910491585731506,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.16502001881599426,
      "rewards/margins": 5.285486698150635,
      "rewards/rejected": -5.450506210327148,
      "step": 450
    },
    {
      "epoch": 0.5687789799072643,
      "grad_norm": 10.8125,
      "learning_rate": 3.420422663269165e-06,
      "log_odds_chosen": 6.765953063964844,
      "log_odds_ratio": -0.26205381751060486,
      "logits/chosen": -0.3317480683326721,
      "logits/rejected": -0.347700834274292,
      "logps/chosen": -0.11256217956542969,
      "logps/rejected": -5.208095550537109,
      "loss": 0.4076,
      "nll_loss": 0.12172554433345795,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11256217956542969,
      "rewards/margins": 5.09553337097168,
      "rewards/rejected": -5.208095550537109,
      "step": 460
    },
    {
      "epoch": 0.5811437403400309,
      "grad_norm": 7.96875,
      "learning_rate": 3.258845103651543e-06,
      "log_odds_chosen": 7.34005880355835,
      "log_odds_ratio": -0.26063185930252075,
      "logits/chosen": -0.33409589529037476,
      "logits/rejected": -0.3848854899406433,
      "logps/chosen": -0.0984729677438736,
      "logps/rejected": -5.6290178298950195,
      "loss": 0.4663,
      "nll_loss": 0.10937750339508057,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.0984729677438736,
      "rewards/margins": 5.530545234680176,
      "rewards/rejected": -5.6290178298950195,
      "step": 470
    },
    {
      "epoch": 0.5935085007727975,
      "grad_norm": 9.8125,
      "learning_rate": 3.098510790637506e-06,
      "log_odds_chosen": 6.773931980133057,
      "log_odds_ratio": -0.3086785674095154,
      "logits/chosen": -0.32566770911216736,
      "logits/rejected": -0.37866756319999695,
      "logps/chosen": -0.12564793229103088,
      "logps/rejected": -5.340775489807129,
      "loss": 0.4019,
      "nll_loss": 0.1339883804321289,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.12564793229103088,
      "rewards/margins": 5.215126991271973,
      "rewards/rejected": -5.340775489807129,
      "step": 480
    },
    {
      "epoch": 0.6058732612055642,
      "grad_norm": 7.28125,
      "learning_rate": 2.939688676224907e-06,
      "log_odds_chosen": 6.174708366394043,
      "log_odds_ratio": -0.41784343123435974,
      "logits/chosen": -0.3641737103462219,
      "logits/rejected": -0.3965590298175812,
      "logps/chosen": -0.1868213266134262,
      "logps/rejected": -4.962578773498535,
      "loss": 0.457,
      "nll_loss": 0.22521409392356873,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.1868213266134262,
      "rewards/margins": 4.775757789611816,
      "rewards/rejected": -4.962578773498535,
      "step": 490
    },
    {
      "epoch": 0.6182380216383307,
      "grad_norm": 9.875,
      "learning_rate": 2.7826451757815653e-06,
      "log_odds_chosen": 7.3394036293029785,
      "log_odds_ratio": -0.24640849232673645,
      "logits/chosen": -0.3706705570220947,
      "logits/rejected": -0.3788830637931824,
      "logps/chosen": -0.1021781712770462,
      "logps/rejected": -5.561215400695801,
      "loss": 0.4106,
      "nll_loss": 0.10798408836126328,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.1021781712770462,
      "rewards/margins": 5.459037780761719,
      "rewards/rejected": -5.561215400695801,
      "step": 500
    },
    {
      "epoch": 0.6306027820710973,
      "grad_norm": 15.875,
      "learning_rate": 2.62764372114811e-06,
      "log_odds_chosen": 6.111435413360596,
      "log_odds_ratio": -0.300571084022522,
      "logits/chosen": -0.36437222361564636,
      "logits/rejected": -0.384086549282074,
      "logps/chosen": -0.18572935461997986,
      "logps/rejected": -4.8279242515563965,
      "loss": 0.4872,
      "nll_loss": 0.2374846488237381,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.18572935461997986,
      "rewards/margins": 4.642194747924805,
      "rewards/rejected": -4.8279242515563965,
      "step": 510
    },
    {
      "epoch": 0.642967542503864,
      "grad_norm": 9.25,
      "learning_rate": 2.4749443187455693e-06,
      "log_odds_chosen": 7.054736137390137,
      "log_odds_ratio": -0.2499779909849167,
      "logits/chosen": -0.3599463105201721,
      "logits/rejected": -0.41123589873313904,
      "logps/chosen": -0.2290632277727127,
      "logps/rejected": -5.664748668670654,
      "loss": 0.4621,
      "nll_loss": 0.3133532404899597,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.2290632277727127,
      "rewards/margins": 5.435685157775879,
      "rewards/rejected": -5.664748668670654,
      "step": 520
    },
    {
      "epoch": 0.6553323029366306,
      "grad_norm": 9.6875,
      "learning_rate": 2.324803113428874e-06,
      "log_odds_chosen": 8.855158805847168,
      "log_odds_ratio": -0.21653752028942108,
      "logits/chosen": -0.3489173352718353,
      "logits/rejected": -0.43390288949012756,
      "logps/chosen": -0.1009451150894165,
      "logps/rejected": -6.972768306732178,
      "loss": 0.3848,
      "nll_loss": 0.11368437111377716,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.1009451150894165,
      "rewards/margins": 6.871823310852051,
      "rewards/rejected": -6.972768306732178,
      "step": 530
    },
    {
      "epoch": 0.6676970633693973,
      "grad_norm": 17.125,
      "learning_rate": 2.177471958817942e-06,
      "log_odds_chosen": 7.061318874359131,
      "log_odds_ratio": -0.2817334234714508,
      "logits/chosen": -0.3172645568847656,
      "logits/rejected": -0.38288062810897827,
      "logps/chosen": -0.09371158480644226,
      "logps/rejected": -5.437645435333252,
      "loss": 0.496,
      "nll_loss": 0.10258449614048004,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.09371158480644226,
      "rewards/margins": 5.343933582305908,
      "rewards/rejected": -5.437645435333252,
      "step": 540
    },
    {
      "epoch": 0.6800618238021638,
      "grad_norm": 11.4375,
      "learning_rate": 2.033197994827084e-06,
      "log_odds_chosen": 7.035434722900391,
      "log_odds_ratio": -0.25783711671829224,
      "logits/chosen": -0.36509639024734497,
      "logits/rejected": -0.4073667526245117,
      "logps/chosen": -0.10131637006998062,
      "logps/rejected": -5.462925910949707,
      "loss": 0.3916,
      "nll_loss": 0.10915403068065643,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.10131637006998062,
      "rewards/margins": 5.36160945892334,
      "rewards/rejected": -5.462925910949707,
      "step": 550
    },
    {
      "epoch": 0.6924265842349304,
      "grad_norm": 11.625,
      "learning_rate": 1.8922232331013697e-06,
      "log_odds_chosen": 6.240553855895996,
      "log_odds_ratio": -0.45034289360046387,
      "logits/chosen": -0.3342943787574768,
      "logits/rejected": -0.3671431541442871,
      "logps/chosen": -0.21692046523094177,
      "logps/rejected": -5.070644855499268,
      "loss": 0.5489,
      "nll_loss": 0.2964481711387634,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.21692046523094177,
      "rewards/margins": 4.853724002838135,
      "rewards/rejected": -5.070644855499268,
      "step": 560
    },
    {
      "epoch": 0.7047913446676971,
      "grad_norm": 16.25,
      "learning_rate": 1.7547841510553995e-06,
      "log_odds_chosen": 7.560661315917969,
      "log_odds_ratio": -0.21453876793384552,
      "logits/chosen": -0.3607560098171234,
      "logits/rejected": -0.41021689772605896,
      "logps/chosen": -0.116365946829319,
      "logps/rejected": -5.9038801193237305,
      "loss": 0.3902,
      "nll_loss": 0.1380702257156372,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.116365946829319,
      "rewards/margins": 5.787513732910156,
      "rewards/rejected": -5.9038801193237305,
      "step": 570
    },
    {
      "epoch": 0.7171561051004637,
      "grad_norm": 10.0,
      "learning_rate": 1.6211112951954391e-06,
      "log_odds_chosen": 6.993834495544434,
      "log_odds_ratio": -0.2905352711677551,
      "logits/chosen": -0.3096144497394562,
      "logits/rejected": -0.364246666431427,
      "logps/chosen": -0.10927136242389679,
      "logps/rejected": -5.500131130218506,
      "loss": 0.4284,
      "nll_loss": 0.11469737440347672,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.10927136242389679,
      "rewards/margins": 5.390860080718994,
      "rewards/rejected": -5.500131130218506,
      "step": 580
    },
    {
      "epoch": 0.7295208655332303,
      "grad_norm": 7.9375,
      "learning_rate": 1.4914288943903196e-06,
      "log_odds_chosen": 7.680525779724121,
      "log_odds_ratio": -0.17404568195343018,
      "logits/chosen": -0.34793198108673096,
      "logits/rejected": -0.38758113980293274,
      "logps/chosen": -0.11340661346912384,
      "logps/rejected": -5.949253082275391,
      "loss": 0.4439,
      "nll_loss": 0.12422819435596466,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.11340661346912384,
      "rewards/margins": 5.835846900939941,
      "rewards/rejected": -5.949253082275391,
      "step": 590
    },
    {
      "epoch": 0.7418856259659969,
      "grad_norm": 9.4375,
      "learning_rate": 1.3659544837398459e-06,
      "log_odds_chosen": 8.277643203735352,
      "log_odds_ratio": -0.23421244323253632,
      "logits/chosen": -0.33900585770606995,
      "logits/rejected": -0.40172091126441956,
      "logps/chosen": -0.2025931179523468,
      "logps/rejected": -6.714430809020996,
      "loss": 0.4434,
      "nll_loss": 0.26020824909210205,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.2025931179523468,
      "rewards/margins": 6.511837959289551,
      "rewards/rejected": -6.714430809020996,
      "step": 600
    },
    {
      "epoch": 0.7542503863987635,
      "grad_norm": 7.90625,
      "learning_rate": 1.2448985396716101e-06,
      "log_odds_chosen": 7.007063865661621,
      "log_odds_ratio": -0.2840166389942169,
      "logits/chosen": -0.3195096552371979,
      "logits/rejected": -0.3758764863014221,
      "logps/chosen": -0.1149110421538353,
      "logps/rejected": -5.493623733520508,
      "loss": 0.3982,
      "nll_loss": 0.12395291030406952,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.1149110421538353,
      "rewards/margins": 5.3787126541137695,
      "rewards/rejected": -5.493623733520508,
      "step": 610
    },
    {
      "epoch": 0.7666151468315301,
      "grad_norm": 6.78125,
      "learning_rate": 1.1284641268783634e-06,
      "log_odds_chosen": 7.604881286621094,
      "log_odds_ratio": -0.2630865275859833,
      "logits/chosen": -0.3634548783302307,
      "logits/rejected": -0.4339476227760315,
      "logps/chosen": -0.11267434060573578,
      "logps/rejected": -6.021006107330322,
      "loss": 0.3734,
      "nll_loss": 0.12431363761425018,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.11267434060573578,
      "rewards/margins": 5.908331871032715,
      "rewards/rejected": -6.021006107330322,
      "step": 620
    },
    {
      "epoch": 0.7789799072642968,
      "grad_norm": 5.96875,
      "learning_rate": 1.0168465576881482e-06,
      "log_odds_chosen": 7.153097629547119,
      "log_odds_ratio": -0.2416454553604126,
      "logits/chosen": -0.3371705114841461,
      "logits/rejected": -0.367400586605072,
      "logps/chosen": -0.12628045678138733,
      "logps/rejected": -5.651804447174072,
      "loss": 0.4343,
      "nll_loss": 0.1415124386548996,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -0.12628045678138733,
      "rewards/margins": 5.525524139404297,
      "rewards/rejected": -5.651804447174072,
      "step": 630
    },
    {
      "epoch": 0.7913446676970634,
      "grad_norm": 13.5625,
      "learning_rate": 9.102330644386023e-07,
      "log_odds_chosen": 7.592588901519775,
      "log_odds_ratio": -0.2542632520198822,
      "logits/chosen": -0.3656562864780426,
      "logits/rejected": -0.41636595129966736,
      "logps/chosen": -0.08560268580913544,
      "logps/rejected": -5.888397216796875,
      "loss": 0.3524,
      "nll_loss": 0.091241754591465,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.08560268580913544,
      "rewards/margins": 5.802794456481934,
      "rewards/rejected": -5.888397216796875,
      "step": 640
    },
    {
      "epoch": 0.80370942812983,
      "grad_norm": 10.6875,
      "learning_rate": 8.088024854050073e-07,
      "log_odds_chosen": 7.490597724914551,
      "log_odds_ratio": -0.2114340364933014,
      "logits/chosen": -0.39008674025535583,
      "logits/rejected": -0.4323447644710541,
      "logps/chosen": -0.0939800813794136,
      "logps/rejected": -5.781932830810547,
      "loss": 0.3448,
      "nll_loss": 0.09702688455581665,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.0939800813794136,
      "rewards/margins": 5.687952518463135,
      "rewards/rejected": -5.781932830810547,
      "step": 650
    },
    {
      "epoch": 0.8160741885625966,
      "grad_norm": 13.8125,
      "learning_rate": 7.127249648089084e-07,
      "log_odds_chosen": 7.167318820953369,
      "log_odds_ratio": -0.2506093382835388,
      "logits/chosen": -0.3160146176815033,
      "logits/rejected": -0.3702705204486847,
      "logps/chosen": -0.11508496105670929,
      "logps/rejected": -5.595463752746582,
      "loss": 0.3517,
      "nll_loss": 0.12279774993658066,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.11508496105670929,
      "rewards/margins": 5.4803786277771,
      "rewards/rejected": -5.595463752746582,
      "step": 660
    },
    {
      "epoch": 0.8284389489953632,
      "grad_norm": 5.4375,
      "learning_rate": 6.221616674105323e-07,
      "log_odds_chosen": 7.596763610839844,
      "log_odds_ratio": -0.16744232177734375,
      "logits/chosen": -0.3435714840888977,
      "logits/rejected": -0.3635823428630829,
      "logps/chosen": -0.10476762056350708,
      "logps/rejected": -5.7375054359436035,
      "loss": 0.3234,
      "nll_loss": 0.11400828510522842,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.10476762056350708,
      "rewards/margins": 5.6327385902404785,
      "rewards/rejected": -5.7375054359436035,
      "step": 670
    },
    {
      "epoch": 0.8408037094281299,
      "grad_norm": 9.6875,
      "learning_rate": 5.372645081637657e-07,
      "log_odds_chosen": 7.392083168029785,
      "log_odds_ratio": -0.24827039241790771,
      "logits/chosen": -0.3297838866710663,
      "logits/rejected": -0.36765509843826294,
      "logps/chosen": -0.10170583426952362,
      "logps/rejected": -5.7545552253723145,
      "loss": 0.3629,
      "nll_loss": 0.10637170076370239,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.10170583426952362,
      "rewards/margins": 5.6528496742248535,
      "rewards/rejected": -5.7545552253723145,
      "step": 680
    },
    {
      "epoch": 0.8531684698608965,
      "grad_norm": 9.5,
      "learning_rate": 4.581758973871608e-07,
      "log_odds_chosen": 6.805171012878418,
      "log_odds_ratio": -0.28382524847984314,
      "logits/chosen": -0.31761232018470764,
      "logits/rejected": -0.35636773705482483,
      "logps/chosen": -0.09772305935621262,
      "logps/rejected": -5.221259117126465,
      "loss": 0.3796,
      "nll_loss": 0.10472699254751205,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.09772305935621262,
      "rewards/margins": 5.123536109924316,
      "rewards/rejected": -5.221259117126465,
      "step": 690
    },
    {
      "epoch": 0.865533230293663,
      "grad_norm": 9.9375,
      "learning_rate": 3.85028501878462e-07,
      "log_odds_chosen": 6.70258092880249,
      "log_odds_ratio": -0.2719319462776184,
      "logits/chosen": -0.34218502044677734,
      "logits/rejected": -0.38877347111701965,
      "logps/chosen": -0.10239502042531967,
      "logps/rejected": -5.217819690704346,
      "loss": 0.4089,
      "nll_loss": 0.10687730461359024,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.10239502042531967,
      "rewards/margins": 5.115423679351807,
      "rewards/rejected": -5.217819690704346,
      "step": 700
    },
    {
      "epoch": 0.8778979907264297,
      "grad_norm": 10.25,
      "learning_rate": 3.179450223733404e-07,
      "log_odds_chosen": 7.523824214935303,
      "log_odds_ratio": -0.2363746464252472,
      "logits/chosen": -0.29321369528770447,
      "logits/rejected": -0.36340442299842834,
      "logps/chosen": -0.12782010436058044,
      "logps/rejected": -5.973371505737305,
      "loss": 0.4016,
      "nll_loss": 0.15740366280078888,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.12782010436058044,
      "rewards/margins": 5.845551013946533,
      "rewards/rejected": -5.973371505737305,
      "step": 710
    },
    {
      "epoch": 0.8902627511591963,
      "grad_norm": 9.6875,
      "learning_rate": 2.5703798772165066e-07,
      "log_odds_chosen": 7.885851860046387,
      "log_odds_ratio": -0.2181587666273117,
      "logits/chosen": -0.33393245935440063,
      "logits/rejected": -0.40647512674331665,
      "logps/chosen": -0.10159681737422943,
      "logps/rejected": -6.100408554077148,
      "loss": 0.3762,
      "nll_loss": 0.11738970130681992,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.10159681737422943,
      "rewards/margins": 5.998810768127441,
      "rewards/rejected": -6.100408554077148,
      "step": 720
    },
    {
      "epoch": 0.9026275115919629,
      "grad_norm": 10.125,
      "learning_rate": 2.0240956612647487e-07,
      "log_odds_chosen": 8.462138175964355,
      "log_odds_ratio": -0.21807698905467987,
      "logits/chosen": -0.34213870763778687,
      "logits/rejected": -0.4005005955696106,
      "logps/chosen": -0.13064655661582947,
      "logps/rejected": -6.555903434753418,
      "loss": 0.3838,
      "nll_loss": 0.15605738759040833,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.13064655661582947,
      "rewards/margins": 6.425257205963135,
      "rewards/rejected": -6.555903434753418,
      "step": 730
    },
    {
      "epoch": 0.9149922720247295,
      "grad_norm": 12.6875,
      "learning_rate": 1.5415139376257557e-07,
      "log_odds_chosen": 6.915780067443848,
      "log_odds_ratio": -0.27733683586120605,
      "logits/chosen": -0.33304262161254883,
      "logits/rejected": -0.3566564917564392,
      "logps/chosen": -0.11249081045389175,
      "logps/rejected": -5.320967197418213,
      "loss": 0.3624,
      "nll_loss": 0.12775245308876038,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -0.11249081045389175,
      "rewards/margins": 5.2084760665893555,
      "rewards/rejected": -5.320967197418213,
      "step": 740
    },
    {
      "epoch": 0.9273570324574961,
      "grad_norm": 13.875,
      "learning_rate": 1.1234442106174702e-07,
      "log_odds_chosen": 8.112470626831055,
      "log_odds_ratio": -0.17907017469406128,
      "logits/chosen": -0.3399219512939453,
      "logits/rejected": -0.3967411518096924,
      "logps/chosen": -0.09002333879470825,
      "logps/rejected": -6.250402927398682,
      "loss": 0.405,
      "nll_loss": 0.09456364065408707,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.09002333879470825,
      "rewards/margins": 6.160379886627197,
      "rewards/rejected": -6.250402927398682,
      "step": 750
    },
    {
      "epoch": 0.9397217928902627,
      "grad_norm": 12.4375,
      "learning_rate": 7.705877692291896e-08,
      "log_odds_chosen": 7.381073951721191,
      "log_odds_ratio": -0.20707659423351288,
      "logits/chosen": -0.33521759510040283,
      "logits/rejected": -0.37959274649620056,
      "logps/chosen": -0.11178772151470184,
      "logps/rejected": -5.718709468841553,
      "loss": 0.3739,
      "nll_loss": 0.12628722190856934,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.11178772151470184,
      "rewards/margins": 5.606922149658203,
      "rewards/rejected": -5.718709468841553,
      "step": 760
    },
    {
      "epoch": 0.9520865533230294,
      "grad_norm": 9.6875,
      "learning_rate": 4.835365107477907e-08,
      "log_odds_chosen": 6.537681579589844,
      "log_odds_ratio": -0.28112491965293884,
      "logits/chosen": -0.32352039217948914,
      "logits/rejected": -0.3757088780403137,
      "logps/chosen": -0.11319409310817719,
      "logps/rejected": -5.078321933746338,
      "loss": 0.4342,
      "nll_loss": 0.1269460767507553,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.11319409310817719,
      "rewards/margins": 4.965128421783447,
      "rewards/rejected": -5.078321933746338,
      "step": 770
    },
    {
      "epoch": 0.964451313755796,
      "grad_norm": 9.25,
      "learning_rate": 2.6277194788258737e-08,
      "log_odds_chosen": 7.285265922546387,
      "log_odds_ratio": -0.34375110268592834,
      "logits/chosen": -0.3152714669704437,
      "logits/rejected": -0.35118311643600464,
      "logps/chosen": -0.19179192185401917,
      "logps/rejected": -5.783249378204346,
      "loss": 0.5394,
      "nll_loss": 0.2578078508377075,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.19179192185401917,
      "rewards/margins": 5.591456890106201,
      "rewards/rejected": -5.783249378204346,
      "step": 780
    },
    {
      "epoch": 0.9768160741885626,
      "grad_norm": 13.875,
      "learning_rate": 1.0866440105421925e-08,
      "log_odds_chosen": 6.224413871765137,
      "log_odds_ratio": -0.32318800687789917,
      "logits/chosen": -0.29942208528518677,
      "logits/rejected": -0.34637898206710815,
      "logps/chosen": -0.12877008318901062,
      "logps/rejected": -4.796613693237305,
      "loss": 0.3981,
      "nll_loss": 0.15550634264945984,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.12877008318901062,
      "rewards/margins": 4.667844295501709,
      "rewards/rejected": -4.796613693237305,
      "step": 790
    },
    {
      "epoch": 0.9891808346213292,
      "grad_norm": 9.5625,
      "learning_rate": 2.1472377202500504e-09,
      "log_odds_chosen": 6.32632303237915,
      "log_odds_ratio": -0.25827693939208984,
      "logits/chosen": -0.34357064962387085,
      "logits/rejected": -0.35584110021591187,
      "logps/chosen": -0.1051005870103836,
      "logps/rejected": -4.862908840179443,
      "loss": 0.3672,
      "nll_loss": 0.10939991474151611,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.1051005870103836,
      "rewards/margins": 4.757808208465576,
      "rewards/rejected": -4.862908840179443,
      "step": 800
    },
    {
      "epoch": 0.9990726429675425,
      "step": 808,
      "total_flos": 0.0,
      "train_loss": 0.411900937852293,
      "train_runtime": 2894.5788,
      "train_samples_per_second": 4.469,
      "train_steps_per_second": 0.279
    }
  ],
  "logging_steps": 10,
  "max_steps": 808,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 100,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": false,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 2,
  "trial_name": null,
  "trial_params": null
}