{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 3.9951721918249117,
  "eval_steps": 500,
  "global_step": 3104,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.012874155133569359,
      "grad_norm": 116.00975162348479,
      "learning_rate": 9.971005154639174e-07,
      "logits/chosen": -0.4733520448207855,
      "logits/rejected": -0.5006790161132812,
      "logps/chosen": -357.125,
      "logps/rejected": -361.4624938964844,
      "loss": 0.6745,
      "rewards/accuracies": 0.3499999940395355,
      "rewards/chosen": -0.2167530059814453,
      "rewards/margins": 0.06282500922679901,
      "rewards/rejected": -0.279653936624527,
      "step": 10
    },
    {
      "epoch": 0.025748310267138717,
      "grad_norm": 110.36198541397852,
      "learning_rate": 9.938788659793814e-07,
      "logits/chosen": -0.4485534727573395,
      "logits/rejected": -0.47062987089157104,
      "logps/chosen": -325.48748779296875,
      "logps/rejected": -322.1000061035156,
      "loss": 0.5977,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.7079528570175171,
      "rewards/margins": 0.3867950439453125,
      "rewards/rejected": -1.094751000404358,
      "step": 20
    },
    {
      "epoch": 0.03862246540070808,
      "grad_norm": 123.52649986895027,
      "learning_rate": 9.906572164948455e-07,
      "logits/chosen": -0.7004241943359375,
      "logits/rejected": -0.7022460699081421,
      "logps/chosen": -400.625,
      "logps/rejected": -400.2749938964844,
      "loss": 0.6372,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.50439453125,
      "rewards/margins": 0.4994445741176605,
      "rewards/rejected": -2.0038084983825684,
      "step": 30
    },
    {
      "epoch": 0.051496620534277435,
      "grad_norm": 98.330089952775,
      "learning_rate": 9.874355670103093e-07,
      "logits/chosen": -0.606005847454071,
      "logits/rejected": -0.628643810749054,
      "logps/chosen": -425.7749938964844,
      "logps/rejected": -373.6000061035156,
      "loss": 0.707,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -1.1126892566680908,
      "rewards/margins": 0.47386473417282104,
      "rewards/rejected": -1.5862548351287842,
      "step": 40
    },
    {
      "epoch": 0.0643707756678468,
      "grad_norm": 108.78565495841251,
      "learning_rate": 9.84213917525773e-07,
      "logits/chosen": -0.47911375761032104,
      "logits/rejected": -0.4762206971645355,
      "logps/chosen": -360.20001220703125,
      "logps/rejected": -352.7250061035156,
      "loss": 0.6266,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": 0.02710571326315403,
      "rewards/margins": 0.42482298612594604,
      "rewards/rejected": -0.3973388671875,
      "step": 50
    },
    {
      "epoch": 0.07724493080141616,
      "grad_norm": 86.86103704117889,
      "learning_rate": 9.80992268041237e-07,
      "logits/chosen": -0.47004395723342896,
      "logits/rejected": -0.4599609375,
      "logps/chosen": -362.01251220703125,
      "logps/rejected": -316.54998779296875,
      "loss": 0.6111,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": 0.381765753030777,
      "rewards/margins": 0.4092163145542145,
      "rewards/rejected": -0.02763671800494194,
      "step": 60
    },
    {
      "epoch": 0.09011908593498552,
      "grad_norm": 112.58148578622469,
      "learning_rate": 9.777706185567009e-07,
      "logits/chosen": -0.47901612520217896,
      "logits/rejected": -0.5212951898574829,
      "logps/chosen": -381.6499938964844,
      "logps/rejected": -362.29998779296875,
      "loss": 0.6341,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": 0.11572265625,
      "rewards/margins": 0.47606199979782104,
      "rewards/rejected": -0.360748291015625,
      "step": 70
    },
    {
      "epoch": 0.10299324106855487,
      "grad_norm": 112.67612692520709,
      "learning_rate": 9.74548969072165e-07,
      "logits/chosen": -0.3992675840854645,
      "logits/rejected": -0.4010772705078125,
      "logps/chosen": -336.23748779296875,
      "logps/rejected": -324.8500061035156,
      "loss": 0.6137,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": 0.11996765434741974,
      "rewards/margins": 0.48876953125,
      "rewards/rejected": -0.3689514100551605,
      "step": 80
    },
    {
      "epoch": 0.11586739620212423,
      "grad_norm": 120.71710686325964,
      "learning_rate": 9.71327319587629e-07,
      "logits/chosen": -0.4455810487270355,
      "logits/rejected": -0.46094971895217896,
      "logps/chosen": -368.4624938964844,
      "logps/rejected": -340.8812561035156,
      "loss": 0.601,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": 0.15057983994483948,
      "rewards/margins": 0.4910644590854645,
      "rewards/rejected": -0.33942872285842896,
      "step": 90
    },
    {
      "epoch": 0.1287415513356936,
      "grad_norm": 94.09238987623964,
      "learning_rate": 9.681056701030927e-07,
      "logits/chosen": -0.4274749755859375,
      "logits/rejected": -0.4272094666957855,
      "logps/chosen": -349.5874938964844,
      "logps/rejected": -326.48748779296875,
      "loss": 0.6003,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": 0.292459100484848,
      "rewards/margins": 0.49652099609375,
      "rewards/rejected": -0.20349006354808807,
      "step": 100
    },
    {
      "epoch": 0.14161570646926294,
      "grad_norm": 115.3027666657839,
      "learning_rate": 9.648840206185567e-07,
      "logits/chosen": -0.4821411073207855,
      "logits/rejected": -0.473165899515152,
      "logps/chosen": -364.82501220703125,
      "logps/rejected": -361.45001220703125,
      "loss": 0.6459,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": 0.7324981689453125,
      "rewards/margins": 0.547314465045929,
      "rewards/rejected": 0.18525390326976776,
      "step": 110
    },
    {
      "epoch": 0.15448986160283232,
      "grad_norm": 100.58566673282141,
      "learning_rate": 9.616623711340205e-07,
      "logits/chosen": -0.4615234434604645,
      "logits/rejected": -0.46809083223342896,
      "logps/chosen": -337.25,
      "logps/rejected": -339.4375,
      "loss": 0.6278,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": 0.8613036870956421,
      "rewards/margins": 0.5265868902206421,
      "rewards/rejected": 0.3347229063510895,
      "step": 120
    },
    {
      "epoch": 0.16736401673640167,
      "grad_norm": 122.76745910102257,
      "learning_rate": 9.584407216494846e-07,
      "logits/chosen": -0.5855712890625,
      "logits/rejected": -0.6137939691543579,
      "logps/chosen": -402.11248779296875,
      "logps/rejected": -360.8500061035156,
      "loss": 0.5982,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": 1.0935547351837158,
      "rewards/margins": 0.5986083745956421,
      "rewards/rejected": 0.495025634765625,
      "step": 130
    },
    {
      "epoch": 0.18023817186997104,
      "grad_norm": 106.63724432186719,
      "learning_rate": 9.552190721649484e-07,
      "logits/chosen": -0.5129760503768921,
      "logits/rejected": -0.5257323980331421,
      "logps/chosen": -352.5874938964844,
      "logps/rejected": -346.38751220703125,
      "loss": 0.6439,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": 0.80364990234375,
      "rewards/margins": 0.48191529512405396,
      "rewards/rejected": 0.32114869356155396,
      "step": 140
    },
    {
      "epoch": 0.1931123270035404,
      "grad_norm": 108.47394171237873,
      "learning_rate": 9.519974226804123e-07,
      "logits/chosen": -0.6437743902206421,
      "logits/rejected": -0.65399169921875,
      "logps/chosen": -382.125,
      "logps/rejected": -388.0249938964844,
      "loss": 0.6183,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": 0.25493162870407104,
      "rewards/margins": 0.700610339641571,
      "rewards/rejected": -0.44627076387405396,
      "step": 150
    },
    {
      "epoch": 0.20598648213710974,
      "grad_norm": 114.07304846157687,
      "learning_rate": 9.487757731958762e-07,
      "logits/chosen": -0.6397460699081421,
      "logits/rejected": -0.612835705280304,
      "logps/chosen": -345.11248779296875,
      "logps/rejected": -337.36248779296875,
      "loss": 0.536,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.21904297173023224,
      "rewards/margins": 0.951220691204071,
      "rewards/rejected": -0.733203113079071,
      "step": 160
    },
    {
      "epoch": 0.21886063727067911,
      "grad_norm": 105.78503442333401,
      "learning_rate": 9.455541237113401e-07,
      "logits/chosen": -0.642993152141571,
      "logits/rejected": -0.634033203125,
      "logps/chosen": -407.07501220703125,
      "logps/rejected": -365.3999938964844,
      "loss": 0.6362,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": 0.16254882514476776,
      "rewards/margins": 0.850903332233429,
      "rewards/rejected": -0.688751220703125,
      "step": 170
    },
    {
      "epoch": 0.23173479240424846,
      "grad_norm": 98.33327737276883,
      "learning_rate": 9.423324742268041e-07,
      "logits/chosen": -0.6112426519393921,
      "logits/rejected": -0.61279296875,
      "logps/chosen": -366.9624938964844,
      "logps/rejected": -348.95001220703125,
      "loss": 0.6003,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": 0.25787353515625,
      "rewards/margins": 0.7079833745956421,
      "rewards/rejected": -0.4504150450229645,
      "step": 180
    },
    {
      "epoch": 0.24460894753781784,
      "grad_norm": 120.00275335834641,
      "learning_rate": 9.39110824742268e-07,
      "logits/chosen": -0.632250964641571,
      "logits/rejected": -0.651928722858429,
      "logps/chosen": -372.3125,
      "logps/rejected": -362.79998779296875,
      "loss": 0.5571,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": 0.01650390587747097,
      "rewards/margins": 0.7537597417831421,
      "rewards/rejected": -0.7377685308456421,
      "step": 190
    },
    {
      "epoch": 0.2574831026713872,
      "grad_norm": 129.37774954834526,
      "learning_rate": 9.358891752577319e-07,
      "logits/chosen": -0.5298446416854858,
      "logits/rejected": -0.5588653683662415,
      "logps/chosen": -385.6000061035156,
      "logps/rejected": -359.0,
      "loss": 0.6273,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.3814453184604645,
      "rewards/margins": 0.734661877155304,
      "rewards/rejected": -1.115972876548767,
      "step": 200
    },
    {
      "epoch": 0.27035725780495656,
      "grad_norm": 85.46600016889145,
      "learning_rate": 9.326675257731958e-07,
      "logits/chosen": -0.5501617193222046,
      "logits/rejected": -0.557861328125,
      "logps/chosen": -375.79998779296875,
      "logps/rejected": -389.61248779296875,
      "loss": 0.5897,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.4184021055698395,
      "rewards/margins": 0.7344604730606079,
      "rewards/rejected": -1.152307152748108,
      "step": 210
    },
    {
      "epoch": 0.2832314129385259,
      "grad_norm": 112.06031531752737,
      "learning_rate": 9.294458762886597e-07,
      "logits/chosen": -0.61865234375,
      "logits/rejected": -0.6116698980331421,
      "logps/chosen": -375.4437561035156,
      "logps/rejected": -348.0,
      "loss": 0.6132,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.16201171278953552,
      "rewards/margins": 0.8055419921875,
      "rewards/rejected": -0.9680572748184204,
      "step": 220
    },
    {
      "epoch": 0.29610556807209526,
      "grad_norm": 131.6342917904006,
      "learning_rate": 9.262242268041238e-07,
      "logits/chosen": -0.6041504144668579,
      "logits/rejected": -0.5826416015625,
      "logps/chosen": -366.8374938964844,
      "logps/rejected": -360.5249938964844,
      "loss": 0.6912,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.23549041152000427,
      "rewards/margins": 0.52764892578125,
      "rewards/rejected": -0.7628723382949829,
      "step": 230
    },
    {
      "epoch": 0.30897972320566464,
      "grad_norm": 90.71845088558837,
      "learning_rate": 9.230025773195877e-07,
      "logits/chosen": -0.47716981172561646,
      "logits/rejected": -0.47523194551467896,
      "logps/chosen": -393.6000061035156,
      "logps/rejected": -380.0,
      "loss": 0.6737,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.11795654147863388,
      "rewards/margins": 0.4956298768520355,
      "rewards/rejected": -0.614117443561554,
      "step": 240
    },
    {
      "epoch": 0.321853878339234,
      "grad_norm": 108.95199143251729,
      "learning_rate": 9.197809278350515e-07,
      "logits/chosen": -0.5445922613143921,
      "logits/rejected": -0.555126965045929,
      "logps/chosen": -348.6499938964844,
      "logps/rejected": -335.07501220703125,
      "loss": 0.5944,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": 0.34312742948532104,
      "rewards/margins": 0.5995849370956421,
      "rewards/rejected": -0.256674200296402,
      "step": 250
    },
    {
      "epoch": 0.33472803347280333,
      "grad_norm": 117.30565370500288,
      "learning_rate": 9.165592783505154e-07,
      "logits/chosen": -0.5303589105606079,
      "logits/rejected": -0.531933605670929,
      "logps/chosen": -383.2875061035156,
      "logps/rejected": -357.375,
      "loss": 0.6335,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": 0.18229980766773224,
      "rewards/margins": 0.515429675579071,
      "rewards/rejected": -0.3332763612270355,
      "step": 260
    },
    {
      "epoch": 0.3476021886063727,
      "grad_norm": 137.214632537694,
      "learning_rate": 9.133376288659793e-07,
      "logits/chosen": -0.49357300996780396,
      "logits/rejected": -0.524523913860321,
      "logps/chosen": -333.70001220703125,
      "logps/rejected": -317.45001220703125,
      "loss": 0.6348,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": 0.3096069395542145,
      "rewards/margins": 0.714184582233429,
      "rewards/rejected": -0.40443724393844604,
      "step": 270
    },
    {
      "epoch": 0.3604763437399421,
      "grad_norm": 120.58309781518359,
      "learning_rate": 9.101159793814432e-07,
      "logits/chosen": -0.547167956829071,
      "logits/rejected": -0.542285144329071,
      "logps/chosen": -345.1625061035156,
      "logps/rejected": -328.2875061035156,
      "loss": 0.61,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": 0.11395873874425888,
      "rewards/margins": 0.7199951410293579,
      "rewards/rejected": -0.6070495843887329,
      "step": 280
    },
    {
      "epoch": 0.3733504988735114,
      "grad_norm": 134.39026346308177,
      "learning_rate": 9.068943298969072e-07,
      "logits/chosen": -0.5337455868721008,
      "logits/rejected": -0.548297107219696,
      "logps/chosen": -339.92498779296875,
      "logps/rejected": -339.0249938964844,
      "loss": 0.7029,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.17380675673484802,
      "rewards/margins": 0.6114867925643921,
      "rewards/rejected": -0.7854369878768921,
      "step": 290
    },
    {
      "epoch": 0.3862246540070808,
      "grad_norm": 102.9863094116885,
      "learning_rate": 9.036726804123711e-07,
      "logits/chosen": -0.522229015827179,
      "logits/rejected": -0.558056652545929,
      "logps/chosen": -399.5,
      "logps/rejected": -381.29998779296875,
      "loss": 0.5618,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.040252685546875,
      "rewards/margins": 0.8593994379043579,
      "rewards/rejected": -0.818774402141571,
      "step": 300
    },
    {
      "epoch": 0.39909880914065016,
      "grad_norm": 75.27946455181142,
      "learning_rate": 9.00451030927835e-07,
      "logits/chosen": -0.594006359577179,
      "logits/rejected": -0.5999511480331421,
      "logps/chosen": -377.1625061035156,
      "logps/rejected": -372.5,
      "loss": 0.5782,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": 0.4391235411167145,
      "rewards/margins": 0.9082275629043579,
      "rewards/rejected": -0.4693969786167145,
      "step": 310
    },
    {
      "epoch": 0.4119729642742195,
      "grad_norm": 109.37234543670532,
      "learning_rate": 8.97229381443299e-07,
      "logits/chosen": -0.559466540813446,
      "logits/rejected": -0.5611419677734375,
      "logps/chosen": -402.92498779296875,
      "logps/rejected": -364.25,
      "loss": 0.6529,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.15833130478858948,
      "rewards/margins": 0.510876476764679,
      "rewards/rejected": -0.3524169921875,
      "step": 320
    },
    {
      "epoch": 0.42484711940778885,
      "grad_norm": 100.35729116251687,
      "learning_rate": 8.940077319587629e-07,
      "logits/chosen": -0.49317628145217896,
      "logits/rejected": -0.510180652141571,
      "logps/chosen": -340.0625,
      "logps/rejected": -362.82501220703125,
      "loss": 0.63,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.11295776069164276,
      "rewards/margins": 0.605303943157196,
      "rewards/rejected": -0.49190062284469604,
      "step": 330
    },
    {
      "epoch": 0.43772127454135823,
      "grad_norm": 118.0041581111788,
      "learning_rate": 8.907860824742268e-07,
      "logits/chosen": -0.5267089605331421,
      "logits/rejected": -0.513476550579071,
      "logps/chosen": -382.75,
      "logps/rejected": -366.42498779296875,
      "loss": 0.6493,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.2607360780239105,
      "rewards/margins": 0.6520751714706421,
      "rewards/rejected": -0.912457287311554,
      "step": 340
    },
    {
      "epoch": 0.4505954296749276,
      "grad_norm": 107.54174475375848,
      "learning_rate": 8.875644329896907e-07,
      "logits/chosen": -0.4590210020542145,
      "logits/rejected": -0.4604553282260895,
      "logps/chosen": -368.58123779296875,
      "logps/rejected": -370.70001220703125,
      "loss": 0.6031,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.18199463188648224,
      "rewards/margins": 0.683074951171875,
      "rewards/rejected": -0.864489734172821,
      "step": 350
    },
    {
      "epoch": 0.4634695848084969,
      "grad_norm": 129.6562462083683,
      "learning_rate": 8.843427835051546e-07,
      "logits/chosen": -0.5956786870956421,
      "logits/rejected": -0.5789794921875,
      "logps/chosen": -377.17498779296875,
      "logps/rejected": -343.67498779296875,
      "loss": 0.6192,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.30079346895217896,
      "rewards/margins": 0.771191418170929,
      "rewards/rejected": -1.0708221197128296,
      "step": 360
    },
    {
      "epoch": 0.4763437399420663,
      "grad_norm": 78.5251638434667,
      "learning_rate": 8.811211340206185e-07,
      "logits/chosen": -0.504956066608429,
      "logits/rejected": -0.5179077386856079,
      "logps/chosen": -404.6499938964844,
      "logps/rejected": -383.6499938964844,
      "loss": 0.6394,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.8715881109237671,
      "rewards/margins": 0.7264404296875,
      "rewards/rejected": -1.597412109375,
      "step": 370
    },
    {
      "epoch": 0.4892178950756357,
      "grad_norm": 115.40461395946903,
      "learning_rate": 8.778994845360824e-07,
      "logits/chosen": -0.4976440370082855,
      "logits/rejected": -0.461700439453125,
      "logps/chosen": -400.70001220703125,
      "logps/rejected": -386.2250061035156,
      "loss": 0.7111,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.793865978717804,
      "rewards/margins": 0.489105224609375,
      "rewards/rejected": -1.283532738685608,
      "step": 380
    },
    {
      "epoch": 0.502092050209205,
      "grad_norm": 86.37675981578106,
      "learning_rate": 8.746778350515463e-07,
      "logits/chosen": -0.46751707792282104,
      "logits/rejected": -0.482351690530777,
      "logps/chosen": -363.92498779296875,
      "logps/rejected": -333.98748779296875,
      "loss": 0.5009,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.16302490234375,
      "rewards/margins": 0.8822997808456421,
      "rewards/rejected": -0.7193847894668579,
      "step": 390
    },
    {
      "epoch": 0.5149662053427744,
      "grad_norm": 81.10071006473835,
      "learning_rate": 8.714561855670102e-07,
      "logits/chosen": -0.5183471441268921,
      "logits/rejected": -0.504687488079071,
      "logps/chosen": -394.875,
      "logps/rejected": -363.125,
      "loss": 0.5786,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.15723876655101776,
      "rewards/margins": 0.764453113079071,
      "rewards/rejected": -0.6068481206893921,
      "step": 400
    },
    {
      "epoch": 0.5278403604763438,
      "grad_norm": 116.45510610225128,
      "learning_rate": 8.682345360824743e-07,
      "logits/chosen": -0.47703248262405396,
      "logits/rejected": -0.48649901151657104,
      "logps/chosen": -344.875,
      "logps/rejected": -347.63751220703125,
      "loss": 0.682,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.3260864317417145,
      "rewards/margins": 0.641326904296875,
      "rewards/rejected": -0.967907726764679,
      "step": 410
    },
    {
      "epoch": 0.5407145156099131,
      "grad_norm": 112.50781335140714,
      "learning_rate": 8.650128865979382e-07,
      "logits/chosen": -0.43253785371780396,
      "logits/rejected": -0.44200438261032104,
      "logps/chosen": -358.8125,
      "logps/rejected": -372.32501220703125,
      "loss": 0.6378,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.4115051329135895,
      "rewards/margins": 0.6463562250137329,
      "rewards/rejected": -1.0579345226287842,
      "step": 420
    },
    {
      "epoch": 0.5535886707434825,
      "grad_norm": 120.49044437617238,
      "learning_rate": 8.617912371134021e-07,
      "logits/chosen": -0.4475952088832855,
      "logits/rejected": -0.41944581270217896,
      "logps/chosen": -357.125,
      "logps/rejected": -344.79998779296875,
      "loss": 0.6222,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.23563233017921448,
      "rewards/margins": 0.6660400629043579,
      "rewards/rejected": -0.9013671875,
      "step": 430
    },
    {
      "epoch": 0.5664628258770518,
      "grad_norm": 68.29113701212785,
      "learning_rate": 8.585695876288659e-07,
      "logits/chosen": -0.4893554747104645,
      "logits/rejected": -0.5026305913925171,
      "logps/chosen": -372.04998779296875,
      "logps/rejected": -373.0,
      "loss": 0.5066,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.12487487494945526,
      "rewards/margins": 0.988940417766571,
      "rewards/rejected": -0.863964855670929,
      "step": 440
    },
    {
      "epoch": 0.5793369810106211,
      "grad_norm": 89.51903868981923,
      "learning_rate": 8.553479381443298e-07,
      "logits/chosen": -0.4515014588832855,
      "logits/rejected": -0.47959595918655396,
      "logps/chosen": -356.125,
      "logps/rejected": -362.67498779296875,
      "loss": 0.6334,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.608264148235321,
      "rewards/margins": 0.724169909954071,
      "rewards/rejected": -0.11616210639476776,
      "step": 450
    },
    {
      "epoch": 0.5922111361441905,
      "grad_norm": 148.43372901427,
      "learning_rate": 8.521262886597937e-07,
      "logits/chosen": -0.529223620891571,
      "logits/rejected": -0.54193115234375,
      "logps/chosen": -421.82501220703125,
      "logps/rejected": -414.875,
      "loss": 0.5713,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": 0.897900402545929,
      "rewards/margins": 0.8597167730331421,
      "rewards/rejected": 0.03883666917681694,
      "step": 460
    },
    {
      "epoch": 0.6050852912777599,
      "grad_norm": 118.08132341606118,
      "learning_rate": 8.489046391752577e-07,
      "logits/chosen": -0.569384753704071,
      "logits/rejected": -0.568408191204071,
      "logps/chosen": -380.57501220703125,
      "logps/rejected": -394.17498779296875,
      "loss": 0.7024,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": 0.3481384217739105,
      "rewards/margins": 0.4853881895542145,
      "rewards/rejected": -0.13858643174171448,
      "step": 470
    },
    {
      "epoch": 0.6179594464113293,
      "grad_norm": 107.65250644411428,
      "learning_rate": 8.456829896907216e-07,
      "logits/chosen": -0.539965808391571,
      "logits/rejected": -0.523364245891571,
      "logps/chosen": -373.3500061035156,
      "logps/rejected": -360.3999938964844,
      "loss": 0.5671,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.508160412311554,
      "rewards/margins": 0.902575671672821,
      "rewards/rejected": -0.3946289122104645,
      "step": 480
    },
    {
      "epoch": 0.6308336015448986,
      "grad_norm": 230.1997356494601,
      "learning_rate": 8.424613402061855e-07,
      "logits/chosen": -0.504443347454071,
      "logits/rejected": -0.5317474603652954,
      "logps/chosen": -366.2875061035156,
      "logps/rejected": -358.7749938964844,
      "loss": 0.5907,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": 0.33641356229782104,
      "rewards/margins": 0.6843627691268921,
      "rewards/rejected": -0.347940057516098,
      "step": 490
    },
    {
      "epoch": 0.643707756678468,
      "grad_norm": 124.54339481905843,
      "learning_rate": 8.392396907216494e-07,
      "logits/chosen": -0.5900634527206421,
      "logits/rejected": -0.597729504108429,
      "logps/chosen": -365.23748779296875,
      "logps/rejected": -349.32501220703125,
      "loss": 0.7723,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.802197277545929,
      "rewards/margins": 0.4662719666957855,
      "rewards/rejected": -1.267785668373108,
      "step": 500
    },
    {
      "epoch": 0.6565819118120373,
      "grad_norm": 132.57625659875265,
      "learning_rate": 8.360180412371134e-07,
      "logits/chosen": -0.5092223882675171,
      "logits/rejected": -0.5284179449081421,
      "logps/chosen": -409.3999938964844,
      "logps/rejected": -370.70001220703125,
      "loss": 0.6064,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.33637696504592896,
      "rewards/margins": 0.8779052495956421,
      "rewards/rejected": -1.2140624523162842,
      "step": 510
    },
    {
      "epoch": 0.6694560669456067,
      "grad_norm": 109.23063994206017,
      "learning_rate": 8.327963917525774e-07,
      "logits/chosen": -0.4924560487270355,
      "logits/rejected": -0.4965576231479645,
      "logps/chosen": -377.0,
      "logps/rejected": -366.48748779296875,
      "loss": 0.5947,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": 0.23482970893383026,
      "rewards/margins": 0.7789062261581421,
      "rewards/rejected": -0.543811023235321,
      "step": 520
    },
    {
      "epoch": 0.682330222079176,
      "grad_norm": 79.28047130384992,
      "learning_rate": 8.295747422680413e-07,
      "logits/chosen": -0.4707702696323395,
      "logits/rejected": -0.494821161031723,
      "logps/chosen": -339.92498779296875,
      "logps/rejected": -329.95001220703125,
      "loss": 0.5824,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": 0.066864013671875,
      "rewards/margins": 0.8806396722793579,
      "rewards/rejected": -0.813763439655304,
      "step": 530
    },
    {
      "epoch": 0.6952043772127454,
      "grad_norm": 80.78969993556497,
      "learning_rate": 8.263530927835051e-07,
      "logits/chosen": -0.565600574016571,
      "logits/rejected": -0.5763915777206421,
      "logps/chosen": -416.6000061035156,
      "logps/rejected": -380.0,
      "loss": 0.6185,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.05002441257238388,
      "rewards/margins": 0.8234497308731079,
      "rewards/rejected": -0.872814953327179,
      "step": 540
    },
    {
      "epoch": 0.7080785323463148,
      "grad_norm": 103.61303464709924,
      "learning_rate": 8.23131443298969e-07,
      "logits/chosen": -0.4691925048828125,
      "logits/rejected": -0.47784423828125,
      "logps/chosen": -354.1000061035156,
      "logps/rejected": -342.17498779296875,
      "loss": 0.5404,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.3255248963832855,
      "rewards/margins": 0.8783203363418579,
      "rewards/rejected": -0.5528930425643921,
      "step": 550
    },
    {
      "epoch": 0.7209526874798842,
      "grad_norm": 121.89695070499967,
      "learning_rate": 8.199097938144329e-07,
      "logits/chosen": -0.4993652403354645,
      "logits/rejected": -0.5168091058731079,
      "logps/chosen": -370.8500061035156,
      "logps/rejected": -361.6000061035156,
      "loss": 0.604,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": 0.34550780057907104,
      "rewards/margins": 0.926831066608429,
      "rewards/rejected": -0.5806182622909546,
      "step": 560
    },
    {
      "epoch": 0.7338268426134535,
      "grad_norm": 110.86609527894251,
      "learning_rate": 8.166881443298968e-07,
      "logits/chosen": -0.5541015863418579,
      "logits/rejected": -0.5427612066268921,
      "logps/chosen": -364.0249938964844,
      "logps/rejected": -360.2250061035156,
      "loss": 0.6308,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.39935302734375,
      "rewards/margins": 0.878186047077179,
      "rewards/rejected": -0.47841185331344604,
      "step": 570
    },
    {
      "epoch": 0.7467009977470228,
      "grad_norm": 110.69885173772958,
      "learning_rate": 8.134664948453608e-07,
      "logits/chosen": -0.50360107421875,
      "logits/rejected": -0.50262451171875,
      "logps/chosen": -397.36248779296875,
      "logps/rejected": -381.8999938964844,
      "loss": 0.6482,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": 0.6587890386581421,
      "rewards/margins": 0.916259765625,
      "rewards/rejected": -0.2574401795864105,
      "step": 580
    },
    {
      "epoch": 0.7595751528805922,
      "grad_norm": 74.35379476153639,
      "learning_rate": 8.102448453608247e-07,
      "logits/chosen": -0.4267639219760895,
      "logits/rejected": -0.45759278535842896,
      "logps/chosen": -381.20001220703125,
      "logps/rejected": -344.88751220703125,
      "loss": 0.5601,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 1.430151343345642,
      "rewards/margins": 0.877062976360321,
      "rewards/rejected": 0.552661120891571,
      "step": 590
    },
    {
      "epoch": 0.7724493080141616,
      "grad_norm": 111.98976593700215,
      "learning_rate": 8.070231958762887e-07,
      "logits/chosen": -0.44500732421875,
      "logits/rejected": -0.42840576171875,
      "logps/chosen": -367.3999938964844,
      "logps/rejected": -347.79998779296875,
      "loss": 0.652,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": 0.9703918695449829,
      "rewards/margins": 0.7120361328125,
      "rewards/rejected": 0.257925420999527,
      "step": 600
    },
    {
      "epoch": 0.7853234631477309,
      "grad_norm": 103.96984933499336,
      "learning_rate": 8.038015463917526e-07,
      "logits/chosen": -0.4539550840854645,
      "logits/rejected": -0.4577895998954773,
      "logps/chosen": -348.79998779296875,
      "logps/rejected": -364.6499938964844,
      "loss": 0.607,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": 0.05695800855755806,
      "rewards/margins": 0.9610961675643921,
      "rewards/rejected": -0.9044739007949829,
      "step": 610
    },
    {
      "epoch": 0.7981976182813003,
      "grad_norm": 83.82621463499343,
      "learning_rate": 8.005798969072165e-07,
      "logits/chosen": -0.44810181856155396,
      "logits/rejected": -0.44012451171875,
      "logps/chosen": -387.7250061035156,
      "logps/rejected": -382.75,
      "loss": 0.7625,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.9187866449356079,
      "rewards/margins": 0.5645202398300171,
      "rewards/rejected": -1.4832274913787842,
      "step": 620
    },
    {
      "epoch": 0.8110717734148697,
      "grad_norm": 136.60454494529628,
      "learning_rate": 7.973582474226804e-07,
      "logits/chosen": -0.53546142578125,
      "logits/rejected": -0.5382217168807983,
      "logps/chosen": -415.1000061035156,
      "logps/rejected": -393.125,
      "loss": 0.6367,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -1.3832824230194092,
      "rewards/margins": 0.589917004108429,
      "rewards/rejected": -1.9736328125,
      "step": 630
    },
    {
      "epoch": 0.823945928548439,
      "grad_norm": 175.185405939881,
      "learning_rate": 7.941365979381443e-07,
      "logits/chosen": -0.522747814655304,
      "logits/rejected": -0.5485290288925171,
      "logps/chosen": -378.4750061035156,
      "logps/rejected": -362.3999938964844,
      "loss": 0.5862,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9291747808456421,
      "rewards/margins": 0.790759265422821,
      "rewards/rejected": -1.7187988758087158,
      "step": 640
    },
    {
      "epoch": 0.8368200836820083,
      "grad_norm": 93.82500243107654,
      "learning_rate": 7.909149484536082e-07,
      "logits/chosen": -0.540942370891571,
      "logits/rejected": -0.540112316608429,
      "logps/chosen": -406.8999938964844,
      "logps/rejected": -394.3999938964844,
      "loss": 0.6338,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.9627624750137329,
      "rewards/margins": 0.6733764410018921,
      "rewards/rejected": -1.6355712413787842,
      "step": 650
    },
    {
      "epoch": 0.8496942388155777,
      "grad_norm": 98.8982278348208,
      "learning_rate": 7.876932989690721e-07,
      "logits/chosen": -0.467041015625,
      "logits/rejected": -0.4739440977573395,
      "logps/chosen": -358.5,
      "logps/rejected": -358.92498779296875,
      "loss": 0.6224,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.10512085258960724,
      "rewards/margins": 0.875134289264679,
      "rewards/rejected": -0.979418933391571,
      "step": 660
    },
    {
      "epoch": 0.8625683939491471,
      "grad_norm": 81.26508421612064,
      "learning_rate": 7.84471649484536e-07,
      "logits/chosen": -0.4732421934604645,
      "logits/rejected": -0.46550291776657104,
      "logps/chosen": -358.42498779296875,
      "logps/rejected": -339.2749938964844,
      "loss": 0.669,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": 0.03513794019818306,
      "rewards/margins": 0.5534912347793579,
      "rewards/rejected": -0.5187774896621704,
      "step": 670
    },
    {
      "epoch": 0.8754425490827165,
      "grad_norm": 96.38179368759864,
      "learning_rate": 7.812499999999999e-07,
      "logits/chosen": -0.502551257610321,
      "logits/rejected": -0.5140380859375,
      "logps/chosen": -387.92498779296875,
      "logps/rejected": -371.48748779296875,
      "loss": 0.5181,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": 0.40931397676467896,
      "rewards/margins": 1.079833984375,
      "rewards/rejected": -0.670947253704071,
      "step": 680
    },
    {
      "epoch": 0.8883167042162858,
      "grad_norm": 101.76727637730922,
      "learning_rate": 7.780283505154638e-07,
      "logits/chosen": -0.40287476778030396,
      "logits/rejected": -0.38833314180374146,
      "logps/chosen": -343.7875061035156,
      "logps/rejected": -352.8125,
      "loss": 0.6171,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.24252013862133026,
      "rewards/margins": 0.8604491949081421,
      "rewards/rejected": -1.102075219154358,
      "step": 690
    },
    {
      "epoch": 0.9011908593498552,
      "grad_norm": 122.00950637295834,
      "learning_rate": 7.748067010309279e-07,
      "logits/chosen": -0.42670899629592896,
      "logits/rejected": -0.44072264432907104,
      "logps/chosen": -361.5,
      "logps/rejected": -335.3500061035156,
      "loss": 0.7121,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": 0.06911621242761612,
      "rewards/margins": 0.7628173828125,
      "rewards/rejected": -0.694226086139679,
      "step": 700
    },
    {
      "epoch": 0.9140650144834245,
      "grad_norm": 127.45981012366637,
      "learning_rate": 7.715850515463918e-07,
      "logits/chosen": -0.4952026307582855,
      "logits/rejected": -0.501721203327179,
      "logps/chosen": -383.5874938964844,
      "logps/rejected": -377.6499938964844,
      "loss": 0.6536,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": 0.18529053032398224,
      "rewards/margins": 0.780322253704071,
      "rewards/rejected": -0.596270740032196,
      "step": 710
    },
    {
      "epoch": 0.9269391696169939,
      "grad_norm": 137.90708858880924,
      "learning_rate": 7.683634020618557e-07,
      "logits/chosen": -0.484619140625,
      "logits/rejected": -0.5116943120956421,
      "logps/chosen": -368.2749938964844,
      "logps/rejected": -354.4750061035156,
      "loss": 0.5936,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": 0.6229461431503296,
      "rewards/margins": 0.652905285358429,
      "rewards/rejected": -0.03063049353659153,
      "step": 720
    },
    {
      "epoch": 0.9398133247505632,
      "grad_norm": 109.52357137017624,
      "learning_rate": 7.651417525773195e-07,
      "logits/chosen": -0.572705090045929,
      "logits/rejected": -0.5957580804824829,
      "logps/chosen": -402.57501220703125,
      "logps/rejected": -377.2250061035156,
      "loss": 0.6558,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": 0.385833740234375,
      "rewards/margins": 0.52545166015625,
      "rewards/rejected": -0.13958129286766052,
      "step": 730
    },
    {
      "epoch": 0.9526874798841326,
      "grad_norm": 133.00416812765525,
      "learning_rate": 7.619201030927834e-07,
      "logits/chosen": -0.4806762635707855,
      "logits/rejected": -0.48743897676467896,
      "logps/chosen": -347.0,
      "logps/rejected": -332.6499938964844,
      "loss": 0.6252,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": 0.3818115293979645,
      "rewards/margins": 0.603198230266571,
      "rewards/rejected": -0.22109374403953552,
      "step": 740
    },
    {
      "epoch": 0.965561635017702,
      "grad_norm": 140.99897302512156,
      "learning_rate": 7.586984536082474e-07,
      "logits/chosen": -0.45958250761032104,
      "logits/rejected": -0.4542907774448395,
      "logps/chosen": -391.1875,
      "logps/rejected": -387.6000061035156,
      "loss": 0.7327,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.12029419094324112,
      "rewards/margins": 0.48200684785842896,
      "rewards/rejected": -0.602307140827179,
      "step": 750
    },
    {
      "epoch": 0.9784357901512714,
      "grad_norm": 131.274271707173,
      "learning_rate": 7.554768041237113e-07,
      "logits/chosen": -0.39448851346969604,
      "logits/rejected": -0.3970092833042145,
      "logps/chosen": -376.48748779296875,
      "logps/rejected": -361.95001220703125,
      "loss": 0.6513,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": 0.16361084580421448,
      "rewards/margins": 0.576507568359375,
      "rewards/rejected": -0.41328126192092896,
      "step": 760
    },
    {
      "epoch": 0.9913099452848407,
      "grad_norm": 73.69391656660149,
      "learning_rate": 7.522551546391752e-07,
      "logits/chosen": -0.412353515625,
      "logits/rejected": -0.41024476289749146,
      "logps/chosen": -370.9750061035156,
      "logps/rejected": -381.3500061035156,
      "loss": 0.6321,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.28642576932907104,
      "rewards/margins": 0.6205993890762329,
      "rewards/rejected": -0.33460694551467896,
      "step": 770
    },
    {
      "epoch": 1.0038622465400708,
      "grad_norm": 7.6687016456195645,
      "learning_rate": 7.490335051546391e-07,
      "logits/chosen": -0.4482797384262085,
      "logits/rejected": -0.45645689964294434,
      "logps/chosen": -348.0769348144531,
      "logps/rejected": -349.5641174316406,
      "loss": 0.4166,
      "rewards/accuracies": 0.7371794581413269,
      "rewards/chosen": 1.3597506284713745,
      "rewards/margins": 1.9606996774673462,
      "rewards/rejected": -0.5973307490348816,
      "step": 780
    },
    {
      "epoch": 1.0167364016736402,
      "grad_norm": 9.230599114177751,
      "learning_rate": 7.45811855670103e-07,
      "logits/chosen": -0.44798582792282104,
      "logits/rejected": -0.4685913026332855,
      "logps/chosen": -367.0249938964844,
      "logps/rejected": -393.32501220703125,
      "loss": 0.067,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 2.8109374046325684,
      "rewards/margins": 6.264452934265137,
      "rewards/rejected": -3.4593749046325684,
      "step": 790
    },
    {
      "epoch": 1.0296105568072096,
      "grad_norm": 16.61098369972291,
      "learning_rate": 7.42590206185567e-07,
      "logits/chosen": -0.5721801519393921,
      "logits/rejected": -0.559155285358429,
      "logps/chosen": -319.0249938964844,
      "logps/rejected": -415.9750061035156,
      "loss": 0.0528,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 2.3069825172424316,
      "rewards/margins": 7.700585842132568,
      "rewards/rejected": -5.3922119140625,
      "step": 800
    },
    {
      "epoch": 1.042484711940779,
      "grad_norm": 22.829431572502866,
      "learning_rate": 7.39368556701031e-07,
      "logits/chosen": -0.627636730670929,
      "logits/rejected": -0.6467529535293579,
      "logps/chosen": -364.125,
      "logps/rejected": -428.7250061035156,
      "loss": 0.0828,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": 1.1227538585662842,
      "rewards/margins": 7.400000095367432,
      "rewards/rejected": -6.280077934265137,
      "step": 810
    },
    {
      "epoch": 1.0553588670743483,
      "grad_norm": 4.36402073822104,
      "learning_rate": 7.361469072164949e-07,
      "logits/chosen": -0.7441650629043579,
      "logits/rejected": -0.72998046875,
      "logps/chosen": -376.57501220703125,
      "logps/rejected": -452.7250061035156,
      "loss": 0.0561,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": 0.10989990085363388,
      "rewards/margins": 8.282812118530273,
      "rewards/rejected": -8.182031631469727,
      "step": 820
    },
    {
      "epoch": 1.0682330222079175,
      "grad_norm": 9.44908390049402,
      "learning_rate": 7.329252577319587e-07,
      "logits/chosen": -0.865307629108429,
      "logits/rejected": -0.8622802495956421,
      "logps/chosen": -369.57501220703125,
      "logps/rejected": -472.8500061035156,
      "loss": 0.0743,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.3714187741279602,
      "rewards/margins": 9.832812309265137,
      "rewards/rejected": -10.209375381469727,
      "step": 830
    },
    {
      "epoch": 1.0811071773414869,
      "grad_norm": 2.5940939792945583,
      "learning_rate": 7.297036082474226e-07,
      "logits/chosen": -0.903515636920929,
      "logits/rejected": -0.876757800579071,
      "logps/chosen": -378.2749938964844,
      "logps/rejected": -444.3999938964844,
      "loss": 0.0859,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.336401343345642,
      "rewards/margins": 9.068163871765137,
      "rewards/rejected": -10.41015625,
      "step": 840
    },
    {
      "epoch": 1.0939813324750562,
      "grad_norm": 4.93864942706677,
      "learning_rate": 7.264819587628865e-07,
      "logits/chosen": -0.8818359375,
      "logits/rejected": -0.883740246295929,
      "logps/chosen": -376.1499938964844,
      "logps/rejected": -451.67498779296875,
      "loss": 0.1076,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -1.2569091320037842,
      "rewards/margins": 8.835156440734863,
      "rewards/rejected": -10.096484184265137,
      "step": 850
    },
    {
      "epoch": 1.1068554876086256,
      "grad_norm": 1.1622572843562746,
      "learning_rate": 7.232603092783504e-07,
      "logits/chosen": -1.0085937976837158,
      "logits/rejected": -0.991894543170929,
      "logps/chosen": -382.2250061035156,
      "logps/rejected": -464.25,
      "loss": 0.0571,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -1.985925316810608,
      "rewards/margins": 9.11328125,
      "rewards/rejected": -11.096875190734863,
      "step": 860
    },
    {
      "epoch": 1.119729642742195,
      "grad_norm": 13.207891586411098,
      "learning_rate": 7.200386597938144e-07,
      "logits/chosen": -0.960888683795929,
      "logits/rejected": -0.9482421875,
      "logps/chosen": -401.70001220703125,
      "logps/rejected": -449.07501220703125,
      "loss": 0.083,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.0959961414337158,
      "rewards/margins": 9.560937881469727,
      "rewards/rejected": -10.660937309265137,
      "step": 870
    },
    {
      "epoch": 1.1326037978757644,
      "grad_norm": 26.57295769679731,
      "learning_rate": 7.168170103092783e-07,
      "logits/chosen": -0.85919189453125,
      "logits/rejected": -0.8564453125,
      "logps/chosen": -392.76251220703125,
      "logps/rejected": -458.79998779296875,
      "loss": 0.069,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.5528320074081421,
      "rewards/margins": 9.028905868530273,
      "rewards/rejected": -9.583593368530273,
      "step": 880
    },
    {
      "epoch": 1.1454779530093337,
      "grad_norm": 5.906974482408663,
      "learning_rate": 7.135953608247423e-07,
      "logits/chosen": -0.826647937297821,
      "logits/rejected": -0.8048461675643921,
      "logps/chosen": -388.04998779296875,
      "logps/rejected": -440.17498779296875,
      "loss": 0.065,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.10826416313648224,
      "rewards/margins": 8.711718559265137,
      "rewards/rejected": -8.819726943969727,
      "step": 890
    },
    {
      "epoch": 1.1583521081429031,
      "grad_norm": 24.927906669623493,
      "learning_rate": 7.103737113402062e-07,
      "logits/chosen": -0.8695923089981079,
      "logits/rejected": -0.856518566608429,
      "logps/chosen": -403.0874938964844,
      "logps/rejected": -490.04998779296875,
      "loss": 0.0603,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.8900390863418579,
      "rewards/margins": 10.569531440734863,
      "rewards/rejected": -11.462499618530273,
      "step": 900
    },
    {
      "epoch": 1.1712262632764725,
      "grad_norm": 4.506062930490332,
      "learning_rate": 7.071520618556701e-07,
      "logits/chosen": -0.9596923589706421,
      "logits/rejected": -0.9530273675918579,
      "logps/chosen": -391.23748779296875,
      "logps/rejected": -456.5249938964844,
      "loss": 0.078,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -2.318310499191284,
      "rewards/margins": 9.502344131469727,
      "rewards/rejected": -11.823827743530273,
      "step": 910
    },
    {
      "epoch": 1.1841004184100419,
      "grad_norm": 22.19667225060241,
      "learning_rate": 7.039304123711341e-07,
      "logits/chosen": -0.8713623285293579,
      "logits/rejected": -0.860217273235321,
      "logps/chosen": -418.04998779296875,
      "logps/rejected": -501.32501220703125,
      "loss": 0.0381,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.0840332508087158,
      "rewards/margins": 10.9453125,
      "rewards/rejected": -12.029687881469727,
      "step": 920
    },
    {
      "epoch": 1.1969745735436113,
      "grad_norm": 3.0097726188152327,
      "learning_rate": 7.007087628865979e-07,
      "logits/chosen": -0.8865722417831421,
      "logits/rejected": -0.886523425579071,
      "logps/chosen": -364.0687561035156,
      "logps/rejected": -432.4375,
      "loss": 0.0699,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.1237884759902954,
      "rewards/margins": 9.000781059265137,
      "rewards/rejected": -10.123632431030273,
      "step": 930
    },
    {
      "epoch": 1.2098487286771806,
      "grad_norm": 122.1654239956535,
      "learning_rate": 6.974871134020618e-07,
      "logits/chosen": -0.9017089605331421,
      "logits/rejected": -0.8979247808456421,
      "logps/chosen": -372.01251220703125,
      "logps/rejected": -438.875,
      "loss": 0.0782,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.824169933795929,
      "rewards/margins": 10.292187690734863,
      "rewards/rejected": -11.118749618530273,
      "step": 940
    },
    {
      "epoch": 1.22272288381075,
      "grad_norm": 0.49727599372317804,
      "learning_rate": 6.942654639175257e-07,
      "logits/chosen": -0.8161590695381165,
      "logits/rejected": -0.813720703125,
      "logps/chosen": -371.48126220703125,
      "logps/rejected": -474.71875,
      "loss": 0.0646,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.615966796875,
      "rewards/margins": 10.194140434265137,
      "rewards/rejected": -10.8134765625,
      "step": 950
    },
    {
      "epoch": 1.2355970389443192,
      "grad_norm": 15.351228026815852,
      "learning_rate": 6.910438144329896e-07,
      "logits/chosen": -0.916015625,
      "logits/rejected": -0.920654296875,
      "logps/chosen": -386.25,
      "logps/rejected": -462.25,
      "loss": 0.0591,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -1.6080443859100342,
      "rewards/margins": 10.084375381469727,
      "rewards/rejected": -11.686718940734863,
      "step": 960
    },
    {
      "epoch": 1.2484711940778888,
      "grad_norm": 73.16765440550557,
      "learning_rate": 6.878221649484535e-07,
      "logits/chosen": -0.9052734375,
      "logits/rejected": -0.92333984375,
      "logps/chosen": -403.8687438964844,
      "logps/rejected": -481.8999938964844,
      "loss": 0.1255,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -3.6683106422424316,
      "rewards/margins": 9.447851181030273,
      "rewards/rejected": -13.109375,
      "step": 970
    },
    {
      "epoch": 1.261345349211458,
      "grad_norm": 1.552151037291679,
      "learning_rate": 6.846005154639176e-07,
      "logits/chosen": -0.928955078125,
      "logits/rejected": -0.9185546636581421,
      "logps/chosen": -415.54998779296875,
      "logps/rejected": -502.17498779296875,
      "loss": 0.0278,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -4.029528617858887,
      "rewards/margins": 11.318750381469727,
      "rewards/rejected": -15.346875190734863,
      "step": 980
    },
    {
      "epoch": 1.2742195043450273,
      "grad_norm": 155.41470997153405,
      "learning_rate": 6.813788659793815e-07,
      "logits/chosen": -0.9188476800918579,
      "logits/rejected": -0.9298309087753296,
      "logps/chosen": -495.7250061035156,
      "logps/rejected": -532.6500244140625,
      "loss": 0.0846,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -3.2472167015075684,
      "rewards/margins": 10.753125190734863,
      "rewards/rejected": -14.003125190734863,
      "step": 990
    },
    {
      "epoch": 1.2870936594785967,
      "grad_norm": 5.116040863492018,
      "learning_rate": 6.781572164948454e-07,
      "logits/chosen": -0.8142944574356079,
      "logits/rejected": -0.812969982624054,
      "logps/chosen": -384.875,
      "logps/rejected": -453.6499938964844,
      "loss": 0.0471,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -2.236328125,
      "rewards/margins": 10.518750190734863,
      "rewards/rejected": -12.74609375,
      "step": 1000
    },
    {
      "epoch": 1.299967814612166,
      "grad_norm": 203.17549940484994,
      "learning_rate": 6.749355670103093e-07,
      "logits/chosen": -0.8174377679824829,
      "logits/rejected": -0.8206436038017273,
      "logps/chosen": -357.5,
      "logps/rejected": -430.3500061035156,
      "loss": 0.0952,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.4882080554962158,
      "rewards/margins": 9.531641006469727,
      "rewards/rejected": -11.017187118530273,
      "step": 1010
    },
    {
      "epoch": 1.3128419697457354,
      "grad_norm": 3.2265856708286482,
      "learning_rate": 6.717139175257731e-07,
      "logits/chosen": -0.848876953125,
      "logits/rejected": -0.840283215045929,
      "logps/chosen": -351.4375,
      "logps/rejected": -410.82501220703125,
      "loss": 0.045,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.6333678960800171,
      "rewards/margins": 9.462499618530273,
      "rewards/rejected": -10.099609375,
      "step": 1020
    },
    {
      "epoch": 1.3257161248793048,
      "grad_norm": 2.48479318064024,
      "learning_rate": 6.68492268041237e-07,
      "logits/chosen": -0.9139159917831421,
      "logits/rejected": -0.905322253704071,
      "logps/chosen": -373.0249938964844,
      "logps/rejected": -462.32501220703125,
      "loss": 0.0602,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.2511962950229645,
      "rewards/margins": 10.485937118530273,
      "rewards/rejected": -10.232812881469727,
      "step": 1030
    },
    {
      "epoch": 1.3385902800128742,
      "grad_norm": 11.744877027581841,
      "learning_rate": 6.65270618556701e-07,
      "logits/chosen": -0.8927978277206421,
      "logits/rejected": -0.901123046875,
      "logps/chosen": -406.7749938964844,
      "logps/rejected": -482.57501220703125,
      "loss": 0.048,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 0.47990721464157104,
      "rewards/margins": 10.22265625,
      "rewards/rejected": -9.742968559265137,
      "step": 1040
    },
    {
      "epoch": 1.3514644351464435,
      "grad_norm": 22.161078888328298,
      "learning_rate": 6.620489690721649e-07,
      "logits/chosen": -0.8483642339706421,
      "logits/rejected": -0.837329089641571,
      "logps/chosen": -371.6875,
      "logps/rejected": -451.1499938964844,
      "loss": 0.0586,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.47760313749313354,
      "rewards/margins": 9.533594131469727,
      "rewards/rejected": -9.053906440734863,
      "step": 1050
    },
    {
      "epoch": 1.364338590280013,
      "grad_norm": 49.17969665142029,
      "learning_rate": 6.588273195876288e-07,
      "logits/chosen": -0.885449230670929,
      "logits/rejected": -0.8734496831893921,
      "logps/chosen": -358.5249938964844,
      "logps/rejected": -416.32501220703125,
      "loss": 0.0744,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.206329345703125,
      "rewards/margins": 9.153905868530273,
      "rewards/rejected": -8.94921875,
      "step": 1060
    },
    {
      "epoch": 1.3772127454135823,
      "grad_norm": 1.3865788823811889,
      "learning_rate": 6.556056701030927e-07,
      "logits/chosen": -0.9468749761581421,
      "logits/rejected": -0.9432128667831421,
      "logps/chosen": -324.04998779296875,
      "logps/rejected": -409.1000061035156,
      "loss": 0.0565,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.23262938857078552,
      "rewards/margins": 9.378125190734863,
      "rewards/rejected": -9.6171875,
      "step": 1070
    },
    {
      "epoch": 1.3900869005471517,
      "grad_norm": 220.96584927000097,
      "learning_rate": 6.523840206185567e-07,
      "logits/chosen": -0.973876953125,
      "logits/rejected": -0.957104504108429,
      "logps/chosen": -389.20001220703125,
      "logps/rejected": -452.5249938964844,
      "loss": 0.0817,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.05045776441693306,
      "rewards/margins": 9.750781059265137,
      "rewards/rejected": -9.705469131469727,
      "step": 1080
    },
    {
      "epoch": 1.4029610556807208,
      "grad_norm": 2.741340879707575,
      "learning_rate": 6.491623711340206e-07,
      "logits/chosen": -0.9906250238418579,
      "logits/rejected": -0.981518566608429,
      "logps/chosen": -397.70001220703125,
      "logps/rejected": -498.95001220703125,
      "loss": 0.0657,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.05647582933306694,
      "rewards/margins": 9.798437118530273,
      "rewards/rejected": -9.861328125,
      "step": 1090
    },
    {
      "epoch": 1.4158352108142904,
      "grad_norm": 5.290171237298657,
      "learning_rate": 6.459407216494846e-07,
      "logits/chosen": -0.9808593988418579,
      "logits/rejected": -0.9601074457168579,
      "logps/chosen": -389.8500061035156,
      "logps/rejected": -472.0,
      "loss": 0.0383,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 0.26752930879592896,
      "rewards/margins": 10.471094131469727,
      "rewards/rejected": -10.202343940734863,
      "step": 1100
    },
    {
      "epoch": 1.4287093659478596,
      "grad_norm": 4.170559858681645,
      "learning_rate": 6.427190721649485e-07,
      "logits/chosen": -0.9833984375,
      "logits/rejected": -0.999316394329071,
      "logps/chosen": -395.375,
      "logps/rejected": -460.20001220703125,
      "loss": 0.0696,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.895581066608429,
      "rewards/margins": 10.228906631469727,
      "rewards/rejected": -11.122655868530273,
      "step": 1110
    },
    {
      "epoch": 1.441583521081429,
      "grad_norm": 75.26395674949998,
      "learning_rate": 6.394974226804123e-07,
      "logits/chosen": -0.964770495891571,
      "logits/rejected": -0.931689441204071,
      "logps/chosen": -374.3999938964844,
      "logps/rejected": -486.3999938964844,
      "loss": 0.0676,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -2.9120116233825684,
      "rewards/margins": 10.5078125,
      "rewards/rejected": -13.420312881469727,
      "step": 1120
    },
    {
      "epoch": 1.4544576762149983,
      "grad_norm": 5.547685368185955,
      "learning_rate": 6.362757731958762e-07,
      "logits/chosen": -0.963085949420929,
      "logits/rejected": -0.958935558795929,
      "logps/chosen": -373.1499938964844,
      "logps/rejected": -480.1000061035156,
      "loss": 0.0545,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -2.208447217941284,
      "rewards/margins": 10.94921875,
      "rewards/rejected": -13.160937309265137,
      "step": 1130
    },
    {
      "epoch": 1.4673318313485677,
      "grad_norm": 10.491415939641222,
      "learning_rate": 6.330541237113401e-07,
      "logits/chosen": -0.9105468988418579,
      "logits/rejected": -0.888720691204071,
      "logps/chosen": -378.5,
      "logps/rejected": -473.0,
      "loss": 0.0594,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -1.3115966320037842,
      "rewards/margins": 10.66796875,
      "rewards/rejected": -11.981249809265137,
      "step": 1140
    },
    {
      "epoch": 1.480205986482137,
      "grad_norm": 5.10762368180044,
      "learning_rate": 6.29832474226804e-07,
      "logits/chosen": -0.921875,
      "logits/rejected": -0.9062744379043579,
      "logps/chosen": -368.5,
      "logps/rejected": -478.4750061035156,
      "loss": 0.0382,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.3436768054962158,
      "rewards/margins": 10.7734375,
      "rewards/rejected": -12.12109375,
      "step": 1150
    },
    {
      "epoch": 1.4930801416157065,
      "grad_norm": 9.900405690491036,
      "learning_rate": 6.26610824742268e-07,
      "logits/chosen": -0.923291027545929,
      "logits/rejected": -0.899157702922821,
      "logps/chosen": -381.17498779296875,
      "logps/rejected": -462.75,
      "loss": 0.0467,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -2.11822509765625,
      "rewards/margins": 10.829687118530273,
      "rewards/rejected": -12.944531440734863,
      "step": 1160
    },
    {
      "epoch": 1.5059542967492758,
      "grad_norm": 6.152643520140975,
      "learning_rate": 6.23389175257732e-07,
      "logits/chosen": -1.017480492591858,
      "logits/rejected": -0.987963855266571,
      "logps/chosen": -414.2749938964844,
      "logps/rejected": -506.70001220703125,
      "loss": 0.0393,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -4.409295558929443,
      "rewards/margins": 10.576562881469727,
      "rewards/rejected": -14.982812881469727,
      "step": 1170
    },
    {
      "epoch": 1.5188284518828452,
      "grad_norm": 10.576249252330557,
      "learning_rate": 6.201675257731959e-07,
      "logits/chosen": -0.940014660358429,
      "logits/rejected": -0.9420166015625,
      "logps/chosen": -388.92498779296875,
      "logps/rejected": -498.70001220703125,
      "loss": 0.0621,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -3.6192870140075684,
      "rewards/margins": 11.6328125,
      "rewards/rejected": -15.253125190734863,
      "step": 1180
    },
    {
      "epoch": 1.5317026070164146,
      "grad_norm": 5.858898211376708,
      "learning_rate": 6.169458762886598e-07,
      "logits/chosen": -0.971435546875,
      "logits/rejected": -0.972216784954071,
      "logps/chosen": -375.6625061035156,
      "logps/rejected": -452.1875,
      "loss": 0.0492,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -3.7707762718200684,
      "rewards/margins": 10.260546684265137,
      "rewards/rejected": -14.032812118530273,
      "step": 1190
    },
    {
      "epoch": 1.544576762149984,
      "grad_norm": 73.088561052754,
      "learning_rate": 6.137242268041237e-07,
      "logits/chosen": -1.0227539539337158,
      "logits/rejected": -0.9654296636581421,
      "logps/chosen": -447.9750061035156,
      "logps/rejected": -547.875,
      "loss": 0.0675,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -4.362378120422363,
      "rewards/margins": 12.967968940734863,
      "rewards/rejected": -17.332813262939453,
      "step": 1200
    },
    {
      "epoch": 1.5574509172835533,
      "grad_norm": 16.087749078441842,
      "learning_rate": 6.105025773195877e-07,
      "logits/chosen": -1.08251953125,
      "logits/rejected": -1.072607398033142,
      "logps/chosen": -428.125,
      "logps/rejected": -541.25,
      "loss": 0.0971,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.455664157867432,
      "rewards/margins": 12.564844131469727,
      "rewards/rejected": -18.026561737060547,
      "step": 1210
    },
    {
      "epoch": 1.5703250724171225,
      "grad_norm": 8.015976908256388,
      "learning_rate": 6.072809278350515e-07,
      "logits/chosen": -0.999829113483429,
      "logits/rejected": -1.026025414466858,
      "logps/chosen": -411.4125061035156,
      "logps/rejected": -488.7749938964844,
      "loss": 0.0738,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -4.011401176452637,
      "rewards/margins": 10.776171684265137,
      "rewards/rejected": -14.782812118530273,
      "step": 1220
    },
    {
      "epoch": 1.583199227550692,
      "grad_norm": 7.496227160228813,
      "learning_rate": 6.040592783505154e-07,
      "logits/chosen": -0.8304198980331421,
      "logits/rejected": -0.83154296875,
      "logps/chosen": -335.81561279296875,
      "logps/rejected": -454.4437561035156,
      "loss": 0.109,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -2.8844237327575684,
      "rewards/margins": 9.571484565734863,
      "rewards/rejected": -12.45703125,
      "step": 1230
    },
    {
      "epoch": 1.5960733826842612,
      "grad_norm": 38.899724862220474,
      "learning_rate": 6.008376288659793e-07,
      "logits/chosen": -0.9081786870956421,
      "logits/rejected": -0.9267822504043579,
      "logps/chosen": -393.25,
      "logps/rejected": -473.04998779296875,
      "loss": 0.0442,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -3.959277391433716,
      "rewards/margins": 11.373437881469727,
      "rewards/rejected": -15.329687118530273,
      "step": 1240
    },
    {
      "epoch": 1.6089475378178308,
      "grad_norm": 56.23438612247682,
      "learning_rate": 5.976159793814432e-07,
      "logits/chosen": -0.971386730670929,
      "logits/rejected": -0.9666992425918579,
      "logps/chosen": -406.29998779296875,
      "logps/rejected": -507.6499938964844,
      "loss": 0.0303,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -2.40478515625,
      "rewards/margins": 11.353124618530273,
      "rewards/rejected": -13.760156631469727,
      "step": 1250
    },
    {
      "epoch": 1.6218216929514,
      "grad_norm": 10.42344864534372,
      "learning_rate": 5.943943298969071e-07,
      "logits/chosen": -0.994335949420929,
      "logits/rejected": -0.992724597454071,
      "logps/chosen": -393.125,
      "logps/rejected": -502.125,
      "loss": 0.0554,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -2.322924852371216,
      "rewards/margins": 11.29296875,
      "rewards/rejected": -13.6171875,
      "step": 1260
    },
    {
      "epoch": 1.6346958480849694,
      "grad_norm": 24.851501417189294,
      "learning_rate": 5.911726804123712e-07,
      "logits/chosen": -0.967041015625,
      "logits/rejected": -0.92529296875,
      "logps/chosen": -361.57501220703125,
      "logps/rejected": -458.0249938964844,
      "loss": 0.0577,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -1.720422387123108,
      "rewards/margins": 11.080469131469727,
      "rewards/rejected": -12.807031631469727,
      "step": 1270
    },
    {
      "epoch": 1.6475700032185387,
      "grad_norm": 1.9460817231647556,
      "learning_rate": 5.879510309278351e-07,
      "logits/chosen": -0.9542480707168579,
      "logits/rejected": -0.949999988079071,
      "logps/chosen": -398.4750061035156,
      "logps/rejected": -488.1000061035156,
      "loss": 0.1017,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -0.5497802495956421,
      "rewards/margins": 11.510156631469727,
      "rewards/rejected": -12.067187309265137,
      "step": 1280
    },
    {
      "epoch": 1.6604441583521081,
      "grad_norm": 22.436709932766853,
      "learning_rate": 5.84729381443299e-07,
      "logits/chosen": -0.920214831829071,
      "logits/rejected": -0.917529284954071,
      "logps/chosen": -394.7250061035156,
      "logps/rejected": -464.8999938964844,
      "loss": 0.0585,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -2.0501708984375,
      "rewards/margins": 11.30859375,
      "rewards/rejected": -13.360937118530273,
      "step": 1290
    },
    {
      "epoch": 1.6733183134856775,
      "grad_norm": 2.922267266125603,
      "learning_rate": 5.815077319587629e-07,
      "logits/chosen": -0.810717761516571,
      "logits/rejected": -0.8197021484375,
      "logps/chosen": -421.3500061035156,
      "logps/rejected": -514.25,
      "loss": 0.0395,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.7827637195587158,
      "rewards/margins": 12.157617568969727,
      "rewards/rejected": -13.939844131469727,
      "step": 1300
    },
    {
      "epoch": 1.6861924686192469,
      "grad_norm": 20.574767289402832,
      "learning_rate": 5.782860824742268e-07,
      "logits/chosen": -0.9783569574356079,
      "logits/rejected": -0.9566406011581421,
      "logps/chosen": -409.6000061035156,
      "logps/rejected": -500.79998779296875,
      "loss": 0.104,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -2.32958984375,
      "rewards/margins": 11.189844131469727,
      "rewards/rejected": -13.518750190734863,
      "step": 1310
    },
    {
      "epoch": 1.6990666237528163,
      "grad_norm": 110.28911944115465,
      "learning_rate": 5.750644329896906e-07,
      "logits/chosen": -0.8671630620956421,
      "logits/rejected": -0.854736328125,
      "logps/chosen": -427.9750061035156,
      "logps/rejected": -514.0750122070312,
      "loss": 0.0576,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -2.6690430641174316,
      "rewards/margins": 10.728124618530273,
      "rewards/rejected": -13.391406059265137,
      "step": 1320
    },
    {
      "epoch": 1.7119407788863856,
      "grad_norm": 29.883782295799218,
      "learning_rate": 5.718427835051546e-07,
      "logits/chosen": -0.869580090045929,
      "logits/rejected": -0.8749023675918579,
      "logps/chosen": -384.375,
      "logps/rejected": -489.75,
      "loss": 0.0525,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -1.936621069908142,
      "rewards/margins": 10.360937118530273,
      "rewards/rejected": -12.296875,
      "step": 1330
    },
    {
      "epoch": 1.724814934019955,
      "grad_norm": 15.825926255473947,
      "learning_rate": 5.686211340206185e-07,
      "logits/chosen": -0.896435558795929,
      "logits/rejected": -0.895214855670929,
      "logps/chosen": -377.75,
      "logps/rejected": -477.75,
      "loss": 0.0428,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.439355492591858,
      "rewards/margins": 10.345312118530273,
      "rewards/rejected": -11.776562690734863,
      "step": 1340
    },
    {
      "epoch": 1.7376890891535242,
      "grad_norm": 8.470103187327458,
      "learning_rate": 5.653994845360824e-07,
      "logits/chosen": -0.949902355670929,
      "logits/rejected": -0.9499267339706421,
      "logps/chosen": -375.375,
      "logps/rejected": -446.95001220703125,
      "loss": 0.0519,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.849633812904358,
      "rewards/margins": 9.740625381469727,
      "rewards/rejected": -11.599218368530273,
      "step": 1350
    },
    {
      "epoch": 1.7505632442870938,
      "grad_norm": 4.5563431355585875,
      "learning_rate": 5.621778350515464e-07,
      "logits/chosen": -0.952929675579071,
      "logits/rejected": -0.928955078125,
      "logps/chosen": -417.29998779296875,
      "logps/rejected": -505.6499938964844,
      "loss": 0.0394,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -2.0886597633361816,
      "rewards/margins": 11.784375190734863,
      "rewards/rejected": -13.874218940734863,
      "step": 1360
    },
    {
      "epoch": 1.763437399420663,
      "grad_norm": 9.103762775061622,
      "learning_rate": 5.589561855670103e-07,
      "logits/chosen": -0.9593261480331421,
      "logits/rejected": -0.9452148675918579,
      "logps/chosen": -423.32501220703125,
      "logps/rejected": -505.6000061035156,
      "loss": 0.049,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -3.0106263160705566,
      "rewards/margins": 10.846094131469727,
      "rewards/rejected": -13.854687690734863,
      "step": 1370
    },
    {
      "epoch": 1.7763115545542325,
      "grad_norm": 1.691523363024812,
      "learning_rate": 5.557345360824743e-07,
      "logits/chosen": -0.845349133014679,
      "logits/rejected": -0.8228515386581421,
      "logps/chosen": -371.42498779296875,
      "logps/rejected": -481.4624938964844,
      "loss": 0.1147,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -2.849853515625,
      "rewards/margins": 10.446874618530273,
      "rewards/rejected": -13.291406631469727,
      "step": 1380
    },
    {
      "epoch": 1.7891857096878017,
      "grad_norm": 14.771932390824606,
      "learning_rate": 5.525128865979382e-07,
      "logits/chosen": -0.84832763671875,
      "logits/rejected": -0.824292004108429,
      "logps/chosen": -376.57501220703125,
      "logps/rejected": -491.3999938964844,
      "loss": 0.065,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.8124511241912842,
      "rewards/margins": 10.788281440734863,
      "rewards/rejected": -12.591405868530273,
      "step": 1390
    },
    {
      "epoch": 1.8020598648213713,
      "grad_norm": 29.299452797327557,
      "learning_rate": 5.492912371134021e-07,
      "logits/chosen": -0.990966796875,
      "logits/rejected": -0.987255871295929,
      "logps/chosen": -412.0,
      "logps/rejected": -511.8500061035156,
      "loss": 0.0333,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -1.2932891845703125,
      "rewards/margins": 10.651562690734863,
      "rewards/rejected": -11.946874618530273,
      "step": 1400
    },
    {
      "epoch": 1.8149340199549404,
      "grad_norm": 100.26192553543316,
      "learning_rate": 5.460695876288659e-07,
      "logits/chosen": -0.902539074420929,
      "logits/rejected": -0.887866199016571,
      "logps/chosen": -407.54998779296875,
      "logps/rejected": -509.25,
      "loss": 0.0592,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.274084448814392,
      "rewards/margins": 11.475781440734863,
      "rewards/rejected": -12.744531631469727,
      "step": 1410
    },
    {
      "epoch": 1.8278081750885098,
      "grad_norm": 35.25113627842931,
      "learning_rate": 5.428479381443298e-07,
      "logits/chosen": -0.954638659954071,
      "logits/rejected": -0.9381347894668579,
      "logps/chosen": -356.9750061035156,
      "logps/rejected": -439.45001220703125,
      "loss": 0.0425,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.6203124523162842,
      "rewards/margins": 10.171875,
      "rewards/rejected": -11.791406631469727,
      "step": 1420
    },
    {
      "epoch": 1.8406823302220792,
      "grad_norm": 0.6778421877980854,
      "learning_rate": 5.396262886597937e-07,
      "logits/chosen": -0.9102538824081421,
      "logits/rejected": -0.8841797113418579,
      "logps/chosen": -375.8999938964844,
      "logps/rejected": -465.1499938964844,
      "loss": 0.0219,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -0.5498291254043579,
      "rewards/margins": 10.825780868530273,
      "rewards/rejected": -11.375,
      "step": 1430
    },
    {
      "epoch": 1.8535564853556485,
      "grad_norm": 21.24551771606512,
      "learning_rate": 5.364046391752577e-07,
      "logits/chosen": -0.9066406488418579,
      "logits/rejected": -0.921826183795929,
      "logps/chosen": -396.8999938964844,
      "logps/rejected": -491.1000061035156,
      "loss": 0.0572,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.164526343345642,
      "rewards/margins": 11.449999809265137,
      "rewards/rejected": -12.615625381469727,
      "step": 1440
    },
    {
      "epoch": 1.866430640489218,
      "grad_norm": 10.543090540151306,
      "learning_rate": 5.331829896907216e-07,
      "logits/chosen": -0.901171863079071,
      "logits/rejected": -0.892016589641571,
      "logps/chosen": -429.82501220703125,
      "logps/rejected": -509.70001220703125,
      "loss": 0.0678,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.7703644037246704,
      "rewards/margins": 10.507031440734863,
      "rewards/rejected": -12.272656440734863,
      "step": 1450
    },
    {
      "epoch": 1.8793047956227873,
      "grad_norm": 0.44776982375157787,
      "learning_rate": 5.299613402061856e-07,
      "logits/chosen": -0.9541991949081421,
      "logits/rejected": -0.9314941167831421,
      "logps/chosen": -406.8999938964844,
      "logps/rejected": -504.7749938964844,
      "loss": 0.0515,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.4205474853515625,
      "rewards/margins": 11.315625190734863,
      "rewards/rejected": -12.733983993530273,
      "step": 1460
    },
    {
      "epoch": 1.8921789507563567,
      "grad_norm": 64.17339300757935,
      "learning_rate": 5.267396907216495e-07,
      "logits/chosen": -0.9212890863418579,
      "logits/rejected": -0.9178222417831421,
      "logps/chosen": -389.32501220703125,
      "logps/rejected": -481.3999938964844,
      "loss": 0.0504,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -2.74786376953125,
      "rewards/margins": 10.51953125,
      "rewards/rejected": -13.26953125,
      "step": 1470
    },
    {
      "epoch": 1.9050531058899258,
      "grad_norm": 2.3569087767370966,
      "learning_rate": 5.235180412371134e-07,
      "logits/chosen": -0.915698230266571,
      "logits/rejected": -0.88592529296875,
      "logps/chosen": -435.17498779296875,
      "logps/rejected": -532.1500244140625,
      "loss": 0.0603,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -2.54071044921875,
      "rewards/margins": 12.057031631469727,
      "rewards/rejected": -14.594531059265137,
      "step": 1480
    },
    {
      "epoch": 1.9179272610234954,
      "grad_norm": 32.07335603062137,
      "learning_rate": 5.202963917525773e-07,
      "logits/chosen": -0.949511706829071,
      "logits/rejected": -0.9261718988418579,
      "logps/chosen": -400.42498779296875,
      "logps/rejected": -488.1499938964844,
      "loss": 0.0346,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.5046019554138184,
      "rewards/margins": 11.435155868530273,
      "rewards/rejected": -13.943750381469727,
      "step": 1490
    },
    {
      "epoch": 1.9308014161570646,
      "grad_norm": 5.079861293118151,
      "learning_rate": 5.170747422680413e-07,
      "logits/chosen": -0.9255126714706421,
      "logits/rejected": -0.9140869379043579,
      "logps/chosen": -393.5,
      "logps/rejected": -460.95001220703125,
      "loss": 0.0647,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -2.2625489234924316,
      "rewards/margins": 11.5546875,
      "rewards/rejected": -13.8203125,
      "step": 1500
    },
    {
      "epoch": 1.9436755712906342,
      "grad_norm": 13.730196804403965,
      "learning_rate": 5.138530927835051e-07,
      "logits/chosen": -0.964599609375,
      "logits/rejected": -0.9576171636581421,
      "logps/chosen": -381.73748779296875,
      "logps/rejected": -464.57501220703125,
      "loss": 0.096,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -1.831457495689392,
      "rewards/margins": 10.2421875,
      "rewards/rejected": -12.07421875,
      "step": 1510
    },
    {
      "epoch": 1.9565497264242033,
      "grad_norm": 1.9292744667507238,
      "learning_rate": 5.10631443298969e-07,
      "logits/chosen": -0.919140636920929,
      "logits/rejected": -0.880786120891571,
      "logps/chosen": -382.9375,
      "logps/rejected": -459.5,
      "loss": 0.055,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.831347644329071,
      "rewards/margins": 10.905468940734863,
      "rewards/rejected": -11.734375,
      "step": 1520
    },
    {
      "epoch": 1.969423881557773,
      "grad_norm": 11.449713018559374,
      "learning_rate": 5.074097938144329e-07,
      "logits/chosen": -0.9642578363418579,
      "logits/rejected": -0.9501708745956421,
      "logps/chosen": -385.29998779296875,
      "logps/rejected": -506.04998779296875,
      "loss": 0.0454,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.8114013671875,
      "rewards/margins": 11.5625,
      "rewards/rejected": -12.375781059265137,
      "step": 1530
    },
    {
      "epoch": 1.982298036691342,
      "grad_norm": 3.350678145456376,
      "learning_rate": 5.041881443298968e-07,
      "logits/chosen": -0.921582043170929,
      "logits/rejected": -0.912402331829071,
      "logps/chosen": -406.29998779296875,
      "logps/rejected": -441.32501220703125,
      "loss": 0.0492,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.100073218345642,
      "rewards/margins": 10.561718940734863,
      "rewards/rejected": -11.6640625,
      "step": 1540
    },
    {
      "epoch": 1.9951721918249115,
      "grad_norm": 54.70846216282573,
      "learning_rate": 5.009664948453608e-07,
      "logits/chosen": -0.922558605670929,
      "logits/rejected": -0.917529284954071,
      "logps/chosen": -389.5249938964844,
      "logps/rejected": -470.75,
      "loss": 0.065,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.89117431640625,
      "rewards/margins": 11.014062881469727,
      "rewards/rejected": -12.916406631469727,
      "step": 1550
    },
    {
      "epoch": 2.0077244930801417,
      "grad_norm": 0.27387321212084004,
      "learning_rate": 4.977448453608248e-07,
      "logits/chosen": -0.9893704652786255,
      "logits/rejected": -0.9759489893913269,
      "logps/chosen": -374.28204345703125,
      "logps/rejected": -480.025634765625,
      "loss": 0.0246,
      "rewards/accuracies": 0.9679487347602844,
      "rewards/chosen": -1.1674273014068604,
      "rewards/margins": 12.660256385803223,
      "rewards/rejected": -13.831730842590332,
      "step": 1560
    },
    {
      "epoch": 2.020598648213711,
      "grad_norm": 16.195642178522633,
      "learning_rate": 4.945231958762887e-07,
      "logits/chosen": -0.8692626953125,
      "logits/rejected": -0.855395495891571,
      "logps/chosen": -364.4125061035156,
      "logps/rejected": -511.4375,
      "loss": 0.0406,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -1.5969727039337158,
      "rewards/margins": 13.655077934265137,
      "rewards/rejected": -15.242968559265137,
      "step": 1570
    },
    {
      "epoch": 2.0334728033472804,
      "grad_norm": 0.011066699938703848,
      "learning_rate": 4.913015463917525e-07,
      "logits/chosen": -0.9894043207168579,
      "logits/rejected": -0.9693359136581421,
      "logps/chosen": -391.7124938964844,
      "logps/rejected": -511.3500061035156,
      "loss": 0.0429,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -2.259082078933716,
      "rewards/margins": 13.080469131469727,
      "rewards/rejected": -15.342187881469727,
      "step": 1580
    },
    {
      "epoch": 2.0463469584808496,
      "grad_norm": 0.30088485969690315,
      "learning_rate": 4.880798969072165e-07,
      "logits/chosen": -0.9302002191543579,
      "logits/rejected": -0.9267578125,
      "logps/chosen": -375.5,
      "logps/rejected": -490.8999938964844,
      "loss": 0.0177,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -2.5155029296875,
      "rewards/margins": 15.078125,
      "rewards/rejected": -17.590625762939453,
      "step": 1590
    },
    {
      "epoch": 2.059221113614419,
      "grad_norm": 3.7766222698166008,
      "learning_rate": 4.848582474226804e-07,
      "logits/chosen": -0.990429699420929,
      "logits/rejected": -0.9600585699081421,
      "logps/chosen": -389.95001220703125,
      "logps/rejected": -542.9000244140625,
      "loss": 0.0144,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -3.7989258766174316,
      "rewards/margins": 15.314062118530273,
      "rewards/rejected": -19.110937118530273,
      "step": 1600
    },
    {
      "epoch": 2.0720952687479883,
      "grad_norm": 1.5230275244877616,
      "learning_rate": 4.816365979381443e-07,
      "logits/chosen": -0.985791027545929,
      "logits/rejected": -0.965527355670929,
      "logps/chosen": -407.125,
      "logps/rejected": -549.5499877929688,
      "loss": 0.0247,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -3.044628858566284,
      "rewards/margins": 15.612500190734863,
      "rewards/rejected": -18.653125762939453,
      "step": 1610
    },
    {
      "epoch": 2.084969423881558,
      "grad_norm": 0.5241685764089725,
      "learning_rate": 4.784149484536082e-07,
      "logits/chosen": -0.8857177495956421,
      "logits/rejected": -0.888867199420929,
      "logps/chosen": -411.79998779296875,
      "logps/rejected": -524.1500244140625,
      "loss": 0.0325,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -5.372949123382568,
      "rewards/margins": 14.409375190734863,
      "rewards/rejected": -19.782812118530273,
      "step": 1620
    },
    {
      "epoch": 2.097843579015127,
      "grad_norm": 14.457877642201927,
      "learning_rate": 4.7519329896907214e-07,
      "logits/chosen": -1.0594604015350342,
      "logits/rejected": -1.0399048328399658,
      "logps/chosen": -423.45001220703125,
      "logps/rejected": -568.0499877929688,
      "loss": 0.0244,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -6.598046779632568,
      "rewards/margins": 15.84375,
      "rewards/rejected": -22.443750381469727,
      "step": 1630
    },
    {
      "epoch": 2.1107177341486967,
      "grad_norm": 18.76607789875824,
      "learning_rate": 4.7197164948453605e-07,
      "logits/chosen": -1.090917944908142,
      "logits/rejected": -1.0798828601837158,
      "logps/chosen": -449.1499938964844,
      "logps/rejected": -559.8499755859375,
      "loss": 0.0335,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -5.502295017242432,
      "rewards/margins": 15.567187309265137,
      "rewards/rejected": -21.0546875,
      "step": 1640
    },
    {
      "epoch": 2.123591889282266,
      "grad_norm": 4.069282822417778,
      "learning_rate": 4.6874999999999996e-07,
      "logits/chosen": -1.083837866783142,
      "logits/rejected": -1.050439476966858,
      "logps/chosen": -424.1499938964844,
      "logps/rejected": -553.5999755859375,
      "loss": 0.0162,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -4.673583984375,
      "rewards/margins": 15.3671875,
      "rewards/rejected": -20.049999237060547,
      "step": 1650
    },
    {
      "epoch": 2.136466044415835,
      "grad_norm": 0.019867844889664508,
      "learning_rate": 4.655283505154639e-07,
      "logits/chosen": -1.0280640125274658,
      "logits/rejected": -1.0054931640625,
      "logps/chosen": -395.95001220703125,
      "logps/rejected": -527.4500122070312,
      "loss": 0.0228,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -4.825781345367432,
      "rewards/margins": 15.746874809265137,
      "rewards/rejected": -20.573436737060547,
      "step": 1660
    },
    {
      "epoch": 2.1493401995494046,
      "grad_norm": 1.405391631885844,
      "learning_rate": 4.6230670103092783e-07,
      "logits/chosen": -1.0292479991912842,
      "logits/rejected": -1.001928687095642,
      "logps/chosen": -449.2250061035156,
      "logps/rejected": -581.4500122070312,
      "loss": 0.0242,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -3.895703077316284,
      "rewards/margins": 15.307812690734863,
      "rewards/rejected": -19.189062118530273,
      "step": 1670
    },
    {
      "epoch": 2.1622143546829737,
      "grad_norm": 0.5519313995694172,
      "learning_rate": 4.590850515463917e-07,
      "logits/chosen": -0.94366455078125,
      "logits/rejected": -0.9423980712890625,
      "logps/chosen": -431.5,
      "logps/rejected": -548.125,
      "loss": 0.0294,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -4.086621284484863,
      "rewards/margins": 14.803906440734863,
      "rewards/rejected": -18.889062881469727,
      "step": 1680
    },
    {
      "epoch": 2.1750885098165433,
      "grad_norm": 0.04917387233529229,
      "learning_rate": 4.5586340206185565e-07,
      "logits/chosen": -1.0073730945587158,
      "logits/rejected": -0.983837902545929,
      "logps/chosen": -393.65625,
      "logps/rejected": -515.4375,
      "loss": 0.0547,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -3.69677734375,
      "rewards/margins": 14.767187118530273,
      "rewards/rejected": -18.470312118530273,
      "step": 1690
    },
    {
      "epoch": 2.1879626649501125,
      "grad_norm": 1.1593323599908398,
      "learning_rate": 4.5264175257731956e-07,
      "logits/chosen": -1.046777367591858,
      "logits/rejected": -1.0193359851837158,
      "logps/chosen": -396.07501220703125,
      "logps/rejected": -525.0750122070312,
      "loss": 0.0483,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.259375095367432,
      "rewards/margins": 14.039843559265137,
      "rewards/rejected": -18.298437118530273,
      "step": 1700
    },
    {
      "epoch": 2.200836820083682,
      "grad_norm": 0.30503012975802324,
      "learning_rate": 4.4942010309278347e-07,
      "logits/chosen": -0.9892578125,
      "logits/rejected": -0.9864746332168579,
      "logps/chosen": -385.6499938964844,
      "logps/rejected": -523.0,
      "loss": 0.0286,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -4.869189262390137,
      "rewards/margins": 15.048437118530273,
      "rewards/rejected": -19.910938262939453,
      "step": 1710
    },
    {
      "epoch": 2.2137109752172512,
      "grad_norm": 0.025660689139265877,
      "learning_rate": 4.4619845360824744e-07,
      "logits/chosen": -0.982128918170929,
      "logits/rejected": -0.9969726800918579,
      "logps/chosen": -435.82501220703125,
      "logps/rejected": -571.4500122070312,
      "loss": 0.035,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -5.135912895202637,
      "rewards/margins": 15.172656059265137,
      "rewards/rejected": -20.3046875,
      "step": 1720
    },
    {
      "epoch": 2.226585130350821,
      "grad_norm": 140.50599875108497,
      "learning_rate": 4.429768041237113e-07,
      "logits/chosen": -0.911694347858429,
      "logits/rejected": -0.8985351324081421,
      "logps/chosen": -404.07501220703125,
      "logps/rejected": -538.2999877929688,
      "loss": 0.0316,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -5.711035251617432,
      "rewards/margins": 14.632031440734863,
      "rewards/rejected": -20.345312118530273,
      "step": 1730
    },
    {
      "epoch": 2.23945928548439,
      "grad_norm": 0.1313501575125143,
      "learning_rate": 4.397551546391752e-07,
      "logits/chosen": -0.933483898639679,
      "logits/rejected": -0.944042980670929,
      "logps/chosen": -427.20001220703125,
      "logps/rejected": -553.0,
      "loss": 0.0365,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -6.276562690734863,
      "rewards/margins": 14.622655868530273,
      "rewards/rejected": -20.8984375,
      "step": 1740
    },
    {
      "epoch": 2.2523334406179596,
      "grad_norm": 0.7371999610473159,
      "learning_rate": 4.3653350515463917e-07,
      "logits/chosen": -0.9942382574081421,
      "logits/rejected": -0.9849609136581421,
      "logps/chosen": -419.70001220703125,
      "logps/rejected": -582.0999755859375,
      "loss": 0.0326,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -6.280077934265137,
      "rewards/margins": 16.040624618530273,
      "rewards/rejected": -22.314062118530273,
      "step": 1750
    },
    {
      "epoch": 2.2652075957515287,
      "grad_norm": 0.129869449494833,
      "learning_rate": 4.333118556701031e-07,
      "logits/chosen": -0.933154284954071,
      "logits/rejected": -0.91748046875,
      "logps/chosen": -430.04998779296875,
      "logps/rejected": -566.4500122070312,
      "loss": 0.042,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -4.696862697601318,
      "rewards/margins": 15.498437881469727,
      "rewards/rejected": -20.201562881469727,
      "step": 1760
    },
    {
      "epoch": 2.2780817508850983,
      "grad_norm": 0.13607345195503057,
      "learning_rate": 4.3009020618556704e-07,
      "logits/chosen": -1.015039086341858,
      "logits/rejected": -0.995849609375,
      "logps/chosen": -451.54998779296875,
      "logps/rejected": -543.4500122070312,
      "loss": 0.0244,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -4.693749904632568,
      "rewards/margins": 14.237500190734863,
      "rewards/rejected": -18.942188262939453,
      "step": 1770
    },
    {
      "epoch": 2.2909559060186675,
      "grad_norm": 0.39155247794928777,
      "learning_rate": 4.268685567010309e-07,
      "logits/chosen": -0.974365234375,
      "logits/rejected": -0.960986316204071,
      "logps/chosen": -401.3999938964844,
      "logps/rejected": -522.4000244140625,
      "loss": 0.0177,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -3.511425733566284,
      "rewards/margins": 15.060937881469727,
      "rewards/rejected": -18.578125,
      "step": 1780
    },
    {
      "epoch": 2.3038300611522367,
      "grad_norm": 0.08585169933868184,
      "learning_rate": 4.236469072164948e-07,
      "logits/chosen": -0.908154308795929,
      "logits/rejected": -0.895312488079071,
      "logps/chosen": -442.70001220703125,
      "logps/rejected": -591.0,
      "loss": 0.0183,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -3.9928956031799316,
      "rewards/margins": 15.760937690734863,
      "rewards/rejected": -19.754688262939453,
      "step": 1790
    },
    {
      "epoch": 2.3167042162858063,
      "grad_norm": 1.0309690238634248,
      "learning_rate": 4.2042525773195877e-07,
      "logits/chosen": -0.9856933355331421,
      "logits/rejected": -0.9870361089706421,
      "logps/chosen": -444.04998779296875,
      "logps/rejected": -564.2999877929688,
      "loss": 0.0266,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -5.415820121765137,
      "rewards/margins": 15.154687881469727,
      "rewards/rejected": -20.568750381469727,
      "step": 1800
    },
    {
      "epoch": 2.329578371419376,
      "grad_norm": 0.3147659868365795,
      "learning_rate": 4.172036082474227e-07,
      "logits/chosen": -0.878552258014679,
      "logits/rejected": -0.864392101764679,
      "logps/chosen": -458.3500061035156,
      "logps/rejected": -572.8499755859375,
      "loss": 0.0491,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -5.186669826507568,
      "rewards/margins": 14.610937118530273,
      "rewards/rejected": -19.798437118530273,
      "step": 1810
    },
    {
      "epoch": 2.342452526552945,
      "grad_norm": 2.027945212441702,
      "learning_rate": 4.1398195876288654e-07,
      "logits/chosen": -0.954394519329071,
      "logits/rejected": -0.9604736566543579,
      "logps/chosen": -451.29998779296875,
      "logps/rejected": -590.4500122070312,
      "loss": 0.0253,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -5.277050971984863,
      "rewards/margins": 14.571874618530273,
      "rewards/rejected": -19.850000381469727,
      "step": 1820
    },
    {
      "epoch": 2.355326681686514,
      "grad_norm": 3.6674065556267053,
      "learning_rate": 4.107603092783505e-07,
      "logits/chosen": -0.9901367425918579,
      "logits/rejected": -0.9849609136581421,
      "logps/chosen": -399.57501220703125,
      "logps/rejected": -510.20001220703125,
      "loss": 0.0399,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -4.4237060546875,
      "rewards/margins": 13.792187690734863,
      "rewards/rejected": -18.212499618530273,
      "step": 1830
    },
    {
      "epoch": 2.3682008368200838,
      "grad_norm": 0.011220754857104063,
      "learning_rate": 4.075386597938144e-07,
      "logits/chosen": -0.9344238042831421,
      "logits/rejected": -0.917895495891571,
      "logps/chosen": -363.42498779296875,
      "logps/rejected": -516.7000122070312,
      "loss": 0.0226,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -4.707543849945068,
      "rewards/margins": 15.096875190734863,
      "rewards/rejected": -19.815624237060547,
      "step": 1840
    },
    {
      "epoch": 2.381074991953653,
      "grad_norm": 1.084213236167663,
      "learning_rate": 4.043170103092783e-07,
      "logits/chosen": -1.003729224205017,
      "logits/rejected": -0.9970871210098267,
      "logps/chosen": -422.0,
      "logps/rejected": -577.9500122070312,
      "loss": 0.018,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -4.445825099945068,
      "rewards/margins": 15.790624618530273,
      "rewards/rejected": -20.237499237060547,
      "step": 1850
    },
    {
      "epoch": 2.3939491470872225,
      "grad_norm": 0.11138110362978013,
      "learning_rate": 4.010953608247423e-07,
      "logits/chosen": -1.015112280845642,
      "logits/rejected": -1.0192139148712158,
      "logps/chosen": -434.6000061035156,
      "logps/rejected": -530.125,
      "loss": 0.0205,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -4.493237495422363,
      "rewards/margins": 15.137499809265137,
      "rewards/rejected": -19.627344131469727,
      "step": 1860
    },
    {
      "epoch": 2.4068233022207917,
      "grad_norm": 0.07209992509377328,
      "learning_rate": 3.9787371134020614e-07,
      "logits/chosen": -1.021484375,
      "logits/rejected": -1.012841820716858,
      "logps/chosen": -408.1875,
      "logps/rejected": -553.0999755859375,
      "loss": 0.0226,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -4.288866996765137,
      "rewards/margins": 15.574999809265137,
      "rewards/rejected": -19.860937118530273,
      "step": 1870
    },
    {
      "epoch": 2.4196974573543613,
      "grad_norm": 26.658700203141652,
      "learning_rate": 3.9465206185567005e-07,
      "logits/chosen": -0.9209960699081421,
      "logits/rejected": -0.8855956792831421,
      "logps/chosen": -436.0,
      "logps/rejected": -571.6749877929688,
      "loss": 0.0188,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -4.043432712554932,
      "rewards/margins": 15.83984375,
      "rewards/rejected": -19.890625,
      "step": 1880
    },
    {
      "epoch": 2.4325716124879304,
      "grad_norm": 10.340676509370857,
      "learning_rate": 3.91430412371134e-07,
      "logits/chosen": -0.9385986328125,
      "logits/rejected": -0.912792980670929,
      "logps/chosen": -400.6000061035156,
      "logps/rejected": -534.9249877929688,
      "loss": 0.0357,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -4.141528129577637,
      "rewards/margins": 14.765625,
      "rewards/rejected": -18.901561737060547,
      "step": 1890
    },
    {
      "epoch": 2.4454457676215,
      "grad_norm": 0.1837428615330857,
      "learning_rate": 3.882087628865979e-07,
      "logits/chosen": -0.942370593547821,
      "logits/rejected": -0.9447265863418579,
      "logps/chosen": -406.9937438964844,
      "logps/rejected": -544.5250244140625,
      "loss": 0.0395,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -3.229174852371216,
      "rewards/margins": 15.078125,
      "rewards/rejected": -18.3046875,
      "step": 1900
    },
    {
      "epoch": 2.458319922755069,
      "grad_norm": 0.06528086713553832,
      "learning_rate": 3.8498711340206184e-07,
      "logits/chosen": -1.0470702648162842,
      "logits/rejected": -1.0382812023162842,
      "logps/chosen": -407.82501220703125,
      "logps/rejected": -526.8499755859375,
      "loss": 0.0141,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -3.409912109375,
      "rewards/margins": 14.985937118530273,
      "rewards/rejected": -18.401561737060547,
      "step": 1910
    },
    {
      "epoch": 2.4711940778886383,
      "grad_norm": 12.890073513058875,
      "learning_rate": 3.8176546391752575e-07,
      "logits/chosen": -1.028564453125,
      "logits/rejected": -1.0325927734375,
      "logps/chosen": -411.04998779296875,
      "logps/rejected": -546.2000122070312,
      "loss": 0.0356,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -3.9456543922424316,
      "rewards/margins": 15.208593368530273,
      "rewards/rejected": -19.146875381469727,
      "step": 1920
    },
    {
      "epoch": 2.484068233022208,
      "grad_norm": 2.4135445503560464,
      "learning_rate": 3.7854381443298966e-07,
      "logits/chosen": -1.056909203529358,
      "logits/rejected": -1.024682641029358,
      "logps/chosen": -431.5249938964844,
      "logps/rejected": -563.25,
      "loss": 0.0294,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -4.433886528015137,
      "rewards/margins": 15.984375,
      "rewards/rejected": -20.426563262939453,
      "step": 1930
    },
    {
      "epoch": 2.4969423881557775,
      "grad_norm": 4.32141413981977,
      "learning_rate": 3.7532216494845357e-07,
      "logits/chosen": -1.046728491783142,
      "logits/rejected": -1.02490234375,
      "logps/chosen": -403.0249938964844,
      "logps/rejected": -542.0499877929688,
      "loss": 0.0218,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -4.49169921875,
      "rewards/margins": 15.235937118530273,
      "rewards/rejected": -19.725000381469727,
      "step": 1940
    },
    {
      "epoch": 2.5098165432893467,
      "grad_norm": 1.4656812942309294,
      "learning_rate": 3.7210051546391753e-07,
      "logits/chosen": -1.004663109779358,
      "logits/rejected": -0.996167004108429,
      "logps/chosen": -426.2124938964844,
      "logps/rejected": -551.2999877929688,
      "loss": 0.0264,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -5.308349609375,
      "rewards/margins": 14.5546875,
      "rewards/rejected": -19.862499237060547,
      "step": 1950
    },
    {
      "epoch": 2.522690698422916,
      "grad_norm": 0.06357133072392389,
      "learning_rate": 3.6887886597938144e-07,
      "logits/chosen": -1.0270264148712158,
      "logits/rejected": -0.9864746332168579,
      "logps/chosen": -407.70001220703125,
      "logps/rejected": -538.8499755859375,
      "loss": 0.0233,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -5.082373142242432,
      "rewards/margins": 14.225000381469727,
      "rewards/rejected": -19.306249618530273,
      "step": 1960
    },
    {
      "epoch": 2.5355648535564854,
      "grad_norm": 0.17219392427815344,
      "learning_rate": 3.656572164948453e-07,
      "logits/chosen": -1.095312476158142,
      "logits/rejected": -1.09765625,
      "logps/chosen": -421.2875061035156,
      "logps/rejected": -564.3499755859375,
      "loss": 0.0244,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -4.148681640625,
      "rewards/margins": 15.556249618530273,
      "rewards/rejected": -19.692188262939453,
      "step": 1970
    },
    {
      "epoch": 2.5484390086900546,
      "grad_norm": 0.02503924044752003,
      "learning_rate": 3.6243556701030926e-07,
      "logits/chosen": -1.0810546875,
      "logits/rejected": -1.093164086341858,
      "logps/chosen": -416.79998779296875,
      "logps/rejected": -556.8499755859375,
      "loss": 0.011,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -4.688818454742432,
      "rewards/margins": 15.245312690734863,
      "rewards/rejected": -19.932811737060547,
      "step": 1980
    },
    {
      "epoch": 2.561313163823624,
      "grad_norm": 0.055260003017054,
      "learning_rate": 3.5921391752577317e-07,
      "logits/chosen": -1.043115258216858,
      "logits/rejected": -1.0253417491912842,
      "logps/chosen": -403.07501220703125,
      "logps/rejected": -537.7750244140625,
      "loss": 0.0178,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -5.153515815734863,
      "rewards/margins": 15.246874809265137,
      "rewards/rejected": -20.393749237060547,
      "step": 1990
    },
    {
      "epoch": 2.5741873189571933,
      "grad_norm": 1.4980599504146637,
      "learning_rate": 3.5599226804123713e-07,
      "logits/chosen": -1.001977562904358,
      "logits/rejected": -0.985791027545929,
      "logps/chosen": -403.82501220703125,
      "logps/rejected": -505.79998779296875,
      "loss": 0.0391,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -4.720019340515137,
      "rewards/margins": 13.964062690734863,
      "rewards/rejected": -18.684375762939453,
      "step": 2000
    },
    {
      "epoch": 2.587061474090763,
      "grad_norm": 85.99081877050921,
      "learning_rate": 3.5277061855670104e-07,
      "logits/chosen": -0.9850708246231079,
      "logits/rejected": -0.9672485589981079,
      "logps/chosen": -401.7875061035156,
      "logps/rejected": -523.0750122070312,
      "loss": 0.0526,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -3.9986329078674316,
      "rewards/margins": 14.18359375,
      "rewards/rejected": -18.178905487060547,
      "step": 2010
    },
    {
      "epoch": 2.599935629224332,
      "grad_norm": 181.98998012124002,
      "learning_rate": 3.495489690721649e-07,
      "logits/chosen": -1.027734398841858,
      "logits/rejected": -1.01318359375,
      "logps/chosen": -444.2749938964844,
      "logps/rejected": -566.0499877929688,
      "loss": 0.0324,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -4.080468654632568,
      "rewards/margins": 14.451562881469727,
      "rewards/rejected": -18.526561737060547,
      "step": 2020
    },
    {
      "epoch": 2.6128097843579017,
      "grad_norm": 3.7256787279315344,
      "learning_rate": 3.4632731958762886e-07,
      "logits/chosen": -1.019921898841858,
      "logits/rejected": -0.9986816644668579,
      "logps/chosen": -391.9375,
      "logps/rejected": -520.6749877929688,
      "loss": 0.0273,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -3.239990234375,
      "rewards/margins": 14.225000381469727,
      "rewards/rejected": -17.467187881469727,
      "step": 2030
    },
    {
      "epoch": 2.625683939491471,
      "grad_norm": 0.6934846653919562,
      "learning_rate": 3.4310567010309277e-07,
      "logits/chosen": -1.037451148033142,
      "logits/rejected": -1.0004150867462158,
      "logps/chosen": -409.70001220703125,
      "logps/rejected": -569.7000122070312,
      "loss": 0.023,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -3.6155762672424316,
      "rewards/margins": 16.117969512939453,
      "rewards/rejected": -19.7265625,
      "step": 2040
    },
    {
      "epoch": 2.63855809462504,
      "grad_norm": 1.084526871715855,
      "learning_rate": 3.398840206185567e-07,
      "logits/chosen": -1.072167992591858,
      "logits/rejected": -1.0823242664337158,
      "logps/chosen": -437.3125,
      "logps/rejected": -534.2249755859375,
      "loss": 0.0156,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -4.259228706359863,
      "rewards/margins": 14.357030868530273,
      "rewards/rejected": -18.607812881469727,
      "step": 2050
    },
    {
      "epoch": 2.6514322497586096,
      "grad_norm": 0.13588939136218478,
      "learning_rate": 3.3666237113402065e-07,
      "logits/chosen": -0.9833984375,
      "logits/rejected": -0.9649413824081421,
      "logps/chosen": -417.70001220703125,
      "logps/rejected": -553.5750122070312,
      "loss": 0.027,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -4.4521484375,
      "rewards/margins": 15.2890625,
      "rewards/rejected": -19.746875762939453,
      "step": 2060
    },
    {
      "epoch": 2.664306404892179,
      "grad_norm": 18.71919913613174,
      "learning_rate": 3.334407216494845e-07,
      "logits/chosen": -0.9654175043106079,
      "logits/rejected": -0.951892077922821,
      "logps/chosen": -411.0,
      "logps/rejected": -526.0499877929688,
      "loss": 0.0241,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -5.281836032867432,
      "rewards/margins": 14.720312118530273,
      "rewards/rejected": -19.996875762939453,
      "step": 2070
    },
    {
      "epoch": 2.6771805600257483,
      "grad_norm": 0.1687335570953075,
      "learning_rate": 3.302190721649484e-07,
      "logits/chosen": -1.0874512195587158,
      "logits/rejected": -1.0557129383087158,
      "logps/chosen": -466.07501220703125,
      "logps/rejected": -609.9000244140625,
      "loss": 0.0229,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -5.810986518859863,
      "rewards/margins": 15.785937309265137,
      "rewards/rejected": -21.600000381469727,
      "step": 2080
    },
    {
      "epoch": 2.6900547151593175,
      "grad_norm": 0.07496715000168404,
      "learning_rate": 3.269974226804124e-07,
      "logits/chosen": -1.061132788658142,
      "logits/rejected": -1.0348632335662842,
      "logps/chosen": -433.5,
      "logps/rejected": -542.5499877929688,
      "loss": 0.0264,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -5.439917087554932,
      "rewards/margins": 14.901562690734863,
      "rewards/rejected": -20.337499618530273,
      "step": 2090
    },
    {
      "epoch": 2.702928870292887,
      "grad_norm": 3.342920359584719,
      "learning_rate": 3.237757731958763e-07,
      "logits/chosen": -1.0269043445587158,
      "logits/rejected": -1.0263183116912842,
      "logps/chosen": -451.07501220703125,
      "logps/rejected": -550.3499755859375,
      "loss": 0.0223,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -5.473876953125,
      "rewards/margins": 14.681249618530273,
      "rewards/rejected": -20.171875,
      "step": 2100
    },
    {
      "epoch": 2.7158030254264562,
      "grad_norm": 0.41399028262679977,
      "learning_rate": 3.205541237113402e-07,
      "logits/chosen": -0.9803222417831421,
      "logits/rejected": -0.970996081829071,
      "logps/chosen": -416.45001220703125,
      "logps/rejected": -556.625,
      "loss": 0.0272,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -4.261034965515137,
      "rewards/margins": 15.196093559265137,
      "rewards/rejected": -19.4609375,
      "step": 2110
    },
    {
      "epoch": 2.728677180560026,
      "grad_norm": 0.08328554014051173,
      "learning_rate": 3.173324742268041e-07,
      "logits/chosen": -1.0111510753631592,
      "logits/rejected": -1.000097632408142,
      "logps/chosen": -394.2250061035156,
      "logps/rejected": -528.5250244140625,
      "loss": 0.0367,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -3.413928270339966,
      "rewards/margins": 14.673437118530273,
      "rewards/rejected": -18.082813262939453,
      "step": 2120
    },
    {
      "epoch": 2.741551335693595,
      "grad_norm": 0.997200008758714,
      "learning_rate": 3.14110824742268e-07,
      "logits/chosen": -0.987011730670929,
      "logits/rejected": -0.9897216558456421,
      "logps/chosen": -425.61248779296875,
      "logps/rejected": -536.1500244140625,
      "loss": 0.0334,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -2.638232469558716,
      "rewards/margins": 13.8203125,
      "rewards/rejected": -16.453125,
      "step": 2130
    },
    {
      "epoch": 2.7544254908271646,
      "grad_norm": 0.07908592939561318,
      "learning_rate": 3.1088917525773193e-07,
      "logits/chosen": -1.01739501953125,
      "logits/rejected": -1.0194091796875,
      "logps/chosen": -412.67498779296875,
      "logps/rejected": -543.7000122070312,
      "loss": 0.0141,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -3.637744188308716,
      "rewards/margins": 14.646875381469727,
      "rewards/rejected": -18.279687881469727,
      "step": 2140
    },
    {
      "epoch": 2.7672996459607337,
      "grad_norm": 0.31312077632852253,
      "learning_rate": 3.076675257731959e-07,
      "logits/chosen": -1.015039086341858,
      "logits/rejected": -0.9869629144668579,
      "logps/chosen": -412.2250061035156,
      "logps/rejected": -571.9000244140625,
      "loss": 0.0135,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -3.989062547683716,
      "rewards/margins": 15.939062118530273,
      "rewards/rejected": -19.928125381469727,
      "step": 2150
    },
    {
      "epoch": 2.7801738010943033,
      "grad_norm": 0.5372266843183616,
      "learning_rate": 3.0444587628865975e-07,
      "logits/chosen": -0.9476562738418579,
      "logits/rejected": -0.9249511957168579,
      "logps/chosen": -433.79998779296875,
      "logps/rejected": -548.9000244140625,
      "loss": 0.0245,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -4.148535251617432,
      "rewards/margins": 15.0546875,
      "rewards/rejected": -19.196874618530273,
      "step": 2160
    },
    {
      "epoch": 2.7930479562278725,
      "grad_norm": 0.21085049221881774,
      "learning_rate": 3.0122422680412366e-07,
      "logits/chosen": -1.0446288585662842,
      "logits/rejected": -1.030908226966858,
      "logps/chosen": -430.75,
      "logps/rejected": -559.0999755859375,
      "loss": 0.0361,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -4.50244140625,
      "rewards/margins": 15.458593368530273,
      "rewards/rejected": -19.962499618530273,
      "step": 2170
    },
    {
      "epoch": 2.8059221113614417,
      "grad_norm": 0.07088843316040844,
      "learning_rate": 2.980025773195876e-07,
      "logits/chosen": -1.0450928211212158,
      "logits/rejected": -1.036767601966858,
      "logps/chosen": -428.4375,
      "logps/rejected": -547.5499877929688,
      "loss": 0.0176,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -4.658154487609863,
      "rewards/margins": 15.4296875,
      "rewards/rejected": -20.071874618530273,
      "step": 2180
    },
    {
      "epoch": 2.8187962664950112,
      "grad_norm": 0.7000631568785479,
      "learning_rate": 2.9478092783505153e-07,
      "logits/chosen": -1.0032227039337158,
      "logits/rejected": -1.0255615711212158,
      "logps/chosen": -431.125,
      "logps/rejected": -551.9000244140625,
      "loss": 0.0408,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -4.879492282867432,
      "rewards/margins": 14.317968368530273,
      "rewards/rejected": -19.201562881469727,
      "step": 2190
    },
    {
      "epoch": 2.831670421628581,
      "grad_norm": 10.34328122014099,
      "learning_rate": 2.915592783505155e-07,
      "logits/chosen": -0.921124279499054,
      "logits/rejected": -0.9173828363418579,
      "logps/chosen": -395.45001220703125,
      "logps/rejected": -533.5999755859375,
      "loss": 0.0529,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -5.427599906921387,
      "rewards/margins": 14.371874809265137,
      "rewards/rejected": -19.810937881469727,
      "step": 2200
    },
    {
      "epoch": 2.84454457676215,
      "grad_norm": 0.08482399874585603,
      "learning_rate": 2.8833762886597935e-07,
      "logits/chosen": -1.068017601966858,
      "logits/rejected": -1.060449242591858,
      "logps/chosen": -420.1000061035156,
      "logps/rejected": -585.4500122070312,
      "loss": 0.0273,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -6.1865234375,
      "rewards/margins": 15.414843559265137,
      "rewards/rejected": -21.610937118530273,
      "step": 2210
    },
    {
      "epoch": 2.857418731895719,
      "grad_norm": 0.3961966887737209,
      "learning_rate": 2.8511597938144326e-07,
      "logits/chosen": -1.036230444908142,
      "logits/rejected": -1.0049316883087158,
      "logps/chosen": -466.8999938964844,
      "logps/rejected": -608.5499877929688,
      "loss": 0.0307,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -6.053515434265137,
      "rewards/margins": 15.393750190734863,
      "rewards/rejected": -21.439062118530273,
      "step": 2220
    },
    {
      "epoch": 2.8702928870292888,
      "grad_norm": 0.09519495396041948,
      "learning_rate": 2.818943298969072e-07,
      "logits/chosen": -1.0175049304962158,
      "logits/rejected": -1.0093994140625,
      "logps/chosen": -416.4750061035156,
      "logps/rejected": -567.5999755859375,
      "loss": 0.0275,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -5.609961032867432,
      "rewards/margins": 15.884374618530273,
      "rewards/rejected": -21.490625381469727,
      "step": 2230
    },
    {
      "epoch": 2.883167042162858,
      "grad_norm": 0.5232996096645688,
      "learning_rate": 2.7867268041237114e-07,
      "logits/chosen": -0.9981933832168579,
      "logits/rejected": -0.9861816167831421,
      "logps/chosen": -430.7250061035156,
      "logps/rejected": -597.5999755859375,
      "loss": 0.0181,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -5.131201267242432,
      "rewards/margins": 17.307811737060547,
      "rewards/rejected": -22.4453125,
      "step": 2240
    },
    {
      "epoch": 2.8960411972964275,
      "grad_norm": 0.7006994780430169,
      "learning_rate": 2.7545103092783505e-07,
      "logits/chosen": -1.0391662120819092,
      "logits/rejected": -1.0439056158065796,
      "logps/chosen": -442.2250061035156,
      "logps/rejected": -572.9000244140625,
      "loss": 0.0318,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -4.493456840515137,
      "rewards/margins": 16.2109375,
      "rewards/rejected": -20.706249237060547,
      "step": 2250
    },
    {
      "epoch": 2.9089153524299967,
      "grad_norm": 180.74744309920393,
      "learning_rate": 2.7222938144329896e-07,
      "logits/chosen": -0.9822998046875,
      "logits/rejected": -0.9486083984375,
      "logps/chosen": -439.6499938964844,
      "logps/rejected": -573.0,
      "loss": 0.0181,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -5.12890625,
      "rewards/margins": 16.4921875,
      "rewards/rejected": -21.623437881469727,
      "step": 2260
    },
    {
      "epoch": 2.9217895075635663,
      "grad_norm": 32.30656704261848,
      "learning_rate": 2.6900773195876287e-07,
      "logits/chosen": -1.033935546875,
      "logits/rejected": -1.022558569908142,
      "logps/chosen": -424.67498779296875,
      "logps/rejected": -589.4500122070312,
      "loss": 0.0319,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -4.392675876617432,
      "rewards/margins": 16.057811737060547,
      "rewards/rejected": -20.462499618530273,
      "step": 2270
    },
    {
      "epoch": 2.9346636626971354,
      "grad_norm": 0.5484870470342832,
      "learning_rate": 2.657860824742268e-07,
      "logits/chosen": -0.999804675579071,
      "logits/rejected": -0.9871581792831421,
      "logps/chosen": -435.17498779296875,
      "logps/rejected": -564.5,
      "loss": 0.0273,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -4.9228515625,
      "rewards/margins": 14.823437690734863,
      "rewards/rejected": -19.743749618530273,
      "step": 2280
    },
    {
      "epoch": 2.947537817830705,
      "grad_norm": 0.06661369102436356,
      "learning_rate": 2.6256443298969074e-07,
      "logits/chosen": -1.0869140625,
      "logits/rejected": -1.0365722179412842,
      "logps/chosen": -414.7749938964844,
      "logps/rejected": -537.1500244140625,
      "loss": 0.0223,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -4.404394626617432,
      "rewards/margins": 16.225780487060547,
      "rewards/rejected": -20.629688262939453,
      "step": 2290
    },
    {
      "epoch": 2.960411972964274,
      "grad_norm": 0.38801027736265253,
      "learning_rate": 2.5934278350515465e-07,
      "logits/chosen": -0.953857421875,
      "logits/rejected": -0.930737316608429,
      "logps/chosen": -417.5249938964844,
      "logps/rejected": -546.25,
      "loss": 0.0278,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -5.3330078125,
      "rewards/margins": 15.092187881469727,
      "rewards/rejected": -20.431249618530273,
      "step": 2300
    },
    {
      "epoch": 2.9732861280978433,
      "grad_norm": 0.5569756223752637,
      "learning_rate": 2.561211340206185e-07,
      "logits/chosen": -0.974780261516571,
      "logits/rejected": -0.959912121295929,
      "logps/chosen": -425.5249938964844,
      "logps/rejected": -586.7000122070312,
      "loss": 0.0455,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.039258003234863,
      "rewards/margins": 16.984375,
      "rewards/rejected": -21.012500762939453,
      "step": 2310
    },
    {
      "epoch": 2.986160283231413,
      "grad_norm": 0.1140239873803737,
      "learning_rate": 2.5289948453608247e-07,
      "logits/chosen": -0.954150378704071,
      "logits/rejected": -0.9415038824081421,
      "logps/chosen": -446.0,
      "logps/rejected": -591.0,
      "loss": 0.0339,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -5.755810737609863,
      "rewards/margins": 16.087499618530273,
      "rewards/rejected": -21.845312118530273,
      "step": 2320
    },
    {
      "epoch": 2.9990344383649825,
      "grad_norm": 0.23176827452269116,
      "learning_rate": 2.496778350515464e-07,
      "logits/chosen": -0.892468273639679,
      "logits/rejected": -0.8869384527206421,
      "logps/chosen": -395.25,
      "logps/rejected": -524.5499877929688,
      "loss": 0.0355,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -4.820703029632568,
      "rewards/margins": 17.042186737060547,
      "rewards/rejected": -21.870311737060547,
      "step": 2330
    },
    {
      "epoch": 3.0115867396202125,
      "grad_norm": 0.04030797300705242,
      "learning_rate": 2.464561855670103e-07,
      "logits/chosen": -0.9333683848381042,
      "logits/rejected": -0.9138621687889099,
      "logps/chosen": -412.23077392578125,
      "logps/rejected": -552.2051391601562,
      "loss": 0.0346,
      "rewards/accuracies": 0.9487179517745972,
      "rewards/chosen": -5.357221603393555,
      "rewards/margins": 15.293269157409668,
      "rewards/rejected": -20.644229888916016,
      "step": 2340
    },
    {
      "epoch": 3.0244608947537817,
      "grad_norm": 0.011972067245460228,
      "learning_rate": 2.432345360824742e-07,
      "logits/chosen": -1.035009741783142,
      "logits/rejected": -0.9994140863418579,
      "logps/chosen": -425.4750061035156,
      "logps/rejected": -578.2000122070312,
      "loss": 0.0218,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -4.936822414398193,
      "rewards/margins": 16.629688262939453,
      "rewards/rejected": -21.5546875,
      "step": 2350
    },
    {
      "epoch": 3.0373350498873513,
      "grad_norm": 0.014209079031048687,
      "learning_rate": 2.4001288659793816e-07,
      "logits/chosen": -0.955126941204071,
      "logits/rejected": -0.945361316204071,
      "logps/chosen": -409.75,
      "logps/rejected": -589.1500244140625,
      "loss": 0.0087,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -4.591943264007568,
      "rewards/margins": 18.920312881469727,
      "rewards/rejected": -23.517187118530273,
      "step": 2360
    },
    {
      "epoch": 3.0502092050209204,
      "grad_norm": 0.003246395439313634,
      "learning_rate": 2.3679123711340205e-07,
      "logits/chosen": -0.972424328327179,
      "logits/rejected": -0.954345703125,
      "logps/chosen": -430.32501220703125,
      "logps/rejected": -595.8499755859375,
      "loss": 0.013,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -5.572167873382568,
      "rewards/margins": 17.579687118530273,
      "rewards/rejected": -23.153125762939453,
      "step": 2370
    },
    {
      "epoch": 3.06308336015449,
      "grad_norm": 0.01783900566903516,
      "learning_rate": 2.3356958762886598e-07,
      "logits/chosen": -0.9012451171875,
      "logits/rejected": -0.8792480230331421,
      "logps/chosen": -434.2749938964844,
      "logps/rejected": -566.9500122070312,
      "loss": 0.0131,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -5.078955173492432,
      "rewards/margins": 15.729687690734863,
      "rewards/rejected": -20.803125381469727,
      "step": 2380
    },
    {
      "epoch": 3.075957515288059,
      "grad_norm": 0.01685872055087948,
      "learning_rate": 2.3034793814432987e-07,
      "logits/chosen": -0.8941406011581421,
      "logits/rejected": -0.877490222454071,
      "logps/chosen": -404.9125061035156,
      "logps/rejected": -555.625,
      "loss": 0.0217,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -5.329394340515137,
      "rewards/margins": 16.1328125,
      "rewards/rejected": -21.4609375,
      "step": 2390
    },
    {
      "epoch": 3.0888316704216288,
      "grad_norm": 0.004614662175876239,
      "learning_rate": 2.271262886597938e-07,
      "logits/chosen": -1.0093262195587158,
      "logits/rejected": -1.0094726085662842,
      "logps/chosen": -451.75,
      "logps/rejected": -598.8499755859375,
      "loss": 0.0216,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -4.370312690734863,
      "rewards/margins": 17.09375,
      "rewards/rejected": -21.471874237060547,
      "step": 2400
    },
    {
      "epoch": 3.101705825555198,
      "grad_norm": 0.010560387649899727,
      "learning_rate": 2.2390463917525774e-07,
      "logits/chosen": -0.912158191204071,
      "logits/rejected": -0.93896484375,
      "logps/chosen": -415.3500061035156,
      "logps/rejected": -563.9749755859375,
      "loss": 0.0182,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -5.893652439117432,
      "rewards/margins": 16.9609375,
      "rewards/rejected": -22.845312118530273,
      "step": 2410
    },
    {
      "epoch": 3.1145799806887675,
      "grad_norm": 0.009728145604973306,
      "learning_rate": 2.2068298969072162e-07,
      "logits/chosen": -0.8603515625,
      "logits/rejected": -0.8588622808456421,
      "logps/chosen": -397.51251220703125,
      "logps/rejected": -570.9249877929688,
      "loss": 0.0088,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -5.475195407867432,
      "rewards/margins": 17.790624618530273,
      "rewards/rejected": -23.262500762939453,
      "step": 2420
    },
    {
      "epoch": 3.1274541358223367,
      "grad_norm": 0.022851496718264388,
      "learning_rate": 2.1746134020618556e-07,
      "logits/chosen": -0.90869140625,
      "logits/rejected": -0.901171863079071,
      "logps/chosen": -409.5,
      "logps/rejected": -572.25,
      "loss": 0.0303,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -6.165625095367432,
      "rewards/margins": 16.734375,
      "rewards/rejected": -22.893749237060547,
      "step": 2430
    },
    {
      "epoch": 3.140328290955906,
      "grad_norm": 0.2577532223041415,
      "learning_rate": 2.1423969072164947e-07,
      "logits/chosen": -0.9032226800918579,
      "logits/rejected": -0.899731457233429,
      "logps/chosen": -430.29998779296875,
      "logps/rejected": -578.6500244140625,
      "loss": 0.0263,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -5.7548828125,
      "rewards/margins": 17.462499618530273,
      "rewards/rejected": -23.220312118530273,
      "step": 2440
    },
    {
      "epoch": 3.1532024460894754,
      "grad_norm": 0.7364005790474251,
      "learning_rate": 2.1101804123711338e-07,
      "logits/chosen": -0.976318359375,
      "logits/rejected": -0.982128918170929,
      "logps/chosen": -452.0,
      "logps/rejected": -611.3499755859375,
      "loss": 0.0087,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -6.922656059265137,
      "rewards/margins": 17.292186737060547,
      "rewards/rejected": -24.209375381469727,
      "step": 2450
    },
    {
      "epoch": 3.1660766012230446,
      "grad_norm": 0.2985041143166853,
      "learning_rate": 2.0779639175257732e-07,
      "logits/chosen": -0.9498962163925171,
      "logits/rejected": -0.9401611089706421,
      "logps/chosen": -484.8500061035156,
      "logps/rejected": -589.2999877929688,
      "loss": 0.035,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -5.613110542297363,
      "rewards/margins": 16.129688262939453,
      "rewards/rejected": -21.748437881469727,
      "step": 2460
    },
    {
      "epoch": 3.178950756356614,
      "grad_norm": 0.2869817908499183,
      "learning_rate": 2.0457474226804123e-07,
      "logits/chosen": -0.969921886920929,
      "logits/rejected": -0.9568847417831421,
      "logps/chosen": -450.20001220703125,
      "logps/rejected": -599.5750122070312,
      "loss": 0.0131,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -6.140234470367432,
      "rewards/margins": 17.448436737060547,
      "rewards/rejected": -23.598438262939453,
      "step": 2470
    },
    {
      "epoch": 3.1918249114901833,
      "grad_norm": 0.04088909996359556,
      "learning_rate": 2.0135309278350516e-07,
      "logits/chosen": -0.958251953125,
      "logits/rejected": -0.948974609375,
      "logps/chosen": -490.25,
      "logps/rejected": -668.0,
      "loss": 0.0225,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -5.629956245422363,
      "rewards/margins": 17.813282012939453,
      "rewards/rejected": -23.443750381469727,
      "step": 2480
    },
    {
      "epoch": 3.204699066623753,
      "grad_norm": 0.08684549219199517,
      "learning_rate": 1.9813144329896905e-07,
      "logits/chosen": -0.9354492425918579,
      "logits/rejected": -0.907482922077179,
      "logps/chosen": -438.79998779296875,
      "logps/rejected": -593.4000244140625,
      "loss": 0.0217,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -6.191357612609863,
      "rewards/margins": 17.426563262939453,
      "rewards/rejected": -23.637500762939453,
      "step": 2490
    },
    {
      "epoch": 3.217573221757322,
      "grad_norm": 0.01725372117673741,
      "learning_rate": 1.9490979381443298e-07,
      "logits/chosen": -0.9265502691268921,
      "logits/rejected": -0.91583251953125,
      "logps/chosen": -365.125,
      "logps/rejected": -559.0499877929688,
      "loss": 0.0216,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -5.943749904632568,
      "rewards/margins": 17.767187118530273,
      "rewards/rejected": -23.715625762939453,
      "step": 2500
    },
    {
      "epoch": 3.2304473768908917,
      "grad_norm": 0.010133221834613732,
      "learning_rate": 1.9168814432989692e-07,
      "logits/chosen": -0.9111572504043579,
      "logits/rejected": -0.9021972417831421,
      "logps/chosen": -419.7749938964844,
      "logps/rejected": -557.8499755859375,
      "loss": 0.0217,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -6.076904296875,
      "rewards/margins": 16.378124237060547,
      "rewards/rejected": -22.446874618530273,
      "step": 2510
    },
    {
      "epoch": 3.243321532024461,
      "grad_norm": 0.024756280732985112,
      "learning_rate": 1.884664948453608e-07,
      "logits/chosen": -0.906176745891571,
      "logits/rejected": -0.9052368402481079,
      "logps/chosen": -460.57501220703125,
      "logps/rejected": -616.2000122070312,
      "loss": 0.0174,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -6.07861328125,
      "rewards/margins": 17.920312881469727,
      "rewards/rejected": -23.996875762939453,
      "step": 2520
    },
    {
      "epoch": 3.2561956871580304,
      "grad_norm": 0.10786257268593104,
      "learning_rate": 1.8524484536082474e-07,
      "logits/chosen": -0.9489501714706421,
      "logits/rejected": -0.9321533441543579,
      "logps/chosen": -427.0,
      "logps/rejected": -575.25,
      "loss": 0.0274,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -6.699999809265137,
      "rewards/margins": 17.05078125,
      "rewards/rejected": -23.767187118530273,
      "step": 2530
    },
    {
      "epoch": 3.2690698422915996,
      "grad_norm": 0.026345085205565732,
      "learning_rate": 1.8202319587628865e-07,
      "logits/chosen": -0.996826171875,
      "logits/rejected": -0.998339831829071,
      "logps/chosen": -443.1000061035156,
      "logps/rejected": -621.7999877929688,
      "loss": 0.013,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -6.483300685882568,
      "rewards/margins": 18.181249618530273,
      "rewards/rejected": -24.65625,
      "step": 2540
    },
    {
      "epoch": 3.281943997425169,
      "grad_norm": 0.04457572944694021,
      "learning_rate": 1.7880154639175256e-07,
      "logits/chosen": -0.869128406047821,
      "logits/rejected": -0.8661864995956421,
      "logps/chosen": -410.375,
      "logps/rejected": -556.5499877929688,
      "loss": 0.0433,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -6.444140434265137,
      "rewards/margins": 17.700000762939453,
      "rewards/rejected": -24.139062881469727,
      "step": 2550
    },
    {
      "epoch": 3.2948181525587383,
      "grad_norm": 0.03949142107866748,
      "learning_rate": 1.7557989690721647e-07,
      "logits/chosen": -0.8691161870956421,
      "logits/rejected": -0.840991199016571,
      "logps/chosen": -453.04998779296875,
      "logps/rejected": -596.4500122070312,
      "loss": 0.042,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -5.3740234375,
      "rewards/margins": 16.5234375,
      "rewards/rejected": -21.896875381469727,
      "step": 2560
    },
    {
      "epoch": 3.3076923076923075,
      "grad_norm": 0.031361210073259906,
      "learning_rate": 1.723582474226804e-07,
      "logits/chosen": -0.9446777105331421,
      "logits/rejected": -0.921069324016571,
      "logps/chosen": -458.07501220703125,
      "logps/rejected": -588.4500122070312,
      "loss": 0.0132,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -6.862548828125,
      "rewards/margins": 16.87890625,
      "rewards/rejected": -23.737499237060547,
      "step": 2570
    },
    {
      "epoch": 3.320566462825877,
      "grad_norm": 0.008891302161788853,
      "learning_rate": 1.6913659793814432e-07,
      "logits/chosen": -0.942626953125,
      "logits/rejected": -0.9102783203125,
      "logps/chosen": -403.2562561035156,
      "logps/rejected": -579.3125,
      "loss": 0.0347,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -5.142187595367432,
      "rewards/margins": 18.44140625,
      "rewards/rejected": -23.569530487060547,
      "step": 2580
    },
    {
      "epoch": 3.3334406179594462,
      "grad_norm": 0.017404878791276196,
      "learning_rate": 1.6591494845360823e-07,
      "logits/chosen": -0.902905285358429,
      "logits/rejected": -0.8969970941543579,
      "logps/chosen": -416.6499938964844,
      "logps/rejected": -574.6500244140625,
      "loss": 0.0132,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -6.773388862609863,
      "rewards/margins": 16.15234375,
      "rewards/rejected": -22.926563262939453,
      "step": 2590
    },
    {
      "epoch": 3.346314773093016,
      "grad_norm": 0.021856131216759354,
      "learning_rate": 1.6269329896907217e-07,
      "logits/chosen": -0.9424804449081421,
      "logits/rejected": -0.927441418170929,
      "logps/chosen": -418.5874938964844,
      "logps/rejected": -579.5750122070312,
      "loss": 0.0303,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -6.185742378234863,
      "rewards/margins": 17.252344131469727,
      "rewards/rejected": -23.4453125,
      "step": 2600
    },
    {
      "epoch": 3.359188928226585,
      "grad_norm": 0.014925207618162201,
      "learning_rate": 1.5947164948453608e-07,
      "logits/chosen": -0.932934582233429,
      "logits/rejected": -0.905834972858429,
      "logps/chosen": -457.17498779296875,
      "logps/rejected": -600.9500122070312,
      "loss": 0.026,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -5.945703029632568,
      "rewards/margins": 16.714061737060547,
      "rewards/rejected": -22.6640625,
      "step": 2610
    },
    {
      "epoch": 3.3720630833601546,
      "grad_norm": 0.005844375059446013,
      "learning_rate": 1.5624999999999999e-07,
      "logits/chosen": -0.8345702886581421,
      "logits/rejected": -0.8366943597793579,
      "logps/chosen": -393.5249938964844,
      "logps/rejected": -576.4500122070312,
      "loss": 0.0347,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -5.985547065734863,
      "rewards/margins": 16.9765625,
      "rewards/rejected": -22.96875,
      "step": 2620
    },
    {
      "epoch": 3.3849372384937237,
      "grad_norm": 109.07022759608414,
      "learning_rate": 1.5302835051546392e-07,
      "logits/chosen": -0.9147682189941406,
      "logits/rejected": -0.895800769329071,
      "logps/chosen": -412.9750061035156,
      "logps/rejected": -585.1500244140625,
      "loss": 0.0378,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -5.943163871765137,
      "rewards/margins": 16.678125381469727,
      "rewards/rejected": -22.629688262939453,
      "step": 2630
    },
    {
      "epoch": 3.3978113936272933,
      "grad_norm": 0.02354853531279238,
      "learning_rate": 1.4980670103092783e-07,
      "logits/chosen": -0.981213390827179,
      "logits/rejected": -0.9670654535293579,
      "logps/chosen": -484.3999938964844,
      "logps/rejected": -592.5999755859375,
      "loss": 0.0176,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -6.2783203125,
      "rewards/margins": 16.71875,
      "rewards/rejected": -23.001562118530273,
      "step": 2640
    },
    {
      "epoch": 3.4106855487608625,
      "grad_norm": 0.023273211343261376,
      "learning_rate": 1.4658505154639174e-07,
      "logits/chosen": -0.9508301019668579,
      "logits/rejected": -0.9560546875,
      "logps/chosen": -414.2250061035156,
      "logps/rejected": -592.4000244140625,
      "loss": 0.0087,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -6.26123046875,
      "rewards/margins": 18.1796875,
      "rewards/rejected": -24.450000762939453,
      "step": 2650
    },
    {
      "epoch": 3.423559703894432,
      "grad_norm": 0.029222841155265352,
      "learning_rate": 1.4336340206185565e-07,
      "logits/chosen": -1.0565917491912842,
      "logits/rejected": -1.0272216796875,
      "logps/chosen": -450.79998779296875,
      "logps/rejected": -590.4000244140625,
      "loss": 0.0216,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -5.90673828125,
      "rewards/margins": 16.371875762939453,
      "rewards/rejected": -22.2890625,
      "step": 2660
    },
    {
      "epoch": 3.4364338590280012,
      "grad_norm": 0.014108471125617151,
      "learning_rate": 1.401417525773196e-07,
      "logits/chosen": -0.9363158941268921,
      "logits/rejected": -0.9248901605606079,
      "logps/chosen": -404.20001220703125,
      "logps/rejected": -574.1500244140625,
      "loss": 0.0389,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -6.414563179016113,
      "rewards/margins": 16.696874618530273,
      "rewards/rejected": -23.107812881469727,
      "step": 2670
    },
    {
      "epoch": 3.449308014161571,
      "grad_norm": 0.023177327589133483,
      "learning_rate": 1.369201030927835e-07,
      "logits/chosen": -0.9132324457168579,
      "logits/rejected": -0.8884521722793579,
      "logps/chosen": -407.0,
      "logps/rejected": -574.5,
      "loss": 0.0224,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -6.184960842132568,
      "rewards/margins": 16.817188262939453,
      "rewards/rejected": -23.001562118530273,
      "step": 2680
    },
    {
      "epoch": 3.46218216929514,
      "grad_norm": 0.008481541035888551,
      "learning_rate": 1.336984536082474e-07,
      "logits/chosen": -0.9932616949081421,
      "logits/rejected": -0.978710949420929,
      "logps/chosen": -453.04998779296875,
      "logps/rejected": -601.9500122070312,
      "loss": 0.0049,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -5.75390625,
      "rewards/margins": 18.299999237060547,
      "rewards/rejected": -24.037500381469727,
      "step": 2690
    },
    {
      "epoch": 3.475056324428709,
      "grad_norm": 0.26185590688201954,
      "learning_rate": 1.3047680412371135e-07,
      "logits/chosen": -0.9244629144668579,
      "logits/rejected": -0.9140380620956421,
      "logps/chosen": -403.98748779296875,
      "logps/rejected": -566.2000122070312,
      "loss": 0.0303,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -5.669335842132568,
      "rewards/margins": 15.934374809265137,
      "rewards/rejected": -21.596874237060547,
      "step": 2700
    },
    {
      "epoch": 3.4879304795622788,
      "grad_norm": 0.13721565109158115,
      "learning_rate": 1.2725515463917523e-07,
      "logits/chosen": -0.8556152582168579,
      "logits/rejected": -0.823779284954071,
      "logps/chosen": -427.8999938964844,
      "logps/rejected": -566.7000122070312,
      "loss": 0.0392,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -5.447851657867432,
      "rewards/margins": 17.535938262939453,
      "rewards/rejected": -22.975000381469727,
      "step": 2710
    },
    {
      "epoch": 3.5008046346958483,
      "grad_norm": 0.0073649770109624054,
      "learning_rate": 1.2403350515463917e-07,
      "logits/chosen": -0.888256847858429,
      "logits/rejected": -0.86865234375,
      "logps/chosen": -408.2749938964844,
      "logps/rejected": -548.5499877929688,
      "loss": 0.048,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -5.692773342132568,
      "rewards/margins": 17.471874237060547,
      "rewards/rejected": -23.168750762939453,
      "step": 2720
    },
    {
      "epoch": 3.5136787898294175,
      "grad_norm": 0.17569853974318173,
      "learning_rate": 1.2081185567010308e-07,
      "logits/chosen": -0.995800793170929,
      "logits/rejected": -0.9750610589981079,
      "logps/chosen": -477.54998779296875,
      "logps/rejected": -588.5999755859375,
      "loss": 0.0188,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -6.515722751617432,
      "rewards/margins": 17.01171875,
      "rewards/rejected": -23.518749237060547,
      "step": 2730
    },
    {
      "epoch": 3.5265529449629867,
      "grad_norm": 0.02761008615338435,
      "learning_rate": 1.17590206185567e-07,
      "logits/chosen": -0.8585449457168579,
      "logits/rejected": -0.8765014410018921,
      "logps/chosen": -437.82501220703125,
      "logps/rejected": -586.0999755859375,
      "loss": 0.0353,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -6.2158203125,
      "rewards/margins": 17.151561737060547,
      "rewards/rejected": -23.360937118530273,
      "step": 2740
    },
    {
      "epoch": 3.5394271000965563,
      "grad_norm": 11.66898424743255,
      "learning_rate": 1.1436855670103092e-07,
      "logits/chosen": -0.9114745855331421,
      "logits/rejected": -0.9142822027206421,
      "logps/chosen": -435.0249938964844,
      "logps/rejected": -567.9749755859375,
      "loss": 0.0276,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -6.442236423492432,
      "rewards/margins": 17.442188262939453,
      "rewards/rejected": -23.887500762939453,
      "step": 2750
    },
    {
      "epoch": 3.5523012552301254,
      "grad_norm": 0.004880119483037049,
      "learning_rate": 1.1114690721649483e-07,
      "logits/chosen": -1.0186278820037842,
      "logits/rejected": -1.0231444835662842,
      "logps/chosen": -427.3500061035156,
      "logps/rejected": -613.0499877929688,
      "loss": 0.0259,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -7.323437690734863,
      "rewards/margins": 18.084375381469727,
      "rewards/rejected": -25.395313262939453,
      "step": 2760
    },
    {
      "epoch": 3.565175410363695,
      "grad_norm": 0.07925246005961702,
      "learning_rate": 1.0792525773195876e-07,
      "logits/chosen": -0.949267566204071,
      "logits/rejected": -0.93017578125,
      "logps/chosen": -414.04998779296875,
      "logps/rejected": -588.3499755859375,
      "loss": 0.0432,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -7.015234470367432,
      "rewards/margins": 17.198436737060547,
      "rewards/rejected": -24.21875,
      "step": 2770
    },
    {
      "epoch": 3.578049565497264,
      "grad_norm": 0.19368073236661892,
      "learning_rate": 1.0470360824742268e-07,
      "logits/chosen": -0.9741576910018921,
      "logits/rejected": -0.9656127691268921,
      "logps/chosen": -451.0375061035156,
      "logps/rejected": -590.7999877929688,
      "loss": 0.0259,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -6.281054496765137,
      "rewards/margins": 16.860937118530273,
      "rewards/rejected": -23.146875381469727,
      "step": 2780
    },
    {
      "epoch": 3.5909237206308338,
      "grad_norm": 0.0023883308566828173,
      "learning_rate": 1.0148195876288659e-07,
      "logits/chosen": -0.9483276605606079,
      "logits/rejected": -0.9399169683456421,
      "logps/chosen": -424.0,
      "logps/rejected": -585.25,
      "loss": 0.026,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -6.641797065734863,
      "rewards/margins": 17.5078125,
      "rewards/rejected": -24.1640625,
      "step": 2790
    },
    {
      "epoch": 3.603797875764403,
      "grad_norm": 0.06892660166670882,
      "learning_rate": 9.826030927835051e-08,
      "logits/chosen": -1.00048828125,
      "logits/rejected": -0.9769287109375,
      "logps/chosen": -482.375,
      "logps/rejected": -592.6500244140625,
      "loss": 0.0216,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -6.117089748382568,
      "rewards/margins": 16.573436737060547,
      "rewards/rejected": -22.678125381469727,
      "step": 2800
    },
    {
      "epoch": 3.6166720308979725,
      "grad_norm": 0.05744424533947675,
      "learning_rate": 9.503865979381443e-08,
      "logits/chosen": -0.894519031047821,
      "logits/rejected": -0.908447265625,
      "logps/chosen": -415.67498779296875,
      "logps/rejected": -574.2999877929688,
      "loss": 0.0216,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -6.658593654632568,
      "rewards/margins": 17.971874237060547,
      "rewards/rejected": -24.634374618530273,
      "step": 2810
    },
    {
      "epoch": 3.6295461860315417,
      "grad_norm": 0.19428536422265558,
      "learning_rate": 9.181701030927835e-08,
      "logits/chosen": -1.0341308116912842,
      "logits/rejected": -1.0309569835662842,
      "logps/chosen": -467.95001220703125,
      "logps/rejected": -612.2000122070312,
      "loss": 0.0389,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -6.307226657867432,
      "rewards/margins": 17.603124618530273,
      "rewards/rejected": -23.903125762939453,
      "step": 2820
    },
    {
      "epoch": 3.642420341165111,
      "grad_norm": 0.19115611112653982,
      "learning_rate": 8.859536082474227e-08,
      "logits/chosen": -0.9891357421875,
      "logits/rejected": -0.9699951410293579,
      "logps/chosen": -465.7250061035156,
      "logps/rejected": -616.0499877929688,
      "loss": 0.0389,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -7.32843017578125,
      "rewards/margins": 16.487499237060547,
      "rewards/rejected": -23.821874618530273,
      "step": 2830
    },
    {
      "epoch": 3.6552944962986804,
      "grad_norm": 0.0054378709746274664,
      "learning_rate": 8.537371134020618e-08,
      "logits/chosen": -0.9260498285293579,
      "logits/rejected": -0.933642566204071,
      "logps/chosen": -429.7749938964844,
      "logps/rejected": -589.6749877929688,
      "loss": 0.0391,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -7.364355564117432,
      "rewards/margins": 17.025781631469727,
      "rewards/rejected": -24.393749237060547,
      "step": 2840
    },
    {
      "epoch": 3.66816865143225,
      "grad_norm": 0.7156365283310772,
      "learning_rate": 8.21520618556701e-08,
      "logits/chosen": -0.91796875,
      "logits/rejected": -0.901538074016571,
      "logps/chosen": -438.5249938964844,
      "logps/rejected": -601.7999877929688,
      "loss": 0.0519,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -6.447070121765137,
      "rewards/margins": 17.762500762939453,
      "rewards/rejected": -24.206249237060547,
      "step": 2850
    },
    {
      "epoch": 3.681042806565819,
      "grad_norm": 0.03899053476143833,
      "learning_rate": 7.893041237113402e-08,
      "logits/chosen": -0.9479004144668579,
      "logits/rejected": -0.9349365234375,
      "logps/chosen": -460.54998779296875,
      "logps/rejected": -621.7999877929688,
      "loss": 0.0311,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -7.09765625,
      "rewards/margins": 18.182811737060547,
      "rewards/rejected": -25.298437118530273,
      "step": 2860
    },
    {
      "epoch": 3.6939169616993883,
      "grad_norm": 0.41126203127443395,
      "learning_rate": 7.570876288659793e-08,
      "logits/chosen": -1.0304687023162842,
      "logits/rejected": -1.018457055091858,
      "logps/chosen": -480.1499938964844,
      "logps/rejected": -617.9000244140625,
      "loss": 0.009,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -6.525000095367432,
      "rewards/margins": 18.549999237060547,
      "rewards/rejected": -25.092187881469727,
      "step": 2870
    },
    {
      "epoch": 3.706791116832958,
      "grad_norm": 0.4478569386073059,
      "learning_rate": 7.248711340206186e-08,
      "logits/chosen": -0.938159167766571,
      "logits/rejected": -0.941943347454071,
      "logps/chosen": -460.8500061035156,
      "logps/rejected": -612.7000122070312,
      "loss": 0.0346,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -7.170507907867432,
      "rewards/margins": 16.8828125,
      "rewards/rejected": -24.060937881469727,
      "step": 2880
    },
    {
      "epoch": 3.719665271966527,
      "grad_norm": 0.01080667980438113,
      "learning_rate": 6.926546391752577e-08,
      "logits/chosen": -0.9442809820175171,
      "logits/rejected": -0.935223400592804,
      "logps/chosen": -422.2250061035156,
      "logps/rejected": -603.4500122070312,
      "loss": 0.0217,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -7.079492092132568,
      "rewards/margins": 18.901561737060547,
      "rewards/rejected": -25.981250762939453,
      "step": 2890
    },
    {
      "epoch": 3.7325394271000967,
      "grad_norm": 0.03399816651851313,
      "learning_rate": 6.604381443298968e-08,
      "logits/chosen": -0.965527355670929,
      "logits/rejected": -0.977587878704071,
      "logps/chosen": -444.4750061035156,
      "logps/rejected": -602.9500122070312,
      "loss": 0.03,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -7.051367282867432,
      "rewards/margins": 17.271875381469727,
      "rewards/rejected": -24.328125,
      "step": 2900
    },
    {
      "epoch": 3.745413582233666,
      "grad_norm": 0.04098255840783047,
      "learning_rate": 6.28221649484536e-08,
      "logits/chosen": -0.9448486566543579,
      "logits/rejected": -0.95050048828125,
      "logps/chosen": -414.125,
      "logps/rejected": -563.75,
      "loss": 0.0144,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -6.292578220367432,
      "rewards/margins": 17.878124237060547,
      "rewards/rejected": -24.176563262939453,
      "step": 2910
    },
    {
      "epoch": 3.7582877373672354,
      "grad_norm": 0.10065251851869494,
      "learning_rate": 5.960051546391753e-08,
      "logits/chosen": -0.9204956293106079,
      "logits/rejected": -0.922607421875,
      "logps/chosen": -433.3500061035156,
      "logps/rejected": -584.4500122070312,
      "loss": 0.041,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -6.477929592132568,
      "rewards/margins": 17.8828125,
      "rewards/rejected": -24.348438262939453,
      "step": 2920
    },
    {
      "epoch": 3.7711618925008046,
      "grad_norm": 0.00807285233362444,
      "learning_rate": 5.637886597938144e-08,
      "logits/chosen": -0.885009765625,
      "logits/rejected": -0.8709716796875,
      "logps/chosen": -433.4750061035156,
      "logps/rejected": -583.4000244140625,
      "loss": 0.0389,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -6.649609565734863,
      "rewards/margins": 17.6484375,
      "rewards/rejected": -24.296875,
      "step": 2930
    },
    {
      "epoch": 3.784036047634374,
      "grad_norm": 0.4957017627720933,
      "learning_rate": 5.3157216494845357e-08,
      "logits/chosen": -0.9778808355331421,
      "logits/rejected": -0.9657226800918579,
      "logps/chosen": -444.82501220703125,
      "logps/rejected": -590.25,
      "loss": 0.0173,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -6.065234184265137,
      "rewards/margins": 17.642187118530273,
      "rewards/rejected": -23.71875,
      "step": 2940
    },
    {
      "epoch": 3.7969102027679433,
      "grad_norm": 0.055678910275804926,
      "learning_rate": 4.993556701030928e-08,
      "logits/chosen": -0.9195556640625,
      "logits/rejected": -0.902905285358429,
      "logps/chosen": -437.20001220703125,
      "logps/rejected": -604.5,
      "loss": 0.0451,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -6.924023628234863,
      "rewards/margins": 16.248437881469727,
      "rewards/rejected": -23.171875,
      "step": 2950
    },
    {
      "epoch": 3.8097843579015125,
      "grad_norm": 0.009201975484736785,
      "learning_rate": 4.67139175257732e-08,
      "logits/chosen": -0.957104504108429,
      "logits/rejected": -0.9645019769668579,
      "logps/chosen": -465.8999938964844,
      "logps/rejected": -607.7000122070312,
      "loss": 0.0267,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -5.574267387390137,
      "rewards/margins": 19.049999237060547,
      "rewards/rejected": -24.634374618530273,
      "step": 2960
    },
    {
      "epoch": 3.822658513035082,
      "grad_norm": 0.011481163313679986,
      "learning_rate": 4.349226804123711e-08,
      "logits/chosen": -0.9524902105331421,
      "logits/rejected": -0.9425293207168579,
      "logps/chosen": -419.1875,
      "logps/rejected": -583.7999877929688,
      "loss": 0.0218,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -6.461523532867432,
      "rewards/margins": 17.15625,
      "rewards/rejected": -23.610937118530273,
      "step": 2970
    },
    {
      "epoch": 3.8355326681686517,
      "grad_norm": 0.448772967049888,
      "learning_rate": 4.027061855670103e-08,
      "logits/chosen": -0.8423095941543579,
      "logits/rejected": -0.84716796875,
      "logps/chosen": -422.95001220703125,
      "logps/rejected": -553.5999755859375,
      "loss": 0.0308,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -6.774609565734863,
      "rewards/margins": 17.057811737060547,
      "rewards/rejected": -23.846874237060547,
      "step": 2980
    },
    {
      "epoch": 3.848406823302221,
      "grad_norm": 0.018685653504530834,
      "learning_rate": 3.704896907216495e-08,
      "logits/chosen": -0.8719238042831421,
      "logits/rejected": -0.865161120891571,
      "logps/chosen": -433.1499938964844,
      "logps/rejected": -584.7000122070312,
      "loss": 0.0401,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -6.331408500671387,
      "rewards/margins": 16.582813262939453,
      "rewards/rejected": -22.91015625,
      "step": 2990
    },
    {
      "epoch": 3.86128097843579,
      "grad_norm": 0.007926715342673765,
      "learning_rate": 3.3827319587628864e-08,
      "logits/chosen": -1.019750952720642,
      "logits/rejected": -1.0137450695037842,
      "logps/chosen": -481.2250061035156,
      "logps/rejected": -622.2000122070312,
      "loss": 0.0173,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -6.560217380523682,
      "rewards/margins": 18.018749237060547,
      "rewards/rejected": -24.575000762939453,
      "step": 3000
    },
    {
      "epoch": 3.8741551335693596,
      "grad_norm": 0.0065490532563155885,
      "learning_rate": 3.060567010309278e-08,
      "logits/chosen": -0.9202514886856079,
      "logits/rejected": -0.8963623046875,
      "logps/chosen": -411.1000061035156,
      "logps/rejected": -550.4000244140625,
      "loss": 0.0267,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -7.885351657867432,
      "rewards/margins": 16.58203125,
      "rewards/rejected": -24.4609375,
      "step": 3010
    },
    {
      "epoch": 3.8870292887029287,
      "grad_norm": 0.01802235010361078,
      "learning_rate": 2.73840206185567e-08,
      "logits/chosen": -1.0900390148162842,
      "logits/rejected": -1.0775146484375,
      "logps/chosen": -432.2250061035156,
      "logps/rejected": -624.7999877929688,
      "loss": 0.013,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -7.279541015625,
      "rewards/margins": 19.043750762939453,
      "rewards/rejected": -26.329687118530273,
      "step": 3020
    },
    {
      "epoch": 3.8999034438364983,
      "grad_norm": 0.004374947890940602,
      "learning_rate": 2.4162371134020617e-08,
      "logits/chosen": -0.9076904058456421,
      "logits/rejected": -0.9190429449081421,
      "logps/chosen": -424.32501220703125,
      "logps/rejected": -582.8499755859375,
      "loss": 0.0173,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -6.466504096984863,
      "rewards/margins": 17.926563262939453,
      "rewards/rejected": -24.399999618530273,
      "step": 3030
    },
    {
      "epoch": 3.9127775989700675,
      "grad_norm": 0.0060379466399444,
      "learning_rate": 2.0940721649484534e-08,
      "logits/chosen": -1.004003882408142,
      "logits/rejected": -1.003271460533142,
      "logps/chosen": -484.04998779296875,
      "logps/rejected": -650.7999877929688,
      "loss": 0.0306,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -6.380663871765137,
      "rewards/margins": 18.453125,
      "rewards/rejected": -24.842187881469727,
      "step": 3040
    },
    {
      "epoch": 3.925651754103637,
      "grad_norm": 1.2095296649438,
      "learning_rate": 1.771907216494845e-08,
      "logits/chosen": -0.9027954339981079,
      "logits/rejected": -0.8821441531181335,
      "logps/chosen": -456.4375,
      "logps/rejected": -596.0250244140625,
      "loss": 0.026,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -6.953515529632568,
      "rewards/margins": 18.3046875,
      "rewards/rejected": -25.2578125,
      "step": 3050
    },
    {
      "epoch": 3.9385259092372062,
      "grad_norm": 0.056068725721234,
      "learning_rate": 1.4497422680412371e-08,
      "logits/chosen": -0.9928649663925171,
      "logits/rejected": -0.989697277545929,
      "logps/chosen": -453.4750061035156,
      "logps/rejected": -609.5499877929688,
      "loss": 0.0309,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -7.050879001617432,
      "rewards/margins": 16.801563262939453,
      "rewards/rejected": -23.850000381469727,
      "step": 3060
    },
    {
      "epoch": 3.951400064370776,
      "grad_norm": 0.28646859373193095,
      "learning_rate": 1.1275773195876288e-08,
      "logits/chosen": -1.015356421470642,
      "logits/rejected": -1.011962890625,
      "logps/chosen": -427.82501220703125,
      "logps/rejected": -580.7000122070312,
      "loss": 0.0173,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -6.504296779632568,
      "rewards/margins": 17.173437118530273,
      "rewards/rejected": -23.674999237060547,
      "step": 3070
    },
    {
      "epoch": 3.964274219504345,
      "grad_norm": 0.23205040355122056,
      "learning_rate": 8.054123711340205e-09,
      "logits/chosen": -1.075097680091858,
      "logits/rejected": -1.074121117591858,
      "logps/chosen": -479.375,
      "logps/rejected": -621.25,
      "loss": 0.0087,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -7.373827934265137,
      "rewards/margins": 17.626562118530273,
      "rewards/rejected": -24.998437881469727,
      "step": 3080
    },
    {
      "epoch": 3.977148374637914,
      "grad_norm": 0.03176959803882333,
      "learning_rate": 4.832474226804124e-09,
      "logits/chosen": -1.0000488758087158,
      "logits/rejected": -0.9969726800918579,
      "logps/chosen": -428.57501220703125,
      "logps/rejected": -571.9500122070312,
      "loss": 0.0173,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -6.996972560882568,
      "rewards/margins": 17.692188262939453,
      "rewards/rejected": -24.689062118530273,
      "step": 3090
    },
    {
      "epoch": 3.9900225297714838,
      "grad_norm": 0.012454681894423131,
      "learning_rate": 1.610824742268041e-09,
      "logits/chosen": -1.0048828125,
      "logits/rejected": -1.0068848133087158,
      "logps/chosen": -428.17498779296875,
      "logps/rejected": -616.25,
      "loss": 0.0173,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -5.966845512390137,
      "rewards/margins": 19.012500762939453,
      "rewards/rejected": -24.996875762939453,
      "step": 3100
    }
  ],
  "logging_steps": 10,
  "max_steps": 3104,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 4,
  "save_steps": 500,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 1,
  "trial_name": null,
  "trial_params": null
}