{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 3.99860529986053,
  "eval_steps": 500,
  "global_step": 9320,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0042913850445231196,
      "grad_norm": 123.28223820003826,
      "learning_rate": 9.990343347639485e-07,
      "logits/chosen": -0.46645814180374146,
      "logits/rejected": -0.4908203184604645,
      "logps/chosen": -369.8999938964844,
      "logps/rejected": -336.25,
      "loss": 0.6198,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.25081557035446167,
      "rewards/margins": 0.2138805389404297,
      "rewards/rejected": -0.46480637788772583,
      "step": 10
    },
    {
      "epoch": 0.008582770089046239,
      "grad_norm": 122.85628104182864,
      "learning_rate": 9.979613733905578e-07,
      "logits/chosen": -0.5171264410018921,
      "logits/rejected": -0.49293214082717896,
      "logps/chosen": -381.7875061035156,
      "logps/rejected": -355.3999938964844,
      "loss": 0.616,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.48773193359375,
      "rewards/margins": 0.5217529535293579,
      "rewards/rejected": -1.0099608898162842,
      "step": 20
    },
    {
      "epoch": 0.012874155133569359,
      "grad_norm": 117.51108293633276,
      "learning_rate": 9.968884120171673e-07,
      "logits/chosen": -0.436453253030777,
      "logits/rejected": -0.4932495057582855,
      "logps/chosen": -357.95001220703125,
      "logps/rejected": -361.2875061035156,
      "loss": 0.6055,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.6916564702987671,
      "rewards/margins": 0.747241199016571,
      "rewards/rejected": -1.4387695789337158,
      "step": 30
    },
    {
      "epoch": 0.017165540178092478,
      "grad_norm": 151.22091963503527,
      "learning_rate": 9.958154506437768e-07,
      "logits/chosen": -0.533123791217804,
      "logits/rejected": -0.559246838092804,
      "logps/chosen": -376.48748779296875,
      "logps/rejected": -337.1000061035156,
      "loss": 0.6231,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.1417968273162842,
      "rewards/margins": 0.6675170660018921,
      "rewards/rejected": -1.8093750476837158,
      "step": 40
    },
    {
      "epoch": 0.021456925222615598,
      "grad_norm": 123.7656817343817,
      "learning_rate": 9.94742489270386e-07,
      "logits/chosen": -0.521679699420929,
      "logits/rejected": -0.548785388469696,
      "logps/chosen": -374.9750061035156,
      "logps/rejected": -351.125,
      "loss": 0.6409,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -1.1759154796600342,
      "rewards/margins": 0.64508056640625,
      "rewards/rejected": -1.821191430091858,
      "step": 50
    },
    {
      "epoch": 0.025748310267138717,
      "grad_norm": 120.53172544442593,
      "learning_rate": 9.936695278969956e-07,
      "logits/chosen": -0.49163818359375,
      "logits/rejected": -0.4847168028354645,
      "logps/chosen": -352.17498779296875,
      "logps/rejected": -360.375,
      "loss": 0.6393,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.9949951171875,
      "rewards/margins": 0.642443835735321,
      "rewards/rejected": -1.6369140148162842,
      "step": 60
    },
    {
      "epoch": 0.03003969531166184,
      "grad_norm": 102.03379010098669,
      "learning_rate": 9.925965665236051e-07,
      "logits/chosen": -0.484405517578125,
      "logits/rejected": -0.4980102479457855,
      "logps/chosen": -356.9624938964844,
      "logps/rejected": -352.1625061035156,
      "loss": 0.5665,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.14703521132469177,
      "rewards/margins": 0.7410827875137329,
      "rewards/rejected": -0.8876098394393921,
      "step": 70
    },
    {
      "epoch": 0.034331080356184956,
      "grad_norm": 122.14870656709334,
      "learning_rate": 9.915236051502144e-07,
      "logits/chosen": -0.43037110567092896,
      "logits/rejected": -0.4543823301792145,
      "logps/chosen": -370.70001220703125,
      "logps/rejected": -349.7250061035156,
      "loss": 0.595,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.05336303636431694,
      "rewards/margins": 0.5692199468612671,
      "rewards/rejected": -0.5160888433456421,
      "step": 80
    },
    {
      "epoch": 0.03862246540070808,
      "grad_norm": 128.38432934049678,
      "learning_rate": 9.90450643776824e-07,
      "logits/chosen": -0.520336925983429,
      "logits/rejected": -0.580810546875,
      "logps/chosen": -338.75,
      "logps/rejected": -324.6625061035156,
      "loss": 0.5676,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": 0.16399535536766052,
      "rewards/margins": 0.7056518793106079,
      "rewards/rejected": -0.541949450969696,
      "step": 90
    },
    {
      "epoch": 0.042913850445231196,
      "grad_norm": 94.4501542954361,
      "learning_rate": 9.893776824034335e-07,
      "logits/chosen": -0.45356446504592896,
      "logits/rejected": -0.4798583984375,
      "logps/chosen": -360.07501220703125,
      "logps/rejected": -344.2749938964844,
      "loss": 0.6084,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": 0.07201538234949112,
      "rewards/margins": 0.6812499761581421,
      "rewards/rejected": -0.6087402105331421,
      "step": 100
    },
    {
      "epoch": 0.04720523548975432,
      "grad_norm": 111.56625191783166,
      "learning_rate": 9.88304721030043e-07,
      "logits/chosen": -0.4960693418979645,
      "logits/rejected": -0.518139660358429,
      "logps/chosen": -335.7124938964844,
      "logps/rejected": -313.32501220703125,
      "loss": 0.5617,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.01784667931497097,
      "rewards/margins": 0.7183471918106079,
      "rewards/rejected": -0.736370861530304,
      "step": 110
    },
    {
      "epoch": 0.051496620534277435,
      "grad_norm": 86.40836010580634,
      "learning_rate": 9.872317596566523e-07,
      "logits/chosen": -0.475289911031723,
      "logits/rejected": -0.514575183391571,
      "logps/chosen": -361.32501220703125,
      "logps/rejected": -335.04998779296875,
      "loss": 0.6201,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.201191708445549,
      "rewards/margins": 0.6319335699081421,
      "rewards/rejected": -0.83392333984375,
      "step": 120
    },
    {
      "epoch": 0.05578800557880056,
      "grad_norm": 108.4407162182439,
      "learning_rate": 9.861587982832618e-07,
      "logits/chosen": -0.4857849180698395,
      "logits/rejected": -0.5697387456893921,
      "logps/chosen": -365.92498779296875,
      "logps/rejected": -368.0,
      "loss": 0.5427,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.5403381586074829,
      "rewards/margins": 0.994189441204071,
      "rewards/rejected": -1.5349609851837158,
      "step": 130
    },
    {
      "epoch": 0.06007939062332368,
      "grad_norm": 91.93070866908043,
      "learning_rate": 9.850858369098713e-07,
      "logits/chosen": -0.571911633014679,
      "logits/rejected": -0.5480591058731079,
      "logps/chosen": -348.875,
      "logps/rejected": -330.5375061035156,
      "loss": 0.5136,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.6399291753768921,
      "rewards/margins": 1.2383301258087158,
      "rewards/rejected": -1.878662109375,
      "step": 140
    },
    {
      "epoch": 0.0643707756678468,
      "grad_norm": 107.23765646095339,
      "learning_rate": 9.840128755364806e-07,
      "logits/chosen": -0.5753418207168579,
      "logits/rejected": -0.6225036382675171,
      "logps/chosen": -336.75,
      "logps/rejected": -314.1499938964844,
      "loss": 0.5343,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": 0.21926268935203552,
      "rewards/margins": 1.053491234779358,
      "rewards/rejected": -0.834228515625,
      "step": 150
    },
    {
      "epoch": 0.06866216071236991,
      "grad_norm": 97.39366534760427,
      "learning_rate": 9.829399141630902e-07,
      "logits/chosen": -0.4952026307582855,
      "logits/rejected": -0.5407348871231079,
      "logps/chosen": -360.0249938964844,
      "logps/rejected": -369.8999938964844,
      "loss": 0.5667,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.79168701171875,
      "rewards/margins": 0.9612060785293579,
      "rewards/rejected": -0.16872557997703552,
      "step": 160
    },
    {
      "epoch": 0.07295354575689304,
      "grad_norm": 84.94364687919794,
      "learning_rate": 9.818669527896995e-07,
      "logits/chosen": -0.5259033441543579,
      "logits/rejected": -0.5442749261856079,
      "logps/chosen": -321.7250061035156,
      "logps/rejected": -324.51251220703125,
      "loss": 0.5968,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": 1.13275146484375,
      "rewards/margins": 0.671069324016571,
      "rewards/rejected": 0.4611450135707855,
      "step": 170
    },
    {
      "epoch": 0.07724493080141616,
      "grad_norm": 76.4469906176523,
      "learning_rate": 9.80793991416309e-07,
      "logits/chosen": -0.5499267578125,
      "logits/rejected": -0.5405212640762329,
      "logps/chosen": -360.4750061035156,
      "logps/rejected": -347.7250061035156,
      "loss": 0.5593,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 1.1092345714569092,
      "rewards/margins": 0.812255859375,
      "rewards/rejected": 0.2970214784145355,
      "step": 180
    },
    {
      "epoch": 0.08153631584593928,
      "grad_norm": 84.08622109044897,
      "learning_rate": 9.797210300429185e-07,
      "logits/chosen": -0.605517566204071,
      "logits/rejected": -0.650634765625,
      "logps/chosen": -374.0625,
      "logps/rejected": -345.7250061035156,
      "loss": 0.5754,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.30867308378219604,
      "rewards/margins": 1.064306616783142,
      "rewards/rejected": -0.755444347858429,
      "step": 190
    },
    {
      "epoch": 0.08582770089046239,
      "grad_norm": 100.50953765509824,
      "learning_rate": 9.786480686695278e-07,
      "logits/chosen": -0.599346935749054,
      "logits/rejected": -0.62518310546875,
      "logps/chosen": -348.75,
      "logps/rejected": -324.20001220703125,
      "loss": 0.6844,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": 0.18247680366039276,
      "rewards/margins": 0.8165283203125,
      "rewards/rejected": -0.632855236530304,
      "step": 200
    },
    {
      "epoch": 0.09011908593498552,
      "grad_norm": 82.11434421339153,
      "learning_rate": 9.775751072961373e-07,
      "logits/chosen": -0.54498291015625,
      "logits/rejected": -0.5819457769393921,
      "logps/chosen": -369.1000061035156,
      "logps/rejected": -355.3500061035156,
      "loss": 0.5911,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.3947997987270355,
      "rewards/margins": 1.1294434070587158,
      "rewards/rejected": -0.734039306640625,
      "step": 210
    },
    {
      "epoch": 0.09441047097950864,
      "grad_norm": 109.34442126475855,
      "learning_rate": 9.765021459227466e-07,
      "logits/chosen": -0.4843994081020355,
      "logits/rejected": -0.5489135980606079,
      "logps/chosen": -317.6499938964844,
      "logps/rejected": -294.0249938964844,
      "loss": 0.5097,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": 1.204687476158142,
      "rewards/margins": 1.0795409679412842,
      "rewards/rejected": 0.12577208876609802,
      "step": 220
    },
    {
      "epoch": 0.09870185602403175,
      "grad_norm": 137.93485882352428,
      "learning_rate": 9.754291845493561e-07,
      "logits/chosen": -0.5229858160018921,
      "logits/rejected": -0.577221691608429,
      "logps/chosen": -371.9750061035156,
      "logps/rejected": -359.3999938964844,
      "loss": 0.53,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": 1.028753638267517,
      "rewards/margins": 0.9674072265625,
      "rewards/rejected": 0.06160888820886612,
      "step": 230
    },
    {
      "epoch": 0.10299324106855487,
      "grad_norm": 112.80341271258577,
      "learning_rate": 9.743562231759657e-07,
      "logits/chosen": -0.572552502155304,
      "logits/rejected": -0.583514392375946,
      "logps/chosen": -369.57501220703125,
      "logps/rejected": -368.125,
      "loss": 0.7429,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.36653441190719604,
      "rewards/margins": 0.748730480670929,
      "rewards/rejected": -0.38227540254592896,
      "step": 240
    },
    {
      "epoch": 0.107284626113078,
      "grad_norm": 125.70540458557501,
      "learning_rate": 9.73283261802575e-07,
      "logits/chosen": -0.5268493890762329,
      "logits/rejected": -0.5599609613418579,
      "logps/chosen": -323.1499938964844,
      "logps/rejected": -312.8500061035156,
      "loss": 0.6057,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": 0.23374633491039276,
      "rewards/margins": 0.94091796875,
      "rewards/rejected": -0.706500232219696,
      "step": 250
    },
    {
      "epoch": 0.11157601115760112,
      "grad_norm": 58.55585095990052,
      "learning_rate": 9.722103004291845e-07,
      "logits/chosen": -0.544238269329071,
      "logits/rejected": -0.611328125,
      "logps/chosen": -319.4125061035156,
      "logps/rejected": -318.3999938964844,
      "loss": 0.4599,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.06166992336511612,
      "rewards/margins": 1.2641417980194092,
      "rewards/rejected": -1.326074242591858,
      "step": 260
    },
    {
      "epoch": 0.11586739620212423,
      "grad_norm": 123.2244445602675,
      "learning_rate": 9.71137339055794e-07,
      "logits/chosen": -0.5576202273368835,
      "logits/rejected": -0.5834106206893921,
      "logps/chosen": -363.0,
      "logps/rejected": -358.5249938964844,
      "loss": 0.7247,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.5901733636856079,
      "rewards/margins": 0.8073974847793579,
      "rewards/rejected": -1.397436499595642,
      "step": 270
    },
    {
      "epoch": 0.12015878124664736,
      "grad_norm": 155.91245588794678,
      "learning_rate": 9.700643776824033e-07,
      "logits/chosen": -0.537121593952179,
      "logits/rejected": -0.563525378704071,
      "logps/chosen": -369.1000061035156,
      "logps/rejected": -365.3500061035156,
      "loss": 0.5173,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.4778243899345398,
      "rewards/margins": 1.3302490711212158,
      "rewards/rejected": -1.8080322742462158,
      "step": 280
    },
    {
      "epoch": 0.12445016629117048,
      "grad_norm": 145.09539210098566,
      "learning_rate": 9.689914163090128e-07,
      "logits/chosen": -0.550463855266571,
      "logits/rejected": -0.563403308391571,
      "logps/chosen": -350.1499938964844,
      "logps/rejected": -333.32501220703125,
      "loss": 0.6547,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.6506713628768921,
      "rewards/margins": 1.0596923828125,
      "rewards/rejected": -1.710913062095642,
      "step": 290
    },
    {
      "epoch": 0.1287415513356936,
      "grad_norm": 110.94655724051188,
      "learning_rate": 9.679184549356223e-07,
      "logits/chosen": -0.5160888433456421,
      "logits/rejected": -0.521374523639679,
      "logps/chosen": -418.7250061035156,
      "logps/rejected": -404.4750061035156,
      "loss": 0.5785,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.046630859375,
      "rewards/margins": 1.01251220703125,
      "rewards/rejected": -1.0598876476287842,
      "step": 300
    },
    {
      "epoch": 0.1330329363802167,
      "grad_norm": 99.15351728439298,
      "learning_rate": 9.668454935622316e-07,
      "logits/chosen": -0.41584473848342896,
      "logits/rejected": -0.46867674589157104,
      "logps/chosen": -352.67498779296875,
      "logps/rejected": -338.125,
      "loss": 0.5692,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.14049682021141052,
      "rewards/margins": 1.0083496570587158,
      "rewards/rejected": -1.1495850086212158,
      "step": 310
    },
    {
      "epoch": 0.13732432142473983,
      "grad_norm": 124.91975033406338,
      "learning_rate": 9.657725321888411e-07,
      "logits/chosen": -0.4778999388217926,
      "logits/rejected": -0.5385467410087585,
      "logps/chosen": -382.07501220703125,
      "logps/rejected": -372.70001220703125,
      "loss": 0.5975,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.22307129204273224,
      "rewards/margins": 1.0057373046875,
      "rewards/rejected": -1.2289307117462158,
      "step": 320
    },
    {
      "epoch": 0.14161570646926294,
      "grad_norm": 116.40483723789796,
      "learning_rate": 9.646995708154507e-07,
      "logits/chosen": -0.500762939453125,
      "logits/rejected": -0.529162585735321,
      "logps/chosen": -356.23748779296875,
      "logps/rejected": -324.17498779296875,
      "loss": 0.6105,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": 0.037841796875,
      "rewards/margins": 0.743518054485321,
      "rewards/rejected": -0.7049926519393921,
      "step": 330
    },
    {
      "epoch": 0.14590709151378609,
      "grad_norm": 116.85502208565543,
      "learning_rate": 9.636266094420602e-07,
      "logits/chosen": -0.47784423828125,
      "logits/rejected": -0.49757081270217896,
      "logps/chosen": -362.54998779296875,
      "logps/rejected": -328.54998779296875,
      "loss": 0.6319,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.04578246921300888,
      "rewards/margins": 0.8777099847793579,
      "rewards/rejected": -0.92218017578125,
      "step": 340
    },
    {
      "epoch": 0.1501984765583092,
      "grad_norm": 114.7419504524314,
      "learning_rate": 9.625536480686695e-07,
      "logits/chosen": -0.6083008050918579,
      "logits/rejected": -0.633898913860321,
      "logps/chosen": -362.875,
      "logps/rejected": -390.3500061035156,
      "loss": 0.6299,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": 0.05139770358800888,
      "rewards/margins": 0.6855224370956421,
      "rewards/rejected": -0.633557140827179,
      "step": 350
    },
    {
      "epoch": 0.15448986160283232,
      "grad_norm": 100.81303221004438,
      "learning_rate": 9.61480686695279e-07,
      "logits/chosen": -0.543261706829071,
      "logits/rejected": -0.5597778558731079,
      "logps/chosen": -350.67498779296875,
      "logps/rejected": -342.75,
      "loss": 0.5379,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": 0.19191284477710724,
      "rewards/margins": 0.8765014410018921,
      "rewards/rejected": -0.6850525140762329,
      "step": 360
    },
    {
      "epoch": 0.15878124664735543,
      "grad_norm": 128.1911073978389,
      "learning_rate": 9.604077253218883e-07,
      "logits/chosen": -0.5430237054824829,
      "logits/rejected": -0.56463623046875,
      "logps/chosen": -359.54998779296875,
      "logps/rejected": -320.29998779296875,
      "loss": 0.6405,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.04185790941119194,
      "rewards/margins": 0.782177746295929,
      "rewards/rejected": -0.824017345905304,
      "step": 370
    },
    {
      "epoch": 0.16307263169187855,
      "grad_norm": 101.11922410312341,
      "learning_rate": 9.593347639484978e-07,
      "logits/chosen": -0.5586913824081421,
      "logits/rejected": -0.576916515827179,
      "logps/chosen": -369.01251220703125,
      "logps/rejected": -352.5,
      "loss": 0.5704,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.02469787560403347,
      "rewards/margins": 1.112890601158142,
      "rewards/rejected": -1.137963891029358,
      "step": 380
    },
    {
      "epoch": 0.16736401673640167,
      "grad_norm": 81.72591212763665,
      "learning_rate": 9.582618025751073e-07,
      "logits/chosen": -0.5726379156112671,
      "logits/rejected": -0.55267333984375,
      "logps/chosen": -403.17498779296875,
      "logps/rejected": -369.8999938964844,
      "loss": 0.5756,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.17046204209327698,
      "rewards/margins": 1.290380835533142,
      "rewards/rejected": -1.461035132408142,
      "step": 390
    },
    {
      "epoch": 0.17165540178092478,
      "grad_norm": 91.69840347728346,
      "learning_rate": 9.571888412017166e-07,
      "logits/chosen": -0.5492798089981079,
      "logits/rejected": -0.545941174030304,
      "logps/chosen": -388.0874938964844,
      "logps/rejected": -356.1499938964844,
      "loss": 0.4508,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": 0.34259033203125,
      "rewards/margins": 1.4014160633087158,
      "rewards/rejected": -1.0582153797149658,
      "step": 400
    },
    {
      "epoch": 0.17594678682544793,
      "grad_norm": 100.70519117112231,
      "learning_rate": 9.561158798283262e-07,
      "logits/chosen": -0.5215393304824829,
      "logits/rejected": -0.561474621295929,
      "logps/chosen": -326.16876220703125,
      "logps/rejected": -326.92498779296875,
      "loss": 0.5462,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.27595216035842896,
      "rewards/margins": 1.1398437023162842,
      "rewards/rejected": -1.415899634361267,
      "step": 410
    },
    {
      "epoch": 0.18023817186997104,
      "grad_norm": 93.74336628274585,
      "learning_rate": 9.550429184549355e-07,
      "logits/chosen": -0.5823608636856079,
      "logits/rejected": -0.574047863483429,
      "logps/chosen": -372.9750061035156,
      "logps/rejected": -382.125,
      "loss": 0.5921,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.36954957246780396,
      "rewards/margins": 1.0365722179412842,
      "rewards/rejected": -1.4060790538787842,
      "step": 420
    },
    {
      "epoch": 0.18452955691449416,
      "grad_norm": 117.78457982524857,
      "learning_rate": 9.53969957081545e-07,
      "logits/chosen": -0.4588623046875,
      "logits/rejected": -0.48617857694625854,
      "logps/chosen": -381.625,
      "logps/rejected": -363.2749938964844,
      "loss": 0.6017,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.17536011338233948,
      "rewards/margins": 0.842358410358429,
      "rewards/rejected": -1.018072485923767,
      "step": 430
    },
    {
      "epoch": 0.18882094195901727,
      "grad_norm": 75.84966571616258,
      "learning_rate": 9.528969957081545e-07,
      "logits/chosen": -0.49233704805374146,
      "logits/rejected": -0.515911877155304,
      "logps/chosen": -350.76251220703125,
      "logps/rejected": -341.3500061035156,
      "loss": 0.4495,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": 0.42026060819625854,
      "rewards/margins": 1.248266577720642,
      "rewards/rejected": -0.8277862668037415,
      "step": 440
    },
    {
      "epoch": 0.1931123270035404,
      "grad_norm": 119.11996145611178,
      "learning_rate": 9.518240343347639e-07,
      "logits/chosen": -0.4456543028354645,
      "logits/rejected": -0.485992431640625,
      "logps/chosen": -380.7749938964844,
      "logps/rejected": -357.29998779296875,
      "loss": 0.4673,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": 0.802136242389679,
      "rewards/margins": 1.6463134288787842,
      "rewards/rejected": -0.8443603515625,
      "step": 450
    },
    {
      "epoch": 0.1974037120480635,
      "grad_norm": 82.4263835338058,
      "learning_rate": 9.507510729613733e-07,
      "logits/chosen": -0.549328625202179,
      "logits/rejected": -0.532366931438446,
      "logps/chosen": -388.5874938964844,
      "logps/rejected": -337.92498779296875,
      "loss": 0.4997,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": 0.844329833984375,
      "rewards/margins": 1.3522460460662842,
      "rewards/rejected": -0.507617175579071,
      "step": 460
    },
    {
      "epoch": 0.20169509709258662,
      "grad_norm": 77.79913068760568,
      "learning_rate": 9.496781115879828e-07,
      "logits/chosen": -0.4855712950229645,
      "logits/rejected": -0.511547863483429,
      "logps/chosen": -365.3500061035156,
      "logps/rejected": -354.7875061035156,
      "loss": 0.5443,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.5145324468612671,
      "rewards/margins": 1.5306885242462158,
      "rewards/rejected": -1.0165526866912842,
      "step": 470
    },
    {
      "epoch": 0.20598648213710974,
      "grad_norm": 100.68132263754269,
      "learning_rate": 9.486051502145921e-07,
      "logits/chosen": -0.47947996854782104,
      "logits/rejected": -0.5260986089706421,
      "logps/chosen": -356.6000061035156,
      "logps/rejected": -340.5,
      "loss": 0.6298,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.3326416015625,
      "rewards/margins": 1.264917016029358,
      "rewards/rejected": -0.9325500726699829,
      "step": 480
    },
    {
      "epoch": 0.21027786718163288,
      "grad_norm": 76.40861266143317,
      "learning_rate": 9.475321888412017e-07,
      "logits/chosen": -0.49945068359375,
      "logits/rejected": -0.562817394733429,
      "logps/chosen": -359.95001220703125,
      "logps/rejected": -350.67498779296875,
      "loss": 0.635,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": 0.824999988079071,
      "rewards/margins": 1.1612060070037842,
      "rewards/rejected": -0.3360351622104645,
      "step": 490
    },
    {
      "epoch": 0.214569252226156,
      "grad_norm": 125.71253555859914,
      "learning_rate": 9.464592274678112e-07,
      "logits/chosen": -0.4984985291957855,
      "logits/rejected": -0.54229736328125,
      "logps/chosen": -378.375,
      "logps/rejected": -372.7250061035156,
      "loss": 0.5321,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 1.1933104991912842,
      "rewards/margins": 1.1146361827850342,
      "rewards/rejected": 0.07810058444738388,
      "step": 500
    },
    {
      "epoch": 0.21886063727067911,
      "grad_norm": 95.40965004395002,
      "learning_rate": 9.453862660944205e-07,
      "logits/chosen": -0.500109851360321,
      "logits/rejected": -0.52734375,
      "logps/chosen": -392.82501220703125,
      "logps/rejected": -351.3500061035156,
      "loss": 0.5787,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": 1.063452124595642,
      "rewards/margins": 1.0601806640625,
      "rewards/rejected": 0.005053711123764515,
      "step": 510
    },
    {
      "epoch": 0.22315202231520223,
      "grad_norm": 74.61364708907215,
      "learning_rate": 9.4431330472103e-07,
      "logits/chosen": -0.48179322481155396,
      "logits/rejected": -0.5057128667831421,
      "logps/chosen": -380.42498779296875,
      "logps/rejected": -344.48748779296875,
      "loss": 0.7151,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": 0.3694091737270355,
      "rewards/margins": 1.1292603015899658,
      "rewards/rejected": -0.7605346441268921,
      "step": 520
    },
    {
      "epoch": 0.22744340735972535,
      "grad_norm": 53.783363403893034,
      "learning_rate": 9.432403433476394e-07,
      "logits/chosen": -0.5444580316543579,
      "logits/rejected": -0.5545409917831421,
      "logps/chosen": -391.92498779296875,
      "logps/rejected": -373.42498779296875,
      "loss": 0.5574,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": 0.779003918170929,
      "rewards/margins": 1.3148682117462158,
      "rewards/rejected": -0.535400390625,
      "step": 530
    },
    {
      "epoch": 0.23173479240424846,
      "grad_norm": 71.56630218026018,
      "learning_rate": 9.421673819742488e-07,
      "logits/chosen": -0.4359497129917145,
      "logits/rejected": -0.47218018770217896,
      "logps/chosen": -359.4624938964844,
      "logps/rejected": -332.79998779296875,
      "loss": 0.5467,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": 0.729156494140625,
      "rewards/margins": 1.1874268054962158,
      "rewards/rejected": -0.4583740234375,
      "step": 540
    },
    {
      "epoch": 0.23602617744877158,
      "grad_norm": 83.66000859870482,
      "learning_rate": 9.410944206008583e-07,
      "logits/chosen": -0.519641101360321,
      "logits/rejected": -0.5178298950195312,
      "logps/chosen": -367.95001220703125,
      "logps/rejected": -364.92498779296875,
      "loss": 0.5371,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": 0.520922839641571,
      "rewards/margins": 0.9896606206893921,
      "rewards/rejected": -0.46781617403030396,
      "step": 550
    },
    {
      "epoch": 0.24031756249329472,
      "grad_norm": 98.16028149637665,
      "learning_rate": 9.400214592274678e-07,
      "logits/chosen": -0.5354369878768921,
      "logits/rejected": -0.539416491985321,
      "logps/chosen": -351.1875,
      "logps/rejected": -366.6499938964844,
      "loss": 0.6552,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.14227294921875,
      "rewards/margins": 0.772296130657196,
      "rewards/rejected": -0.6302856206893921,
      "step": 560
    },
    {
      "epoch": 0.24460894753781784,
      "grad_norm": 120.3807602820233,
      "learning_rate": 9.389484978540773e-07,
      "logits/chosen": -0.505786120891571,
      "logits/rejected": -0.5287231206893921,
      "logps/chosen": -407.95001220703125,
      "logps/rejected": -365.3500061035156,
      "loss": 0.5672,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": 0.16014404594898224,
      "rewards/margins": 1.027227759361267,
      "rewards/rejected": -0.8665100336074829,
      "step": 570
    },
    {
      "epoch": 0.24890033258234096,
      "grad_norm": 94.05463646589862,
      "learning_rate": 9.378755364806866e-07,
      "logits/chosen": -0.4578491151332855,
      "logits/rejected": -0.503631591796875,
      "logps/chosen": -347.625,
      "logps/rejected": -342.67498779296875,
      "loss": 0.6144,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": 0.19006958603858948,
      "rewards/margins": 1.177221655845642,
      "rewards/rejected": -0.986157238483429,
      "step": 580
    },
    {
      "epoch": 0.25319171762686404,
      "grad_norm": 96.0622685208306,
      "learning_rate": 9.368025751072961e-07,
      "logits/chosen": -0.45648193359375,
      "logits/rejected": -0.4850097596645355,
      "logps/chosen": -377.875,
      "logps/rejected": -350.625,
      "loss": 0.528,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": 0.43720704317092896,
      "rewards/margins": 1.2394530773162842,
      "rewards/rejected": -0.802227795124054,
      "step": 590
    },
    {
      "epoch": 0.2574831026713872,
      "grad_norm": 116.19988652027885,
      "learning_rate": 9.357296137339056e-07,
      "logits/chosen": -0.5290161371231079,
      "logits/rejected": -0.538250744342804,
      "logps/chosen": -322.32501220703125,
      "logps/rejected": -310.375,
      "loss": 0.5809,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": 0.8603149652481079,
      "rewards/margins": 1.279638648033142,
      "rewards/rejected": -0.4193115234375,
      "step": 600
    },
    {
      "epoch": 0.26177448771591033,
      "grad_norm": 117.07665088552639,
      "learning_rate": 9.346566523605149e-07,
      "logits/chosen": -0.5156494379043579,
      "logits/rejected": -0.4985107481479645,
      "logps/chosen": -346.42498779296875,
      "logps/rejected": -361.25,
      "loss": 0.6903,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.5416504144668579,
      "rewards/margins": 0.9817138910293579,
      "rewards/rejected": -0.4396118223667145,
      "step": 610
    },
    {
      "epoch": 0.2660658727604334,
      "grad_norm": 111.00597088432458,
      "learning_rate": 9.335836909871244e-07,
      "logits/chosen": -0.491018682718277,
      "logits/rejected": -0.5232299566268921,
      "logps/chosen": -379.125,
      "logps/rejected": -347.5249938964844,
      "loss": 0.6917,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": 0.4049438536167145,
      "rewards/margins": 0.8136962652206421,
      "rewards/rejected": -0.40876466035842896,
      "step": 620
    },
    {
      "epoch": 0.27035725780495656,
      "grad_norm": 96.2197512844947,
      "learning_rate": 9.325107296137338e-07,
      "logits/chosen": -0.5179535150527954,
      "logits/rejected": -0.52593994140625,
      "logps/chosen": -353.63751220703125,
      "logps/rejected": -387.6499938964844,
      "loss": 0.6251,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.011566162109375,
      "rewards/margins": 0.927441418170929,
      "rewards/rejected": -0.9385620355606079,
      "step": 630
    },
    {
      "epoch": 0.27464864284947965,
      "grad_norm": 69.60649548304919,
      "learning_rate": 9.314377682403433e-07,
      "logits/chosen": -0.41236573457717896,
      "logits/rejected": -0.4709274172782898,
      "logps/chosen": -361.8374938964844,
      "logps/rejected": -336.67498779296875,
      "loss": 0.58,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.27885740995407104,
      "rewards/margins": 1.119421362876892,
      "rewards/rejected": -0.8396850824356079,
      "step": 640
    },
    {
      "epoch": 0.2789400278940028,
      "grad_norm": 87.23889104729567,
      "learning_rate": 9.303648068669528e-07,
      "logits/chosen": -0.47505491971969604,
      "logits/rejected": -0.516845703125,
      "logps/chosen": -343.88751220703125,
      "logps/rejected": -319.07501220703125,
      "loss": 0.5413,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": 0.35932618379592896,
      "rewards/margins": 1.234075903892517,
      "rewards/rejected": -0.87445068359375,
      "step": 650
    },
    {
      "epoch": 0.2832314129385259,
      "grad_norm": 136.43154228077452,
      "learning_rate": 9.292918454935622e-07,
      "logits/chosen": -0.4903015196323395,
      "logits/rejected": -0.517138659954071,
      "logps/chosen": -381.8500061035156,
      "logps/rejected": -368.0625,
      "loss": 0.5993,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.6085571050643921,
      "rewards/margins": 1.2699706554412842,
      "rewards/rejected": -0.660449206829071,
      "step": 660
    },
    {
      "epoch": 0.287522797983049,
      "grad_norm": 115.65279317320193,
      "learning_rate": 9.282188841201717e-07,
      "logits/chosen": -0.5132201910018921,
      "logits/rejected": -0.549694836139679,
      "logps/chosen": -323.3374938964844,
      "logps/rejected": -322.7250061035156,
      "loss": 0.6454,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.16215820610523224,
      "rewards/margins": 0.9600830078125,
      "rewards/rejected": -0.798248291015625,
      "step": 670
    },
    {
      "epoch": 0.29181418302757217,
      "grad_norm": 102.45490493033104,
      "learning_rate": 9.27145922746781e-07,
      "logits/chosen": -0.49029541015625,
      "logits/rejected": -0.52154541015625,
      "logps/chosen": -375.3374938964844,
      "logps/rejected": -358.13751220703125,
      "loss": 0.5784,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.48070067167282104,
      "rewards/margins": 1.1212646961212158,
      "rewards/rejected": -0.6394012570381165,
      "step": 680
    },
    {
      "epoch": 0.29610556807209526,
      "grad_norm": 71.07473549836888,
      "learning_rate": 9.260729613733905e-07,
      "logits/chosen": -0.4853271543979645,
      "logits/rejected": -0.5065673589706421,
      "logps/chosen": -378.17498779296875,
      "logps/rejected": -362.2250061035156,
      "loss": 0.7095,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": 0.543322741985321,
      "rewards/margins": 0.8026123046875,
      "rewards/rejected": -0.25929564237594604,
      "step": 690
    },
    {
      "epoch": 0.3003969531166184,
      "grad_norm": 109.60981194919961,
      "learning_rate": 9.25e-07,
      "logits/chosen": -0.42185670137405396,
      "logits/rejected": -0.4507690370082855,
      "logps/chosen": -368.70001220703125,
      "logps/rejected": -329.07501220703125,
      "loss": 0.5381,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": 0.49433594942092896,
      "rewards/margins": 1.210058569908142,
      "rewards/rejected": -0.715771496295929,
      "step": 700
    },
    {
      "epoch": 0.3046883381611415,
      "grad_norm": 89.04505013063321,
      "learning_rate": 9.239270386266093e-07,
      "logits/chosen": -0.434173583984375,
      "logits/rejected": -0.469259649515152,
      "logps/chosen": -347.57501220703125,
      "logps/rejected": -329.6499938964844,
      "loss": 0.6525,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": 0.39166259765625,
      "rewards/margins": 0.98681640625,
      "rewards/rejected": -0.5951949954032898,
      "step": 710
    },
    {
      "epoch": 0.30897972320566464,
      "grad_norm": 101.2686169602893,
      "learning_rate": 9.228540772532189e-07,
      "logits/chosen": -0.5307220220565796,
      "logits/rejected": -0.516613781452179,
      "logps/chosen": -356.42498779296875,
      "logps/rejected": -339.45001220703125,
      "loss": 0.5437,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": 0.6387084722518921,
      "rewards/margins": 0.982983410358429,
      "rewards/rejected": -0.34521484375,
      "step": 720
    },
    {
      "epoch": 0.3132711082501877,
      "grad_norm": 138.20748334338063,
      "learning_rate": 9.217811158798283e-07,
      "logits/chosen": -0.4175048768520355,
      "logits/rejected": -0.43641358613967896,
      "logps/chosen": -372.1499938964844,
      "logps/rejected": -371.79998779296875,
      "loss": 0.6532,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": 0.24131163954734802,
      "rewards/margins": 0.812853991985321,
      "rewards/rejected": -0.571728527545929,
      "step": 730
    },
    {
      "epoch": 0.31756249329471087,
      "grad_norm": 55.41267518981706,
      "learning_rate": 9.207081545064377e-07,
      "logits/chosen": -0.42187803983688354,
      "logits/rejected": -0.4542236328125,
      "logps/chosen": -334.5625,
      "logps/rejected": -346.6000061035156,
      "loss": 0.4551,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": 0.4122070372104645,
      "rewards/margins": 1.5115845203399658,
      "rewards/rejected": -1.0981566905975342,
      "step": 740
    },
    {
      "epoch": 0.321853878339234,
      "grad_norm": 115.35651982235808,
      "learning_rate": 9.196351931330472e-07,
      "logits/chosen": -0.4146179258823395,
      "logits/rejected": -0.4374542236328125,
      "logps/chosen": -337.20001220703125,
      "logps/rejected": -355.57501220703125,
      "loss": 0.5936,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": 0.11771850287914276,
      "rewards/margins": 1.1624023914337158,
      "rewards/rejected": -1.044854760169983,
      "step": 750
    },
    {
      "epoch": 0.3261452633837571,
      "grad_norm": 97.98417602949746,
      "learning_rate": 9.185622317596566e-07,
      "logits/chosen": -0.47843629121780396,
      "logits/rejected": -0.46956175565719604,
      "logps/chosen": -345.2250061035156,
      "logps/rejected": -328.6499938964844,
      "loss": 0.5544,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.4692626893520355,
      "rewards/margins": 1.1520264148712158,
      "rewards/rejected": -0.6825164556503296,
      "step": 760
    },
    {
      "epoch": 0.33043664842828024,
      "grad_norm": 104.16665949708845,
      "learning_rate": 9.17489270386266e-07,
      "logits/chosen": -0.568432629108429,
      "logits/rejected": -0.575335681438446,
      "logps/chosen": -380.3999938964844,
      "logps/rejected": -346.29998779296875,
      "loss": 0.4675,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": 0.012725830078125,
      "rewards/margins": 1.4672119617462158,
      "rewards/rejected": -1.4542236328125,
      "step": 770
    },
    {
      "epoch": 0.33472803347280333,
      "grad_norm": 119.56617553801557,
      "learning_rate": 9.164163090128754e-07,
      "logits/chosen": -0.553271472454071,
      "logits/rejected": -0.57843017578125,
      "logps/chosen": -332.13751220703125,
      "logps/rejected": -339.375,
      "loss": 0.556,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": 0.04550781100988388,
      "rewards/margins": 1.414697289466858,
      "rewards/rejected": -1.367919921875,
      "step": 780
    },
    {
      "epoch": 0.3390194185173265,
      "grad_norm": 98.99743825504981,
      "learning_rate": 9.15343347639485e-07,
      "logits/chosen": -0.5143066644668579,
      "logits/rejected": -0.5286620855331421,
      "logps/chosen": -375.32501220703125,
      "logps/rejected": -368.9624938964844,
      "loss": 0.5585,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.15742187201976776,
      "rewards/margins": 1.386144995689392,
      "rewards/rejected": -1.5444519519805908,
      "step": 790
    },
    {
      "epoch": 0.34331080356184956,
      "grad_norm": 75.37385349647795,
      "learning_rate": 9.142703862660945e-07,
      "logits/chosen": -0.5247802734375,
      "logits/rejected": -0.52978515625,
      "logps/chosen": -348.98748779296875,
      "logps/rejected": -346.36248779296875,
      "loss": 0.5873,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.5354980230331421,
      "rewards/margins": 1.0202515125274658,
      "rewards/rejected": -0.4848083555698395,
      "step": 800
    },
    {
      "epoch": 0.3476021886063727,
      "grad_norm": 90.6096235292697,
      "learning_rate": 9.131974248927038e-07,
      "logits/chosen": -0.4453979432582855,
      "logits/rejected": -0.47803956270217896,
      "logps/chosen": -373.7749938964844,
      "logps/rejected": -375.6499938964844,
      "loss": 0.5592,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.302276611328125,
      "rewards/margins": 1.0711669921875,
      "rewards/rejected": -0.768969714641571,
      "step": 810
    },
    {
      "epoch": 0.35189357365089585,
      "grad_norm": 99.82672620444873,
      "learning_rate": 9.121244635193133e-07,
      "logits/chosen": -0.5496581792831421,
      "logits/rejected": -0.542236328125,
      "logps/chosen": -374.7749938964844,
      "logps/rejected": -350.9750061035156,
      "loss": 0.4969,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.16860350966453552,
      "rewards/margins": 1.4805419445037842,
      "rewards/rejected": -1.6498534679412842,
      "step": 820
    },
    {
      "epoch": 0.35618495869541894,
      "grad_norm": 94.16289003441854,
      "learning_rate": 9.110515021459227e-07,
      "logits/chosen": -0.48774415254592896,
      "logits/rejected": -0.490966796875,
      "logps/chosen": -340.625,
      "logps/rejected": -351.2749938964844,
      "loss": 0.7307,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.72174072265625,
      "rewards/margins": 0.8165343999862671,
      "rewards/rejected": -1.5394287109375,
      "step": 830
    },
    {
      "epoch": 0.3604763437399421,
      "grad_norm": 104.945667417502,
      "learning_rate": 9.099785407725321e-07,
      "logits/chosen": -0.501965343952179,
      "logits/rejected": -0.529003918170929,
      "logps/chosen": -343.36248779296875,
      "logps/rejected": -337.7250061035156,
      "loss": 0.5847,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.06689453125,
      "rewards/margins": 1.0524413585662842,
      "rewards/rejected": -1.1195068359375,
      "step": 840
    },
    {
      "epoch": 0.3647677287844652,
      "grad_norm": 141.3638974601805,
      "learning_rate": 9.089055793991416e-07,
      "logits/chosen": -0.4378768801689148,
      "logits/rejected": -0.46462708711624146,
      "logps/chosen": -334.07501220703125,
      "logps/rejected": -331.79998779296875,
      "loss": 0.6944,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": 0.35230714082717896,
      "rewards/margins": 0.8697754144668579,
      "rewards/rejected": -0.5178467035293579,
      "step": 850
    },
    {
      "epoch": 0.3690591138289883,
      "grad_norm": 98.86332296676377,
      "learning_rate": 9.07832618025751e-07,
      "logits/chosen": -0.4075004458427429,
      "logits/rejected": -0.42329102754592896,
      "logps/chosen": -327.0874938964844,
      "logps/rejected": -337.57501220703125,
      "loss": 0.5143,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 1.0659058094024658,
      "rewards/margins": 1.3506591320037842,
      "rewards/rejected": -0.2843566834926605,
      "step": 860
    },
    {
      "epoch": 0.3733504988735114,
      "grad_norm": 143.52865264201142,
      "learning_rate": 9.067596566523604e-07,
      "logits/chosen": -0.47900390625,
      "logits/rejected": -0.5025634765625,
      "logps/chosen": -325.375,
      "logps/rejected": -306.1499938964844,
      "loss": 0.6415,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": 0.839221179485321,
      "rewards/margins": 0.882495105266571,
      "rewards/rejected": -0.04320068284869194,
      "step": 870
    },
    {
      "epoch": 0.37764188391803455,
      "grad_norm": 108.44675985314916,
      "learning_rate": 9.056866952789699e-07,
      "logits/chosen": -0.492462158203125,
      "logits/rejected": -0.5123046636581421,
      "logps/chosen": -371.01251220703125,
      "logps/rejected": -341.8999938964844,
      "loss": 0.5498,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": 0.5734497308731079,
      "rewards/margins": 1.064050316810608,
      "rewards/rejected": -0.49089354276657104,
      "step": 880
    },
    {
      "epoch": 0.3819332689625577,
      "grad_norm": 85.83065193263236,
      "learning_rate": 9.046137339055794e-07,
      "logits/chosen": -0.47416383028030396,
      "logits/rejected": -0.5217592120170593,
      "logps/chosen": -376.5874938964844,
      "logps/rejected": -326.8999938964844,
      "loss": 0.6264,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": 0.06551513820886612,
      "rewards/margins": 0.9853760004043579,
      "rewards/rejected": -0.919665515422821,
      "step": 890
    },
    {
      "epoch": 0.3862246540070808,
      "grad_norm": 112.8340748218402,
      "learning_rate": 9.035407725321889e-07,
      "logits/chosen": -0.5336059331893921,
      "logits/rejected": -0.5383239984512329,
      "logps/chosen": -380.0,
      "logps/rejected": -365.75,
      "loss": 0.7112,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.01059570349752903,
      "rewards/margins": 1.0574462413787842,
      "rewards/rejected": -1.068023681640625,
      "step": 900
    },
    {
      "epoch": 0.3905160390516039,
      "grad_norm": 104.60729787920864,
      "learning_rate": 9.024678111587982e-07,
      "logits/chosen": -0.4833007752895355,
      "logits/rejected": -0.5191894769668579,
      "logps/chosen": -317.51251220703125,
      "logps/rejected": -328.86248779296875,
      "loss": 0.487,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": 0.4936767518520355,
      "rewards/margins": 1.44757080078125,
      "rewards/rejected": -0.952838122844696,
      "step": 910
    },
    {
      "epoch": 0.394807424096127,
      "grad_norm": 134.56374098328996,
      "learning_rate": 9.013948497854077e-07,
      "logits/chosen": -0.560375988483429,
      "logits/rejected": -0.5675293207168579,
      "logps/chosen": -396.6000061035156,
      "logps/rejected": -352.8500061035156,
      "loss": 0.6012,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": 0.537890613079071,
      "rewards/margins": 1.114013671875,
      "rewards/rejected": -0.5757080316543579,
      "step": 920
    },
    {
      "epoch": 0.39909880914065016,
      "grad_norm": 89.61196386462339,
      "learning_rate": 9.003218884120171e-07,
      "logits/chosen": -0.47153931856155396,
      "logits/rejected": -0.49626463651657104,
      "logps/chosen": -390.3999938964844,
      "logps/rejected": -380.67498779296875,
      "loss": 0.5459,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.09894714504480362,
      "rewards/margins": 1.4040648937225342,
      "rewards/rejected": -1.304443359375,
      "step": 930
    },
    {
      "epoch": 0.40339019418517325,
      "grad_norm": 82.28967332012682,
      "learning_rate": 8.992489270386265e-07,
      "logits/chosen": -0.524951159954071,
      "logits/rejected": -0.5859740972518921,
      "logps/chosen": -372.20001220703125,
      "logps/rejected": -372.5,
      "loss": 0.5709,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.4099975526332855,
      "rewards/margins": 1.3982422351837158,
      "rewards/rejected": -0.987438976764679,
      "step": 940
    },
    {
      "epoch": 0.4076815792296964,
      "grad_norm": 134.56483869217476,
      "learning_rate": 8.981759656652361e-07,
      "logits/chosen": -0.49836426973342896,
      "logits/rejected": -0.5463012456893921,
      "logps/chosen": -369.9750061035156,
      "logps/rejected": -345.125,
      "loss": 0.592,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": 0.2688537538051605,
      "rewards/margins": 1.27587890625,
      "rewards/rejected": -1.006103515625,
      "step": 950
    },
    {
      "epoch": 0.4119729642742195,
      "grad_norm": 141.27056328687465,
      "learning_rate": 8.971030042918455e-07,
      "logits/chosen": -0.43629151582717896,
      "logits/rejected": -0.47950440645217896,
      "logps/chosen": -383.5249938964844,
      "logps/rejected": -338.57501220703125,
      "loss": 0.731,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": 0.39167481660842896,
      "rewards/margins": 1.1161620616912842,
      "rewards/rejected": -0.7248779535293579,
      "step": 960
    },
    {
      "epoch": 0.4162643493187426,
      "grad_norm": 103.51897784625359,
      "learning_rate": 8.960300429184549e-07,
      "logits/chosen": -0.498382568359375,
      "logits/rejected": -0.516064465045929,
      "logps/chosen": -372.82501220703125,
      "logps/rejected": -358.0249938964844,
      "loss": 0.597,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": 0.18765106797218323,
      "rewards/margins": 1.342736840248108,
      "rewards/rejected": -1.1550414562225342,
      "step": 970
    },
    {
      "epoch": 0.42055573436326577,
      "grad_norm": 152.1051504836179,
      "learning_rate": 8.949570815450643e-07,
      "logits/chosen": -0.4586181640625,
      "logits/rejected": -0.49034422636032104,
      "logps/chosen": -309.75,
      "logps/rejected": -317.76251220703125,
      "loss": 0.6408,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": 0.04051513597369194,
      "rewards/margins": 1.1795654296875,
      "rewards/rejected": -1.1399352550506592,
      "step": 980
    },
    {
      "epoch": 0.42484711940778885,
      "grad_norm": 123.64678035860396,
      "learning_rate": 8.938841201716738e-07,
      "logits/chosen": -0.46697998046875,
      "logits/rejected": -0.475045770406723,
      "logps/chosen": -411.11248779296875,
      "logps/rejected": -369.6499938964844,
      "loss": 0.6128,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.15746459364891052,
      "rewards/margins": 1.188378930091858,
      "rewards/rejected": -1.03192138671875,
      "step": 990
    },
    {
      "epoch": 0.429138504452312,
      "grad_norm": 95.25459373166133,
      "learning_rate": 8.928111587982832e-07,
      "logits/chosen": -0.507373034954071,
      "logits/rejected": -0.551562488079071,
      "logps/chosen": -374.1000061035156,
      "logps/rejected": -324.29998779296875,
      "loss": 0.622,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.04002685472369194,
      "rewards/margins": 1.050103783607483,
      "rewards/rejected": -1.089263916015625,
      "step": 1000
    },
    {
      "epoch": 0.4334298894968351,
      "grad_norm": 115.34806962949547,
      "learning_rate": 8.917381974248926e-07,
      "logits/chosen": -0.5554443597793579,
      "logits/rejected": -0.5665038824081421,
      "logps/chosen": -417.0,
      "logps/rejected": -361.29998779296875,
      "loss": 0.5668,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": 0.2359161376953125,
      "rewards/margins": 1.0950195789337158,
      "rewards/rejected": -0.8587890863418579,
      "step": 1010
    },
    {
      "epoch": 0.43772127454135823,
      "grad_norm": 91.53553385211777,
      "learning_rate": 8.906652360515021e-07,
      "logits/chosen": -0.43243408203125,
      "logits/rejected": -0.46153563261032104,
      "logps/chosen": -378.0,
      "logps/rejected": -372.67498779296875,
      "loss": 0.5821,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.08709716796875,
      "rewards/margins": 1.1092040538787842,
      "rewards/rejected": -1.0213196277618408,
      "step": 1020
    },
    {
      "epoch": 0.4420126595858813,
      "grad_norm": 109.75710674669165,
      "learning_rate": 8.895922746781116e-07,
      "logits/chosen": -0.506561279296875,
      "logits/rejected": -0.5144714117050171,
      "logps/chosen": -371.67498779296875,
      "logps/rejected": -349.0874938964844,
      "loss": 0.5648,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.05256347730755806,
      "rewards/margins": 1.103796362876892,
      "rewards/rejected": -1.0504150390625,
      "step": 1030
    },
    {
      "epoch": 0.44630404463040446,
      "grad_norm": 93.91961654807437,
      "learning_rate": 8.88519313304721e-07,
      "logits/chosen": -0.47314453125,
      "logits/rejected": -0.5016113519668579,
      "logps/chosen": -386.92498779296875,
      "logps/rejected": -379.04998779296875,
      "loss": 0.5926,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": 0.16387939453125,
      "rewards/margins": 1.212377905845642,
      "rewards/rejected": -1.049780249595642,
      "step": 1040
    },
    {
      "epoch": 0.4505954296749276,
      "grad_norm": 90.52997225614212,
      "learning_rate": 8.874463519313305e-07,
      "logits/chosen": -0.44517821073532104,
      "logits/rejected": -0.4933105409145355,
      "logps/chosen": -395.3500061035156,
      "logps/rejected": -342.1000061035156,
      "loss": 0.6056,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.11962280422449112,
      "rewards/margins": 1.24884033203125,
      "rewards/rejected": -1.1293456554412842,
      "step": 1050
    },
    {
      "epoch": 0.4548868147194507,
      "grad_norm": 74.04010129210053,
      "learning_rate": 8.863733905579399e-07,
      "logits/chosen": -0.41621702909469604,
      "logits/rejected": -0.4658203125,
      "logps/chosen": -338.4750061035156,
      "logps/rejected": -331.67498779296875,
      "loss": 0.6029,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": 0.12017822265625,
      "rewards/margins": 1.0793945789337158,
      "rewards/rejected": -0.9591614007949829,
      "step": 1060
    },
    {
      "epoch": 0.45917819976397384,
      "grad_norm": 74.4564060165688,
      "learning_rate": 8.853004291845493e-07,
      "logits/chosen": -0.546459972858429,
      "logits/rejected": -0.611010730266571,
      "logps/chosen": -357.73748779296875,
      "logps/rejected": -333.5,
      "loss": 0.4912,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.0178680419921875,
      "rewards/margins": 1.5290710926055908,
      "rewards/rejected": -1.5110962390899658,
      "step": 1070
    },
    {
      "epoch": 0.4634695848084969,
      "grad_norm": 98.24596655627826,
      "learning_rate": 8.842274678111587e-07,
      "logits/chosen": -0.5308777093887329,
      "logits/rejected": -0.537243664264679,
      "logps/chosen": -367.5625,
      "logps/rejected": -351.375,
      "loss": 0.4987,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4163879454135895,
      "rewards/margins": 1.558447241783142,
      "rewards/rejected": -1.9744141101837158,
      "step": 1080
    },
    {
      "epoch": 0.46776096985302007,
      "grad_norm": 131.9870699092233,
      "learning_rate": 8.831545064377682e-07,
      "logits/chosen": -0.538891613483429,
      "logits/rejected": -0.588305652141571,
      "logps/chosen": -412.07501220703125,
      "logps/rejected": -380.20001220703125,
      "loss": 0.7044,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.9598938226699829,
      "rewards/margins": 0.9187988042831421,
      "rewards/rejected": -1.878564476966858,
      "step": 1090
    },
    {
      "epoch": 0.47205235489754316,
      "grad_norm": 115.58651937418213,
      "learning_rate": 8.820815450643776e-07,
      "logits/chosen": -0.510485827922821,
      "logits/rejected": -0.5465087890625,
      "logps/chosen": -356.1875,
      "logps/rejected": -373.9624938964844,
      "loss": 0.5985,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.9252868890762329,
      "rewards/margins": 1.1397216320037842,
      "rewards/rejected": -2.063671827316284,
      "step": 1100
    },
    {
      "epoch": 0.4763437399420663,
      "grad_norm": 159.2697928214491,
      "learning_rate": 8.81008583690987e-07,
      "logits/chosen": -0.4638732969760895,
      "logits/rejected": -0.48553466796875,
      "logps/chosen": -364.3500061035156,
      "logps/rejected": -321.57501220703125,
      "loss": 0.6099,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.53936767578125,
      "rewards/margins": 0.917163074016571,
      "rewards/rejected": -1.4571044445037842,
      "step": 1110
    },
    {
      "epoch": 0.48063512498658945,
      "grad_norm": 95.382031633549,
      "learning_rate": 8.799356223175966e-07,
      "logits/chosen": -0.497894287109375,
      "logits/rejected": -0.504931628704071,
      "logps/chosen": -342.6000061035156,
      "logps/rejected": -355.7749938964844,
      "loss": 0.6026,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": 0.2616943418979645,
      "rewards/margins": 1.0916259288787842,
      "rewards/rejected": -0.830029308795929,
      "step": 1120
    },
    {
      "epoch": 0.48492651003111253,
      "grad_norm": 71.83647371428191,
      "learning_rate": 8.788626609442059e-07,
      "logits/chosen": -0.512347400188446,
      "logits/rejected": -0.5821593999862671,
      "logps/chosen": -355.9375,
      "logps/rejected": -348.76251220703125,
      "loss": 0.5979,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.4642578065395355,
      "rewards/margins": 1.021484375,
      "rewards/rejected": -0.5578094720840454,
      "step": 1130
    },
    {
      "epoch": 0.4892178950756357,
      "grad_norm": 91.19027065620813,
      "learning_rate": 8.777896995708154e-07,
      "logits/chosen": -0.532092273235321,
      "logits/rejected": -0.549420177936554,
      "logps/chosen": -361.04998779296875,
      "logps/rejected": -364.2250061035156,
      "loss": 0.585,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": 0.675488293170929,
      "rewards/margins": 1.165808081626892,
      "rewards/rejected": -0.4899353086948395,
      "step": 1140
    },
    {
      "epoch": 0.49350928012015877,
      "grad_norm": 115.32812364907116,
      "learning_rate": 8.767167381974249e-07,
      "logits/chosen": -0.41058349609375,
      "logits/rejected": -0.4657699465751648,
      "logps/chosen": -350.4750061035156,
      "logps/rejected": -356.11248779296875,
      "loss": 0.6066,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.3958496153354645,
      "rewards/margins": 1.008276343345642,
      "rewards/rejected": -0.612597644329071,
      "step": 1150
    },
    {
      "epoch": 0.4978006651646819,
      "grad_norm": 78.0197602345477,
      "learning_rate": 8.756437768240343e-07,
      "logits/chosen": -0.48851317167282104,
      "logits/rejected": -0.509521484375,
      "logps/chosen": -377.0874938964844,
      "logps/rejected": -352.57501220703125,
      "loss": 0.5534,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": 0.22884520888328552,
      "rewards/margins": 1.447045922279358,
      "rewards/rejected": -1.21844482421875,
      "step": 1160
    },
    {
      "epoch": 0.502092050209205,
      "grad_norm": 98.64277713762311,
      "learning_rate": 8.745708154506437e-07,
      "logits/chosen": -0.522113025188446,
      "logits/rejected": -0.574328601360321,
      "logps/chosen": -376.1499938964844,
      "logps/rejected": -376.4750061035156,
      "loss": 0.5435,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": 0.15298843383789062,
      "rewards/margins": 1.590429663658142,
      "rewards/rejected": -1.437719702720642,
      "step": 1170
    },
    {
      "epoch": 0.5063834352537281,
      "grad_norm": 106.05493802876313,
      "learning_rate": 8.734978540772531e-07,
      "logits/chosen": -0.445648193359375,
      "logits/rejected": -0.46367186307907104,
      "logps/chosen": -381.70001220703125,
      "logps/rejected": -372.5,
      "loss": 0.7092,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.11246337741613388,
      "rewards/margins": 1.025366187095642,
      "rewards/rejected": -1.136804223060608,
      "step": 1180
    },
    {
      "epoch": 0.5106748202982513,
      "grad_norm": 115.26043535385436,
      "learning_rate": 8.724248927038627e-07,
      "logits/chosen": -0.39453125,
      "logits/rejected": -0.41694337129592896,
      "logps/chosen": -327.5375061035156,
      "logps/rejected": -354.375,
      "loss": 0.5654,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.74749755859375,
      "rewards/margins": 1.164697289466858,
      "rewards/rejected": -0.41730958223342896,
      "step": 1190
    },
    {
      "epoch": 0.5149662053427744,
      "grad_norm": 113.80958925768223,
      "learning_rate": 8.713519313304721e-07,
      "logits/chosen": -0.493438720703125,
      "logits/rejected": -0.534472644329071,
      "logps/chosen": -381.01251220703125,
      "logps/rejected": -353.70001220703125,
      "loss": 0.5776,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": 0.4181884825229645,
      "rewards/margins": 0.985180675983429,
      "rewards/rejected": -0.5670410394668579,
      "step": 1200
    },
    {
      "epoch": 0.5192575903872975,
      "grad_norm": 64.87761934202179,
      "learning_rate": 8.702789699570815e-07,
      "logits/chosen": -0.5803467035293579,
      "logits/rejected": -0.5775787234306335,
      "logps/chosen": -354.9750061035156,
      "logps/rejected": -345.7749938964844,
      "loss": 0.6283,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.09605102241039276,
      "rewards/margins": 0.893798828125,
      "rewards/rejected": -0.7971435785293579,
      "step": 1210
    },
    {
      "epoch": 0.5235489754318207,
      "grad_norm": 75.79881167922929,
      "learning_rate": 8.69206008583691e-07,
      "logits/chosen": -0.524340808391571,
      "logits/rejected": -0.605224609375,
      "logps/chosen": -403.6499938964844,
      "logps/rejected": -350.7749938964844,
      "loss": 0.5637,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": 0.03439941257238388,
      "rewards/margins": 1.2552368640899658,
      "rewards/rejected": -1.220422387123108,
      "step": 1220
    },
    {
      "epoch": 0.5278403604763438,
      "grad_norm": 200.8530863955299,
      "learning_rate": 8.681330472103003e-07,
      "logits/chosen": -0.55548095703125,
      "logits/rejected": -0.5864623785018921,
      "logps/chosen": -353.67498779296875,
      "logps/rejected": -305.29998779296875,
      "loss": 0.6713,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": 0.28569334745407104,
      "rewards/margins": 1.134545922279358,
      "rewards/rejected": -0.8502563238143921,
      "step": 1230
    },
    {
      "epoch": 0.5321317455208668,
      "grad_norm": 97.55383490779244,
      "learning_rate": 8.670600858369098e-07,
      "logits/chosen": -0.45197755098342896,
      "logits/rejected": -0.49714356660842896,
      "logps/chosen": -329.5,
      "logps/rejected": -329.73748779296875,
      "loss": 0.5312,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": 0.22989502549171448,
      "rewards/margins": 1.3463256359100342,
      "rewards/rejected": -1.1160156726837158,
      "step": 1240
    },
    {
      "epoch": 0.5364231305653899,
      "grad_norm": 85.32964636123666,
      "learning_rate": 8.659871244635193e-07,
      "logits/chosen": -0.4923034608364105,
      "logits/rejected": -0.5574706792831421,
      "logps/chosen": -346.5375061035156,
      "logps/rejected": -351.48748779296875,
      "loss": 0.5077,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.21806640923023224,
      "rewards/margins": 1.7045409679412842,
      "rewards/rejected": -1.487463355064392,
      "step": 1250
    },
    {
      "epoch": 0.5407145156099131,
      "grad_norm": 122.6983432711232,
      "learning_rate": 8.649141630901287e-07,
      "logits/chosen": -0.551116943359375,
      "logits/rejected": -0.5929931402206421,
      "logps/chosen": -394.88751220703125,
      "logps/rejected": -373.61248779296875,
      "loss": 0.6315,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.910290539264679,
      "rewards/margins": 1.1508057117462158,
      "rewards/rejected": -2.061230421066284,
      "step": 1260
    },
    {
      "epoch": 0.5450059006544362,
      "grad_norm": 79.73902746855065,
      "learning_rate": 8.638412017167382e-07,
      "logits/chosen": -0.47783201932907104,
      "logits/rejected": -0.5023193359375,
      "logps/chosen": -375.625,
      "logps/rejected": -367.07501220703125,
      "loss": 0.511,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.6886962652206421,
      "rewards/margins": 1.309088110923767,
      "rewards/rejected": -1.99755859375,
      "step": 1270
    },
    {
      "epoch": 0.5492972856989593,
      "grad_norm": 76.48265299898162,
      "learning_rate": 8.627682403433476e-07,
      "logits/chosen": -0.5455322265625,
      "logits/rejected": -0.5739501714706421,
      "logps/chosen": -408.25,
      "logps/rejected": -354.9624938964844,
      "loss": 0.6486,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.799389660358429,
      "rewards/margins": 0.9174438714981079,
      "rewards/rejected": -1.7162597179412842,
      "step": 1280
    },
    {
      "epoch": 0.5535886707434825,
      "grad_norm": 91.0745408997436,
      "learning_rate": 8.616952789699571e-07,
      "logits/chosen": -0.4973998963832855,
      "logits/rejected": -0.573132336139679,
      "logps/chosen": -378.2250061035156,
      "logps/rejected": -350.32501220703125,
      "loss": 0.5263,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.21334533393383026,
      "rewards/margins": 1.448999047279358,
      "rewards/rejected": -1.661865234375,
      "step": 1290
    },
    {
      "epoch": 0.5578800557880056,
      "grad_norm": 153.60675520341502,
      "learning_rate": 8.606223175965665e-07,
      "logits/chosen": -0.43608397245407104,
      "logits/rejected": -0.4373107850551605,
      "logps/chosen": -340.5874938964844,
      "logps/rejected": -347.82501220703125,
      "loss": 0.643,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.29754638671875,
      "rewards/margins": 1.2620849609375,
      "rewards/rejected": -1.560156226158142,
      "step": 1300
    },
    {
      "epoch": 0.5621714408325287,
      "grad_norm": 132.43446554600487,
      "learning_rate": 8.595493562231759e-07,
      "logits/chosen": -0.4892822206020355,
      "logits/rejected": -0.526074230670929,
      "logps/chosen": -365.92498779296875,
      "logps/rejected": -343.42498779296875,
      "loss": 0.7188,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.16191406548023224,
      "rewards/margins": 0.955456554889679,
      "rewards/rejected": -1.1182372570037842,
      "step": 1310
    },
    {
      "epoch": 0.5664628258770518,
      "grad_norm": 71.70573005978581,
      "learning_rate": 8.584763948497854e-07,
      "logits/chosen": -0.543383777141571,
      "logits/rejected": -0.541186511516571,
      "logps/chosen": -360.95001220703125,
      "logps/rejected": -371.45001220703125,
      "loss": 0.6053,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.3445373475551605,
      "rewards/margins": 0.891772449016571,
      "rewards/rejected": -0.5476318597793579,
      "step": 1320
    },
    {
      "epoch": 0.570754210921575,
      "grad_norm": 127.82314313548048,
      "learning_rate": 8.574034334763947e-07,
      "logits/chosen": -0.48707884550094604,
      "logits/rejected": -0.5375640988349915,
      "logps/chosen": -365.1875,
      "logps/rejected": -366.67498779296875,
      "loss": 0.5814,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.08065185695886612,
      "rewards/margins": 1.303247094154358,
      "rewards/rejected": -1.3848145008087158,
      "step": 1330
    },
    {
      "epoch": 0.575045595966098,
      "grad_norm": 107.88477863428349,
      "learning_rate": 8.563304721030042e-07,
      "logits/chosen": -0.526837170124054,
      "logits/rejected": -0.546130359172821,
      "logps/chosen": -352.82501220703125,
      "logps/rejected": -352.32501220703125,
      "loss": 0.5981,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.06484375149011612,
      "rewards/margins": 1.154577612876892,
      "rewards/rejected": -1.220068335533142,
      "step": 1340
    },
    {
      "epoch": 0.5793369810106211,
      "grad_norm": 82.1472115579418,
      "learning_rate": 8.552575107296138e-07,
      "logits/chosen": -0.634167492389679,
      "logits/rejected": -0.673510730266571,
      "logps/chosen": -403.0874938964844,
      "logps/rejected": -376.0249938964844,
      "loss": 0.6074,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.0027954101096838713,
      "rewards/margins": 1.2101562023162842,
      "rewards/rejected": -1.2136719226837158,
      "step": 1350
    },
    {
      "epoch": 0.5836283660551443,
      "grad_norm": 94.92143754062582,
      "learning_rate": 8.541845493562231e-07,
      "logits/chosen": -0.49714356660842896,
      "logits/rejected": -0.5233856439590454,
      "logps/chosen": -333.3374938964844,
      "logps/rejected": -327.25,
      "loss": 0.5949,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.05092773586511612,
      "rewards/margins": 1.311303734779358,
      "rewards/rejected": -1.361291527748108,
      "step": 1360
    },
    {
      "epoch": 0.5879197510996674,
      "grad_norm": 94.34744735913843,
      "learning_rate": 8.531115879828326e-07,
      "logits/chosen": -0.5505005121231079,
      "logits/rejected": -0.576000988483429,
      "logps/chosen": -370.6000061035156,
      "logps/rejected": -357.29998779296875,
      "loss": 0.6575,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.18602295219898224,
      "rewards/margins": 1.278955101966858,
      "rewards/rejected": -1.464624047279358,
      "step": 1370
    },
    {
      "epoch": 0.5922111361441905,
      "grad_norm": 127.09428801599846,
      "learning_rate": 8.52038626609442e-07,
      "logits/chosen": -0.4859985411167145,
      "logits/rejected": -0.5118652582168579,
      "logps/chosen": -337.5874938964844,
      "logps/rejected": -351.2875061035156,
      "loss": 0.6361,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.3132003843784332,
      "rewards/margins": 1.212499976158142,
      "rewards/rejected": -1.5251861810684204,
      "step": 1380
    },
    {
      "epoch": 0.5965025211887136,
      "grad_norm": 64.80417600604287,
      "learning_rate": 8.509656652360515e-07,
      "logits/chosen": -0.5366455316543579,
      "logits/rejected": -0.576123058795929,
      "logps/chosen": -373.79998779296875,
      "logps/rejected": -379.1875,
      "loss": 0.5529,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.0067962645553052425,
      "rewards/margins": 1.2827637195587158,
      "rewards/rejected": -1.2896331548690796,
      "step": 1390
    },
    {
      "epoch": 0.6007939062332368,
      "grad_norm": 97.83397912112129,
      "learning_rate": 8.498927038626609e-07,
      "logits/chosen": -0.52276611328125,
      "logits/rejected": -0.5662841796875,
      "logps/chosen": -363.375,
      "logps/rejected": -359.625,
      "loss": 0.6168,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": 0.23829345405101776,
      "rewards/margins": 1.1818726062774658,
      "rewards/rejected": -0.9430511593818665,
      "step": 1400
    },
    {
      "epoch": 0.6050852912777599,
      "grad_norm": 112.3779143546819,
      "learning_rate": 8.488197424892703e-07,
      "logits/chosen": -0.4321044981479645,
      "logits/rejected": -0.46345824003219604,
      "logps/chosen": -372.20623779296875,
      "logps/rejected": -337.1875,
      "loss": 0.5327,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.12424316257238388,
      "rewards/margins": 1.40032958984375,
      "rewards/rejected": -1.2748901844024658,
      "step": 1410
    },
    {
      "epoch": 0.609376676322283,
      "grad_norm": 72.4529126410346,
      "learning_rate": 8.477467811158799e-07,
      "logits/chosen": -0.4897705018520355,
      "logits/rejected": -0.52484130859375,
      "logps/chosen": -327.79998779296875,
      "logps/rejected": -329.67498779296875,
      "loss": 0.6019,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.06064758449792862,
      "rewards/margins": 1.131445288658142,
      "rewards/rejected": -1.071081519126892,
      "step": 1420
    },
    {
      "epoch": 0.6136680613668062,
      "grad_norm": 97.63216460404503,
      "learning_rate": 8.466738197424892e-07,
      "logits/chosen": -0.48497313261032104,
      "logits/rejected": -0.5263916254043579,
      "logps/chosen": -357.67498779296875,
      "logps/rejected": -331.375,
      "loss": 0.6787,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": 0.07767333835363388,
      "rewards/margins": 1.00714111328125,
      "rewards/rejected": -0.929492175579071,
      "step": 1430
    },
    {
      "epoch": 0.6179594464113293,
      "grad_norm": 57.5331067814223,
      "learning_rate": 8.456008583690987e-07,
      "logits/chosen": -0.5661865472793579,
      "logits/rejected": -0.6107543706893921,
      "logps/chosen": -346.6625061035156,
      "logps/rejected": -352.7250061035156,
      "loss": 0.6395,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.14810791611671448,
      "rewards/margins": 1.1807372570037842,
      "rewards/rejected": -1.32977294921875,
      "step": 1440
    },
    {
      "epoch": 0.6222508314558524,
      "grad_norm": 69.50617792157094,
      "learning_rate": 8.445278969957082e-07,
      "logits/chosen": -0.4361205995082855,
      "logits/rejected": -0.4748779237270355,
      "logps/chosen": -355.75,
      "logps/rejected": -347.5249938964844,
      "loss": 0.5356,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.3649047911167145,
      "rewards/margins": 1.450170874595642,
      "rewards/rejected": -1.0839111804962158,
      "step": 1450
    },
    {
      "epoch": 0.6265422165003754,
      "grad_norm": 68.13949614065807,
      "learning_rate": 8.434549356223175e-07,
      "logits/chosen": -0.57635498046875,
      "logits/rejected": -0.6020752191543579,
      "logps/chosen": -358.9750061035156,
      "logps/rejected": -343.17498779296875,
      "loss": 0.609,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.4049972593784332,
      "rewards/margins": 1.356420874595642,
      "rewards/rejected": -0.9520202875137329,
      "step": 1460
    },
    {
      "epoch": 0.6308336015448986,
      "grad_norm": 99.54853354184904,
      "learning_rate": 8.42381974248927e-07,
      "logits/chosen": -0.54022216796875,
      "logits/rejected": -0.5438476800918579,
      "logps/chosen": -365.73748779296875,
      "logps/rejected": -353.42498779296875,
      "loss": 0.6873,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.16910400986671448,
      "rewards/margins": 0.8871093988418579,
      "rewards/rejected": -0.71746826171875,
      "step": 1470
    },
    {
      "epoch": 0.6351249865894217,
      "grad_norm": 83.2495790704316,
      "learning_rate": 8.413090128755364e-07,
      "logits/chosen": -0.550823986530304,
      "logits/rejected": -0.5851806402206421,
      "logps/chosen": -374.54998779296875,
      "logps/rejected": -344.04998779296875,
      "loss": 0.7087,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.4762206971645355,
      "rewards/margins": 0.963916003704071,
      "rewards/rejected": -1.440100073814392,
      "step": 1480
    },
    {
      "epoch": 0.6394163716339448,
      "grad_norm": 91.46916470585484,
      "learning_rate": 8.402360515021459e-07,
      "logits/chosen": -0.52392578125,
      "logits/rejected": -0.5739990472793579,
      "logps/chosen": -378.57501220703125,
      "logps/rejected": -373.42498779296875,
      "loss": 0.5314,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.6462768316268921,
      "rewards/margins": 1.498559594154358,
      "rewards/rejected": -2.1455078125,
      "step": 1490
    },
    {
      "epoch": 0.643707756678468,
      "grad_norm": 130.29345838295373,
      "learning_rate": 8.391630901287554e-07,
      "logits/chosen": -0.42219239473342896,
      "logits/rejected": -0.46629637479782104,
      "logps/chosen": -351.5874938964844,
      "logps/rejected": -351.07501220703125,
      "loss": 0.6083,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.30372923612594604,
      "rewards/margins": 1.2931396961212158,
      "rewards/rejected": -1.5969970226287842,
      "step": 1500
    },
    {
      "epoch": 0.6479991417229911,
      "grad_norm": 87.98382679498306,
      "learning_rate": 8.380901287553648e-07,
      "logits/chosen": -0.4512939453125,
      "logits/rejected": -0.45451050996780396,
      "logps/chosen": -372.92498779296875,
      "logps/rejected": -350.5249938964844,
      "loss": 0.6227,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.4544677734375,
      "rewards/margins": 1.4044189453125,
      "rewards/rejected": -1.8577392101287842,
      "step": 1510
    },
    {
      "epoch": 0.6522905267675142,
      "grad_norm": 69.90835845853988,
      "learning_rate": 8.370171673819743e-07,
      "logits/chosen": -0.47783201932907104,
      "logits/rejected": -0.48542481660842896,
      "logps/chosen": -374.0249938964844,
      "logps/rejected": -346.0,
      "loss": 0.6425,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.5445556640625,
      "rewards/margins": 1.402075171470642,
      "rewards/rejected": -1.945703148841858,
      "step": 1520
    },
    {
      "epoch": 0.6565819118120373,
      "grad_norm": 74.9862455300779,
      "learning_rate": 8.359442060085836e-07,
      "logits/chosen": -0.4558349549770355,
      "logits/rejected": -0.4989990293979645,
      "logps/chosen": -363.4750061035156,
      "logps/rejected": -364.17498779296875,
      "loss": 0.5358,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.29169923067092896,
      "rewards/margins": 1.52032470703125,
      "rewards/rejected": -1.8122985363006592,
      "step": 1530
    },
    {
      "epoch": 0.6608732968565605,
      "grad_norm": 66.70997081258484,
      "learning_rate": 8.348712446351931e-07,
      "logits/chosen": -0.49382323026657104,
      "logits/rejected": -0.509033203125,
      "logps/chosen": -343.6875,
      "logps/rejected": -349.9750061035156,
      "loss": 0.5414,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.3521972596645355,
      "rewards/margins": 1.3031494617462158,
      "rewards/rejected": -1.6546630859375,
      "step": 1540
    },
    {
      "epoch": 0.6651646819010836,
      "grad_norm": 92.37399732783172,
      "learning_rate": 8.337982832618026e-07,
      "logits/chosen": -0.49000245332717896,
      "logits/rejected": -0.5064147710800171,
      "logps/chosen": -354.7124938964844,
      "logps/rejected": -343.42498779296875,
      "loss": 0.5505,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.3075927793979645,
      "rewards/margins": 1.5106933116912842,
      "rewards/rejected": -1.818933129310608,
      "step": 1550
    },
    {
      "epoch": 0.6694560669456067,
      "grad_norm": 107.0779843087361,
      "learning_rate": 8.327253218884119e-07,
      "logits/chosen": -0.4959472715854645,
      "logits/rejected": -0.548205554485321,
      "logps/chosen": -389.79998779296875,
      "logps/rejected": -370.70001220703125,
      "loss": 0.6266,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.4276229739189148,
      "rewards/margins": 1.133325219154358,
      "rewards/rejected": -1.56103515625,
      "step": 1560
    },
    {
      "epoch": 0.6737474519901299,
      "grad_norm": 71.25240887690347,
      "learning_rate": 8.316523605150214e-07,
      "logits/chosen": -0.49489134550094604,
      "logits/rejected": -0.545239269733429,
      "logps/chosen": -385.0,
      "logps/rejected": -357.11248779296875,
      "loss": 0.5734,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.11707153171300888,
      "rewards/margins": 1.5208251476287842,
      "rewards/rejected": -1.63751220703125,
      "step": 1570
    },
    {
      "epoch": 0.678038837034653,
      "grad_norm": 133.35487990624426,
      "learning_rate": 8.305793991416309e-07,
      "logits/chosen": -0.4952392578125,
      "logits/rejected": -0.561535656452179,
      "logps/chosen": -379.625,
      "logps/rejected": -416.5249938964844,
      "loss": 0.6352,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.49039918184280396,
      "rewards/margins": 1.2760498523712158,
      "rewards/rejected": -1.7668273448944092,
      "step": 1580
    },
    {
      "epoch": 0.682330222079176,
      "grad_norm": 102.2630137778118,
      "learning_rate": 8.295064377682403e-07,
      "logits/chosen": -0.4881347715854645,
      "logits/rejected": -0.5366576910018921,
      "logps/chosen": -355.0,
      "logps/rejected": -347.4750061035156,
      "loss": 0.5743,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.4215942323207855,
      "rewards/margins": 1.5192992687225342,
      "rewards/rejected": -1.940954566001892,
      "step": 1590
    },
    {
      "epoch": 0.6866216071236991,
      "grad_norm": 124.30570547005117,
      "learning_rate": 8.284334763948498e-07,
      "logits/chosen": -0.41212159395217896,
      "logits/rejected": -0.4427490234375,
      "logps/chosen": -348.6499938964844,
      "logps/rejected": -353.42498779296875,
      "loss": 0.6294,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.49810487031936646,
      "rewards/margins": 1.0903809070587158,
      "rewards/rejected": -1.588433861732483,
      "step": 1600
    },
    {
      "epoch": 0.6909129921682223,
      "grad_norm": 77.9879074432851,
      "learning_rate": 8.273605150214592e-07,
      "logits/chosen": -0.43266600370407104,
      "logits/rejected": -0.48729246854782104,
      "logps/chosen": -352.07501220703125,
      "logps/rejected": -370.04998779296875,
      "loss": 0.6644,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.3375000059604645,
      "rewards/margins": 0.8544921875,
      "rewards/rejected": -1.192346215248108,
      "step": 1610
    },
    {
      "epoch": 0.6952043772127454,
      "grad_norm": 96.49390855559005,
      "learning_rate": 8.262875536480687e-07,
      "logits/chosen": -0.5049682855606079,
      "logits/rejected": -0.5408996343612671,
      "logps/chosen": -370.70001220703125,
      "logps/rejected": -356.6499938964844,
      "loss": 0.4994,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.23572388291358948,
      "rewards/margins": 1.4106934070587158,
      "rewards/rejected": -1.644799828529358,
      "step": 1620
    },
    {
      "epoch": 0.6994957622572685,
      "grad_norm": 115.16569613535562,
      "learning_rate": 8.25214592274678e-07,
      "logits/chosen": -0.49749755859375,
      "logits/rejected": -0.565502941608429,
      "logps/chosen": -397.07501220703125,
      "logps/rejected": -389.3999938964844,
      "loss": 0.5624,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.4298950135707855,
      "rewards/margins": 1.4709961414337158,
      "rewards/rejected": -1.9002685546875,
      "step": 1630
    },
    {
      "epoch": 0.7037871473017917,
      "grad_norm": 127.40252022777491,
      "learning_rate": 8.241416309012875e-07,
      "logits/chosen": -0.5443359613418579,
      "logits/rejected": -0.5760253667831421,
      "logps/chosen": -359.57501220703125,
      "logps/rejected": -360.7124938964844,
      "loss": 0.5496,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.5360199213027954,
      "rewards/margins": 1.4149658679962158,
      "rewards/rejected": -1.950537085533142,
      "step": 1640
    },
    {
      "epoch": 0.7080785323463148,
      "grad_norm": 60.46521285483948,
      "learning_rate": 8.23068669527897e-07,
      "logits/chosen": -0.50640869140625,
      "logits/rejected": -0.509936511516571,
      "logps/chosen": -352.01251220703125,
      "logps/rejected": -361.70001220703125,
      "loss": 0.4966,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.4793640077114105,
      "rewards/margins": 1.878198266029358,
      "rewards/rejected": -2.358996629714966,
      "step": 1650
    },
    {
      "epoch": 0.7123699173908379,
      "grad_norm": 182.34033105607347,
      "learning_rate": 8.219957081545064e-07,
      "logits/chosen": -0.583239734172821,
      "logits/rejected": -0.583789050579071,
      "logps/chosen": -404.70001220703125,
      "logps/rejected": -401.42498779296875,
      "loss": 0.6094,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.755078136920929,
      "rewards/margins": 1.395166039466858,
      "rewards/rejected": -2.150158643722534,
      "step": 1660
    },
    {
      "epoch": 0.716661302435361,
      "grad_norm": 104.64828868762461,
      "learning_rate": 8.209227467811159e-07,
      "logits/chosen": -0.521435558795929,
      "logits/rejected": -0.5520263910293579,
      "logps/chosen": -370.7250061035156,
      "logps/rejected": -359.20001220703125,
      "loss": 0.6848,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.4616332948207855,
      "rewards/margins": 1.199731469154358,
      "rewards/rejected": -1.661767601966858,
      "step": 1670
    },
    {
      "epoch": 0.7209526874798842,
      "grad_norm": 77.17303575841277,
      "learning_rate": 8.198497854077253e-07,
      "logits/chosen": -0.46148985624313354,
      "logits/rejected": -0.55828857421875,
      "logps/chosen": -365.6000061035156,
      "logps/rejected": -342.07501220703125,
      "loss": 0.6922,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.05918274074792862,
      "rewards/margins": 1.3850586414337158,
      "rewards/rejected": -1.444067358970642,
      "step": 1680
    },
    {
      "epoch": 0.7252440725244073,
      "grad_norm": 64.4697217979389,
      "learning_rate": 8.187768240343347e-07,
      "logits/chosen": -0.42725831270217896,
      "logits/rejected": -0.48114013671875,
      "logps/chosen": -356.75,
      "logps/rejected": -330.01251220703125,
      "loss": 0.4518,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": 0.17878417670726776,
      "rewards/margins": 1.6595947742462158,
      "rewards/rejected": -1.481781005859375,
      "step": 1690
    },
    {
      "epoch": 0.7295354575689303,
      "grad_norm": 149.1032910167046,
      "learning_rate": 8.177038626609442e-07,
      "logits/chosen": -0.406646728515625,
      "logits/rejected": -0.4510498046875,
      "logps/chosen": -356.3500061035156,
      "logps/rejected": -367.5249938964844,
      "loss": 0.5885,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.11580810695886612,
      "rewards/margins": 1.317968726158142,
      "rewards/rejected": -1.4335448741912842,
      "step": 1700
    },
    {
      "epoch": 0.7338268426134535,
      "grad_norm": 102.8164344382787,
      "learning_rate": 8.166309012875536e-07,
      "logits/chosen": -0.447854608297348,
      "logits/rejected": -0.4827026426792145,
      "logps/chosen": -363.8999938964844,
      "logps/rejected": -340.8999938964844,
      "loss": 0.6358,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.10651855170726776,
      "rewards/margins": 1.2876708507537842,
      "rewards/rejected": -1.394189476966858,
      "step": 1710
    },
    {
      "epoch": 0.7381182276579766,
      "grad_norm": 153.69365216307173,
      "learning_rate": 8.155579399141631e-07,
      "logits/chosen": -0.41840821504592896,
      "logits/rejected": -0.4554687440395355,
      "logps/chosen": -387.3125,
      "logps/rejected": -365.875,
      "loss": 0.6182,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.20069579780101776,
      "rewards/margins": 1.3489501476287842,
      "rewards/rejected": -1.5494384765625,
      "step": 1720
    },
    {
      "epoch": 0.7424096127024997,
      "grad_norm": 76.04781640711542,
      "learning_rate": 8.144849785407724e-07,
      "logits/chosen": -0.4141906797885895,
      "logits/rejected": -0.4426940977573395,
      "logps/chosen": -367.42498779296875,
      "logps/rejected": -331.04998779296875,
      "loss": 0.5223,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.08599853515625,
      "rewards/margins": 1.4048278331756592,
      "rewards/rejected": -1.4912109375,
      "step": 1730
    },
    {
      "epoch": 0.7467009977470228,
      "grad_norm": 68.23310240077483,
      "learning_rate": 8.13412017167382e-07,
      "logits/chosen": -0.46959227323532104,
      "logits/rejected": -0.48597413301467896,
      "logps/chosen": -374.54998779296875,
      "logps/rejected": -384.8999938964844,
      "loss": 0.5471,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.08927001804113388,
      "rewards/margins": 1.407373070716858,
      "rewards/rejected": -1.496606469154358,
      "step": 1740
    },
    {
      "epoch": 0.750992382791546,
      "grad_norm": 107.96260731375123,
      "learning_rate": 8.123390557939915e-07,
      "logits/chosen": -0.4058776795864105,
      "logits/rejected": -0.45594483613967896,
      "logps/chosen": -375.7250061035156,
      "logps/rejected": -334.6499938964844,
      "loss": 0.6711,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.3480224609375,
      "rewards/margins": 1.2222411632537842,
      "rewards/rejected": -1.570587158203125,
      "step": 1750
    },
    {
      "epoch": 0.7552837678360691,
      "grad_norm": 129.23873009661452,
      "learning_rate": 8.112660944206008e-07,
      "logits/chosen": -0.41258543729782104,
      "logits/rejected": -0.407440185546875,
      "logps/chosen": -363.3374938964844,
      "logps/rejected": -338.7749938964844,
      "loss": 0.55,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.14633789658546448,
      "rewards/margins": 1.4525146484375,
      "rewards/rejected": -1.598779320716858,
      "step": 1760
    },
    {
      "epoch": 0.7595751528805922,
      "grad_norm": 48.144126354391204,
      "learning_rate": 8.101931330472103e-07,
      "logits/chosen": -0.361968994140625,
      "logits/rejected": -0.3884033262729645,
      "logps/chosen": -338.45001220703125,
      "logps/rejected": -314.1000061035156,
      "loss": 0.6883,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": 0.23784179985523224,
      "rewards/margins": 1.268212914466858,
      "rewards/rejected": -1.0308105945587158,
      "step": 1770
    },
    {
      "epoch": 0.7638665379251154,
      "grad_norm": 95.56446959520261,
      "learning_rate": 8.091201716738197e-07,
      "logits/chosen": -0.31524658203125,
      "logits/rejected": -0.340383917093277,
      "logps/chosen": -319.8500061035156,
      "logps/rejected": -321.0,
      "loss": 0.5326,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.3859619200229645,
      "rewards/margins": 1.397973656654358,
      "rewards/rejected": -1.0114624500274658,
      "step": 1780
    },
    {
      "epoch": 0.7681579229696385,
      "grad_norm": 103.09494934406682,
      "learning_rate": 8.080472103004291e-07,
      "logits/chosen": -0.4416259825229645,
      "logits/rejected": -0.47047120332717896,
      "logps/chosen": -357.9750061035156,
      "logps/rejected": -350.92498779296875,
      "loss": 0.6076,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.05386962741613388,
      "rewards/margins": 1.2465698719024658,
      "rewards/rejected": -1.300622582435608,
      "step": 1790
    },
    {
      "epoch": 0.7724493080141616,
      "grad_norm": 145.80884265242486,
      "learning_rate": 8.069742489270386e-07,
      "logits/chosen": -0.42521363496780396,
      "logits/rejected": -0.4398437440395355,
      "logps/chosen": -377.8500061035156,
      "logps/rejected": -367.67498779296875,
      "loss": 0.5738,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.4979003965854645,
      "rewards/margins": 1.4534118175506592,
      "rewards/rejected": -1.952612280845642,
      "step": 1800
    },
    {
      "epoch": 0.7767406930586847,
      "grad_norm": 101.16487539387255,
      "learning_rate": 8.05901287553648e-07,
      "logits/chosen": -0.49574583768844604,
      "logits/rejected": -0.5306152105331421,
      "logps/chosen": -385.5249938964844,
      "logps/rejected": -367.57501220703125,
      "loss": 0.5431,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.5192779302597046,
      "rewards/margins": 1.4374268054962158,
      "rewards/rejected": -1.9562499523162842,
      "step": 1810
    },
    {
      "epoch": 0.7810320781032078,
      "grad_norm": 84.4411142242806,
      "learning_rate": 8.048283261802575e-07,
      "logits/chosen": -0.4796142578125,
      "logits/rejected": -0.508770763874054,
      "logps/chosen": -365.86248779296875,
      "logps/rejected": -356.3500061035156,
      "loss": 0.6657,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.883410632610321,
      "rewards/margins": 1.223120093345642,
      "rewards/rejected": -2.1060791015625,
      "step": 1820
    },
    {
      "epoch": 0.7853234631477309,
      "grad_norm": 115.57764002933129,
      "learning_rate": 8.037553648068669e-07,
      "logits/chosen": -0.546875,
      "logits/rejected": -0.5829498171806335,
      "logps/chosen": -359.7250061035156,
      "logps/rejected": -337.57501220703125,
      "loss": 0.7296,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -1.026031494140625,
      "rewards/margins": 0.8546386957168579,
      "rewards/rejected": -1.880517601966858,
      "step": 1830
    },
    {
      "epoch": 0.789614848192254,
      "grad_norm": 103.0460534756151,
      "learning_rate": 8.026824034334764e-07,
      "logits/chosen": -0.5229126214981079,
      "logits/rejected": -0.5728759765625,
      "logps/chosen": -403.4750061035156,
      "logps/rejected": -356.04998779296875,
      "loss": 0.6826,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.888287365436554,
      "rewards/margins": 0.9383544921875,
      "rewards/rejected": -1.827490210533142,
      "step": 1840
    },
    {
      "epoch": 0.7939062332367771,
      "grad_norm": 58.14013738451265,
      "learning_rate": 8.016094420600859e-07,
      "logits/chosen": -0.4228149354457855,
      "logits/rejected": -0.47142332792282104,
      "logps/chosen": -379.82501220703125,
      "logps/rejected": -374.45001220703125,
      "loss": 0.5898,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.415771484375,
      "rewards/margins": 1.040673851966858,
      "rewards/rejected": -1.455438256263733,
      "step": 1850
    },
    {
      "epoch": 0.7981976182813003,
      "grad_norm": 64.72324046460037,
      "learning_rate": 8.005364806866952e-07,
      "logits/chosen": -0.48001402616500854,
      "logits/rejected": -0.5206695795059204,
      "logps/chosen": -355.51251220703125,
      "logps/rejected": -329.92498779296875,
      "loss": 0.5486,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.14082030951976776,
      "rewards/margins": 1.267333984375,
      "rewards/rejected": -1.126745581626892,
      "step": 1860
    },
    {
      "epoch": 0.8024890033258234,
      "grad_norm": 92.66296531949637,
      "learning_rate": 7.994635193133047e-07,
      "logits/chosen": -0.4834960997104645,
      "logits/rejected": -0.52276611328125,
      "logps/chosen": -346.875,
      "logps/rejected": -360.1000061035156,
      "loss": 0.6322,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.29844969511032104,
      "rewards/margins": 1.1924560070037842,
      "rewards/rejected": -1.491418480873108,
      "step": 1870
    },
    {
      "epoch": 0.8067803883703465,
      "grad_norm": 155.25372862771923,
      "learning_rate": 7.983905579399141e-07,
      "logits/chosen": -0.48237305879592896,
      "logits/rejected": -0.513842761516571,
      "logps/chosen": -398.125,
      "logps/rejected": -375.82501220703125,
      "loss": 0.6616,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.661267101764679,
      "rewards/margins": 1.28704833984375,
      "rewards/rejected": -1.948278784751892,
      "step": 1880
    },
    {
      "epoch": 0.8110717734148697,
      "grad_norm": 96.48468679812369,
      "learning_rate": 7.973175965665235e-07,
      "logits/chosen": -0.5263671875,
      "logits/rejected": -0.551501452922821,
      "logps/chosen": -352.61248779296875,
      "logps/rejected": -361.2749938964844,
      "loss": 0.5677,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.6576477289199829,
      "rewards/margins": 1.3949706554412842,
      "rewards/rejected": -2.052685499191284,
      "step": 1890
    },
    {
      "epoch": 0.8153631584593928,
      "grad_norm": 117.26416137288112,
      "learning_rate": 7.96244635193133e-07,
      "logits/chosen": -0.4586242735385895,
      "logits/rejected": -0.4781738221645355,
      "logps/chosen": -352.3500061035156,
      "logps/rejected": -356.07501220703125,
      "loss": 0.5319,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.47181397676467896,
      "rewards/margins": 1.3197510242462158,
      "rewards/rejected": -1.7908935546875,
      "step": 1900
    },
    {
      "epoch": 0.8196545435039159,
      "grad_norm": 75.35991358014043,
      "learning_rate": 7.951716738197425e-07,
      "logits/chosen": -0.5106903314590454,
      "logits/rejected": -0.535693347454071,
      "logps/chosen": -395.38751220703125,
      "logps/rejected": -396.625,
      "loss": 0.5579,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.4886108338832855,
      "rewards/margins": 1.457763671875,
      "rewards/rejected": -1.94580078125,
      "step": 1910
    },
    {
      "epoch": 0.823945928548439,
      "grad_norm": 87.36390962579817,
      "learning_rate": 7.940987124463519e-07,
      "logits/chosen": -0.43651121854782104,
      "logits/rejected": -0.463156133890152,
      "logps/chosen": -334.88751220703125,
      "logps/rejected": -323.82501220703125,
      "loss": 0.6742,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.5482543706893921,
      "rewards/margins": 0.8956054449081421,
      "rewards/rejected": -1.4434630870819092,
      "step": 1920
    },
    {
      "epoch": 0.8282373135929622,
      "grad_norm": 69.40186273752217,
      "learning_rate": 7.930257510729613e-07,
      "logits/chosen": -0.4944824278354645,
      "logits/rejected": -0.5515899658203125,
      "logps/chosen": -351.7250061035156,
      "logps/rejected": -350.42498779296875,
      "loss": 0.4568,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.10759429633617401,
      "rewards/margins": 1.702294945716858,
      "rewards/rejected": -1.808691382408142,
      "step": 1930
    },
    {
      "epoch": 0.8325286986374852,
      "grad_norm": 104.89991735780596,
      "learning_rate": 7.919527896995708e-07,
      "logits/chosen": -0.418426513671875,
      "logits/rejected": -0.44764405488967896,
      "logps/chosen": -341.04998779296875,
      "logps/rejected": -335.6499938964844,
      "loss": 0.5415,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.15447387099266052,
      "rewards/margins": 1.450292944908142,
      "rewards/rejected": -1.604248046875,
      "step": 1940
    },
    {
      "epoch": 0.8368200836820083,
      "grad_norm": 82.68903649771558,
      "learning_rate": 7.908798283261803e-07,
      "logits/chosen": -0.4495605528354645,
      "logits/rejected": -0.510424792766571,
      "logps/chosen": -373.3374938964844,
      "logps/rejected": -364.67498779296875,
      "loss": 0.5789,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.779125988483429,
      "rewards/margins": 1.6708984375,
      "rewards/rejected": -2.449951171875,
      "step": 1950
    },
    {
      "epoch": 0.8411114687265315,
      "grad_norm": 86.21352102169897,
      "learning_rate": 7.898068669527896e-07,
      "logits/chosen": -0.46674805879592896,
      "logits/rejected": -0.528363049030304,
      "logps/chosen": -367.42498779296875,
      "logps/rejected": -361.70001220703125,
      "loss": 0.6793,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.8564208745956421,
      "rewards/margins": 1.278417944908142,
      "rewards/rejected": -2.133984327316284,
      "step": 1960
    },
    {
      "epoch": 0.8454028537710546,
      "grad_norm": 54.69051017258,
      "learning_rate": 7.887339055793991e-07,
      "logits/chosen": -0.4278625547885895,
      "logits/rejected": -0.43818360567092896,
      "logps/chosen": -364.5249938964844,
      "logps/rejected": -342.13751220703125,
      "loss": 0.6952,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.8060302734375,
      "rewards/margins": 1.024389624595642,
      "rewards/rejected": -1.830468773841858,
      "step": 1970
    },
    {
      "epoch": 0.8496942388155777,
      "grad_norm": 148.37339661044908,
      "learning_rate": 7.876609442060086e-07,
      "logits/chosen": -0.4754638671875,
      "logits/rejected": -0.551074206829071,
      "logps/chosen": -358.0,
      "logps/rejected": -356.6499938964844,
      "loss": 0.6452,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.5104004144668579,
      "rewards/margins": 1.115087866783142,
      "rewards/rejected": -1.6243164539337158,
      "step": 1980
    },
    {
      "epoch": 0.8539856238601008,
      "grad_norm": 116.66459040759774,
      "learning_rate": 7.86587982832618e-07,
      "logits/chosen": -0.4542480409145355,
      "logits/rejected": -0.5120605230331421,
      "logps/chosen": -351.17498779296875,
      "logps/rejected": -348.70001220703125,
      "loss": 0.5138,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.41557615995407104,
      "rewards/margins": 1.316796898841858,
      "rewards/rejected": -1.7315795421600342,
      "step": 1990
    },
    {
      "epoch": 0.858277008904624,
      "grad_norm": 74.51727230697634,
      "learning_rate": 7.855150214592274e-07,
      "logits/chosen": -0.5084472894668579,
      "logits/rejected": -0.5223144292831421,
      "logps/chosen": -350.5375061035156,
      "logps/rejected": -353.45001220703125,
      "loss": 0.6133,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.742321789264679,
      "rewards/margins": 1.1166503429412842,
      "rewards/rejected": -1.8595459461212158,
      "step": 2000
    },
    {
      "epoch": 0.8625683939491471,
      "grad_norm": 76.42518524272083,
      "learning_rate": 7.844420600858369e-07,
      "logits/chosen": -0.40562134981155396,
      "logits/rejected": -0.4638671875,
      "logps/chosen": -378.8500061035156,
      "logps/rejected": -348.5249938964844,
      "loss": 0.5948,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.3991332948207855,
      "rewards/margins": 1.241552710533142,
      "rewards/rejected": -1.6391723155975342,
      "step": 2010
    },
    {
      "epoch": 0.8668597789936702,
      "grad_norm": 54.51765241834094,
      "learning_rate": 7.833690987124463e-07,
      "logits/chosen": -0.4284210205078125,
      "logits/rejected": -0.4304443299770355,
      "logps/chosen": -362.2749938964844,
      "logps/rejected": -367.2749938964844,
      "loss": 0.5515,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.05325622484087944,
      "rewards/margins": 1.4757080078125,
      "rewards/rejected": -1.5295226573944092,
      "step": 2020
    },
    {
      "epoch": 0.8711511640381934,
      "grad_norm": 138.1342111756548,
      "learning_rate": 7.822961373390557e-07,
      "logits/chosen": -0.40978699922561646,
      "logits/rejected": -0.4723144471645355,
      "logps/chosen": -319.73748779296875,
      "logps/rejected": -332.95001220703125,
      "loss": 0.6562,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.10593261569738388,
      "rewards/margins": 1.3680419921875,
      "rewards/rejected": -1.4743163585662842,
      "step": 2030
    },
    {
      "epoch": 0.8754425490827165,
      "grad_norm": 91.78491639079485,
      "learning_rate": 7.812231759656652e-07,
      "logits/chosen": -0.4817871153354645,
      "logits/rejected": -0.528088390827179,
      "logps/chosen": -396.75,
      "logps/rejected": -370.88751220703125,
      "loss": 0.5513,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.0025268555618822575,
      "rewards/margins": 1.516137719154358,
      "rewards/rejected": -1.5186767578125,
      "step": 2040
    },
    {
      "epoch": 0.8797339341272395,
      "grad_norm": 77.17492445581468,
      "learning_rate": 7.801502145922745e-07,
      "logits/chosen": -0.4148010313510895,
      "logits/rejected": -0.4875854551792145,
      "logps/chosen": -358.92498779296875,
      "logps/rejected": -361.42498779296875,
      "loss": 0.6086,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.23957519233226776,
      "rewards/margins": 1.202734351158142,
      "rewards/rejected": -1.441613793373108,
      "step": 2050
    },
    {
      "epoch": 0.8840253191717626,
      "grad_norm": 135.0296706180856,
      "learning_rate": 7.790772532188841e-07,
      "logits/chosen": -0.47888487577438354,
      "logits/rejected": -0.521099865436554,
      "logps/chosen": -373.7749938964844,
      "logps/rejected": -352.8500061035156,
      "loss": 0.6476,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.2934204041957855,
      "rewards/margins": 1.2010009288787842,
      "rewards/rejected": -1.494726538658142,
      "step": 2060
    },
    {
      "epoch": 0.8883167042162858,
      "grad_norm": 108.33533325666654,
      "learning_rate": 7.780042918454936e-07,
      "logits/chosen": -0.44842529296875,
      "logits/rejected": -0.4959869384765625,
      "logps/chosen": -334.82501220703125,
      "logps/rejected": -344.9750061035156,
      "loss": 0.5891,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.3254150450229645,
      "rewards/margins": 1.216406226158142,
      "rewards/rejected": -1.539819359779358,
      "step": 2070
    },
    {
      "epoch": 0.8926080892608089,
      "grad_norm": 119.50061579026615,
      "learning_rate": 7.76931330472103e-07,
      "logits/chosen": -0.4915771484375,
      "logits/rejected": -0.5180877447128296,
      "logps/chosen": -416.32501220703125,
      "logps/rejected": -357.8500061035156,
      "loss": 0.7027,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.12856444716453552,
      "rewards/margins": 0.9456542730331421,
      "rewards/rejected": -1.075097680091858,
      "step": 2080
    },
    {
      "epoch": 0.896899474305332,
      "grad_norm": 94.56096315743068,
      "learning_rate": 7.758583690987124e-07,
      "logits/chosen": -0.41645509004592896,
      "logits/rejected": -0.44414061307907104,
      "logps/chosen": -364.8500061035156,
      "logps/rejected": -338.7749938964844,
      "loss": 0.608,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": 0.17025145888328552,
      "rewards/margins": 1.036157250404358,
      "rewards/rejected": -0.8646789789199829,
      "step": 2090
    },
    {
      "epoch": 0.9011908593498552,
      "grad_norm": 127.52232658232568,
      "learning_rate": 7.747854077253218e-07,
      "logits/chosen": -0.49885255098342896,
      "logits/rejected": -0.5464843511581421,
      "logps/chosen": -360.8125,
      "logps/rejected": -368.57501220703125,
      "loss": 0.6526,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.13181762397289276,
      "rewards/margins": 1.148290991783142,
      "rewards/rejected": -1.015893578529358,
      "step": 2100
    },
    {
      "epoch": 0.9054822443943783,
      "grad_norm": 92.20078512328853,
      "learning_rate": 7.737124463519313e-07,
      "logits/chosen": -0.5203918218612671,
      "logits/rejected": -0.545550525188446,
      "logps/chosen": -364.67498779296875,
      "logps/rejected": -353.82501220703125,
      "loss": 0.6857,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": 0.07219238579273224,
      "rewards/margins": 0.6644226312637329,
      "rewards/rejected": -0.591845691204071,
      "step": 2110
    },
    {
      "epoch": 0.9097736294389014,
      "grad_norm": 61.334335744281304,
      "learning_rate": 7.726394849785407e-07,
      "logits/chosen": -0.48918455839157104,
      "logits/rejected": -0.563916027545929,
      "logps/chosen": -366.875,
      "logps/rejected": -357.4750061035156,
      "loss": 0.5989,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.024749755859375,
      "rewards/margins": 1.312280297279358,
      "rewards/rejected": -1.3367919921875,
      "step": 2120
    },
    {
      "epoch": 0.9140650144834245,
      "grad_norm": 79.2160070511654,
      "learning_rate": 7.715665236051501e-07,
      "logits/chosen": -0.4615112245082855,
      "logits/rejected": -0.516741931438446,
      "logps/chosen": -358.8374938964844,
      "logps/rejected": -371.32501220703125,
      "loss": 0.5299,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.07028808444738388,
      "rewards/margins": 1.395654320716858,
      "rewards/rejected": -1.4654541015625,
      "step": 2130
    },
    {
      "epoch": 0.9183563995279477,
      "grad_norm": 99.46895603219903,
      "learning_rate": 7.704935622317597e-07,
      "logits/chosen": -0.44951170682907104,
      "logits/rejected": -0.49359130859375,
      "logps/chosen": -306.76873779296875,
      "logps/rejected": -319.4375,
      "loss": 0.6028,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.11284790188074112,
      "rewards/margins": 1.2532470226287842,
      "rewards/rejected": -1.364446997642517,
      "step": 2140
    },
    {
      "epoch": 0.9226477845724708,
      "grad_norm": 101.25106091610708,
      "learning_rate": 7.69420600858369e-07,
      "logits/chosen": -0.5082947015762329,
      "logits/rejected": -0.5229736566543579,
      "logps/chosen": -396.375,
      "logps/rejected": -371.2749938964844,
      "loss": 0.6378,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.20824280381202698,
      "rewards/margins": 1.2848389148712158,
      "rewards/rejected": -1.4934203624725342,
      "step": 2150
    },
    {
      "epoch": 0.9269391696169939,
      "grad_norm": 79.54297809921317,
      "learning_rate": 7.683476394849785e-07,
      "logits/chosen": -0.4613037109375,
      "logits/rejected": -0.4837402403354645,
      "logps/chosen": -338.20001220703125,
      "logps/rejected": -326.7749938964844,
      "loss": 0.6613,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": 0.09599609673023224,
      "rewards/margins": 1.33660888671875,
      "rewards/rejected": -1.240136742591858,
      "step": 2160
    },
    {
      "epoch": 0.931230554661517,
      "grad_norm": 59.944492669875906,
      "learning_rate": 7.67274678111588e-07,
      "logits/chosen": -0.47618407011032104,
      "logits/rejected": -0.518261730670929,
      "logps/chosen": -356.1875,
      "logps/rejected": -334.70001220703125,
      "loss": 0.6136,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.14970092475414276,
      "rewards/margins": 1.50927734375,
      "rewards/rejected": -1.359765648841858,
      "step": 2170
    },
    {
      "epoch": 0.9355219397060401,
      "grad_norm": 115.99307866573162,
      "learning_rate": 7.662017167381974e-07,
      "logits/chosen": -0.5174560546875,
      "logits/rejected": -0.568493664264679,
      "logps/chosen": -359.6000061035156,
      "logps/rejected": -344.51251220703125,
      "loss": 0.59,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.625903308391571,
      "rewards/margins": 1.3516113758087158,
      "rewards/rejected": -0.726550281047821,
      "step": 2180
    },
    {
      "epoch": 0.9398133247505632,
      "grad_norm": 93.0545576745169,
      "learning_rate": 7.651287553648068e-07,
      "logits/chosen": -0.4970703125,
      "logits/rejected": -0.572436511516571,
      "logps/chosen": -389.3500061035156,
      "logps/rejected": -326.7250061035156,
      "loss": 0.5889,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": 0.589404284954071,
      "rewards/margins": 1.3962280750274658,
      "rewards/rejected": -0.80706787109375,
      "step": 2190
    },
    {
      "epoch": 0.9441047097950863,
      "grad_norm": 112.3176831541809,
      "learning_rate": 7.640557939914162e-07,
      "logits/chosen": -0.469583123922348,
      "logits/rejected": -0.4869018495082855,
      "logps/chosen": -389.61248779296875,
      "logps/rejected": -341.45001220703125,
      "loss": 0.5473,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.7899566888809204,
      "rewards/margins": 1.5120117664337158,
      "rewards/rejected": -0.7217773199081421,
      "step": 2200
    },
    {
      "epoch": 0.9483960948396095,
      "grad_norm": 69.84713721388559,
      "learning_rate": 7.629828326180258e-07,
      "logits/chosen": -0.504223644733429,
      "logits/rejected": -0.5489257574081421,
      "logps/chosen": -337.61248779296875,
      "logps/rejected": -321.5249938964844,
      "loss": 0.4556,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": 0.3408203125,
      "rewards/margins": 1.6074097156524658,
      "rewards/rejected": -1.266626000404358,
      "step": 2210
    },
    {
      "epoch": 0.9526874798841326,
      "grad_norm": 92.65696311334176,
      "learning_rate": 7.619098712446352e-07,
      "logits/chosen": -0.46807557344436646,
      "logits/rejected": -0.566113293170929,
      "logps/chosen": -371.13751220703125,
      "logps/rejected": -353.70001220703125,
      "loss": 0.5619,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.577130138874054,
      "rewards/margins": 1.8107421398162842,
      "rewards/rejected": -2.3877930641174316,
      "step": 2220
    },
    {
      "epoch": 0.9569788649286557,
      "grad_norm": 113.47943811133267,
      "learning_rate": 7.608369098712446e-07,
      "logits/chosen": -0.4912170469760895,
      "logits/rejected": -0.5322510004043579,
      "logps/chosen": -370.5,
      "logps/rejected": -378.20001220703125,
      "loss": 0.5809,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.962756335735321,
      "rewards/margins": 1.6916015148162842,
      "rewards/rejected": -2.654296875,
      "step": 2230
    },
    {
      "epoch": 0.9612702499731789,
      "grad_norm": 80.58967592092004,
      "learning_rate": 7.597639484978541e-07,
      "logits/chosen": -0.3969230651855469,
      "logits/rejected": -0.4379516541957855,
      "logps/chosen": -355.4624938964844,
      "logps/rejected": -356.5,
      "loss": 0.5291,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.7849975824356079,
      "rewards/margins": 1.7169921398162842,
      "rewards/rejected": -2.504565477371216,
      "step": 2240
    },
    {
      "epoch": 0.965561635017702,
      "grad_norm": 97.03344726050717,
      "learning_rate": 7.586909871244634e-07,
      "logits/chosen": -0.5103515386581421,
      "logits/rejected": -0.5685790777206421,
      "logps/chosen": -361.4750061035156,
      "logps/rejected": -376.1499938964844,
      "loss": 0.5653,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.8719482421875,
      "rewards/margins": 1.3838989734649658,
      "rewards/rejected": -2.2578492164611816,
      "step": 2250
    },
    {
      "epoch": 0.9698530200622251,
      "grad_norm": 88.956038779029,
      "learning_rate": 7.576180257510729e-07,
      "logits/chosen": -0.48927003145217896,
      "logits/rejected": -0.5119994878768921,
      "logps/chosen": -346.35626220703125,
      "logps/rejected": -337.32501220703125,
      "loss": 0.7,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.8078247308731079,
      "rewards/margins": 1.037023901939392,
      "rewards/rejected": -1.846215844154358,
      "step": 2260
    },
    {
      "epoch": 0.9741444051067482,
      "grad_norm": 124.32061280539118,
      "learning_rate": 7.565450643776824e-07,
      "logits/chosen": -0.46381837129592896,
      "logits/rejected": -0.511340320110321,
      "logps/chosen": -397.375,
      "logps/rejected": -395.95001220703125,
      "loss": 0.6108,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.768206775188446,
      "rewards/margins": 1.4190185070037842,
      "rewards/rejected": -2.1888184547424316,
      "step": 2270
    },
    {
      "epoch": 0.9784357901512714,
      "grad_norm": 100.19724839414987,
      "learning_rate": 7.554721030042917e-07,
      "logits/chosen": -0.518725574016571,
      "logits/rejected": -0.5394287109375,
      "logps/chosen": -377.625,
      "logps/rejected": -361.1625061035156,
      "loss": 0.5221,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.678759753704071,
      "rewards/margins": 1.383544921875,
      "rewards/rejected": -2.0615234375,
      "step": 2280
    },
    {
      "epoch": 0.9827271751957944,
      "grad_norm": 69.13476580861254,
      "learning_rate": 7.543991416309013e-07,
      "logits/chosen": -0.46104735136032104,
      "logits/rejected": -0.5124450922012329,
      "logps/chosen": -357.625,
      "logps/rejected": -345.4125061035156,
      "loss": 0.6403,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.6890014410018921,
      "rewards/margins": 1.289648413658142,
      "rewards/rejected": -1.977746605873108,
      "step": 2290
    },
    {
      "epoch": 0.9870185602403175,
      "grad_norm": 130.45723858358505,
      "learning_rate": 7.533261802575107e-07,
      "logits/chosen": -0.530078113079071,
      "logits/rejected": -0.5547851324081421,
      "logps/chosen": -397.5,
      "logps/rejected": -366.70001220703125,
      "loss": 0.576,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.22761230170726776,
      "rewards/margins": 1.387853980064392,
      "rewards/rejected": -1.6165587902069092,
      "step": 2300
    },
    {
      "epoch": 0.9913099452848407,
      "grad_norm": 99.00884454952215,
      "learning_rate": 7.522532188841202e-07,
      "logits/chosen": -0.5212036371231079,
      "logits/rejected": -0.5637451410293579,
      "logps/chosen": -342.1499938964844,
      "logps/rejected": -327.17498779296875,
      "loss": 0.5438,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.2813476622104645,
      "rewards/margins": 1.3585937023162842,
      "rewards/rejected": -1.6410033702850342,
      "step": 2310
    },
    {
      "epoch": 0.9956013303293638,
      "grad_norm": 128.79776121372566,
      "learning_rate": 7.511802575107296e-07,
      "logits/chosen": -0.5853271484375,
      "logits/rejected": -0.5942627191543579,
      "logps/chosen": -374.32501220703125,
      "logps/rejected": -333.9750061035156,
      "loss": 0.5645,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.25792235136032104,
      "rewards/margins": 1.2442138195037842,
      "rewards/rejected": -1.5037353038787842,
      "step": 2320
    },
    {
      "epoch": 0.9998927153738869,
      "grad_norm": 91.00148901713771,
      "learning_rate": 7.50107296137339e-07,
      "logits/chosen": -0.536328136920929,
      "logits/rejected": -0.5860840082168579,
      "logps/chosen": -390.6000061035156,
      "logps/rejected": -365.29998779296875,
      "loss": 0.6829,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.3775573670864105,
      "rewards/margins": 1.2346923351287842,
      "rewards/rejected": -1.6121337413787842,
      "step": 2330
    },
    {
      "epoch": 1.0038622465400708,
      "grad_norm": 4.621264339534561,
      "learning_rate": 7.490343347639485e-07,
      "logits/chosen": -0.48371514678001404,
      "logits/rejected": -0.5052523016929626,
      "logps/chosen": -355.5,
      "logps/rejected": -418.7837829589844,
      "loss": 0.0335,
      "rewards/accuracies": 0.9864864945411682,
      "rewards/chosen": 2.5475611686706543,
      "rewards/margins": 7.206925868988037,
      "rewards/rejected": -4.657094478607178,
      "step": 2340
    },
    {
      "epoch": 1.008153631584594,
      "grad_norm": 0.9073196202676276,
      "learning_rate": 7.479613733905578e-07,
      "logits/chosen": -0.5647217035293579,
      "logits/rejected": -0.612353503704071,
      "logps/chosen": -368.5,
      "logps/rejected": -420.5,
      "loss": 0.0346,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": 1.9235718250274658,
      "rewards/margins": 7.966796875,
      "rewards/rejected": -6.041406154632568,
      "step": 2350
    },
    {
      "epoch": 1.012445016629117,
      "grad_norm": 1.0015719207867788,
      "learning_rate": 7.468884120171673e-07,
      "logits/chosen": -0.5823119878768921,
      "logits/rejected": -0.608288586139679,
      "logps/chosen": -339.4624938964844,
      "logps/rejected": -381.1499938964844,
      "loss": 0.0433,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 2.1738572120666504,
      "rewards/margins": 8.417187690734863,
      "rewards/rejected": -6.245312690734863,
      "step": 2360
    },
    {
      "epoch": 1.0167364016736402,
      "grad_norm": 13.85385602579631,
      "learning_rate": 7.458154506437769e-07,
      "logits/chosen": -0.668505847454071,
      "logits/rejected": -0.6723388433456421,
      "logps/chosen": -391.6499938964844,
      "logps/rejected": -438.3999938964844,
      "loss": 0.0606,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 1.41058349609375,
      "rewards/margins": 8.915234565734863,
      "rewards/rejected": -7.505859375,
      "step": 2370
    },
    {
      "epoch": 1.0210277867181632,
      "grad_norm": 5.570589963103772,
      "learning_rate": 7.447424892703862e-07,
      "logits/chosen": -0.655468761920929,
      "logits/rejected": -0.694384753704071,
      "logps/chosen": -377.4750061035156,
      "logps/rejected": -426.3500061035156,
      "loss": 0.0413,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 1.406591773033142,
      "rewards/margins": 9.630468368530273,
      "rewards/rejected": -8.219141006469727,
      "step": 2380
    },
    {
      "epoch": 1.0253191717626864,
      "grad_norm": 9.930833208056239,
      "learning_rate": 7.436695278969957e-07,
      "logits/chosen": -0.7109375,
      "logits/rejected": -0.726855456829071,
      "logps/chosen": -364.1000061035156,
      "logps/rejected": -416.32501220703125,
      "loss": 0.0589,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": 0.509033203125,
      "rewards/margins": 9.190625190734863,
      "rewards/rejected": -8.672656059265137,
      "step": 2390
    },
    {
      "epoch": 1.0296105568072096,
      "grad_norm": 41.13654226096193,
      "learning_rate": 7.425965665236051e-07,
      "logits/chosen": -0.733203113079071,
      "logits/rejected": -0.800097644329071,
      "logps/chosen": -365.1499938964844,
      "logps/rejected": -417.95001220703125,
      "loss": 0.075,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 0.7457641363143921,
      "rewards/margins": 8.888671875,
      "rewards/rejected": -8.137890815734863,
      "step": 2400
    },
    {
      "epoch": 1.0339019418517326,
      "grad_norm": 1.104155581549493,
      "learning_rate": 7.415236051502145e-07,
      "logits/chosen": -0.6716552972793579,
      "logits/rejected": -0.704174816608429,
      "logps/chosen": -339.875,
      "logps/rejected": -439.4624938964844,
      "loss": 0.0258,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": 1.308508276939392,
      "rewards/margins": 9.708984375,
      "rewards/rejected": -8.400390625,
      "step": 2410
    },
    {
      "epoch": 1.0381933268962558,
      "grad_norm": 26.46691059125149,
      "learning_rate": 7.40450643776824e-07,
      "logits/chosen": -0.76165771484375,
      "logits/rejected": -0.8182128667831421,
      "logps/chosen": -372.82501220703125,
      "logps/rejected": -425.5249938964844,
      "loss": 0.048,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 0.5227111577987671,
      "rewards/margins": 9.0546875,
      "rewards/rejected": -8.536718368530273,
      "step": 2420
    },
    {
      "epoch": 1.042484711940779,
      "grad_norm": 3.660143258678806,
      "learning_rate": 7.393776824034334e-07,
      "logits/chosen": -0.663159191608429,
      "logits/rejected": -0.6626983880996704,
      "logps/chosen": -371.0375061035156,
      "logps/rejected": -435.79998779296875,
      "loss": 0.0334,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": 0.49080199003219604,
      "rewards/margins": 9.974218368530273,
      "rewards/rejected": -9.483593940734863,
      "step": 2430
    },
    {
      "epoch": 1.046776096985302,
      "grad_norm": 16.94590830302066,
      "learning_rate": 7.38304721030043e-07,
      "logits/chosen": -0.7198852300643921,
      "logits/rejected": -0.743847668170929,
      "logps/chosen": -380.01251220703125,
      "logps/rejected": -457.1499938964844,
      "loss": 0.0944,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.004931640811264515,
      "rewards/margins": 10.082812309265137,
      "rewards/rejected": -10.091405868530273,
      "step": 2440
    },
    {
      "epoch": 1.0510674820298251,
      "grad_norm": 3.835280816722687,
      "learning_rate": 7.372317596566523e-07,
      "logits/chosen": -0.703417956829071,
      "logits/rejected": -0.734082043170929,
      "logps/chosen": -337.1625061035156,
      "logps/rejected": -419.5,
      "loss": 0.0361,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.538134753704071,
      "rewards/margins": 9.961718559265137,
      "rewards/rejected": -9.419530868530273,
      "step": 2450
    },
    {
      "epoch": 1.0553588670743483,
      "grad_norm": 10.659711615249932,
      "learning_rate": 7.361587982832618e-07,
      "logits/chosen": -0.7720702886581421,
      "logits/rejected": -0.8095947504043579,
      "logps/chosen": -367.1000061035156,
      "logps/rejected": -464.45001220703125,
      "loss": 0.0267,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -1.0674927234649658,
      "rewards/margins": 11.157031059265137,
      "rewards/rejected": -12.225781440734863,
      "step": 2460
    },
    {
      "epoch": 1.0596502521188713,
      "grad_norm": 56.6233849490534,
      "learning_rate": 7.350858369098713e-07,
      "logits/chosen": -0.8584960699081421,
      "logits/rejected": -0.881787121295929,
      "logps/chosen": -401.8999938964844,
      "logps/rejected": -454.8500061035156,
      "loss": 0.0473,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -1.2240722179412842,
      "rewards/margins": 10.826562881469727,
      "rewards/rejected": -12.045312881469727,
      "step": 2470
    },
    {
      "epoch": 1.0639416371633945,
      "grad_norm": 2.250184231795305,
      "learning_rate": 7.340128755364806e-07,
      "logits/chosen": -0.8057616949081421,
      "logits/rejected": -0.811328113079071,
      "logps/chosen": -404.04998779296875,
      "logps/rejected": -483.6499938964844,
      "loss": 0.0411,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.6454101800918579,
      "rewards/margins": 10.669921875,
      "rewards/rejected": -11.316015243530273,
      "step": 2480
    },
    {
      "epoch": 1.0682330222079175,
      "grad_norm": 32.256317713873734,
      "learning_rate": 7.329399141630901e-07,
      "logits/chosen": -0.677734375,
      "logits/rejected": -0.6978210210800171,
      "logps/chosen": -384.125,
      "logps/rejected": -435.79998779296875,
      "loss": 0.0614,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.56732177734375,
      "rewards/margins": 10.059374809265137,
      "rewards/rejected": -10.625391006469727,
      "step": 2490
    },
    {
      "epoch": 1.0725244072524407,
      "grad_norm": 14.208372299625701,
      "learning_rate": 7.318669527896995e-07,
      "logits/chosen": -0.7776855230331421,
      "logits/rejected": -0.8174804449081421,
      "logps/chosen": -392.67498779296875,
      "logps/rejected": -452.82501220703125,
      "loss": 0.0166,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -0.559033215045929,
      "rewards/margins": 10.008593559265137,
      "rewards/rejected": -10.560155868530273,
      "step": 2500
    },
    {
      "epoch": 1.076815792296964,
      "grad_norm": 56.13443787511979,
      "learning_rate": 7.307939914163089e-07,
      "logits/chosen": -0.7345947027206421,
      "logits/rejected": -0.7589111328125,
      "logps/chosen": -356.2749938964844,
      "logps/rejected": -463.375,
      "loss": 0.0297,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -0.47059327363967896,
      "rewards/margins": 11.346094131469727,
      "rewards/rejected": -11.814844131469727,
      "step": 2510
    },
    {
      "epoch": 1.0811071773414869,
      "grad_norm": 0.7670041871093395,
      "learning_rate": 7.297210300429184e-07,
      "logits/chosen": -0.759960949420929,
      "logits/rejected": -0.7905517816543579,
      "logps/chosen": -404.04998779296875,
      "logps/rejected": -461.25,
      "loss": 0.0157,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09202270209789276,
      "rewards/margins": 10.5234375,
      "rewards/rejected": -10.622655868530273,
      "step": 2520
    },
    {
      "epoch": 1.08539856238601,
      "grad_norm": 32.136216847856105,
      "learning_rate": 7.286480686695279e-07,
      "logits/chosen": -0.83544921875,
      "logits/rejected": -0.8470703363418579,
      "logps/chosen": -391.5874938964844,
      "logps/rejected": -446.0,
      "loss": 0.0428,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.1182861328125,
      "rewards/margins": 9.594531059265137,
      "rewards/rejected": -9.71484375,
      "step": 2530
    },
    {
      "epoch": 1.0896899474305333,
      "grad_norm": 24.666337072380923,
      "learning_rate": 7.275751072961374e-07,
      "logits/chosen": -0.6993652582168579,
      "logits/rejected": -0.7181152105331421,
      "logps/chosen": -401.79998779296875,
      "logps/rejected": -431.79998779296875,
      "loss": 0.0965,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -0.25605469942092896,
      "rewards/margins": 9.896875381469727,
      "rewards/rejected": -10.151562690734863,
      "step": 2540
    },
    {
      "epoch": 1.0939813324750562,
      "grad_norm": 34.22705197223228,
      "learning_rate": 7.265021459227467e-07,
      "logits/chosen": -0.7691650390625,
      "logits/rejected": -0.7947998046875,
      "logps/chosen": -367.95001220703125,
      "logps/rejected": -439.1499938964844,
      "loss": 0.0501,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.16741943359375,
      "rewards/margins": 9.961718559265137,
      "rewards/rejected": -9.801562309265137,
      "step": 2550
    },
    {
      "epoch": 1.0982727175195794,
      "grad_norm": 0.9318919309568936,
      "learning_rate": 7.254291845493562e-07,
      "logits/chosen": -0.7223876714706421,
      "logits/rejected": -0.7237182855606079,
      "logps/chosen": -371.9125061035156,
      "logps/rejected": -426.875,
      "loss": 0.0725,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.3539977967739105,
      "rewards/margins": 9.237109184265137,
      "rewards/rejected": -9.59375,
      "step": 2560
    },
    {
      "epoch": 1.1025641025641026,
      "grad_norm": 13.10894288906175,
      "learning_rate": 7.243562231759657e-07,
      "logits/chosen": -0.753588855266571,
      "logits/rejected": -0.758007824420929,
      "logps/chosen": -381.2250061035156,
      "logps/rejected": -431.54998779296875,
      "loss": 0.0484,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": 0.561230480670929,
      "rewards/margins": 9.284375190734863,
      "rewards/rejected": -8.720703125,
      "step": 2570
    },
    {
      "epoch": 1.1068554876086256,
      "grad_norm": 2.7797157178374077,
      "learning_rate": 7.23283261802575e-07,
      "logits/chosen": -0.760424792766571,
      "logits/rejected": -0.8070312738418579,
      "logps/chosen": -382.125,
      "logps/rejected": -431.8999938964844,
      "loss": 0.0269,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": 0.2987060546875,
      "rewards/margins": 9.362500190734863,
      "rewards/rejected": -9.059765815734863,
      "step": 2580
    },
    {
      "epoch": 1.1111468726531488,
      "grad_norm": 40.30443027517919,
      "learning_rate": 7.222103004291845e-07,
      "logits/chosen": -0.780566394329071,
      "logits/rejected": -0.824511706829071,
      "logps/chosen": -368.9750061035156,
      "logps/rejected": -463.79998779296875,
      "loss": 0.0367,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -0.04778594896197319,
      "rewards/margins": 9.537890434265137,
      "rewards/rejected": -9.589062690734863,
      "step": 2590
    },
    {
      "epoch": 1.115438257697672,
      "grad_norm": 2.91081604718025,
      "learning_rate": 7.21137339055794e-07,
      "logits/chosen": -0.7046874761581421,
      "logits/rejected": -0.7578369379043579,
      "logps/chosen": -374.54998779296875,
      "logps/rejected": -418.57501220703125,
      "loss": 0.0397,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.2928222715854645,
      "rewards/margins": 9.178906440734863,
      "rewards/rejected": -9.471875190734863,
      "step": 2600
    },
    {
      "epoch": 1.119729642742195,
      "grad_norm": 2.302094926567975,
      "learning_rate": 7.200643776824034e-07,
      "logits/chosen": -0.687207043170929,
      "logits/rejected": -0.7332519292831421,
      "logps/chosen": -363.75,
      "logps/rejected": -423.75,
      "loss": 0.0252,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": 0.654296875,
      "rewards/margins": 9.546093940734863,
      "rewards/rejected": -8.88671875,
      "step": 2610
    },
    {
      "epoch": 1.1240210277867182,
      "grad_norm": 7.734768592289183,
      "learning_rate": 7.189914163090129e-07,
      "logits/chosen": -0.7929443120956421,
      "logits/rejected": -0.801708996295929,
      "logps/chosen": -345.3500061035156,
      "logps/rejected": -417.5,
      "loss": 0.0188,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": 0.852954089641571,
      "rewards/margins": 10.467187881469727,
      "rewards/rejected": -9.613672256469727,
      "step": 2620
    },
    {
      "epoch": 1.1283124128312414,
      "grad_norm": 14.90668423249433,
      "learning_rate": 7.179184549356223e-07,
      "logits/chosen": -0.836474597454071,
      "logits/rejected": -0.8541504144668579,
      "logps/chosen": -360.2749938964844,
      "logps/rejected": -462.9750061035156,
      "loss": 0.0247,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -0.9029785394668579,
      "rewards/margins": 11.117968559265137,
      "rewards/rejected": -12.0078125,
      "step": 2630
    },
    {
      "epoch": 1.1326037978757644,
      "grad_norm": 13.572432917845044,
      "learning_rate": 7.168454935622317e-07,
      "logits/chosen": -0.8270508050918579,
      "logits/rejected": -0.8109375238418579,
      "logps/chosen": -395.67498779296875,
      "logps/rejected": -472.67498779296875,
      "loss": 0.0707,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.8949340581893921,
      "rewards/margins": 10.668749809265137,
      "rewards/rejected": -11.560937881469727,
      "step": 2640
    },
    {
      "epoch": 1.1368951829202876,
      "grad_norm": 2.3916939826930452,
      "learning_rate": 7.157725321888411e-07,
      "logits/chosen": -0.758618175983429,
      "logits/rejected": -0.784423828125,
      "logps/chosen": -371.45001220703125,
      "logps/rejected": -457.7250061035156,
      "loss": 0.0294,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -0.5729125738143921,
      "rewards/margins": 10.409375190734863,
      "rewards/rejected": -10.973437309265137,
      "step": 2650
    },
    {
      "epoch": 1.1411865679648105,
      "grad_norm": 24.57237073545116,
      "learning_rate": 7.146995708154506e-07,
      "logits/chosen": -0.83740234375,
      "logits/rejected": -0.8505859375,
      "logps/chosen": -376.79998779296875,
      "logps/rejected": -433.07501220703125,
      "loss": 0.0265,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -0.10926513373851776,
      "rewards/margins": 10.55859375,
      "rewards/rejected": -10.669530868530273,
      "step": 2660
    },
    {
      "epoch": 1.1454779530093337,
      "grad_norm": 12.976040034990163,
      "learning_rate": 7.136266094420601e-07,
      "logits/chosen": -0.801525890827179,
      "logits/rejected": -0.822338879108429,
      "logps/chosen": -344.45001220703125,
      "logps/rejected": -405.79998779296875,
      "loss": 0.0549,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -0.20175781846046448,
      "rewards/margins": 9.954687118530273,
      "rewards/rejected": -10.162500381469727,
      "step": 2670
    },
    {
      "epoch": 1.149769338053857,
      "grad_norm": 21.156465403687218,
      "learning_rate": 7.125536480686694e-07,
      "logits/chosen": -0.8708251714706421,
      "logits/rejected": -0.8871093988418579,
      "logps/chosen": -357.32501220703125,
      "logps/rejected": -441.32501220703125,
      "loss": 0.0365,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -1.3420898914337158,
      "rewards/margins": 10.209375381469727,
      "rewards/rejected": -11.5546875,
      "step": 2680
    },
    {
      "epoch": 1.15406072309838,
      "grad_norm": 23.06986252897006,
      "learning_rate": 7.11480686695279e-07,
      "logits/chosen": -0.8985351324081421,
      "logits/rejected": -0.9410156011581421,
      "logps/chosen": -357.17498779296875,
      "logps/rejected": -445.2749938964844,
      "loss": 0.0467,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.244421362876892,
      "rewards/margins": 10.967187881469727,
      "rewards/rejected": -12.209375381469727,
      "step": 2690
    },
    {
      "epoch": 1.1583521081429031,
      "grad_norm": 29.63571617594808,
      "learning_rate": 7.104077253218884e-07,
      "logits/chosen": -0.806121826171875,
      "logits/rejected": -0.8232666254043579,
      "logps/chosen": -328.04998779296875,
      "logps/rejected": -427.32501220703125,
      "loss": 0.059,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.07687988132238388,
      "rewards/margins": 9.431249618530273,
      "rewards/rejected": -9.350000381469727,
      "step": 2700
    },
    {
      "epoch": 1.1626434931874263,
      "grad_norm": 3.533641603488294,
      "learning_rate": 7.093347639484978e-07,
      "logits/chosen": -0.7093261480331421,
      "logits/rejected": -0.755688488483429,
      "logps/chosen": -367.9750061035156,
      "logps/rejected": -465.70001220703125,
      "loss": 0.0668,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.6290038824081421,
      "rewards/margins": 11.038281440734863,
      "rewards/rejected": -11.672266006469727,
      "step": 2710
    },
    {
      "epoch": 1.1669348782319493,
      "grad_norm": 22.985807536024577,
      "learning_rate": 7.082618025751073e-07,
      "logits/chosen": -0.7316039800643921,
      "logits/rejected": -0.7901366949081421,
      "logps/chosen": -362.3500061035156,
      "logps/rejected": -455.7250061035156,
      "loss": 0.0454,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.3712524473667145,
      "rewards/margins": 10.564844131469727,
      "rewards/rejected": -10.193750381469727,
      "step": 2720
    },
    {
      "epoch": 1.1712262632764725,
      "grad_norm": 7.632039371329289,
      "learning_rate": 7.071888412017167e-07,
      "logits/chosen": -0.7713378667831421,
      "logits/rejected": -0.841259777545929,
      "logps/chosen": -341.63751220703125,
      "logps/rejected": -420.32501220703125,
      "loss": 0.0352,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.0281982421875,
      "rewards/margins": 9.876562118530273,
      "rewards/rejected": -9.853906631469727,
      "step": 2730
    },
    {
      "epoch": 1.1755176483209957,
      "grad_norm": 39.23964632567033,
      "learning_rate": 7.061158798283261e-07,
      "logits/chosen": -0.7262207269668579,
      "logits/rejected": -0.739184558391571,
      "logps/chosen": -312.9750061035156,
      "logps/rejected": -401.9750061035156,
      "loss": 0.0599,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.04046630859375,
      "rewards/margins": 9.716796875,
      "rewards/rejected": -9.757421493530273,
      "step": 2740
    },
    {
      "epoch": 1.1798090333655187,
      "grad_norm": 0.3362127037141487,
      "learning_rate": 7.050429184549355e-07,
      "logits/chosen": -0.7503417730331421,
      "logits/rejected": -0.8472656011581421,
      "logps/chosen": -337.8999938964844,
      "logps/rejected": -438.82501220703125,
      "loss": 0.0379,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -1.022039771080017,
      "rewards/margins": 10.184374809265137,
      "rewards/rejected": -11.204687118530273,
      "step": 2750
    },
    {
      "epoch": 1.1841004184100419,
      "grad_norm": 4.074310779489148,
      "learning_rate": 7.03969957081545e-07,
      "logits/chosen": -0.7583984136581421,
      "logits/rejected": -0.826367199420929,
      "logps/chosen": -373.38751220703125,
      "logps/rejected": -436.4750061035156,
      "loss": 0.0284,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -0.9810546636581421,
      "rewards/margins": 11.284765243530273,
      "rewards/rejected": -12.265625,
      "step": 2760
    },
    {
      "epoch": 1.1883918034545649,
      "grad_norm": 0.877756042896416,
      "learning_rate": 7.028969957081546e-07,
      "logits/chosen": -0.8500000238418579,
      "logits/rejected": -0.886425793170929,
      "logps/chosen": -390.92498779296875,
      "logps/rejected": -472.9750061035156,
      "loss": 0.0278,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.941174328327179,
      "rewards/margins": 11.620312690734863,
      "rewards/rejected": -12.567187309265137,
      "step": 2770
    },
    {
      "epoch": 1.192683188499088,
      "grad_norm": 26.86733553765975,
      "learning_rate": 7.018240343347639e-07,
      "logits/chosen": -0.8299316167831421,
      "logits/rejected": -0.893872082233429,
      "logps/chosen": -350.4125061035156,
      "logps/rejected": -431.0249938964844,
      "loss": 0.052,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -1.0543701648712158,
      "rewards/margins": 10.25,
      "rewards/rejected": -11.300000190734863,
      "step": 2780
    },
    {
      "epoch": 1.1969745735436113,
      "grad_norm": 7.857861314661304,
      "learning_rate": 7.007510729613734e-07,
      "logits/chosen": -0.8516601324081421,
      "logits/rejected": -0.8866211175918579,
      "logps/chosen": -349.23748779296875,
      "logps/rejected": -412.4750061035156,
      "loss": 0.0736,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.607196033000946,
      "rewards/margins": 9.149999618530273,
      "rewards/rejected": -9.756250381469727,
      "step": 2790
    },
    {
      "epoch": 1.2012659585881342,
      "grad_norm": 0.5826627106052477,
      "learning_rate": 6.996781115879828e-07,
      "logits/chosen": -0.841796875,
      "logits/rejected": -0.8597656488418579,
      "logps/chosen": -314.7250061035156,
      "logps/rejected": -390.5249938964844,
      "loss": 0.0503,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 0.20391845703125,
      "rewards/margins": 9.140625,
      "rewards/rejected": -8.932812690734863,
      "step": 2800
    },
    {
      "epoch": 1.2055573436326574,
      "grad_norm": 26.811612128313687,
      "learning_rate": 6.986051502145922e-07,
      "logits/chosen": -0.888720691204071,
      "logits/rejected": -0.911083996295929,
      "logps/chosen": -356.6000061035156,
      "logps/rejected": -426.61248779296875,
      "loss": 0.078,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.18930664658546448,
      "rewards/margins": 9.575780868530273,
      "rewards/rejected": -9.387890815734863,
      "step": 2810
    },
    {
      "epoch": 1.2098487286771806,
      "grad_norm": 4.121814312645254,
      "learning_rate": 6.975321888412017e-07,
      "logits/chosen": -0.8480468988418579,
      "logits/rejected": -0.8599914312362671,
      "logps/chosen": -387.5249938964844,
      "logps/rejected": -451.125,
      "loss": 0.0353,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.3008178770542145,
      "rewards/margins": 10.2890625,
      "rewards/rejected": -9.989843368530273,
      "step": 2820
    },
    {
      "epoch": 1.2141401137217036,
      "grad_norm": 4.5248474916663675,
      "learning_rate": 6.964592274678111e-07,
      "logits/chosen": -0.8492676019668579,
      "logits/rejected": -0.8797851800918579,
      "logps/chosen": -394.7250061035156,
      "logps/rejected": -451.4750061035156,
      "loss": 0.0225,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -0.06778259575366974,
      "rewards/margins": 10.164843559265137,
      "rewards/rejected": -10.228906631469727,
      "step": 2830
    },
    {
      "epoch": 1.2184314987662268,
      "grad_norm": 5.541312160188442,
      "learning_rate": 6.953862660944206e-07,
      "logits/chosen": -0.841796875,
      "logits/rejected": -0.866406261920929,
      "logps/chosen": -361.45001220703125,
      "logps/rejected": -453.0249938964844,
      "loss": 0.02,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": 0.06345214694738388,
      "rewards/margins": 11.374218940734863,
      "rewards/rejected": -11.317187309265137,
      "step": 2840
    },
    {
      "epoch": 1.22272288381075,
      "grad_norm": 26.377008401638317,
      "learning_rate": 6.9431330472103e-07,
      "logits/chosen": -0.931640625,
      "logits/rejected": -0.928515613079071,
      "logps/chosen": -437.8500061035156,
      "logps/rejected": -502.75,
      "loss": 0.017,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.22867432236671448,
      "rewards/margins": 11.522656440734863,
      "rewards/rejected": -11.743749618530273,
      "step": 2850
    },
    {
      "epoch": 1.227014268855273,
      "grad_norm": 6.870812485403802,
      "learning_rate": 6.932403433476395e-07,
      "logits/chosen": -0.9039551019668579,
      "logits/rejected": -0.937548816204071,
      "logps/chosen": -363.34375,
      "logps/rejected": -436.57501220703125,
      "loss": 0.0165,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.0352540016174316,
      "rewards/margins": 11.083593368530273,
      "rewards/rejected": -13.119531631469727,
      "step": 2860
    },
    {
      "epoch": 1.2313056538997962,
      "grad_norm": 87.44948971323274,
      "learning_rate": 6.921673819742489e-07,
      "logits/chosen": -0.8780273199081421,
      "logits/rejected": -0.889208972454071,
      "logps/chosen": -390.54998779296875,
      "logps/rejected": -473.8999938964844,
      "loss": 0.0812,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.065527319908142,
      "rewards/margins": 11.532812118530273,
      "rewards/rejected": -12.608593940734863,
      "step": 2870
    },
    {
      "epoch": 1.2355970389443192,
      "grad_norm": 22.858618765707323,
      "learning_rate": 6.910944206008583e-07,
      "logits/chosen": -0.8177734613418579,
      "logits/rejected": -0.817578136920929,
      "logps/chosen": -357.6000061035156,
      "logps/rejected": -454.625,
      "loss": 0.0265,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -0.5521606206893921,
      "rewards/margins": 11.212499618530273,
      "rewards/rejected": -11.760937690734863,
      "step": 2880
    },
    {
      "epoch": 1.2398884239888424,
      "grad_norm": 3.405664659904639,
      "learning_rate": 6.900214592274678e-07,
      "logits/chosen": -0.8048340082168579,
      "logits/rejected": -0.8421386480331421,
      "logps/chosen": -355.9375,
      "logps/rejected": -461.82501220703125,
      "loss": 0.0509,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.609814465045929,
      "rewards/margins": 11.509765625,
      "rewards/rejected": -12.110937118530273,
      "step": 2890
    },
    {
      "epoch": 1.2441798090333656,
      "grad_norm": 37.88277843642416,
      "learning_rate": 6.889484978540772e-07,
      "logits/chosen": -0.8687988519668579,
      "logits/rejected": -0.910351574420929,
      "logps/chosen": -385.32501220703125,
      "logps/rejected": -450.70001220703125,
      "loss": 0.0426,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.856909155845642,
      "rewards/margins": 10.971875190734863,
      "rewards/rejected": -12.822656631469727,
      "step": 2900
    },
    {
      "epoch": 1.2484711940778888,
      "grad_norm": 44.66449596518931,
      "learning_rate": 6.878755364806866e-07,
      "logits/chosen": -0.89892578125,
      "logits/rejected": -0.953320324420929,
      "logps/chosen": -368.20001220703125,
      "logps/rejected": -439.25,
      "loss": 0.0538,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -2.863537549972534,
      "rewards/margins": 11.170312881469727,
      "rewards/rejected": -14.048437118530273,
      "step": 2910
    },
    {
      "epoch": 1.2527625791224117,
      "grad_norm": 1.6280211293743867,
      "learning_rate": 6.868025751072962e-07,
      "logits/chosen": -0.8168090581893921,
      "logits/rejected": -0.8503051996231079,
      "logps/chosen": -364.5375061035156,
      "logps/rejected": -471.04998779296875,
      "loss": 0.0899,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -2.455078125,
      "rewards/margins": 10.997655868530273,
      "rewards/rejected": -13.453125,
      "step": 2920
    },
    {
      "epoch": 1.257053964166935,
      "grad_norm": 40.97709466941475,
      "learning_rate": 6.857296137339056e-07,
      "logits/chosen": -0.8005126714706421,
      "logits/rejected": -0.830078125,
      "logps/chosen": -403.0375061035156,
      "logps/rejected": -475.95001220703125,
      "loss": 0.0342,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.4419463872909546,
      "rewards/margins": 11.62109375,
      "rewards/rejected": -13.057031631469727,
      "step": 2930
    },
    {
      "epoch": 1.261345349211458,
      "grad_norm": 2.715054569843987,
      "learning_rate": 6.84656652360515e-07,
      "logits/chosen": -0.7897888422012329,
      "logits/rejected": -0.8424316644668579,
      "logps/chosen": -382.54998779296875,
      "logps/rejected": -462.29998779296875,
      "loss": 0.0314,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -1.0203368663787842,
      "rewards/margins": 11.227343559265137,
      "rewards/rejected": -12.250781059265137,
      "step": 2940
    },
    {
      "epoch": 1.265636734255981,
      "grad_norm": 1.910852165802857,
      "learning_rate": 6.835836909871244e-07,
      "logits/chosen": -0.8514159917831421,
      "logits/rejected": -0.892333984375,
      "logps/chosen": -379.1000061035156,
      "logps/rejected": -437.8999938964844,
      "loss": 0.0381,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -1.2767517566680908,
      "rewards/margins": 10.62109375,
      "rewards/rejected": -11.896093368530273,
      "step": 2950
    },
    {
      "epoch": 1.2699281193005043,
      "grad_norm": 263.47155980737926,
      "learning_rate": 6.825107296137339e-07,
      "logits/chosen": -0.870880126953125,
      "logits/rejected": -0.881787121295929,
      "logps/chosen": -404.375,
      "logps/rejected": -486.82501220703125,
      "loss": 0.0672,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.812725841999054,
      "rewards/margins": 11.595312118530273,
      "rewards/rejected": -12.4140625,
      "step": 2960
    },
    {
      "epoch": 1.2742195043450273,
      "grad_norm": 2.6453361628214336,
      "learning_rate": 6.814377682403433e-07,
      "logits/chosen": -0.7888427972793579,
      "logits/rejected": -0.8486083745956421,
      "logps/chosen": -358.7749938964844,
      "logps/rejected": -451.125,
      "loss": 0.047,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.951403796672821,
      "rewards/margins": 10.165624618530273,
      "rewards/rejected": -11.111719131469727,
      "step": 2970
    },
    {
      "epoch": 1.2785108893895505,
      "grad_norm": 40.41025891003457,
      "learning_rate": 6.803648068669527e-07,
      "logits/chosen": -0.8348144292831421,
      "logits/rejected": -0.9014648199081421,
      "logps/chosen": -410.2875061035156,
      "logps/rejected": -471.2250061035156,
      "loss": 0.0277,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -0.4649108946323395,
      "rewards/margins": 11.67578125,
      "rewards/rejected": -12.141406059265137,
      "step": 2980
    },
    {
      "epoch": 1.2828022744340735,
      "grad_norm": 36.50133661236316,
      "learning_rate": 6.792918454935622e-07,
      "logits/chosen": -0.8729492425918579,
      "logits/rejected": -0.9267578125,
      "logps/chosen": -335.4750061035156,
      "logps/rejected": -458.2749938964844,
      "loss": 0.0395,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.5681518316268921,
      "rewards/margins": 10.342187881469727,
      "rewards/rejected": -10.915624618530273,
      "step": 2990
    },
    {
      "epoch": 1.2870936594785967,
      "grad_norm": 7.674718113692602,
      "learning_rate": 6.782188841201717e-07,
      "logits/chosen": -0.843505859375,
      "logits/rejected": -0.88232421875,
      "logps/chosen": -367.125,
      "logps/rejected": -440.1499938964844,
      "loss": 0.0229,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -0.21315917372703552,
      "rewards/margins": 10.846875190734863,
      "rewards/rejected": -11.069531440734863,
      "step": 3000
    },
    {
      "epoch": 1.2913850445231199,
      "grad_norm": 8.023624970801535,
      "learning_rate": 6.771459227467811e-07,
      "logits/chosen": -0.8708251714706421,
      "logits/rejected": -0.882397472858429,
      "logps/chosen": -405.5,
      "logps/rejected": -483.8999938964844,
      "loss": 0.0212,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -0.44740599393844604,
      "rewards/margins": 10.907812118530273,
      "rewards/rejected": -11.353124618530273,
      "step": 3010
    },
    {
      "epoch": 1.295676429567643,
      "grad_norm": 0.8484430855200427,
      "learning_rate": 6.760729613733906e-07,
      "logits/chosen": -0.9527343511581421,
      "logits/rejected": -0.948535144329071,
      "logps/chosen": -393.7749938964844,
      "logps/rejected": -492.95001220703125,
      "loss": 0.0578,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -1.077123999595642,
      "rewards/margins": 11.541406631469727,
      "rewards/rejected": -12.615625381469727,
      "step": 3020
    },
    {
      "epoch": 1.299967814612166,
      "grad_norm": 20.097868125405526,
      "learning_rate": 6.75e-07,
      "logits/chosen": -0.934765636920929,
      "logits/rejected": -0.9468749761581421,
      "logps/chosen": -411.625,
      "logps/rejected": -464.32501220703125,
      "loss": 0.0561,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.880200207233429,
      "rewards/margins": 11.565625190734863,
      "rewards/rejected": -12.44921875,
      "step": 3030
    },
    {
      "epoch": 1.3042591996566892,
      "grad_norm": 49.158517994013735,
      "learning_rate": 6.739270386266094e-07,
      "logits/chosen": -0.887744128704071,
      "logits/rejected": -0.9627929925918579,
      "logps/chosen": -361.1625061035156,
      "logps/rejected": -412.04998779296875,
      "loss": 0.0607,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.02607421949505806,
      "rewards/margins": 9.708593368530273,
      "rewards/rejected": -9.742968559265137,
      "step": 3040
    },
    {
      "epoch": 1.3085505847012122,
      "grad_norm": 1.6205881909254827,
      "learning_rate": 6.728540772532188e-07,
      "logits/chosen": -0.953320324420929,
      "logits/rejected": -0.996142566204071,
      "logps/chosen": -388.92498779296875,
      "logps/rejected": -451.75,
      "loss": 0.0361,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.4447387754917145,
      "rewards/margins": 11.506250381469727,
      "rewards/rejected": -11.071093559265137,
      "step": 3050
    },
    {
      "epoch": 1.3128419697457354,
      "grad_norm": 19.552662385739566,
      "learning_rate": 6.717811158798283e-07,
      "logits/chosen": -0.8965820074081421,
      "logits/rejected": -0.960742175579071,
      "logps/chosen": -360.82501220703125,
      "logps/rejected": -480.125,
      "loss": 0.034,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.39521485567092896,
      "rewards/margins": 11.825780868530273,
      "rewards/rejected": -12.221094131469727,
      "step": 3060
    },
    {
      "epoch": 1.3171333547902586,
      "grad_norm": 1.4849512594661396,
      "learning_rate": 6.707081545064377e-07,
      "logits/chosen": -0.965869128704071,
      "logits/rejected": -0.987500011920929,
      "logps/chosen": -361.6000061035156,
      "logps/rejected": -458.6000061035156,
      "loss": 0.0443,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -2.022216796875,
      "rewards/margins": 11.507031440734863,
      "rewards/rejected": -13.5234375,
      "step": 3070
    },
    {
      "epoch": 1.3214247398347818,
      "grad_norm": 27.460062468197606,
      "learning_rate": 6.696351931330472e-07,
      "logits/chosen": -0.976757824420929,
      "logits/rejected": -1.0224120616912842,
      "logps/chosen": -406.95001220703125,
      "logps/rejected": -455.54998779296875,
      "loss": 0.0478,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.4090332984924316,
      "rewards/margins": 12.051172256469727,
      "rewards/rejected": -14.462499618530273,
      "step": 3080
    },
    {
      "epoch": 1.3257161248793048,
      "grad_norm": 0.6392965356522042,
      "learning_rate": 6.685622317596567e-07,
      "logits/chosen": -0.9383544921875,
      "logits/rejected": -1.003076195716858,
      "logps/chosen": -386.1000061035156,
      "logps/rejected": -482.25,
      "loss": 0.0547,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -3.5667967796325684,
      "rewards/margins": 12.789843559265137,
      "rewards/rejected": -16.360937118530273,
      "step": 3090
    },
    {
      "epoch": 1.330007509923828,
      "grad_norm": 5.355188149665065,
      "learning_rate": 6.67489270386266e-07,
      "logits/chosen": -0.96826171875,
      "logits/rejected": -1.0000488758087158,
      "logps/chosen": -414.5,
      "logps/rejected": -471.54998779296875,
      "loss": 0.0351,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -3.867480516433716,
      "rewards/margins": 11.431249618530273,
      "rewards/rejected": -15.298437118530273,
      "step": 3100
    },
    {
      "epoch": 1.334298894968351,
      "grad_norm": 10.6816822027675,
      "learning_rate": 6.664163090128755e-07,
      "logits/chosen": -0.9092773199081421,
      "logits/rejected": -0.967236340045929,
      "logps/chosen": -373.2749938964844,
      "logps/rejected": -477.1499938964844,
      "loss": 0.0548,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.1260008811950684,
      "rewards/margins": 12.357812881469727,
      "rewards/rejected": -14.482812881469727,
      "step": 3110
    },
    {
      "epoch": 1.3385902800128742,
      "grad_norm": 55.84987914140638,
      "learning_rate": 6.65343347639485e-07,
      "logits/chosen": -0.827197253704071,
      "logits/rejected": -0.886914074420929,
      "logps/chosen": -347.2875061035156,
      "logps/rejected": -417.79998779296875,
      "loss": 0.0744,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.746289074420929,
      "rewards/margins": 10.245312690734863,
      "rewards/rejected": -10.993749618530273,
      "step": 3120
    },
    {
      "epoch": 1.3428816650573974,
      "grad_norm": 12.504389988513624,
      "learning_rate": 6.642703862660944e-07,
      "logits/chosen": -0.792041003704071,
      "logits/rejected": -0.8616943359375,
      "logps/chosen": -328.125,
      "logps/rejected": -442.8500061035156,
      "loss": 0.0528,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.819012463092804,
      "rewards/margins": 10.645312309265137,
      "rewards/rejected": -11.44921875,
      "step": 3130
    },
    {
      "epoch": 1.3471730501019203,
      "grad_norm": 5.10389712400187,
      "learning_rate": 6.631974248927038e-07,
      "logits/chosen": -0.886157214641571,
      "logits/rejected": -0.914990246295929,
      "logps/chosen": -364.1499938964844,
      "logps/rejected": -447.95001220703125,
      "loss": 0.1306,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.21928711235523224,
      "rewards/margins": 10.199609756469727,
      "rewards/rejected": -10.417187690734863,
      "step": 3140
    },
    {
      "epoch": 1.3514644351464435,
      "grad_norm": 13.306530408934037,
      "learning_rate": 6.621244635193132e-07,
      "logits/chosen": -0.86669921875,
      "logits/rejected": -0.884228527545929,
      "logps/chosen": -357.40625,
      "logps/rejected": -459.75,
      "loss": 0.0239,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.02045898512005806,
      "rewards/margins": 12.219531059265137,
      "rewards/rejected": -12.235937118530273,
      "step": 3150
    },
    {
      "epoch": 1.3557558201909665,
      "grad_norm": 2.1072482395091865,
      "learning_rate": 6.610515021459228e-07,
      "logits/chosen": -0.919812023639679,
      "logits/rejected": -0.9695800542831421,
      "logps/chosen": -341.9125061035156,
      "logps/rejected": -443.375,
      "loss": 0.0403,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.7022949457168579,
      "rewards/margins": 10.909375190734863,
      "rewards/rejected": -11.614843368530273,
      "step": 3160
    },
    {
      "epoch": 1.3600472052354897,
      "grad_norm": 5.629703901927869,
      "learning_rate": 6.599785407725322e-07,
      "logits/chosen": -0.8363281488418579,
      "logits/rejected": -0.877307116985321,
      "logps/chosen": -352.4750061035156,
      "logps/rejected": -442.8500061035156,
      "loss": 0.025,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -0.4149536192417145,
      "rewards/margins": 11.703125,
      "rewards/rejected": -12.123437881469727,
      "step": 3170
    },
    {
      "epoch": 1.364338590280013,
      "grad_norm": 4.860320544615607,
      "learning_rate": 6.589055793991416e-07,
      "logits/chosen": -0.9039306640625,
      "logits/rejected": -0.988476574420929,
      "logps/chosen": -349.57501220703125,
      "logps/rejected": -444.6499938964844,
      "loss": 0.0475,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -1.4019286632537842,
      "rewards/margins": 10.744531631469727,
      "rewards/rejected": -12.150781631469727,
      "step": 3180
    },
    {
      "epoch": 1.3686299753245361,
      "grad_norm": 4.037837096221678,
      "learning_rate": 6.578326180257511e-07,
      "logits/chosen": -0.901611328125,
      "logits/rejected": -0.9452148675918579,
      "logps/chosen": -370.3500061035156,
      "logps/rejected": -462.1000061035156,
      "loss": 0.0933,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -1.109643578529358,
      "rewards/margins": 11.450780868530273,
      "rewards/rejected": -12.568750381469727,
      "step": 3190
    },
    {
      "epoch": 1.372921360369059,
      "grad_norm": 18.095349464306317,
      "learning_rate": 6.567596566523604e-07,
      "logits/chosen": -0.799792468547821,
      "logits/rejected": -0.849194347858429,
      "logps/chosen": -356.54998779296875,
      "logps/rejected": -469.3500061035156,
      "loss": 0.0607,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -1.1903808116912842,
      "rewards/margins": 11.115625381469727,
      "rewards/rejected": -12.303906440734863,
      "step": 3200
    },
    {
      "epoch": 1.3772127454135823,
      "grad_norm": 35.88434976990525,
      "learning_rate": 6.556866952789699e-07,
      "logits/chosen": -0.8663085699081421,
      "logits/rejected": -0.925793468952179,
      "logps/chosen": -365.61248779296875,
      "logps/rejected": -432.5249938964844,
      "loss": 0.0286,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -1.6905395984649658,
      "rewards/margins": 10.071093559265137,
      "rewards/rejected": -11.756250381469727,
      "step": 3210
    },
    {
      "epoch": 1.3815041304581053,
      "grad_norm": 73.23609788848009,
      "learning_rate": 6.546137339055794e-07,
      "logits/chosen": -0.8587890863418579,
      "logits/rejected": -0.9124511480331421,
      "logps/chosen": -362.0,
      "logps/rejected": -456.5,
      "loss": 0.0763,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -1.5350341796875,
      "rewards/margins": 11.740625381469727,
      "rewards/rejected": -13.271093368530273,
      "step": 3220
    },
    {
      "epoch": 1.3857955155026285,
      "grad_norm": 0.7790833607907854,
      "learning_rate": 6.535407725321889e-07,
      "logits/chosen": -0.820599377155304,
      "logits/rejected": -0.851245105266571,
      "logps/chosen": -367.2875061035156,
      "logps/rejected": -446.92498779296875,
      "loss": 0.0627,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.3724365234375,
      "rewards/margins": 10.6796875,
      "rewards/rejected": -12.052343368530273,
      "step": 3230
    },
    {
      "epoch": 1.3900869005471517,
      "grad_norm": 9.925110141823033,
      "learning_rate": 6.524678111587983e-07,
      "logits/chosen": -0.8824462890625,
      "logits/rejected": -0.91156005859375,
      "logps/chosen": -360.9750061035156,
      "logps/rejected": -452.95001220703125,
      "loss": 0.0441,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.72637939453125,
      "rewards/margins": 11.247655868530273,
      "rewards/rejected": -11.969922065734863,
      "step": 3240
    },
    {
      "epoch": 1.3943782855916746,
      "grad_norm": 11.862398847089578,
      "learning_rate": 6.513948497854077e-07,
      "logits/chosen": -0.8667358160018921,
      "logits/rejected": -0.925537109375,
      "logps/chosen": -352.95001220703125,
      "logps/rejected": -479.57501220703125,
      "loss": 0.0464,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.000732421875,
      "rewards/margins": 11.036328315734863,
      "rewards/rejected": -12.036718368530273,
      "step": 3250
    },
    {
      "epoch": 1.3986696706361978,
      "grad_norm": 52.12055214740879,
      "learning_rate": 6.503218884120172e-07,
      "logits/chosen": -0.904101550579071,
      "logits/rejected": -0.958691418170929,
      "logps/chosen": -352.2749938964844,
      "logps/rejected": -464.79998779296875,
      "loss": 0.033,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -1.6461181640625,
      "rewards/margins": 11.440625190734863,
      "rewards/rejected": -13.096875190734863,
      "step": 3260
    },
    {
      "epoch": 1.4029610556807208,
      "grad_norm": 2.460546072361694,
      "learning_rate": 6.492489270386266e-07,
      "logits/chosen": -0.893994152545929,
      "logits/rejected": -0.940869152545929,
      "logps/chosen": -386.6499938964844,
      "logps/rejected": -484.5249938964844,
      "loss": 0.0189,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.015515089035034,
      "rewards/margins": 11.568750381469727,
      "rewards/rejected": -13.580469131469727,
      "step": 3270
    },
    {
      "epoch": 1.407252440725244,
      "grad_norm": 73.79064857220027,
      "learning_rate": 6.48175965665236e-07,
      "logits/chosen": -0.993408203125,
      "logits/rejected": -1.011474609375,
      "logps/chosen": -360.23748779296875,
      "logps/rejected": -452.7250061035156,
      "loss": 0.0474,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -2.589679002761841,
      "rewards/margins": 10.752344131469727,
      "rewards/rejected": -13.3515625,
      "step": 3280
    },
    {
      "epoch": 1.4115438257697672,
      "grad_norm": 2.4879505586418,
      "learning_rate": 6.471030042918455e-07,
      "logits/chosen": -0.9061523675918579,
      "logits/rejected": -0.927441418170929,
      "logps/chosen": -363.4750061035156,
      "logps/rejected": -469.0,
      "loss": 0.0743,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -2.492077589035034,
      "rewards/margins": 11.246874809265137,
      "rewards/rejected": -13.743749618530273,
      "step": 3290
    },
    {
      "epoch": 1.4158352108142904,
      "grad_norm": 20.67310797004625,
      "learning_rate": 6.460300429184548e-07,
      "logits/chosen": -0.8389511108398438,
      "logits/rejected": -0.887011706829071,
      "logps/chosen": -342.45001220703125,
      "logps/rejected": -440.70001220703125,
      "loss": 0.0588,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -2.3333740234375,
      "rewards/margins": 11.0390625,
      "rewards/rejected": -13.37890625,
      "step": 3300
    },
    {
      "epoch": 1.4201265958588134,
      "grad_norm": 27.45922221621303,
      "learning_rate": 6.449570815450644e-07,
      "logits/chosen": -0.848681628704071,
      "logits/rejected": -0.859667956829071,
      "logps/chosen": -399.2749938964844,
      "logps/rejected": -490.1000061035156,
      "loss": 0.0208,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -1.4686768054962158,
      "rewards/margins": 12.849218368530273,
      "rewards/rejected": -14.309374809265137,
      "step": 3310
    },
    {
      "epoch": 1.4244179809033366,
      "grad_norm": 100.86499383327056,
      "learning_rate": 6.438841201716739e-07,
      "logits/chosen": -0.788226306438446,
      "logits/rejected": -0.841064453125,
      "logps/chosen": -361.6499938964844,
      "logps/rejected": -470.13751220703125,
      "loss": 0.09,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -2.818530321121216,
      "rewards/margins": 11.14453125,
      "rewards/rejected": -13.959375381469727,
      "step": 3320
    },
    {
      "epoch": 1.4287093659478596,
      "grad_norm": 15.527156967901591,
      "learning_rate": 6.428111587982832e-07,
      "logits/chosen": -0.914257824420929,
      "logits/rejected": -0.981640636920929,
      "logps/chosen": -441.42498779296875,
      "logps/rejected": -521.8499755859375,
      "loss": 0.0553,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.0823700428009033,
      "rewards/margins": 13.349218368530273,
      "rewards/rejected": -15.4296875,
      "step": 3330
    },
    {
      "epoch": 1.4330007509923828,
      "grad_norm": 5.138597307024332,
      "learning_rate": 6.417381974248927e-07,
      "logits/chosen": -0.915576159954071,
      "logits/rejected": -0.934326171875,
      "logps/chosen": -408.82501220703125,
      "logps/rejected": -495.8999938964844,
      "loss": 0.023,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -1.8404541015625,
      "rewards/margins": 12.379687309265137,
      "rewards/rejected": -14.21875,
      "step": 3340
    },
    {
      "epoch": 1.437292136036906,
      "grad_norm": 30.739236838663548,
      "learning_rate": 6.406652360515021e-07,
      "logits/chosen": -0.907763659954071,
      "logits/rejected": -0.9356933832168579,
      "logps/chosen": -383.23748779296875,
      "logps/rejected": -465.42498779296875,
      "loss": 0.0697,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.17578125,
      "rewards/margins": 11.319531440734863,
      "rewards/rejected": -12.500781059265137,
      "step": 3350
    },
    {
      "epoch": 1.441583521081429,
      "grad_norm": 72.15085958322028,
      "learning_rate": 6.395922746781116e-07,
      "logits/chosen": -0.8552001714706421,
      "logits/rejected": -0.8736327886581421,
      "logps/chosen": -395.13751220703125,
      "logps/rejected": -495.2250061035156,
      "loss": 0.0325,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.1392700672149658,
      "rewards/margins": 11.385156631469727,
      "rewards/rejected": -12.53125,
      "step": 3360
    },
    {
      "epoch": 1.4458749061259522,
      "grad_norm": 0.08641280978616944,
      "learning_rate": 6.385193133047209e-07,
      "logits/chosen": -0.9839843511581421,
      "logits/rejected": -1.024804711341858,
      "logps/chosen": -364.42498779296875,
      "logps/rejected": -436.8500061035156,
      "loss": 0.0635,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.2257232666015625,
      "rewards/margins": 10.419530868530273,
      "rewards/rejected": -11.639062881469727,
      "step": 3370
    },
    {
      "epoch": 1.4501662911704754,
      "grad_norm": 82.77325381607612,
      "learning_rate": 6.374463519313304e-07,
      "logits/chosen": -0.952587902545929,
      "logits/rejected": -1.029150366783142,
      "logps/chosen": -361.875,
      "logps/rejected": -440.1000061035156,
      "loss": 0.149,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -1.9668457508087158,
      "rewards/margins": 10.737500190734863,
      "rewards/rejected": -12.696874618530273,
      "step": 3380
    },
    {
      "epoch": 1.4544576762149983,
      "grad_norm": 11.201184849575593,
      "learning_rate": 6.3637339055794e-07,
      "logits/chosen": -0.8238525390625,
      "logits/rejected": -0.865795910358429,
      "logps/chosen": -362.76251220703125,
      "logps/rejected": -442.82501220703125,
      "loss": 0.0466,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.965649425983429,
      "rewards/margins": 10.416796684265137,
      "rewards/rejected": -11.377344131469727,
      "step": 3390
    },
    {
      "epoch": 1.4587490612595215,
      "grad_norm": 28.645591372671245,
      "learning_rate": 6.353004291845493e-07,
      "logits/chosen": -0.8511962890625,
      "logits/rejected": -0.904370129108429,
      "logps/chosen": -375.67498779296875,
      "logps/rejected": -420.375,
      "loss": 0.0492,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.15196533501148224,
      "rewards/margins": 9.792187690734863,
      "rewards/rejected": -9.948437690734863,
      "step": 3400
    },
    {
      "epoch": 1.4630404463040447,
      "grad_norm": 18.72589477248601,
      "learning_rate": 6.342274678111588e-07,
      "logits/chosen": -0.884960949420929,
      "logits/rejected": -0.93994140625,
      "logps/chosen": -362.95001220703125,
      "logps/rejected": -443.375,
      "loss": 0.0542,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": 0.6876281499862671,
      "rewards/margins": 10.577343940734863,
      "rewards/rejected": -9.887890815734863,
      "step": 3410
    },
    {
      "epoch": 1.4673318313485677,
      "grad_norm": 37.42507430124475,
      "learning_rate": 6.331545064377683e-07,
      "logits/chosen": -0.9524902105331421,
      "logits/rejected": -0.952258288860321,
      "logps/chosen": -366.8999938964844,
      "logps/rejected": -467.70001220703125,
      "loss": 0.0419,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.40147703886032104,
      "rewards/margins": 10.142969131469727,
      "rewards/rejected": -10.545312881469727,
      "step": 3420
    },
    {
      "epoch": 1.471623216393091,
      "grad_norm": 4.108013106392496,
      "learning_rate": 6.320815450643776e-07,
      "logits/chosen": -0.9612792730331421,
      "logits/rejected": -0.9871581792831421,
      "logps/chosen": -379.13751220703125,
      "logps/rejected": -471.95001220703125,
      "loss": 0.0425,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.687084972858429,
      "rewards/margins": 11.103124618530273,
      "rewards/rejected": -11.796093940734863,
      "step": 3430
    },
    {
      "epoch": 1.4759146014376139,
      "grad_norm": 11.298846121119066,
      "learning_rate": 6.310085836909871e-07,
      "logits/chosen": -0.996289074420929,
      "logits/rejected": -1.025488257408142,
      "logps/chosen": -388.0249938964844,
      "logps/rejected": -453.70001220703125,
      "loss": 0.0398,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.0525145530700684,
      "rewards/margins": 11.0234375,
      "rewards/rejected": -13.068750381469727,
      "step": 3440
    },
    {
      "epoch": 1.480205986482137,
      "grad_norm": 9.573571871020905,
      "learning_rate": 6.299356223175965e-07,
      "logits/chosen": -1.032128930091858,
      "logits/rejected": -1.0539062023162842,
      "logps/chosen": -383.125,
      "logps/rejected": -461.1000061035156,
      "loss": 0.0311,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -1.744970679283142,
      "rewards/margins": 11.297656059265137,
      "rewards/rejected": -13.029687881469727,
      "step": 3450
    },
    {
      "epoch": 1.4844973715266603,
      "grad_norm": 6.539920807912932,
      "learning_rate": 6.288626609442059e-07,
      "logits/chosen": -0.959033191204071,
      "logits/rejected": -0.9933105707168579,
      "logps/chosen": -379.45001220703125,
      "logps/rejected": -457.1000061035156,
      "loss": 0.0382,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.4843261241912842,
      "rewards/margins": 11.4921875,
      "rewards/rejected": -12.977343559265137,
      "step": 3460
    },
    {
      "epoch": 1.4887887565711835,
      "grad_norm": 45.07064251767986,
      "learning_rate": 6.277896995708153e-07,
      "logits/chosen": -1.010156273841858,
      "logits/rejected": -1.030859351158142,
      "logps/chosen": -374.8999938964844,
      "logps/rejected": -456.45001220703125,
      "loss": 0.0535,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.559667944908142,
      "rewards/margins": 10.966405868530273,
      "rewards/rejected": -12.53125,
      "step": 3470
    },
    {
      "epoch": 1.4930801416157065,
      "grad_norm": 21.773530122368797,
      "learning_rate": 6.267167381974249e-07,
      "logits/chosen": -0.9640868902206421,
      "logits/rejected": -0.977099597454071,
      "logps/chosen": -355.75,
      "logps/rejected": -444.29998779296875,
      "loss": 0.0326,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.6719635128974915,
      "rewards/margins": 11.660937309265137,
      "rewards/rejected": -12.331250190734863,
      "step": 3480
    },
    {
      "epoch": 1.4973715266602297,
      "grad_norm": 22.277100761238152,
      "learning_rate": 6.256437768240344e-07,
      "logits/chosen": -1.01171875,
      "logits/rejected": -1.056396484375,
      "logps/chosen": -359.45001220703125,
      "logps/rejected": -472.875,
      "loss": 0.0345,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.42729490995407104,
      "rewards/margins": 12.108593940734863,
      "rewards/rejected": -12.538281440734863,
      "step": 3490
    },
    {
      "epoch": 1.5016629117047526,
      "grad_norm": 3.5957062667518778,
      "learning_rate": 6.245708154506437e-07,
      "logits/chosen": -0.9813476800918579,
      "logits/rejected": -1.0085937976837158,
      "logps/chosen": -367.88751220703125,
      "logps/rejected": -451.1499938964844,
      "loss": 0.0721,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -0.3477416932582855,
      "rewards/margins": 11.082812309265137,
      "rewards/rejected": -11.432031631469727,
      "step": 3500
    },
    {
      "epoch": 1.5059542967492758,
      "grad_norm": 4.917160245072736,
      "learning_rate": 6.234978540772532e-07,
      "logits/chosen": -1.0276367664337158,
      "logits/rejected": -1.061425805091858,
      "logps/chosen": -369.32501220703125,
      "logps/rejected": -471.125,
      "loss": 0.0404,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.6171020269393921,
      "rewards/margins": 11.725000381469727,
      "rewards/rejected": -12.339062690734863,
      "step": 3510
    },
    {
      "epoch": 1.510245681793799,
      "grad_norm": 124.9174010662355,
      "learning_rate": 6.224248927038627e-07,
      "logits/chosen": -0.9134765863418579,
      "logits/rejected": -0.9443359375,
      "logps/chosen": -380.32501220703125,
      "logps/rejected": -466.95001220703125,
      "loss": 0.0731,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -1.318139672279358,
      "rewards/margins": 11.22265625,
      "rewards/rejected": -12.540624618530273,
      "step": 3520
    },
    {
      "epoch": 1.5145370668383222,
      "grad_norm": 39.454257922741654,
      "learning_rate": 6.21351931330472e-07,
      "logits/chosen": -0.896533191204071,
      "logits/rejected": -0.97119140625,
      "logps/chosen": -381.79998779296875,
      "logps/rejected": -469.20001220703125,
      "loss": 0.0599,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.8893616199493408,
      "rewards/margins": 11.817187309265137,
      "rewards/rejected": -13.703125,
      "step": 3530
    },
    {
      "epoch": 1.5188284518828452,
      "grad_norm": 16.748630995113967,
      "learning_rate": 6.202789699570815e-07,
      "logits/chosen": -1.1181640625,
      "logits/rejected": -1.141015648841858,
      "logps/chosen": -393.0249938964844,
      "logps/rejected": -480.07501220703125,
      "loss": 0.0285,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -1.777185082435608,
      "rewards/margins": 11.828906059265137,
      "rewards/rejected": -13.604687690734863,
      "step": 3540
    },
    {
      "epoch": 1.5231198369273682,
      "grad_norm": 27.06889714766912,
      "learning_rate": 6.19206008583691e-07,
      "logits/chosen": -1.0304687023162842,
      "logits/rejected": -1.0901367664337158,
      "logps/chosen": -376.75,
      "logps/rejected": -456.32501220703125,
      "loss": 0.0399,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -1.6439940929412842,
      "rewards/margins": 11.532812118530273,
      "rewards/rejected": -13.173437118530273,
      "step": 3550
    },
    {
      "epoch": 1.5274112219718914,
      "grad_norm": 12.346708553639452,
      "learning_rate": 6.181330472103004e-07,
      "logits/chosen": -0.93798828125,
      "logits/rejected": -1.017333984375,
      "logps/chosen": -358.48748779296875,
      "logps/rejected": -449.20001220703125,
      "loss": 0.043,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.921130359172821,
      "rewards/margins": 10.507031440734863,
      "rewards/rejected": -11.431249618530273,
      "step": 3560
    },
    {
      "epoch": 1.5317026070164146,
      "grad_norm": 17.240386928961588,
      "learning_rate": 6.170600858369098e-07,
      "logits/chosen": -1.028466820716858,
      "logits/rejected": -1.0939452648162842,
      "logps/chosen": -388.1499938964844,
      "logps/rejected": -466.79998779296875,
      "loss": 0.0443,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.1668701171875,
      "rewards/margins": 11.296875,
      "rewards/rejected": -11.134374618530273,
      "step": 3570
    },
    {
      "epoch": 1.5359939920609378,
      "grad_norm": 7.925542237496129,
      "learning_rate": 6.159871244635193e-07,
      "logits/chosen": -1.050878882408142,
      "logits/rejected": -1.0861327648162842,
      "logps/chosen": -350.23748779296875,
      "logps/rejected": -453.6000061035156,
      "loss": 0.021,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -0.07758788764476776,
      "rewards/margins": 10.716405868530273,
      "rewards/rejected": -10.796875,
      "step": 3580
    },
    {
      "epoch": 1.5402853771054608,
      "grad_norm": 52.56466961019095,
      "learning_rate": 6.149141630901288e-07,
      "logits/chosen": -1.0403320789337158,
      "logits/rejected": -1.057226538658142,
      "logps/chosen": -328.20001220703125,
      "logps/rejected": -420.54998779296875,
      "loss": 0.1081,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -1.152563452720642,
      "rewards/margins": 10.215039253234863,
      "rewards/rejected": -11.369336128234863,
      "step": 3590
    },
    {
      "epoch": 1.544576762149984,
      "grad_norm": 20.27197244135147,
      "learning_rate": 6.138412017167381e-07,
      "logits/chosen": -0.971923828125,
      "logits/rejected": -1.02294921875,
      "logps/chosen": -353.73748779296875,
      "logps/rejected": -447.57501220703125,
      "loss": 0.0199,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.41986083984375,
      "rewards/margins": 11.297656059265137,
      "rewards/rejected": -10.87890625,
      "step": 3600
    },
    {
      "epoch": 1.548868147194507,
      "grad_norm": 15.161298138441703,
      "learning_rate": 6.127682403433476e-07,
      "logits/chosen": -1.034204125404358,
      "logits/rejected": -1.080810546875,
      "logps/chosen": -364.7250061035156,
      "logps/rejected": -451.57501220703125,
      "loss": 0.0516,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": 1.055029273033142,
      "rewards/margins": 10.393750190734863,
      "rewards/rejected": -9.3359375,
      "step": 3610
    },
    {
      "epoch": 1.5531595322390301,
      "grad_norm": 5.131590755196448,
      "learning_rate": 6.11695278969957e-07,
      "logits/chosen": -1.0348632335662842,
      "logits/rejected": -1.0750000476837158,
      "logps/chosen": -351.03125,
      "logps/rejected": -450.57501220703125,
      "loss": 0.0249,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.06328125298023224,
      "rewards/margins": 10.162500381469727,
      "rewards/rejected": -10.220703125,
      "step": 3620
    },
    {
      "epoch": 1.5574509172835533,
      "grad_norm": 26.73930536393512,
      "learning_rate": 6.106223175965665e-07,
      "logits/chosen": -0.9801880121231079,
      "logits/rejected": -0.9995361566543579,
      "logps/chosen": -342.5375061035156,
      "logps/rejected": -418.0,
      "loss": 0.062,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.30524903535842896,
      "rewards/margins": 9.898046493530273,
      "rewards/rejected": -10.20703125,
      "step": 3630
    },
    {
      "epoch": 1.5617423023280765,
      "grad_norm": 4.241567091111774,
      "learning_rate": 6.09549356223176e-07,
      "logits/chosen": -1.0172851085662842,
      "logits/rejected": -1.042456030845642,
      "logps/chosen": -374.82501220703125,
      "logps/rejected": -455.45001220703125,
      "loss": 0.0434,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.5003417730331421,
      "rewards/margins": 11.211718559265137,
      "rewards/rejected": -11.712499618530273,
      "step": 3640
    },
    {
      "epoch": 1.5660336873725995,
      "grad_norm": 11.55841738570082,
      "learning_rate": 6.084763948497854e-07,
      "logits/chosen": -0.9365234375,
      "logits/rejected": -0.9844726324081421,
      "logps/chosen": -385.5249938964844,
      "logps/rejected": -474.95001220703125,
      "loss": 0.107,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -1.064538598060608,
      "rewards/margins": 10.694531440734863,
      "rewards/rejected": -11.749218940734863,
      "step": 3650
    },
    {
      "epoch": 1.5703250724171225,
      "grad_norm": 18.012969322287542,
      "learning_rate": 6.074034334763948e-07,
      "logits/chosen": -1.0526854991912842,
      "logits/rejected": -1.085205078125,
      "logps/chosen": -367.3125,
      "logps/rejected": -459.32501220703125,
      "loss": 0.0513,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -1.0918700695037842,
      "rewards/margins": 11.31640625,
      "rewards/rejected": -12.410937309265137,
      "step": 3660
    },
    {
      "epoch": 1.5746164574616457,
      "grad_norm": 7.8670591121164835,
      "learning_rate": 6.063304721030042e-07,
      "logits/chosen": -0.9852660894393921,
      "logits/rejected": -1.0500609874725342,
      "logps/chosen": -376.9624938964844,
      "logps/rejected": -484.0249938964844,
      "loss": 0.0781,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.945361316204071,
      "rewards/margins": 12.44921875,
      "rewards/rejected": -13.397656440734863,
      "step": 3670
    },
    {
      "epoch": 1.578907842506169,
      "grad_norm": 10.443051178099875,
      "learning_rate": 6.052575107296137e-07,
      "logits/chosen": -0.99615478515625,
      "logits/rejected": -1.068457007408142,
      "logps/chosen": -412.875,
      "logps/rejected": -499.8999938964844,
      "loss": 0.0289,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -1.360693335533142,
      "rewards/margins": 11.412500381469727,
      "rewards/rejected": -12.767969131469727,
      "step": 3680
    },
    {
      "epoch": 1.583199227550692,
      "grad_norm": 5.296962907358093,
      "learning_rate": 6.041845493562231e-07,
      "logits/chosen": -1.0727050304412842,
      "logits/rejected": -1.1242187023162842,
      "logps/chosen": -350.5249938964844,
      "logps/rejected": -441.82501220703125,
      "loss": 0.0386,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -1.6430175304412842,
      "rewards/margins": 11.55859375,
      "rewards/rejected": -13.203125,
      "step": 3690
    },
    {
      "epoch": 1.587490612595215,
      "grad_norm": 2.230846412563901,
      "learning_rate": 6.031115879828325e-07,
      "logits/chosen": -0.994189441204071,
      "logits/rejected": -1.039453148841858,
      "logps/chosen": -363.2250061035156,
      "logps/rejected": -476.1000061035156,
      "loss": 0.0633,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -1.7603027820587158,
      "rewards/margins": 10.814453125,
      "rewards/rejected": -12.578125,
      "step": 3700
    },
    {
      "epoch": 1.5917819976397383,
      "grad_norm": 11.68472547219547,
      "learning_rate": 6.020386266094421e-07,
      "logits/chosen": -1.108496069908142,
      "logits/rejected": -1.1541016101837158,
      "logps/chosen": -369.45001220703125,
      "logps/rejected": -497.75,
      "loss": 0.0088,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.591455101966858,
      "rewards/margins": 12.387499809265137,
      "rewards/rejected": -13.973437309265137,
      "step": 3710
    },
    {
      "epoch": 1.5960733826842612,
      "grad_norm": 15.161777954421922,
      "learning_rate": 6.009656652360515e-07,
      "logits/chosen": -1.1013672351837158,
      "logits/rejected": -1.1129882335662842,
      "logps/chosen": -419.3999938964844,
      "logps/rejected": -487.6499938964844,
      "loss": 0.0336,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -1.280175805091858,
      "rewards/margins": 11.493749618530273,
      "rewards/rejected": -12.779687881469727,
      "step": 3720
    },
    {
      "epoch": 1.6003647677287844,
      "grad_norm": 46.747152388521656,
      "learning_rate": 5.998927038626609e-07,
      "logits/chosen": -1.000585913658142,
      "logits/rejected": -1.0385253429412842,
      "logps/chosen": -369.0,
      "logps/rejected": -435.29998779296875,
      "loss": 0.0755,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -1.6717529296875,
      "rewards/margins": 11.267578125,
      "rewards/rejected": -12.938281059265137,
      "step": 3730
    },
    {
      "epoch": 1.6046561527733076,
      "grad_norm": 4.610886513347477,
      "learning_rate": 5.988197424892704e-07,
      "logits/chosen": -1.032128930091858,
      "logits/rejected": -1.055566430091858,
      "logps/chosen": -352.0,
      "logps/rejected": -488.95001220703125,
      "loss": 0.0438,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -1.308349609375,
      "rewards/margins": 12.499218940734863,
      "rewards/rejected": -13.817187309265137,
      "step": 3740
    },
    {
      "epoch": 1.6089475378178308,
      "grad_norm": 2.12022158001016,
      "learning_rate": 5.977467811158798e-07,
      "logits/chosen": -1.066015601158142,
      "logits/rejected": -1.093017578125,
      "logps/chosen": -365.5,
      "logps/rejected": -434.1499938964844,
      "loss": 0.0374,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -1.0483887195587158,
      "rewards/margins": 10.621874809265137,
      "rewards/rejected": -11.671093940734863,
      "step": 3750
    },
    {
      "epoch": 1.6132389228623538,
      "grad_norm": 7.3791654757765,
      "learning_rate": 5.966738197424892e-07,
      "logits/chosen": -1.073974609375,
      "logits/rejected": -1.0671173334121704,
      "logps/chosen": -380.57501220703125,
      "logps/rejected": -484.5,
      "loss": 0.0901,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.2013671398162842,
      "rewards/margins": 11.805078506469727,
      "rewards/rejected": -13.010937690734863,
      "step": 3760
    },
    {
      "epoch": 1.6175303079068768,
      "grad_norm": 6.726731150209308,
      "learning_rate": 5.956008583690986e-07,
      "logits/chosen": -0.9701172113418579,
      "logits/rejected": -0.9835449457168579,
      "logps/chosen": -362.1000061035156,
      "logps/rejected": -495.2250061035156,
      "loss": 0.0408,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.4636474549770355,
      "rewards/margins": 12.257031440734863,
      "rewards/rejected": -12.715624809265137,
      "step": 3770
    },
    {
      "epoch": 1.6218216929514,
      "grad_norm": 3.283700623134968,
      "learning_rate": 5.945278969957081e-07,
      "logits/chosen": -1.0281982421875,
      "logits/rejected": -1.0763671398162842,
      "logps/chosen": -373.25,
      "logps/rejected": -461.5249938964844,
      "loss": 0.0363,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -1.326806664466858,
      "rewards/margins": 10.408594131469727,
      "rewards/rejected": -11.733593940734863,
      "step": 3780
    },
    {
      "epoch": 1.6261130779959232,
      "grad_norm": 46.263555191190605,
      "learning_rate": 5.934549356223176e-07,
      "logits/chosen": -1.0485351085662842,
      "logits/rejected": -1.0465819835662842,
      "logps/chosen": -334.1000061035156,
      "logps/rejected": -428.2250061035156,
      "loss": 0.0653,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.558642566204071,
      "rewards/margins": 10.5,
      "rewards/rejected": -11.052343368530273,
      "step": 3790
    },
    {
      "epoch": 1.6304044630404464,
      "grad_norm": 70.58914327261067,
      "learning_rate": 5.92381974248927e-07,
      "logits/chosen": -0.9837890863418579,
      "logits/rejected": -0.9887939691543579,
      "logps/chosen": -373.67498779296875,
      "logps/rejected": -460.2250061035156,
      "loss": 0.0614,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.5447021722793579,
      "rewards/margins": 10.692187309265137,
      "rewards/rejected": -11.242968559265137,
      "step": 3800
    },
    {
      "epoch": 1.6346958480849694,
      "grad_norm": 9.551501688796025,
      "learning_rate": 5.913090128755365e-07,
      "logits/chosen": -0.9583495855331421,
      "logits/rejected": -1.0110352039337158,
      "logps/chosen": -355.875,
      "logps/rejected": -438.8999938964844,
      "loss": 0.0437,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.19427490234375,
      "rewards/margins": 10.16796875,
      "rewards/rejected": -10.365625381469727,
      "step": 3810
    },
    {
      "epoch": 1.6389872331294926,
      "grad_norm": 9.735697658639353,
      "learning_rate": 5.902360515021459e-07,
      "logits/chosen": -0.991137683391571,
      "logits/rejected": -1.012597680091858,
      "logps/chosen": -402.5,
      "logps/rejected": -464.6499938964844,
      "loss": 0.1226,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.716552734375,
      "rewards/margins": 10.024999618530273,
      "rewards/rejected": -10.742968559265137,
      "step": 3820
    },
    {
      "epoch": 1.6432786181740155,
      "grad_norm": 134.6426067063096,
      "learning_rate": 5.891630901287553e-07,
      "logits/chosen": -0.966601550579071,
      "logits/rejected": -0.9903320074081421,
      "logps/chosen": -344.29998779296875,
      "logps/rejected": -436.4750061035156,
      "loss": 0.0521,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.16999511420726776,
      "rewards/margins": 10.494531631469727,
      "rewards/rejected": -10.666406631469727,
      "step": 3830
    },
    {
      "epoch": 1.6475700032185387,
      "grad_norm": 2.0498518632404643,
      "learning_rate": 5.880901287553648e-07,
      "logits/chosen": -0.9258788824081421,
      "logits/rejected": -0.9781738519668579,
      "logps/chosen": -395.3500061035156,
      "logps/rejected": -452.125,
      "loss": 0.0418,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.5118042230606079,
      "rewards/margins": 9.500781059265137,
      "rewards/rejected": -10.008593559265137,
      "step": 3840
    },
    {
      "epoch": 1.651861388263062,
      "grad_norm": 0.11991663763700955,
      "learning_rate": 5.870171673819742e-07,
      "logits/chosen": -0.877124011516571,
      "logits/rejected": -0.90020751953125,
      "logps/chosen": -331.8125,
      "logps/rejected": -450.54998779296875,
      "loss": 0.04,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -0.05183105543255806,
      "rewards/margins": 11.319531440734863,
      "rewards/rejected": -11.373437881469727,
      "step": 3850
    },
    {
      "epoch": 1.6561527733075851,
      "grad_norm": 21.277396660430856,
      "learning_rate": 5.859442060085836e-07,
      "logits/chosen": -0.9910644292831421,
      "logits/rejected": -1.045507788658142,
      "logps/chosen": -400.4750061035156,
      "logps/rejected": -477.95001220703125,
      "loss": 0.0675,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 0.4636291563510895,
      "rewards/margins": 11.4921875,
      "rewards/rejected": -11.025781631469727,
      "step": 3860
    },
    {
      "epoch": 1.6604441583521081,
      "grad_norm": 5.9248262913392695,
      "learning_rate": 5.848712446351931e-07,
      "logits/chosen": -0.88916015625,
      "logits/rejected": -0.940991222858429,
      "logps/chosen": -360.45001220703125,
      "logps/rejected": -454.17498779296875,
      "loss": 0.0577,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.8077758550643921,
      "rewards/margins": 10.555468559265137,
      "rewards/rejected": -11.360156059265137,
      "step": 3870
    },
    {
      "epoch": 1.664735543396631,
      "grad_norm": 5.636344279914434,
      "learning_rate": 5.837982832618026e-07,
      "logits/chosen": -0.971386730670929,
      "logits/rejected": -1.02880859375,
      "logps/chosen": -363.82501220703125,
      "logps/rejected": -435.125,
      "loss": 0.1098,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.787792980670929,
      "rewards/margins": 11.44140625,
      "rewards/rejected": -12.232812881469727,
      "step": 3880
    },
    {
      "epoch": 1.6690269284411543,
      "grad_norm": 24.121842854189637,
      "learning_rate": 5.82725321888412e-07,
      "logits/chosen": -0.909912109375,
      "logits/rejected": -0.9408935308456421,
      "logps/chosen": -340.95001220703125,
      "logps/rejected": -436.4750061035156,
      "loss": 0.1015,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -0.77166748046875,
      "rewards/margins": 11.192968368530273,
      "rewards/rejected": -11.967968940734863,
      "step": 3890
    },
    {
      "epoch": 1.6733183134856775,
      "grad_norm": 13.088354166874673,
      "learning_rate": 5.816523605150214e-07,
      "logits/chosen": -0.985644519329071,
      "logits/rejected": -1.042333960533142,
      "logps/chosen": -377.04998779296875,
      "logps/rejected": -444.8999938964844,
      "loss": 0.0334,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -1.29119873046875,
      "rewards/margins": 10.6875,
      "rewards/rejected": -11.982812881469727,
      "step": 3900
    },
    {
      "epoch": 1.6776096985302007,
      "grad_norm": 10.340463453342215,
      "learning_rate": 5.805793991416309e-07,
      "logits/chosen": -0.94921875,
      "logits/rejected": -1.012304663658142,
      "logps/chosen": -359.6000061035156,
      "logps/rejected": -472.0249938964844,
      "loss": 0.0473,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.7431091070175171,
      "rewards/margins": 11.539843559265137,
      "rewards/rejected": -12.279687881469727,
      "step": 3910
    },
    {
      "epoch": 1.681901083574724,
      "grad_norm": 1.4418473392006272,
      "learning_rate": 5.795064377682402e-07,
      "logits/chosen": -0.917773425579071,
      "logits/rejected": -0.9466797113418579,
      "logps/chosen": -340.67498779296875,
      "logps/rejected": -426.3999938964844,
      "loss": 0.0836,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.956042468547821,
      "rewards/margins": 10.712499618530273,
      "rewards/rejected": -11.675000190734863,
      "step": 3920
    },
    {
      "epoch": 1.6861924686192469,
      "grad_norm": 3.427322931850794,
      "learning_rate": 5.784334763948497e-07,
      "logits/chosen": -0.878613293170929,
      "logits/rejected": -0.9514404535293579,
      "logps/chosen": -389.4624938964844,
      "logps/rejected": -471.32501220703125,
      "loss": 0.0432,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.8802734613418579,
      "rewards/margins": 10.985937118530273,
      "rewards/rejected": -11.862500190734863,
      "step": 3930
    },
    {
      "epoch": 1.6904838536637699,
      "grad_norm": 49.213936872673436,
      "learning_rate": 5.773605150214593e-07,
      "logits/chosen": -0.987988293170929,
      "logits/rejected": -1.0217773914337158,
      "logps/chosen": -400.29998779296875,
      "logps/rejected": -487.95001220703125,
      "loss": 0.0239,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.957409679889679,
      "rewards/margins": 11.645312309265137,
      "rewards/rejected": -12.603124618530273,
      "step": 3940
    },
    {
      "epoch": 1.694775238708293,
      "grad_norm": 1.7707211425103353,
      "learning_rate": 5.762875536480687e-07,
      "logits/chosen": -0.9774414300918579,
      "logits/rejected": -0.999707043170929,
      "logps/chosen": -378.54998779296875,
      "logps/rejected": -487.3999938964844,
      "loss": 0.0141,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.3688598573207855,
      "rewards/margins": 11.663281440734863,
      "rewards/rejected": -12.035937309265137,
      "step": 3950
    },
    {
      "epoch": 1.6990666237528163,
      "grad_norm": 68.9402764828647,
      "learning_rate": 5.752145922746781e-07,
      "logits/chosen": -0.8727051019668579,
      "logits/rejected": -0.893115222454071,
      "logps/chosen": -366.42498779296875,
      "logps/rejected": -456.1000061035156,
      "loss": 0.0269,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -1.109838843345642,
      "rewards/margins": 11.327343940734863,
      "rewards/rejected": -12.435937881469727,
      "step": 3960
    },
    {
      "epoch": 1.7033580087973395,
      "grad_norm": 20.086796671432815,
      "learning_rate": 5.741416309012875e-07,
      "logits/chosen": -1.0078125,
      "logits/rejected": -1.0475585460662842,
      "logps/chosen": -386.54998779296875,
      "logps/rejected": -482.5,
      "loss": 0.0103,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.840234398841858,
      "rewards/margins": 11.315625190734863,
      "rewards/rejected": -13.162500381469727,
      "step": 3970
    },
    {
      "epoch": 1.7076493938418624,
      "grad_norm": 9.260172986644193,
      "learning_rate": 5.73068669527897e-07,
      "logits/chosen": -0.951904296875,
      "logits/rejected": -0.9915527105331421,
      "logps/chosen": -344.6000061035156,
      "logps/rejected": -448.07501220703125,
      "loss": 0.0668,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.832019031047821,
      "rewards/margins": 11.274999618530273,
      "rewards/rejected": -12.108593940734863,
      "step": 3980
    },
    {
      "epoch": 1.7119407788863856,
      "grad_norm": 57.237223091711535,
      "learning_rate": 5.719957081545064e-07,
      "logits/chosen": -0.9855591058731079,
      "logits/rejected": -1.035546898841858,
      "logps/chosen": -370.125,
      "logps/rejected": -513.4500122070312,
      "loss": 0.0264,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -1.373773217201233,
      "rewards/margins": 12.010156631469727,
      "rewards/rejected": -13.376562118530273,
      "step": 3990
    },
    {
      "epoch": 1.7162321639309086,
      "grad_norm": 15.018330304073203,
      "learning_rate": 5.709227467811158e-07,
      "logits/chosen": -0.9442383050918579,
      "logits/rejected": -1.0,
      "logps/chosen": -394.20001220703125,
      "logps/rejected": -492.1000061035156,
      "loss": 0.0157,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.021313428878784,
      "rewards/margins": 11.879687309265137,
      "rewards/rejected": -13.903905868530273,
      "step": 4000
    },
    {
      "epoch": 1.7205235489754318,
      "grad_norm": 122.68808198378277,
      "learning_rate": 5.698497854077253e-07,
      "logits/chosen": -0.907177746295929,
      "logits/rejected": -0.9401489496231079,
      "logps/chosen": -370.54998779296875,
      "logps/rejected": -454.57501220703125,
      "loss": 0.051,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.528466820716858,
      "rewards/margins": 11.520312309265137,
      "rewards/rejected": -13.050000190734863,
      "step": 4010
    },
    {
      "epoch": 1.724814934019955,
      "grad_norm": 0.4396574274383947,
      "learning_rate": 5.687768240343346e-07,
      "logits/chosen": -0.947509765625,
      "logits/rejected": -0.9839843511581421,
      "logps/chosen": -391.54998779296875,
      "logps/rejected": -472.2250061035156,
      "loss": 0.0231,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -1.2883422374725342,
      "rewards/margins": 11.306249618530273,
      "rewards/rejected": -12.596875190734863,
      "step": 4020
    },
    {
      "epoch": 1.7291063190644782,
      "grad_norm": 26.73275191558095,
      "learning_rate": 5.677038626609442e-07,
      "logits/chosen": -0.9722656011581421,
      "logits/rejected": -1.000341773033142,
      "logps/chosen": -409.45001220703125,
      "logps/rejected": -471.70001220703125,
      "loss": 0.0404,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -1.661981225013733,
      "rewards/margins": 11.647656440734863,
      "rewards/rejected": -13.30859375,
      "step": 4030
    },
    {
      "epoch": 1.7333977041090012,
      "grad_norm": 19.19872470819934,
      "learning_rate": 5.666309012875537e-07,
      "logits/chosen": -1.033203125,
      "logits/rejected": -1.0773437023162842,
      "logps/chosen": -358.5,
      "logps/rejected": -469.75,
      "loss": 0.0255,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -1.284204125404358,
      "rewards/margins": 10.657031059265137,
      "rewards/rejected": -11.944531440734863,
      "step": 4040
    },
    {
      "epoch": 1.7376890891535242,
      "grad_norm": 1.3548188306470739,
      "learning_rate": 5.655579399141631e-07,
      "logits/chosen": -1.0725586414337158,
      "logits/rejected": -1.125097632408142,
      "logps/chosen": -421.2250061035156,
      "logps/rejected": -485.6499938964844,
      "loss": 0.0546,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -0.903643786907196,
      "rewards/margins": 10.552343368530273,
      "rewards/rejected": -11.461718559265137,
      "step": 4050
    },
    {
      "epoch": 1.7419804741980474,
      "grad_norm": 2.8207139503507674,
      "learning_rate": 5.644849785407725e-07,
      "logits/chosen": -0.896472156047821,
      "logits/rejected": -0.9592040777206421,
      "logps/chosen": -353.0375061035156,
      "logps/rejected": -467.29998779296875,
      "loss": 0.0245,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -1.0071289539337158,
      "rewards/margins": 11.774218559265137,
      "rewards/rejected": -12.777734756469727,
      "step": 4060
    },
    {
      "epoch": 1.7462718592425706,
      "grad_norm": 1.86076650332812,
      "learning_rate": 5.634120171673819e-07,
      "logits/chosen": -0.9498046636581421,
      "logits/rejected": -0.9789062738418579,
      "logps/chosen": -376.95001220703125,
      "logps/rejected": -482.0249938964844,
      "loss": 0.0653,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.7867920398712158,
      "rewards/margins": 11.701562881469727,
      "rewards/rejected": -13.493749618530273,
      "step": 4070
    },
    {
      "epoch": 1.7505632442870938,
      "grad_norm": 3.286991435004404,
      "learning_rate": 5.623390557939914e-07,
      "logits/chosen": -1.005712866783142,
      "logits/rejected": -1.0442383289337158,
      "logps/chosen": -414.75,
      "logps/rejected": -502.95001220703125,
      "loss": 0.078,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -1.6180908679962158,
      "rewards/margins": 12.618749618530273,
      "rewards/rejected": -14.239062309265137,
      "step": 4080
    },
    {
      "epoch": 1.7548546293316167,
      "grad_norm": 2.500263047383862,
      "learning_rate": 5.612660944206008e-07,
      "logits/chosen": -0.954516589641571,
      "logits/rejected": -1.022070288658142,
      "logps/chosen": -368.92498779296875,
      "logps/rejected": -479.1000061035156,
      "loss": 0.0418,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -1.1765015125274658,
      "rewards/margins": 13.39453125,
      "rewards/rejected": -14.567187309265137,
      "step": 4090
    },
    {
      "epoch": 1.75914601437614,
      "grad_norm": 29.826078732562603,
      "learning_rate": 5.601931330472103e-07,
      "logits/chosen": -0.98291015625,
      "logits/rejected": -0.9886718988418579,
      "logps/chosen": -371.57501220703125,
      "logps/rejected": -465.04998779296875,
      "loss": 0.0271,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -1.32659912109375,
      "rewards/margins": 11.499218940734863,
      "rewards/rejected": -12.832812309265137,
      "step": 4100
    },
    {
      "epoch": 1.763437399420663,
      "grad_norm": 46.447264070681186,
      "learning_rate": 5.591201716738198e-07,
      "logits/chosen": -0.9245361089706421,
      "logits/rejected": -0.9839843511581421,
      "logps/chosen": -362.2749938964844,
      "logps/rejected": -447.6000061035156,
      "loss": 0.0716,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.5927245616912842,
      "rewards/margins": 11.297656059265137,
      "rewards/rejected": -12.885156631469727,
      "step": 4110
    },
    {
      "epoch": 1.767728784465186,
      "grad_norm": 5.575681277057093,
      "learning_rate": 5.580472103004291e-07,
      "logits/chosen": -0.874438464641571,
      "logits/rejected": -0.939697265625,
      "logps/chosen": -405.75,
      "logps/rejected": -490.3999938964844,
      "loss": 0.0468,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.0489501953125,
      "rewards/margins": 11.47265625,
      "rewards/rejected": -12.51953125,
      "step": 4120
    },
    {
      "epoch": 1.7720201695097093,
      "grad_norm": 4.935040382025751,
      "learning_rate": 5.569742489270386e-07,
      "logits/chosen": -0.8921874761581421,
      "logits/rejected": -0.8934081792831421,
      "logps/chosen": -359.79998779296875,
      "logps/rejected": -451.20001220703125,
      "loss": 0.0474,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.152868628501892,
      "rewards/margins": 11.1875,
      "rewards/rejected": -12.335156440734863,
      "step": 4130
    },
    {
      "epoch": 1.7763115545542325,
      "grad_norm": 0.5506212259709995,
      "learning_rate": 5.559012875536481e-07,
      "logits/chosen": -0.8636718988418579,
      "logits/rejected": -0.9234863519668579,
      "logps/chosen": -365.92498779296875,
      "logps/rejected": -463.32501220703125,
      "loss": 0.042,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -1.297882080078125,
      "rewards/margins": 11.146093368530273,
      "rewards/rejected": -12.446874618530273,
      "step": 4140
    },
    {
      "epoch": 1.7806029395987555,
      "grad_norm": 49.99298736633369,
      "learning_rate": 5.548283261802574e-07,
      "logits/chosen": -1.000390648841858,
      "logits/rejected": -1.0189208984375,
      "logps/chosen": -345.0,
      "logps/rejected": -493.0,
      "loss": 0.0256,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -1.4110839366912842,
      "rewards/margins": 12.52734375,
      "rewards/rejected": -13.935937881469727,
      "step": 4150
    },
    {
      "epoch": 1.7848943246432785,
      "grad_norm": 53.297165842946654,
      "learning_rate": 5.537553648068669e-07,
      "logits/chosen": -1.02392578125,
      "logits/rejected": -1.0719726085662842,
      "logps/chosen": -365.42498779296875,
      "logps/rejected": -472.54998779296875,
      "loss": 0.0625,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.6956055164337158,
      "rewards/margins": 11.735156059265137,
      "rewards/rejected": -13.427343368530273,
      "step": 4160
    },
    {
      "epoch": 1.7891857096878017,
      "grad_norm": 80.7522111175801,
      "learning_rate": 5.526824034334763e-07,
      "logits/chosen": -1.0426757335662842,
      "logits/rejected": -1.087158203125,
      "logps/chosen": -398.6000061035156,
      "logps/rejected": -478.875,
      "loss": 0.0849,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -1.6728302240371704,
      "rewards/margins": 10.279687881469727,
      "rewards/rejected": -11.959375381469727,
      "step": 4170
    },
    {
      "epoch": 1.7934770947323249,
      "grad_norm": 1.1018441662589793,
      "learning_rate": 5.516094420600859e-07,
      "logits/chosen": -0.9812988042831421,
      "logits/rejected": -0.989453136920929,
      "logps/chosen": -428.8500061035156,
      "logps/rejected": -489.79998779296875,
      "loss": 0.0142,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -1.0172851085662842,
      "rewards/margins": 12.43359375,
      "rewards/rejected": -13.443750381469727,
      "step": 4180
    },
    {
      "epoch": 1.797768479776848,
      "grad_norm": 20.682326648490672,
      "learning_rate": 5.505364806866953e-07,
      "logits/chosen": -0.9356689453125,
      "logits/rejected": -0.963574230670929,
      "logps/chosen": -376.3125,
      "logps/rejected": -464.9750061035156,
      "loss": 0.0691,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.13818359375,
      "rewards/margins": 11.571874618530273,
      "rewards/rejected": -12.719531059265137,
      "step": 4190
    },
    {
      "epoch": 1.8020598648213713,
      "grad_norm": 15.379090636705824,
      "learning_rate": 5.494635193133047e-07,
      "logits/chosen": -0.9769042730331421,
      "logits/rejected": -1.007421851158142,
      "logps/chosen": -361.82501220703125,
      "logps/rejected": -473.3500061035156,
      "loss": 0.0377,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -1.09783935546875,
      "rewards/margins": 11.612500190734863,
      "rewards/rejected": -12.715624809265137,
      "step": 4200
    },
    {
      "epoch": 1.8063512498658942,
      "grad_norm": 52.329435507561556,
      "learning_rate": 5.483905579399142e-07,
      "logits/chosen": -0.890820324420929,
      "logits/rejected": -0.906689465045929,
      "logps/chosen": -368.79998779296875,
      "logps/rejected": -457.2250061035156,
      "loss": 0.0449,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.0990631580352783,
      "rewards/margins": 11.224218368530273,
      "rewards/rejected": -12.311718940734863,
      "step": 4210
    },
    {
      "epoch": 1.8106426349104172,
      "grad_norm": 158.0503270116596,
      "learning_rate": 5.473175965665235e-07,
      "logits/chosen": -0.909716784954071,
      "logits/rejected": -0.9754638671875,
      "logps/chosen": -381.9125061035156,
      "logps/rejected": -447.86248779296875,
      "loss": 0.0552,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.0349609851837158,
      "rewards/margins": 11.165624618530273,
      "rewards/rejected": -12.204687118530273,
      "step": 4220
    },
    {
      "epoch": 1.8149340199549404,
      "grad_norm": 23.085347892645842,
      "learning_rate": 5.46244635193133e-07,
      "logits/chosen": -0.9601806402206421,
      "logits/rejected": -0.9806152582168579,
      "logps/chosen": -381.38751220703125,
      "logps/rejected": -460.0249938964844,
      "loss": 0.0516,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.6144530773162842,
      "rewards/margins": 11.489062309265137,
      "rewards/rejected": -13.094531059265137,
      "step": 4230
    },
    {
      "epoch": 1.8192254049994636,
      "grad_norm": 0.639237582939514,
      "learning_rate": 5.451716738197425e-07,
      "logits/chosen": -0.9354492425918579,
      "logits/rejected": -0.983105480670929,
      "logps/chosen": -378.01251220703125,
      "logps/rejected": -445.8500061035156,
      "loss": 0.035,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.6176636219024658,
      "rewards/margins": 12.065625190734863,
      "rewards/rejected": -13.68359375,
      "step": 4240
    },
    {
      "epoch": 1.8235167900439868,
      "grad_norm": 9.146186731241539,
      "learning_rate": 5.440987124463518e-07,
      "logits/chosen": -0.9319823980331421,
      "logits/rejected": -0.9876464605331421,
      "logps/chosen": -380.2749938964844,
      "logps/rejected": -500.3999938964844,
      "loss": 0.0651,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -2.169995069503784,
      "rewards/margins": 12.806249618530273,
      "rewards/rejected": -14.9765625,
      "step": 4250
    },
    {
      "epoch": 1.8278081750885098,
      "grad_norm": 51.85960284001404,
      "learning_rate": 5.430257510729614e-07,
      "logits/chosen": -0.9583984613418579,
      "logits/rejected": -0.996142566204071,
      "logps/chosen": -376.13751220703125,
      "logps/rejected": -488.6499938964844,
      "loss": 0.0546,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -1.7899901866912842,
      "rewards/margins": 13.244531631469727,
      "rewards/rejected": -15.040624618530273,
      "step": 4260
    },
    {
      "epoch": 1.832099560133033,
      "grad_norm": 10.827144951742474,
      "learning_rate": 5.419527896995708e-07,
      "logits/chosen": -0.955761730670929,
      "logits/rejected": -1.014550805091858,
      "logps/chosen": -384.61248779296875,
      "logps/rejected": -473.20001220703125,
      "loss": 0.0168,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -1.629003882408142,
      "rewards/margins": 12.381250381469727,
      "rewards/rejected": -14.006250381469727,
      "step": 4270
    },
    {
      "epoch": 1.836390945177556,
      "grad_norm": 3.245345263573117,
      "learning_rate": 5.408798283261803e-07,
      "logits/chosen": -0.989245593547821,
      "logits/rejected": -1.0054931640625,
      "logps/chosen": -362.4750061035156,
      "logps/rejected": -434.79998779296875,
      "loss": 0.0464,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -2.1982178688049316,
      "rewards/margins": 10.431249618530273,
      "rewards/rejected": -12.629687309265137,
      "step": 4280
    },
    {
      "epoch": 1.8406823302220792,
      "grad_norm": 10.487556838008045,
      "learning_rate": 5.398068669527897e-07,
      "logits/chosen": -0.935302734375,
      "logits/rejected": -0.9648193120956421,
      "logps/chosen": -410.6000061035156,
      "logps/rejected": -499.6000061035156,
      "loss": 0.0371,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -1.293371558189392,
      "rewards/margins": 12.314844131469727,
      "rewards/rejected": -13.608593940734863,
      "step": 4290
    },
    {
      "epoch": 1.8449737152666024,
      "grad_norm": 18.537853125563366,
      "learning_rate": 5.387339055793991e-07,
      "logits/chosen": -0.972973644733429,
      "logits/rejected": -0.996337890625,
      "logps/chosen": -399.29998779296875,
      "logps/rejected": -497.6499938964844,
      "loss": 0.0357,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -1.343481421470642,
      "rewards/margins": 12.47265625,
      "rewards/rejected": -13.809374809265137,
      "step": 4300
    },
    {
      "epoch": 1.8492651003111256,
      "grad_norm": 153.06839854682727,
      "learning_rate": 5.376609442060086e-07,
      "logits/chosen": -0.8988037109375,
      "logits/rejected": -0.92950439453125,
      "logps/chosen": -404.375,
      "logps/rejected": -467.1499938964844,
      "loss": 0.0655,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.0414550304412842,
      "rewards/margins": 11.288281440734863,
      "rewards/rejected": -12.322656631469727,
      "step": 4310
    },
    {
      "epoch": 1.8535564853556485,
      "grad_norm": 27.06877025444218,
      "learning_rate": 5.365879828326179e-07,
      "logits/chosen": -0.9356933832168579,
      "logits/rejected": -0.9637206792831421,
      "logps/chosen": -369.78125,
      "logps/rejected": -456.79998779296875,
      "loss": 0.0597,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -2.0084471702575684,
      "rewards/margins": 10.43359375,
      "rewards/rejected": -12.454687118530273,
      "step": 4320
    },
    {
      "epoch": 1.8578478704001715,
      "grad_norm": 4.310579860258788,
      "learning_rate": 5.355150214592274e-07,
      "logits/chosen": -0.849682629108429,
      "logits/rejected": -0.8754638433456421,
      "logps/chosen": -372.0249938964844,
      "logps/rejected": -495.25,
      "loss": 0.0687,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -1.2420165538787842,
      "rewards/margins": 12.567187309265137,
      "rewards/rejected": -13.809374809265137,
      "step": 4330
    },
    {
      "epoch": 1.8621392554446947,
      "grad_norm": 10.135312676352479,
      "learning_rate": 5.34442060085837e-07,
      "logits/chosen": -0.9974609613418579,
      "logits/rejected": -1.062475562095642,
      "logps/chosen": -348.67498779296875,
      "logps/rejected": -440.3500061035156,
      "loss": 0.053,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.0508790016174316,
      "rewards/margins": 11.01953125,
      "rewards/rejected": -13.072656631469727,
      "step": 4340
    },
    {
      "epoch": 1.866430640489218,
      "grad_norm": 36.928863505563584,
      "learning_rate": 5.333690987124463e-07,
      "logits/chosen": -0.9471191167831421,
      "logits/rejected": -0.977783203125,
      "logps/chosen": -384.7250061035156,
      "logps/rejected": -516.2999877929688,
      "loss": 0.0308,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -1.8770020008087158,
      "rewards/margins": 12.849218368530273,
      "rewards/rejected": -14.7265625,
      "step": 4350
    },
    {
      "epoch": 1.8707220255337411,
      "grad_norm": 0.4141237719958023,
      "learning_rate": 5.322961373390558e-07,
      "logits/chosen": -0.913818359375,
      "logits/rejected": -0.9593750238418579,
      "logps/chosen": -373.9125061035156,
      "logps/rejected": -487.375,
      "loss": 0.0333,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -1.8375732898712158,
      "rewards/margins": 11.971094131469727,
      "rewards/rejected": -13.809374809265137,
      "step": 4360
    },
    {
      "epoch": 1.875013410578264,
      "grad_norm": 56.450189376473574,
      "learning_rate": 5.312231759656652e-07,
      "logits/chosen": -1.008691430091858,
      "logits/rejected": -1.0224609375,
      "logps/chosen": -385.95001220703125,
      "logps/rejected": -460.79998779296875,
      "loss": 0.041,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -2.0730834007263184,
      "rewards/margins": 12.158594131469727,
      "rewards/rejected": -14.232030868530273,
      "step": 4370
    },
    {
      "epoch": 1.8793047956227873,
      "grad_norm": 9.54783905701276,
      "learning_rate": 5.301502145922746e-07,
      "logits/chosen": -0.8971191644668579,
      "logits/rejected": -0.9556640386581421,
      "logps/chosen": -413.9750061035156,
      "logps/rejected": -502.32501220703125,
      "loss": 0.0272,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.6408447027206421,
      "rewards/margins": 12.224218368530273,
      "rewards/rejected": -12.864843368530273,
      "step": 4380
    },
    {
      "epoch": 1.8835961806673103,
      "grad_norm": 182.77364675505413,
      "learning_rate": 5.290772532188841e-07,
      "logits/chosen": -0.9560546875,
      "logits/rejected": -0.9826904535293579,
      "logps/chosen": -392.4750061035156,
      "logps/rejected": -480.5,
      "loss": 0.0504,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -1.87359619140625,
      "rewards/margins": 10.739062309265137,
      "rewards/rejected": -12.615625381469727,
      "step": 4390
    },
    {
      "epoch": 1.8878875657118335,
      "grad_norm": 20.390810817068484,
      "learning_rate": 5.280042918454935e-07,
      "logits/chosen": -0.83056640625,
      "logits/rejected": -0.853564441204071,
      "logps/chosen": -348.23126220703125,
      "logps/rejected": -479.79998779296875,
      "loss": 0.0394,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.3259766101837158,
      "rewards/margins": 12.457812309265137,
      "rewards/rejected": -13.792187690734863,
      "step": 4400
    },
    {
      "epoch": 1.8921789507563567,
      "grad_norm": 15.337774109502499,
      "learning_rate": 5.26931330472103e-07,
      "logits/chosen": -0.859326183795929,
      "logits/rejected": -0.894482433795929,
      "logps/chosen": -384.23748779296875,
      "logps/rejected": -491.2250061035156,
      "loss": 0.0464,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -2.2955565452575684,
      "rewards/margins": 11.717187881469727,
      "rewards/rejected": -14.010937690734863,
      "step": 4410
    },
    {
      "epoch": 1.8964703358008799,
      "grad_norm": 2.265968307575372,
      "learning_rate": 5.258583690987124e-07,
      "logits/chosen": -0.9061034917831421,
      "logits/rejected": -0.9704345464706421,
      "logps/chosen": -355.1499938964844,
      "logps/rejected": -474.125,
      "loss": 0.0127,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.347851514816284,
      "rewards/margins": 11.787500381469727,
      "rewards/rejected": -14.135937690734863,
      "step": 4420
    },
    {
      "epoch": 1.9007617208454028,
      "grad_norm": 32.705082452578196,
      "learning_rate": 5.247854077253219e-07,
      "logits/chosen": -0.835253894329071,
      "logits/rejected": -0.8897949457168579,
      "logps/chosen": -373.875,
      "logps/rejected": -454.67498779296875,
      "loss": 0.079,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -2.26690673828125,
      "rewards/margins": 11.690625190734863,
      "rewards/rejected": -13.959375381469727,
      "step": 4430
    },
    {
      "epoch": 1.9050531058899258,
      "grad_norm": 46.704049108499824,
      "learning_rate": 5.237124463519314e-07,
      "logits/chosen": -0.892626941204071,
      "logits/rejected": -0.9200439453125,
      "logps/chosen": -360.875,
      "logps/rejected": -448.29998779296875,
      "loss": 0.1067,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.8506591320037842,
      "rewards/margins": 11.145312309265137,
      "rewards/rejected": -12.996874809265137,
      "step": 4440
    },
    {
      "epoch": 1.909344490934449,
      "grad_norm": 3.7170229703324456,
      "learning_rate": 5.226394849785407e-07,
      "logits/chosen": -0.91796875,
      "logits/rejected": -0.937060534954071,
      "logps/chosen": -381.0,
      "logps/rejected": -479.1000061035156,
      "loss": 0.0612,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.321240186691284,
      "rewards/margins": 11.30078125,
      "rewards/rejected": -13.6171875,
      "step": 4450
    },
    {
      "epoch": 1.9136358759789722,
      "grad_norm": 3.153634344334266,
      "learning_rate": 5.215665236051502e-07,
      "logits/chosen": -0.9183105230331421,
      "logits/rejected": -0.9302978515625,
      "logps/chosen": -371.17498779296875,
      "logps/rejected": -504.3500061035156,
      "loss": 0.0337,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.168798804283142,
      "rewards/margins": 12.172266006469727,
      "rewards/rejected": -13.350781440734863,
      "step": 4460
    },
    {
      "epoch": 1.9179272610234954,
      "grad_norm": 41.25008649238425,
      "learning_rate": 5.204935622317596e-07,
      "logits/chosen": -0.8514648675918579,
      "logits/rejected": -0.9063965082168579,
      "logps/chosen": -392.125,
      "logps/rejected": -449.70001220703125,
      "loss": 0.0424,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.37053221464157104,
      "rewards/margins": 11.278124809265137,
      "rewards/rejected": -11.649999618530273,
      "step": 4470
    },
    {
      "epoch": 1.9222186460680184,
      "grad_norm": 20.23026179444304,
      "learning_rate": 5.19420600858369e-07,
      "logits/chosen": -0.890380859375,
      "logits/rejected": -0.947509765625,
      "logps/chosen": -388.75,
      "logps/rejected": -474.25,
      "loss": 0.054,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.42601317167282104,
      "rewards/margins": 11.646875381469727,
      "rewards/rejected": -12.080469131469727,
      "step": 4480
    },
    {
      "epoch": 1.9265100311125416,
      "grad_norm": 1.3408718104792696,
      "learning_rate": 5.183476394849786e-07,
      "logits/chosen": -0.913647472858429,
      "logits/rejected": -0.9691406488418579,
      "logps/chosen": -397.07501220703125,
      "logps/rejected": -463.54998779296875,
      "loss": 0.0443,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.03182373195886612,
      "rewards/margins": 12.149999618530273,
      "rewards/rejected": -12.184765815734863,
      "step": 4490
    },
    {
      "epoch": 1.9308014161570646,
      "grad_norm": 53.11739706779699,
      "learning_rate": 5.17274678111588e-07,
      "logits/chosen": -0.913134753704071,
      "logits/rejected": -0.930957019329071,
      "logps/chosen": -394.70001220703125,
      "logps/rejected": -491.95001220703125,
      "loss": 0.057,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.619213879108429,
      "rewards/margins": 12.110156059265137,
      "rewards/rejected": -12.73046875,
      "step": 4500
    },
    {
      "epoch": 1.9350928012015878,
      "grad_norm": 2.944999759554218,
      "learning_rate": 5.162017167381975e-07,
      "logits/chosen": -1.0154297351837158,
      "logits/rejected": -1.0544922351837158,
      "logps/chosen": -347.7250061035156,
      "logps/rejected": -461.5,
      "loss": 0.0699,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.9320068359375,
      "rewards/margins": 12.104296684265137,
      "rewards/rejected": -13.038281440734863,
      "step": 4510
    },
    {
      "epoch": 1.939384186246111,
      "grad_norm": 44.9848920404682,
      "learning_rate": 5.151287553648068e-07,
      "logits/chosen": -0.966625988483429,
      "logits/rejected": -0.982177734375,
      "logps/chosen": -353.8500061035156,
      "logps/rejected": -448.95001220703125,
      "loss": 0.0719,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -1.562841773033142,
      "rewards/margins": 10.638280868530273,
      "rewards/rejected": -12.199999809265137,
      "step": 4520
    },
    {
      "epoch": 1.9436755712906342,
      "grad_norm": 2.4273690078595576,
      "learning_rate": 5.140557939914163e-07,
      "logits/chosen": -0.930004894733429,
      "logits/rejected": -0.971606433391571,
      "logps/chosen": -385.17498779296875,
      "logps/rejected": -453.0,
      "loss": 0.0359,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.6266112327575684,
      "rewards/margins": 11.353124618530273,
      "rewards/rejected": -13.970312118530273,
      "step": 4530
    },
    {
      "epoch": 1.9479669563351572,
      "grad_norm": 0.563126903276404,
      "learning_rate": 5.129828326180258e-07,
      "logits/chosen": -0.9853515625,
      "logits/rejected": -1.020410180091858,
      "logps/chosen": -380.9750061035156,
      "logps/rejected": -449.375,
      "loss": 0.0585,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.808874487876892,
      "rewards/margins": 12.053125381469727,
      "rewards/rejected": -13.865625381469727,
      "step": 4540
    },
    {
      "epoch": 1.9522583413796801,
      "grad_norm": 70.01156196901671,
      "learning_rate": 5.119098712446351e-07,
      "logits/chosen": -0.983105480670929,
      "logits/rejected": -1.0392272472381592,
      "logps/chosen": -354.8125,
      "logps/rejected": -465.32501220703125,
      "loss": 0.0366,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.0246825218200684,
      "rewards/margins": 11.590624809265137,
      "rewards/rejected": -13.615625381469727,
      "step": 4550
    },
    {
      "epoch": 1.9565497264242033,
      "grad_norm": 96.2369459024813,
      "learning_rate": 5.108369098712446e-07,
      "logits/chosen": -1.011962890625,
      "logits/rejected": -1.036596655845642,
      "logps/chosen": -365.82501220703125,
      "logps/rejected": -493.6000061035156,
      "loss": 0.0822,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -2.084564208984375,
      "rewards/margins": 11.507031440734863,
      "rewards/rejected": -13.598437309265137,
      "step": 4560
    },
    {
      "epoch": 1.9608411114687265,
      "grad_norm": 5.208717013449673,
      "learning_rate": 5.09763948497854e-07,
      "logits/chosen": -0.9501708745956421,
      "logits/rejected": -0.993847668170929,
      "logps/chosen": -339.9125061035156,
      "logps/rejected": -462.2749938964844,
      "loss": 0.0325,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -2.009857177734375,
      "rewards/margins": 11.40625,
      "rewards/rejected": -13.421093940734863,
      "step": 4570
    },
    {
      "epoch": 1.9651324965132497,
      "grad_norm": 0.8034446024792199,
      "learning_rate": 5.086909871244635e-07,
      "logits/chosen": -0.998779296875,
      "logits/rejected": -1.0099608898162842,
      "logps/chosen": -392.20001220703125,
      "logps/rejected": -483.17498779296875,
      "loss": 0.0594,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.5322265625,
      "rewards/margins": 11.840624809265137,
      "rewards/rejected": -13.367968559265137,
      "step": 4580
    },
    {
      "epoch": 1.969423881557773,
      "grad_norm": 4.328983885472126,
      "learning_rate": 5.07618025751073e-07,
      "logits/chosen": -0.982617199420929,
      "logits/rejected": -1.0090820789337158,
      "logps/chosen": -441.17498779296875,
      "logps/rejected": -502.9750061035156,
      "loss": 0.0302,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.317587375640869,
      "rewards/margins": 12.170702934265137,
      "rewards/rejected": -14.484375,
      "step": 4590
    },
    {
      "epoch": 1.973715266602296,
      "grad_norm": 27.71457389608174,
      "learning_rate": 5.065450643776824e-07,
      "logits/chosen": -1.0812499523162842,
      "logits/rejected": -1.1186034679412842,
      "logps/chosen": -413.375,
      "logps/rejected": -475.6499938964844,
      "loss": 0.0504,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -1.806396484375,
      "rewards/margins": 11.99609375,
      "rewards/rejected": -13.814062118530273,
      "step": 4600
    },
    {
      "epoch": 1.9780066516468189,
      "grad_norm": 47.444585000623135,
      "learning_rate": 5.054721030042918e-07,
      "logits/chosen": -0.9796386957168579,
      "logits/rejected": -1.038671851158142,
      "logps/chosen": -400.9750061035156,
      "logps/rejected": -486.29998779296875,
      "loss": 0.0708,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -2.5377440452575684,
      "rewards/margins": 12.588281631469727,
      "rewards/rejected": -15.128125190734863,
      "step": 4610
    },
    {
      "epoch": 1.982298036691342,
      "grad_norm": 1.710670740867975,
      "learning_rate": 5.043991416309012e-07,
      "logits/chosen": -0.931591808795929,
      "logits/rejected": -0.978710949420929,
      "logps/chosen": -395.42498779296875,
      "logps/rejected": -472.07501220703125,
      "loss": 0.0716,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.3408629894256592,
      "rewards/margins": 12.425390243530273,
      "rewards/rejected": -13.76953125,
      "step": 4620
    },
    {
      "epoch": 1.9865894217358653,
      "grad_norm": 90.63523783921438,
      "learning_rate": 5.033261802575107e-07,
      "logits/chosen": -0.903759777545929,
      "logits/rejected": -0.9393340945243835,
      "logps/chosen": -399.6000061035156,
      "logps/rejected": -520.0750122070312,
      "loss": 0.0275,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -1.478979468345642,
      "rewards/margins": 13.239843368530273,
      "rewards/rejected": -14.727343559265137,
      "step": 4630
    },
    {
      "epoch": 1.9908808067803885,
      "grad_norm": 30.711067654319294,
      "learning_rate": 5.022532188841202e-07,
      "logits/chosen": -0.8616698980331421,
      "logits/rejected": -0.9175781011581421,
      "logps/chosen": -388.2875061035156,
      "logps/rejected": -472.42498779296875,
      "loss": 0.0317,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -1.0430176258087158,
      "rewards/margins": 12.157812118530273,
      "rewards/rejected": -13.208593368530273,
      "step": 4640
    },
    {
      "epoch": 1.9951721918249115,
      "grad_norm": 0.3829447074715143,
      "learning_rate": 5.011802575107296e-07,
      "logits/chosen": -0.959277331829071,
      "logits/rejected": -1.0222656726837158,
      "logps/chosen": -396.8999938964844,
      "logps/rejected": -496.54998779296875,
      "loss": 0.0585,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.042028784751892,
      "rewards/margins": 12.974218368530273,
      "rewards/rejected": -14.018750190734863,
      "step": 4650
    },
    {
      "epoch": 1.9994635768694347,
      "grad_norm": 41.67127946798679,
      "learning_rate": 5.001072961373391e-07,
      "logits/chosen": -0.886279284954071,
      "logits/rejected": -0.9300781488418579,
      "logps/chosen": -347.3125,
      "logps/rejected": -449.25,
      "loss": 0.0342,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -1.505346655845642,
      "rewards/margins": 11.438281059265137,
      "rewards/rejected": -12.946874618530273,
      "step": 4660
    },
    {
      "epoch": 2.0034331080356185,
      "grad_norm": 0.027779198802832493,
      "learning_rate": 4.990343347639485e-07,
      "logits/chosen": -0.9422508478164673,
      "logits/rejected": -0.962890625,
      "logps/chosen": -377.70269775390625,
      "logps/rejected": -489.1351318359375,
      "loss": 0.0216,
      "rewards/accuracies": 0.9797297120094299,
      "rewards/chosen": -0.5486037731170654,
      "rewards/margins": 13.966216087341309,
      "rewards/rejected": -14.513513565063477,
      "step": 4670
    },
    {
      "epoch": 2.0077244930801417,
      "grad_norm": 0.058596251501053756,
      "learning_rate": 4.979613733905579e-07,
      "logits/chosen": -0.9541015625,
      "logits/rejected": -0.979443371295929,
      "logps/chosen": -383.4750061035156,
      "logps/rejected": -476.6000061035156,
      "loss": 0.0132,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -1.5182373523712158,
      "rewards/margins": 13.623437881469727,
      "rewards/rejected": -15.140625,
      "step": 4680
    },
    {
      "epoch": 2.012015878124665,
      "grad_norm": 0.08219661419120212,
      "learning_rate": 4.968884120171674e-07,
      "logits/chosen": -0.985546886920929,
      "logits/rejected": -1.0363280773162842,
      "logps/chosen": -419.82501220703125,
      "logps/rejected": -545.5499877929688,
      "loss": 0.0004,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.5763061046600342,
      "rewards/margins": 15.018750190734863,
      "rewards/rejected": -16.584375381469727,
      "step": 4690
    },
    {
      "epoch": 2.016307263169188,
      "grad_norm": 1.6454054767946809,
      "learning_rate": 4.958154506437768e-07,
      "logits/chosen": -0.954028308391571,
      "logits/rejected": -1.0149657726287842,
      "logps/chosen": -411.4375,
      "logps/rejected": -562.375,
      "loss": 0.0121,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.0797362327575684,
      "rewards/margins": 16.078907012939453,
      "rewards/rejected": -18.160938262939453,
      "step": 4700
    },
    {
      "epoch": 2.020598648213711,
      "grad_norm": 0.14648838332714317,
      "learning_rate": 4.947424892703862e-07,
      "logits/chosen": -0.9322265386581421,
      "logits/rejected": -0.9351562261581421,
      "logps/chosen": -367.88751220703125,
      "logps/rejected": -512.5250244140625,
      "loss": 0.0133,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.308837890625,
      "rewards/margins": 16.378124237060547,
      "rewards/rejected": -18.676563262939453,
      "step": 4710
    },
    {
      "epoch": 2.024890033258234,
      "grad_norm": 0.6465582591461517,
      "learning_rate": 4.936695278969956e-07,
      "logits/chosen": -0.9916015863418579,
      "logits/rejected": -1.0476562976837158,
      "logps/chosen": -438.875,
      "logps/rejected": -554.5999755859375,
      "loss": 0.0082,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.7232422828674316,
      "rewards/margins": 15.8359375,
      "rewards/rejected": -18.568750381469727,
      "step": 4720
    },
    {
      "epoch": 2.0291814183027572,
      "grad_norm": 0.3091010197952818,
      "learning_rate": 4.925965665236052e-07,
      "logits/chosen": -0.8946288824081421,
      "logits/rejected": -0.9400390386581421,
      "logps/chosen": -411.75,
      "logps/rejected": -561.25,
      "loss": 0.0242,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -2.6024413108825684,
      "rewards/margins": 16.043750762939453,
      "rewards/rejected": -18.659374237060547,
      "step": 4730
    },
    {
      "epoch": 2.0334728033472804,
      "grad_norm": 2.768302372276932,
      "learning_rate": 4.915236051502146e-07,
      "logits/chosen": -0.9747680425643921,
      "logits/rejected": -1.01904296875,
      "logps/chosen": -323.54998779296875,
      "logps/rejected": -495.8999938964844,
      "loss": 0.0136,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -1.592260718345642,
      "rewards/margins": 15.161718368530273,
      "rewards/rejected": -16.748437881469727,
      "step": 4740
    },
    {
      "epoch": 2.0377641883918036,
      "grad_norm": 0.9396057921861978,
      "learning_rate": 4.90450643776824e-07,
      "logits/chosen": -0.9452148675918579,
      "logits/rejected": -0.968579113483429,
      "logps/chosen": -371.375,
      "logps/rejected": -514.5,
      "loss": 0.0064,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.461010694503784,
      "rewards/margins": 14.9296875,
      "rewards/rejected": -17.3984375,
      "step": 4750
    },
    {
      "epoch": 2.0420555734363264,
      "grad_norm": 1.3859603812649923,
      "learning_rate": 4.893776824034335e-07,
      "logits/chosen": -1.015234351158142,
      "logits/rejected": -1.0478026866912842,
      "logps/chosen": -363.04998779296875,
      "logps/rejected": -488.4750061035156,
      "loss": 0.0086,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.0025877952575684,
      "rewards/margins": 15.129687309265137,
      "rewards/rejected": -17.135936737060547,
      "step": 4760
    },
    {
      "epoch": 2.0463469584808496,
      "grad_norm": 0.009066003312516537,
      "learning_rate": 4.883047210300429e-07,
      "logits/chosen": -1.040869116783142,
      "logits/rejected": -1.0402100086212158,
      "logps/chosen": -383.125,
      "logps/rejected": -523.0999755859375,
      "loss": 0.0098,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.7064208984375,
      "rewards/margins": 15.251562118530273,
      "rewards/rejected": -17.9375,
      "step": 4770
    },
    {
      "epoch": 2.0506383435253728,
      "grad_norm": 0.1620939336388554,
      "learning_rate": 4.872317596566523e-07,
      "logits/chosen": -1.003271460533142,
      "logits/rejected": -1.0250976085662842,
      "logps/chosen": -364.5249938964844,
      "logps/rejected": -512.9500122070312,
      "loss": 0.011,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -3.374316453933716,
      "rewards/margins": 15.456250190734863,
      "rewards/rejected": -18.8359375,
      "step": 4780
    },
    {
      "epoch": 2.054929728569896,
      "grad_norm": 10.107963912563209,
      "learning_rate": 4.861587982832618e-07,
      "logits/chosen": -1.0659668445587158,
      "logits/rejected": -1.1134765148162842,
      "logps/chosen": -382.70001220703125,
      "logps/rejected": -531.9749755859375,
      "loss": 0.0054,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.9286742210388184,
      "rewards/margins": 15.485937118530273,
      "rewards/rejected": -18.424999237060547,
      "step": 4790
    },
    {
      "epoch": 2.059221113614419,
      "grad_norm": 0.12108622323493068,
      "learning_rate": 4.850858369098712e-07,
      "logits/chosen": -0.99560546875,
      "logits/rejected": -1.0546875,
      "logps/chosen": -365.625,
      "logps/rejected": -497.04998779296875,
      "loss": 0.0108,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -3.7535157203674316,
      "rewards/margins": 15.518750190734863,
      "rewards/rejected": -19.270313262939453,
      "step": 4800
    },
    {
      "epoch": 2.0635124986589424,
      "grad_norm": 0.6044017396242726,
      "learning_rate": 4.840128755364807e-07,
      "logits/chosen": -1.090429663658142,
      "logits/rejected": -1.139746069908142,
      "logps/chosen": -378.4750061035156,
      "logps/rejected": -505.17498779296875,
      "loss": 0.0257,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -3.446582078933716,
      "rewards/margins": 16.165624618530273,
      "rewards/rejected": -19.604686737060547,
      "step": 4810
    },
    {
      "epoch": 2.067803883703465,
      "grad_norm": 4.8369341786667635,
      "learning_rate": 4.829399141630901e-07,
      "logits/chosen": -1.0584716796875,
      "logits/rejected": -1.1092529296875,
      "logps/chosen": -381.9750061035156,
      "logps/rejected": -534.6500244140625,
      "loss": 0.0058,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -3.72216796875,
      "rewards/margins": 15.753125190734863,
      "rewards/rejected": -19.487499237060547,
      "step": 4820
    },
    {
      "epoch": 2.0720952687479883,
      "grad_norm": 0.5817973223502714,
      "learning_rate": 4.818669527896996e-07,
      "logits/chosen": -0.994000256061554,
      "logits/rejected": -1.0557861328125,
      "logps/chosen": -393.0,
      "logps/rejected": -497.6000061035156,
      "loss": 0.0061,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -3.760449171066284,
      "rewards/margins": 14.862500190734863,
      "rewards/rejected": -18.615625381469727,
      "step": 4830
    },
    {
      "epoch": 2.0763866537925115,
      "grad_norm": 0.056653641243303195,
      "learning_rate": 4.80793991416309e-07,
      "logits/chosen": -1.0192382335662842,
      "logits/rejected": -1.075781226158142,
      "logps/chosen": -387.3999938964844,
      "logps/rejected": -539.25,
      "loss": 0.0116,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -3.884777784347534,
      "rewards/margins": 16.787500381469727,
      "rewards/rejected": -20.673437118530273,
      "step": 4840
    },
    {
      "epoch": 2.0806780388370347,
      "grad_norm": 0.17191243721182908,
      "learning_rate": 4.797210300429184e-07,
      "logits/chosen": -1.0615234375,
      "logits/rejected": -1.1252930164337158,
      "logps/chosen": -362.875,
      "logps/rejected": -514.9500122070312,
      "loss": 0.0113,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -3.2962889671325684,
      "rewards/margins": 16.417186737060547,
      "rewards/rejected": -19.709375381469727,
      "step": 4850
    },
    {
      "epoch": 2.084969423881558,
      "grad_norm": 0.19566869513875954,
      "learning_rate": 4.786480686695279e-07,
      "logits/chosen": -1.043701171875,
      "logits/rejected": -1.080322265625,
      "logps/chosen": -386.67498779296875,
      "logps/rejected": -515.0,
      "loss": 0.0064,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -3.559497117996216,
      "rewards/margins": 16.100000381469727,
      "rewards/rejected": -19.668750762939453,
      "step": 4860
    },
    {
      "epoch": 2.0892608089260807,
      "grad_norm": 0.35072865560749955,
      "learning_rate": 4.775751072961373e-07,
      "logits/chosen": -1.0676758289337158,
      "logits/rejected": -1.09375,
      "logps/chosen": -384.625,
      "logps/rejected": -528.6500244140625,
      "loss": 0.022,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -3.4744873046875,
      "rewards/margins": 16.159374237060547,
      "rewards/rejected": -19.626562118530273,
      "step": 4870
    },
    {
      "epoch": 2.093552193970604,
      "grad_norm": 0.07825584574057835,
      "learning_rate": 4.7650214592274674e-07,
      "logits/chosen": -1.0342528820037842,
      "logits/rejected": -1.1032226085662842,
      "logps/chosen": -388.5375061035156,
      "logps/rejected": -513.7750244140625,
      "loss": 0.0136,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.6719727516174316,
      "rewards/margins": 16.390625,
      "rewards/rejected": -19.065624237060547,
      "step": 4880
    },
    {
      "epoch": 2.097843579015127,
      "grad_norm": 0.16214672396271004,
      "learning_rate": 4.754291845493562e-07,
      "logits/chosen": -0.9888671636581421,
      "logits/rejected": -1.033544898033142,
      "logps/chosen": -379.45001220703125,
      "logps/rejected": -511.54998779296875,
      "loss": 0.011,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.792724609375,
      "rewards/margins": 16.501562118530273,
      "rewards/rejected": -19.293750762939453,
      "step": 4890
    },
    {
      "epoch": 2.1021349640596503,
      "grad_norm": 2.293165816140267,
      "learning_rate": 4.743562231759656e-07,
      "logits/chosen": -0.9765380620956421,
      "logits/rejected": -1.0322265625,
      "logps/chosen": -395.6499938964844,
      "logps/rejected": -535.5499877929688,
      "loss": 0.0278,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -3.7255616188049316,
      "rewards/margins": 15.540624618530273,
      "rewards/rejected": -19.267187118530273,
      "step": 4900
    },
    {
      "epoch": 2.1064263491041735,
      "grad_norm": 0.0350920407522629,
      "learning_rate": 4.732832618025751e-07,
      "logits/chosen": -1.08203125,
      "logits/rejected": -1.118261694908142,
      "logps/chosen": -395.5249938964844,
      "logps/rejected": -561.7999877929688,
      "loss": 0.0094,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -4.230554103851318,
      "rewards/margins": 17.887500762939453,
      "rewards/rejected": -22.103124618530273,
      "step": 4910
    },
    {
      "epoch": 2.1107177341486967,
      "grad_norm": 1.3768287495370228,
      "learning_rate": 4.7221030042918455e-07,
      "logits/chosen": -1.1409180164337158,
      "logits/rejected": -1.157128930091858,
      "logps/chosen": -386.45001220703125,
      "logps/rejected": -521.7000122070312,
      "loss": 0.0062,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -4.490136623382568,
      "rewards/margins": 15.6640625,
      "rewards/rejected": -20.142187118530273,
      "step": 4920
    },
    {
      "epoch": 2.1150091191932194,
      "grad_norm": 14.770899356614224,
      "learning_rate": 4.7113733905579396e-07,
      "logits/chosen": -1.069970726966858,
      "logits/rejected": -1.1149413585662842,
      "logps/chosen": -421.54998779296875,
      "logps/rejected": -579.8499755859375,
      "loss": 0.0062,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -4.514746189117432,
      "rewards/margins": 16.623437881469727,
      "rewards/rejected": -21.134374618530273,
      "step": 4930
    },
    {
      "epoch": 2.1193005042377426,
      "grad_norm": 0.05859768547809522,
      "learning_rate": 4.700643776824034e-07,
      "logits/chosen": -1.067480444908142,
      "logits/rejected": -1.104589819908142,
      "logps/chosen": -404.45001220703125,
      "logps/rejected": -542.9000244140625,
      "loss": 0.0047,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -3.472973585128784,
      "rewards/margins": 17.254688262939453,
      "rewards/rejected": -20.729686737060547,
      "step": 4940
    },
    {
      "epoch": 2.123591889282266,
      "grad_norm": 0.1989401785399322,
      "learning_rate": 4.6899141630901283e-07,
      "logits/chosen": -1.057226538658142,
      "logits/rejected": -1.135473608970642,
      "logps/chosen": -408.67498779296875,
      "logps/rejected": -546.5999755859375,
      "loss": 0.0113,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -3.691210985183716,
      "rewards/margins": 15.300000190734863,
      "rewards/rejected": -18.989063262939453,
      "step": 4950
    },
    {
      "epoch": 2.127883274326789,
      "grad_norm": 0.721478379214845,
      "learning_rate": 4.679184549356223e-07,
      "logits/chosen": -0.971240222454071,
      "logits/rejected": -1.0159423351287842,
      "logps/chosen": -373.375,
      "logps/rejected": -529.9000244140625,
      "loss": 0.0344,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -3.3294434547424316,
      "rewards/margins": 15.7265625,
      "rewards/rejected": -19.065624237060547,
      "step": 4960
    },
    {
      "epoch": 2.1321746593713122,
      "grad_norm": 0.6134904165378215,
      "learning_rate": 4.6684549356223176e-07,
      "logits/chosen": -1.086328148841858,
      "logits/rejected": -1.100341796875,
      "logps/chosen": -390.07501220703125,
      "logps/rejected": -495.04998779296875,
      "loss": 0.0208,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -2.837127685546875,
      "rewards/margins": 14.7734375,
      "rewards/rejected": -17.604686737060547,
      "step": 4970
    },
    {
      "epoch": 2.136466044415835,
      "grad_norm": 0.17165449036213082,
      "learning_rate": 4.6577253218884117e-07,
      "logits/chosen": -0.9689697027206421,
      "logits/rejected": -0.9990234375,
      "logps/chosen": -393.5249938964844,
      "logps/rejected": -532.2999877929688,
      "loss": 0.0175,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -3.0843505859375,
      "rewards/margins": 16.454687118530273,
      "rewards/rejected": -19.535938262939453,
      "step": 4980
    },
    {
      "epoch": 2.140757429460358,
      "grad_norm": 10.715440711865096,
      "learning_rate": 4.6469957081545064e-07,
      "logits/chosen": -0.990234375,
      "logits/rejected": -1.05419921875,
      "logps/chosen": -400.1499938964844,
      "logps/rejected": -514.0999755859375,
      "loss": 0.0118,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.9560546875,
      "rewards/margins": 14.573437690734863,
      "rewards/rejected": -17.526561737060547,
      "step": 4990
    },
    {
      "epoch": 2.1450488145048814,
      "grad_norm": 0.21734046220221115,
      "learning_rate": 4.6362660944206005e-07,
      "logits/chosen": -1.0558593273162842,
      "logits/rejected": -1.113867163658142,
      "logps/chosen": -396.3500061035156,
      "logps/rejected": -535.2999877929688,
      "loss": 0.009,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.804003953933716,
      "rewards/margins": 15.469531059265137,
      "rewards/rejected": -18.270313262939453,
      "step": 5000
    },
    {
      "epoch": 2.1493401995494046,
      "grad_norm": 0.07366536026877506,
      "learning_rate": 4.625536480686695e-07,
      "logits/chosen": -1.0662109851837158,
      "logits/rejected": -1.101318359375,
      "logps/chosen": -373.2749938964844,
      "logps/rejected": -526.5499877929688,
      "loss": 0.0141,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -3.2818846702575684,
      "rewards/margins": 16.060937881469727,
      "rewards/rejected": -19.34375,
      "step": 5010
    },
    {
      "epoch": 2.153631584593928,
      "grad_norm": 0.053383814454467625,
      "learning_rate": 4.61480686695279e-07,
      "logits/chosen": -0.997802734375,
      "logits/rejected": -1.0464355945587158,
      "logps/chosen": -388.0249938964844,
      "logps/rejected": -541.5250244140625,
      "loss": 0.0101,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -3.4193358421325684,
      "rewards/margins": 15.7421875,
      "rewards/rejected": -19.157812118530273,
      "step": 5020
    },
    {
      "epoch": 2.157922969638451,
      "grad_norm": 30.554006644635155,
      "learning_rate": 4.604077253218884e-07,
      "logits/chosen": -1.010839819908142,
      "logits/rejected": -1.05712890625,
      "logps/chosen": -389.9750061035156,
      "logps/rejected": -565.0999755859375,
      "loss": 0.0221,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -3.3442139625549316,
      "rewards/margins": 16.832813262939453,
      "rewards/rejected": -20.174999237060547,
      "step": 5030
    },
    {
      "epoch": 2.1622143546829737,
      "grad_norm": 0.36406288211520504,
      "learning_rate": 4.5933476394849785e-07,
      "logits/chosen": -1.0473144054412842,
      "logits/rejected": -1.095190405845642,
      "logps/chosen": -410.92498779296875,
      "logps/rejected": -543.8499755859375,
      "loss": 0.0163,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -3.646679639816284,
      "rewards/margins": 15.607812881469727,
      "rewards/rejected": -19.265625,
      "step": 5040
    },
    {
      "epoch": 2.166505739727497,
      "grad_norm": 1.3264477997536612,
      "learning_rate": 4.5826180257510726e-07,
      "logits/chosen": -1.056982398033142,
      "logits/rejected": -1.09619140625,
      "logps/chosen": -370.2749938964844,
      "logps/rejected": -482.70001220703125,
      "loss": 0.0094,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -3.526660203933716,
      "rewards/margins": 16.501562118530273,
      "rewards/rejected": -20.032812118530273,
      "step": 5050
    },
    {
      "epoch": 2.17079712477202,
      "grad_norm": 0.127756197175878,
      "learning_rate": 4.5718884120171667e-07,
      "logits/chosen": -0.9688720703125,
      "logits/rejected": -1.0232422351837158,
      "logps/chosen": -389.0375061035156,
      "logps/rejected": -525.3499755859375,
      "loss": 0.0176,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -3.3698363304138184,
      "rewards/margins": 16.307811737060547,
      "rewards/rejected": -19.676563262939453,
      "step": 5060
    },
    {
      "epoch": 2.1750885098165433,
      "grad_norm": 0.1689204957253295,
      "learning_rate": 4.561158798283262e-07,
      "logits/chosen": -0.90478515625,
      "logits/rejected": -0.9461425542831421,
      "logps/chosen": -384.6499938964844,
      "logps/rejected": -550.0,
      "loss": 0.0178,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -3.3662109375,
      "rewards/margins": 16.768749237060547,
      "rewards/rejected": -20.134374618530273,
      "step": 5070
    },
    {
      "epoch": 2.1793798948610665,
      "grad_norm": 11.691878545403311,
      "learning_rate": 4.550429184549356e-07,
      "logits/chosen": -0.985058605670929,
      "logits/rejected": -1.033203125,
      "logps/chosen": -390.25,
      "logps/rejected": -538.2750244140625,
      "loss": 0.0134,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -4.363915920257568,
      "rewards/margins": 17.29296875,
      "rewards/rejected": -21.662500381469727,
      "step": 5080
    },
    {
      "epoch": 2.1836712799055897,
      "grad_norm": 83.96859492226662,
      "learning_rate": 4.5396995708154506e-07,
      "logits/chosen": -0.990966796875,
      "logits/rejected": -1.010656714439392,
      "logps/chosen": -397.82501220703125,
      "logps/rejected": -598.0999755859375,
      "loss": 0.0095,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -4.075976371765137,
      "rewards/margins": 17.720312118530273,
      "rewards/rejected": -21.792186737060547,
      "step": 5090
    },
    {
      "epoch": 2.1879626649501125,
      "grad_norm": 8.167130568299799,
      "learning_rate": 4.528969957081545e-07,
      "logits/chosen": -0.9888671636581421,
      "logits/rejected": -1.0208008289337158,
      "logps/chosen": -380.75,
      "logps/rejected": -501.67498779296875,
      "loss": 0.0178,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -3.988964796066284,
      "rewards/margins": 15.824999809265137,
      "rewards/rejected": -19.814062118530273,
      "step": 5100
    },
    {
      "epoch": 2.1922540499946357,
      "grad_norm": 0.7514487648816531,
      "learning_rate": 4.518240343347639e-07,
      "logits/chosen": -1.0299866199493408,
      "logits/rejected": -1.071069359779358,
      "logps/chosen": -364.875,
      "logps/rejected": -514.1500244140625,
      "loss": 0.0045,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.192889213562012,
      "rewards/margins": 16.693750381469727,
      "rewards/rejected": -20.887500762939453,
      "step": 5110
    },
    {
      "epoch": 2.196545435039159,
      "grad_norm": 1.9850796369896155,
      "learning_rate": 4.507510729613734e-07,
      "logits/chosen": -0.922009289264679,
      "logits/rejected": -0.9329589605331421,
      "logps/chosen": -437.57501220703125,
      "logps/rejected": -537.3499755859375,
      "loss": 0.0248,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -2.6697020530700684,
      "rewards/margins": 16.026561737060547,
      "rewards/rejected": -18.685937881469727,
      "step": 5120
    },
    {
      "epoch": 2.200836820083682,
      "grad_norm": 0.7025867833273719,
      "learning_rate": 4.496781115879828e-07,
      "logits/chosen": -0.9795166254043579,
      "logits/rejected": -1.039697289466858,
      "logps/chosen": -404.375,
      "logps/rejected": -516.2999877929688,
      "loss": 0.0048,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.909472703933716,
      "rewards/margins": 16.010936737060547,
      "rewards/rejected": -18.928125381469727,
      "step": 5130
    },
    {
      "epoch": 2.2051282051282053,
      "grad_norm": 0.3500906087515865,
      "learning_rate": 4.486051502145923e-07,
      "logits/chosen": -0.983715832233429,
      "logits/rejected": -1.0534851551055908,
      "logps/chosen": -432.54998779296875,
      "logps/rejected": -539.0999755859375,
      "loss": 0.0012,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.522509813308716,
      "rewards/margins": 16.248437881469727,
      "rewards/rejected": -18.770313262939453,
      "step": 5140
    },
    {
      "epoch": 2.209419590172728,
      "grad_norm": 0.04524861441466119,
      "learning_rate": 4.475321888412017e-07,
      "logits/chosen": -1.0263671875,
      "logits/rejected": -1.0385253429412842,
      "logps/chosen": -406.7250061035156,
      "logps/rejected": -550.4249877929688,
      "loss": 0.0134,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.9786133766174316,
      "rewards/margins": 17.032812118530273,
      "rewards/rejected": -19.995311737060547,
      "step": 5150
    },
    {
      "epoch": 2.2137109752172512,
      "grad_norm": 8.03434705267353,
      "learning_rate": 4.464592274678111e-07,
      "logits/chosen": -1.008935570716858,
      "logits/rejected": -1.0579102039337158,
      "logps/chosen": -387.7749938964844,
      "logps/rejected": -529.75,
      "loss": 0.0424,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.853222608566284,
      "rewards/margins": 16.379688262939453,
      "rewards/rejected": -19.231250762939453,
      "step": 5160
    },
    {
      "epoch": 2.2180023602617744,
      "grad_norm": 47.28819703651963,
      "learning_rate": 4.453862660944206e-07,
      "logits/chosen": -0.909069836139679,
      "logits/rejected": -0.9488281011581421,
      "logps/chosen": -407.1499938964844,
      "logps/rejected": -536.7999877929688,
      "loss": 0.0195,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -3.4315428733825684,
      "rewards/margins": 16.923437118530273,
      "rewards/rejected": -20.364063262939453,
      "step": 5170
    },
    {
      "epoch": 2.2222937453062976,
      "grad_norm": 2.083238998806373,
      "learning_rate": 4.4431330472103003e-07,
      "logits/chosen": -1.0880858898162842,
      "logits/rejected": -1.1287109851837158,
      "logps/chosen": -425.8500061035156,
      "logps/rejected": -554.5,
      "loss": 0.0368,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -3.27490234375,
      "rewards/margins": 16.768749237060547,
      "rewards/rejected": -20.03125,
      "step": 5180
    },
    {
      "epoch": 2.226585130350821,
      "grad_norm": 0.4925850847248279,
      "learning_rate": 4.432403433476395e-07,
      "logits/chosen": -0.9459228515625,
      "logits/rejected": -1.0033690929412842,
      "logps/chosen": -367.92498779296875,
      "logps/rejected": -504.875,
      "loss": 0.0234,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -2.6727662086486816,
      "rewards/margins": 15.846094131469727,
      "rewards/rejected": -18.506250381469727,
      "step": 5190
    },
    {
      "epoch": 2.230876515395344,
      "grad_norm": 5.7630945979082995,
      "learning_rate": 4.421673819742489e-07,
      "logits/chosen": -1.0013427734375,
      "logits/rejected": -1.067285180091858,
      "logps/chosen": -375.3999938964844,
      "logps/rejected": -500.6000061035156,
      "loss": 0.0143,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.674633741378784,
      "rewards/margins": 15.443750381469727,
      "rewards/rejected": -18.118749618530273,
      "step": 5200
    },
    {
      "epoch": 2.235167900439867,
      "grad_norm": 1.5679725384583572,
      "learning_rate": 4.410944206008583e-07,
      "logits/chosen": -1.095800757408142,
      "logits/rejected": -1.1016113758087158,
      "logps/chosen": -387.1000061035156,
      "logps/rejected": -550.4500122070312,
      "loss": 0.0047,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -3.0596680641174316,
      "rewards/margins": 16.259374618530273,
      "rewards/rejected": -19.3359375,
      "step": 5210
    },
    {
      "epoch": 2.23945928548439,
      "grad_norm": 0.03225714681207087,
      "learning_rate": 4.4002145922746783e-07,
      "logits/chosen": -0.998339831829071,
      "logits/rejected": -1.0480468273162842,
      "logps/chosen": -378.42498779296875,
      "logps/rejected": -541.9000244140625,
      "loss": 0.0065,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -3.501708984375,
      "rewards/margins": 16.284374237060547,
      "rewards/rejected": -19.776561737060547,
      "step": 5220
    },
    {
      "epoch": 2.243750670528913,
      "grad_norm": 0.004920872976934373,
      "learning_rate": 4.3894849785407724e-07,
      "logits/chosen": -1.080175757408142,
      "logits/rejected": -1.0818359851837158,
      "logps/chosen": -406.82501220703125,
      "logps/rejected": -576.5999755859375,
      "loss": 0.0108,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -3.13037109375,
      "rewards/margins": 17.79296875,
      "rewards/rejected": -20.9296875,
      "step": 5230
    },
    {
      "epoch": 2.2480420555734364,
      "grad_norm": 12.726439792503966,
      "learning_rate": 4.378755364806867e-07,
      "logits/chosen": -1.1072266101837158,
      "logits/rejected": -1.160498023033142,
      "logps/chosen": -433.75,
      "logps/rejected": -555.3499755859375,
      "loss": 0.0144,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -3.2865967750549316,
      "rewards/margins": 17.217187881469727,
      "rewards/rejected": -20.495311737060547,
      "step": 5240
    },
    {
      "epoch": 2.2523334406179596,
      "grad_norm": 0.6472053356818118,
      "learning_rate": 4.368025751072961e-07,
      "logits/chosen": -1.0504882335662842,
      "logits/rejected": -1.091406226158142,
      "logps/chosen": -404.7250061035156,
      "logps/rejected": -580.2999877929688,
      "loss": 0.0132,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -4.839276313781738,
      "rewards/margins": 17.854686737060547,
      "rewards/rejected": -22.690624237060547,
      "step": 5250
    },
    {
      "epoch": 2.256624825662483,
      "grad_norm": 2.120632098194414,
      "learning_rate": 4.3572961373390553e-07,
      "logits/chosen": -1.0519530773162842,
      "logits/rejected": -1.104736328125,
      "logps/chosen": -399.57501220703125,
      "logps/rejected": -565.5,
      "loss": 0.0104,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -4.434326171875,
      "rewards/margins": 17.124217987060547,
      "rewards/rejected": -21.556249618530273,
      "step": 5260
    },
    {
      "epoch": 2.2609162107070055,
      "grad_norm": 0.2266722644435845,
      "learning_rate": 4.3465665236051504e-07,
      "logits/chosen": -1.061865210533142,
      "logits/rejected": -1.102636694908142,
      "logps/chosen": -398.70001220703125,
      "logps/rejected": -555.4500122070312,
      "loss": 0.0272,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -3.9124999046325684,
      "rewards/margins": 15.9765625,
      "rewards/rejected": -19.873437881469727,
      "step": 5270
    },
    {
      "epoch": 2.2652075957515287,
      "grad_norm": 0.47840375623469705,
      "learning_rate": 4.3358369098712445e-07,
      "logits/chosen": -1.0869140625,
      "logits/rejected": -1.1140625476837158,
      "logps/chosen": -404.2250061035156,
      "logps/rejected": -554.1500244140625,
      "loss": 0.0048,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -3.31884765625,
      "rewards/margins": 16.293750762939453,
      "rewards/rejected": -19.607812881469727,
      "step": 5280
    },
    {
      "epoch": 2.269498980796052,
      "grad_norm": 1.3972790081288071,
      "learning_rate": 4.3251072961373387e-07,
      "logits/chosen": -1.121191382408142,
      "logits/rejected": -1.1702148914337158,
      "logps/chosen": -427.67498779296875,
      "logps/rejected": -543.2750244140625,
      "loss": 0.005,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -3.4876952171325684,
      "rewards/margins": 17.021875381469727,
      "rewards/rejected": -20.515625,
      "step": 5290
    },
    {
      "epoch": 2.273790365840575,
      "grad_norm": 7.753159743807861,
      "learning_rate": 4.3143776824034333e-07,
      "logits/chosen": -1.1396484375,
      "logits/rejected": -1.14892578125,
      "logps/chosen": -373.95001220703125,
      "logps/rejected": -522.2999877929688,
      "loss": 0.0053,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -3.731201171875,
      "rewards/margins": 16.629688262939453,
      "rewards/rejected": -20.356250762939453,
      "step": 5300
    },
    {
      "epoch": 2.2780817508850983,
      "grad_norm": 1.5625678087647874,
      "learning_rate": 4.3036480686695274e-07,
      "logits/chosen": -1.0247681140899658,
      "logits/rejected": -1.0874755382537842,
      "logps/chosen": -422.82501220703125,
      "logps/rejected": -543.0999755859375,
      "loss": 0.0174,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -3.6404786109924316,
      "rewards/margins": 16.012500762939453,
      "rewards/rejected": -19.657032012939453,
      "step": 5310
    },
    {
      "epoch": 2.282373135929621,
      "grad_norm": 0.6229183782132712,
      "learning_rate": 4.2929184549356226e-07,
      "logits/chosen": -1.095800757408142,
      "logits/rejected": -1.1505858898162842,
      "logps/chosen": -404.9750061035156,
      "logps/rejected": -536.1500244140625,
      "loss": 0.0142,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -3.1294922828674316,
      "rewards/margins": 17.264062881469727,
      "rewards/rejected": -20.396875381469727,
      "step": 5320
    },
    {
      "epoch": 2.2866645209741443,
      "grad_norm": 0.3087413270696727,
      "learning_rate": 4.2821888412017167e-07,
      "logits/chosen": -1.1193358898162842,
      "logits/rejected": -1.1549804210662842,
      "logps/chosen": -417.92498779296875,
      "logps/rejected": -546.6500244140625,
      "loss": 0.0041,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.2455811500549316,
      "rewards/margins": 15.850000381469727,
      "rewards/rejected": -19.106250762939453,
      "step": 5330
    },
    {
      "epoch": 2.2909559060186675,
      "grad_norm": 0.003652175168591188,
      "learning_rate": 4.271459227467811e-07,
      "logits/chosen": -1.068945288658142,
      "logits/rejected": -1.1414062976837158,
      "logps/chosen": -389.875,
      "logps/rejected": -503.75,
      "loss": 0.021,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.663818359375,
      "rewards/margins": 15.785937309265137,
      "rewards/rejected": -18.453125,
      "step": 5340
    },
    {
      "epoch": 2.2952472910631907,
      "grad_norm": 3.884799768529656,
      "learning_rate": 4.2607296137339054e-07,
      "logits/chosen": -1.068457007408142,
      "logits/rejected": -1.152734398841858,
      "logps/chosen": -364.63751220703125,
      "logps/rejected": -527.4749755859375,
      "loss": 0.0133,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -1.917626976966858,
      "rewards/margins": 16.182811737060547,
      "rewards/rejected": -18.096874237060547,
      "step": 5350
    },
    {
      "epoch": 2.299538676107714,
      "grad_norm": 0.16399496948065465,
      "learning_rate": 4.2499999999999995e-07,
      "logits/chosen": -1.111328125,
      "logits/rejected": -1.1243164539337158,
      "logps/chosen": -390.75,
      "logps/rejected": -509.1499938964844,
      "loss": 0.0096,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.883837938308716,
      "rewards/margins": 14.521875381469727,
      "rewards/rejected": -17.409374237060547,
      "step": 5360
    },
    {
      "epoch": 2.3038300611522367,
      "grad_norm": 0.12459157726055184,
      "learning_rate": 4.2392703862660947e-07,
      "logits/chosen": -1.094482421875,
      "logits/rejected": -1.155908226966858,
      "logps/chosen": -411.2250061035156,
      "logps/rejected": -474.8500061035156,
      "loss": 0.0143,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -1.8063476085662842,
      "rewards/margins": 15.239062309265137,
      "rewards/rejected": -17.051563262939453,
      "step": 5370
    },
    {
      "epoch": 2.30812144619676,
      "grad_norm": 0.041684749572121406,
      "learning_rate": 4.228540772532189e-07,
      "logits/chosen": -1.0705077648162842,
      "logits/rejected": -1.136132836341858,
      "logps/chosen": -394.7875061035156,
      "logps/rejected": -525.5999755859375,
      "loss": 0.0066,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.0628418922424316,
      "rewards/margins": 16.475000381469727,
      "rewards/rejected": -18.53125,
      "step": 5380
    },
    {
      "epoch": 2.312412831241283,
      "grad_norm": 0.4458624637924813,
      "learning_rate": 4.217811158798283e-07,
      "logits/chosen": -1.0709717273712158,
      "logits/rejected": -1.086755394935608,
      "logps/chosen": -363.6000061035156,
      "logps/rejected": -522.0,
      "loss": 0.0266,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -2.9620361328125,
      "rewards/margins": 15.334375381469727,
      "rewards/rejected": -18.293750762939453,
      "step": 5390
    },
    {
      "epoch": 2.3167042162858063,
      "grad_norm": 1.0838703671622913,
      "learning_rate": 4.2070815450643776e-07,
      "logits/chosen": -1.082421898841858,
      "logits/rejected": -1.133544921875,
      "logps/chosen": -370.04998779296875,
      "logps/rejected": -534.5,
      "loss": 0.0331,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -3.82958984375,
      "rewards/margins": 16.793750762939453,
      "rewards/rejected": -20.628124237060547,
      "step": 5400
    },
    {
      "epoch": 2.3209956013303294,
      "grad_norm": 0.26152986942580675,
      "learning_rate": 4.1963519313304717e-07,
      "logits/chosen": -1.1138184070587158,
      "logits/rejected": -1.167504906654358,
      "logps/chosen": -381.4750061035156,
      "logps/rejected": -542.2000122070312,
      "loss": 0.006,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.64007568359375,
      "rewards/margins": 17.375,
      "rewards/rejected": -20.034374237060547,
      "step": 5410
    },
    {
      "epoch": 2.3252869863748526,
      "grad_norm": 0.0019311125967350841,
      "learning_rate": 4.185622317596567e-07,
      "logits/chosen": -1.09033203125,
      "logits/rejected": -1.116943359375,
      "logps/chosen": -377.29998779296875,
      "logps/rejected": -535.7000122070312,
      "loss": 0.0201,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.8272461891174316,
      "rewards/margins": 16.582813262939453,
      "rewards/rejected": -19.418750762939453,
      "step": 5420
    },
    {
      "epoch": 2.329578371419376,
      "grad_norm": 14.480264934693706,
      "learning_rate": 4.174892703862661e-07,
      "logits/chosen": -1.1448242664337158,
      "logits/rejected": -1.1791503429412842,
      "logps/chosen": -411.42498779296875,
      "logps/rejected": -538.5750122070312,
      "loss": 0.0064,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -3.0055909156799316,
      "rewards/margins": 17.0546875,
      "rewards/rejected": -20.056249618530273,
      "step": 5430
    },
    {
      "epoch": 2.3338697564638986,
      "grad_norm": 0.3593446635187225,
      "learning_rate": 4.164163090128755e-07,
      "logits/chosen": -1.120361328125,
      "logits/rejected": -1.1204102039337158,
      "logps/chosen": -378.70001220703125,
      "logps/rejected": -557.125,
      "loss": 0.0092,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -3.4407105445861816,
      "rewards/margins": 16.876562118530273,
      "rewards/rejected": -20.3203125,
      "step": 5440
    },
    {
      "epoch": 2.338161141508422,
      "grad_norm": 0.007330612517640106,
      "learning_rate": 4.1534334763948497e-07,
      "logits/chosen": -1.137109398841858,
      "logits/rejected": -1.191015601158142,
      "logps/chosen": -412.57501220703125,
      "logps/rejected": -534.9500122070312,
      "loss": 0.0132,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -3.130859375,
      "rewards/margins": 16.567188262939453,
      "rewards/rejected": -19.681249618530273,
      "step": 5450
    },
    {
      "epoch": 2.342452526552945,
      "grad_norm": 0.8087353389458987,
      "learning_rate": 4.142703862660944e-07,
      "logits/chosen": -1.201416015625,
      "logits/rejected": -1.24951171875,
      "logps/chosen": -420.0249938964844,
      "logps/rejected": -536.7999877929688,
      "loss": 0.01,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -4.246923923492432,
      "rewards/margins": 16.125,
      "rewards/rejected": -20.373437881469727,
      "step": 5460
    },
    {
      "epoch": 2.346743911597468,
      "grad_norm": 4.755790898336294,
      "learning_rate": 4.131974248927038e-07,
      "logits/chosen": -1.124902367591858,
      "logits/rejected": -1.1564452648162842,
      "logps/chosen": -385.4750061035156,
      "logps/rejected": -515.7999877929688,
      "loss": 0.0134,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -3.160717725753784,
      "rewards/margins": 16.515625,
      "rewards/rejected": -19.671875,
      "step": 5470
    },
    {
      "epoch": 2.3510352966419914,
      "grad_norm": 0.16957193415368896,
      "learning_rate": 4.121244635193133e-07,
      "logits/chosen": -1.09423828125,
      "logits/rejected": -1.149072289466858,
      "logps/chosen": -375.8999938964844,
      "logps/rejected": -542.7999877929688,
      "loss": 0.0088,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -4.269189357757568,
      "rewards/margins": 16.610937118530273,
      "rewards/rejected": -20.887500762939453,
      "step": 5480
    },
    {
      "epoch": 2.355326681686514,
      "grad_norm": 0.5155251966851477,
      "learning_rate": 4.110515021459227e-07,
      "logits/chosen": -1.1798827648162842,
      "logits/rejected": -1.2258789539337158,
      "logps/chosen": -426.07501220703125,
      "logps/rejected": -551.4500122070312,
      "loss": 0.0051,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -3.3642578125,
      "rewards/margins": 17.207813262939453,
      "rewards/rejected": -20.578125,
      "step": 5490
    },
    {
      "epoch": 2.3596180667310374,
      "grad_norm": 0.03925810644275777,
      "learning_rate": 4.099785407725322e-07,
      "logits/chosen": -1.156152367591858,
      "logits/rejected": -1.206152319908142,
      "logps/chosen": -389.29998779296875,
      "logps/rejected": -517.4749755859375,
      "loss": 0.0229,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -3.204296827316284,
      "rewards/margins": 15.995312690734863,
      "rewards/rejected": -19.196874618530273,
      "step": 5500
    },
    {
      "epoch": 2.3639094517755606,
      "grad_norm": 0.06456187153106611,
      "learning_rate": 4.089055793991416e-07,
      "logits/chosen": -1.136621117591858,
      "logits/rejected": -1.1979491710662842,
      "logps/chosen": -427.25,
      "logps/rejected": -528.25,
      "loss": 0.0198,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -3.3629393577575684,
      "rewards/margins": 15.651562690734863,
      "rewards/rejected": -19.009374618530273,
      "step": 5510
    },
    {
      "epoch": 2.3682008368200838,
      "grad_norm": 44.63832046339587,
      "learning_rate": 4.07832618025751e-07,
      "logits/chosen": -1.137451171875,
      "logits/rejected": -1.160986304283142,
      "logps/chosen": -403.5,
      "logps/rejected": -565.4500122070312,
      "loss": 0.0017,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.7186279296875,
      "rewards/margins": 17.582813262939453,
      "rewards/rejected": -20.3046875,
      "step": 5520
    },
    {
      "epoch": 2.372492221864607,
      "grad_norm": 0.7412019086772551,
      "learning_rate": 4.067596566523605e-07,
      "logits/chosen": -1.215966820716858,
      "logits/rejected": -1.234960913658142,
      "logps/chosen": -396.4750061035156,
      "logps/rejected": -546.5,
      "loss": 0.0002,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.4859375953674316,
      "rewards/margins": 17.003124237060547,
      "rewards/rejected": -20.496875762939453,
      "step": 5530
    },
    {
      "epoch": 2.3767836069091297,
      "grad_norm": 0.8700084075218637,
      "learning_rate": 4.0568669527896993e-07,
      "logits/chosen": -1.1442382335662842,
      "logits/rejected": -1.1884276866912842,
      "logps/chosen": -399.4750061035156,
      "logps/rejected": -557.9500122070312,
      "loss": 0.0136,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -3.8752808570861816,
      "rewards/margins": 16.350000381469727,
      "rewards/rejected": -20.225000381469727,
      "step": 5540
    },
    {
      "epoch": 2.381074991953653,
      "grad_norm": 10.853439155676176,
      "learning_rate": 4.046137339055794e-07,
      "logits/chosen": -1.1411621570587158,
      "logits/rejected": -1.2041504383087158,
      "logps/chosen": -448.5249938964844,
      "logps/rejected": -552.6500244140625,
      "loss": 0.0081,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.804003953933716,
      "rewards/margins": 16.563282012939453,
      "rewards/rejected": -20.364063262939453,
      "step": 5550
    },
    {
      "epoch": 2.385366376998176,
      "grad_norm": 0.2803492463732435,
      "learning_rate": 4.035407725321888e-07,
      "logits/chosen": -1.1279785633087158,
      "logits/rejected": -1.171777367591858,
      "logps/chosen": -408.92498779296875,
      "logps/rejected": -555.4500122070312,
      "loss": 0.0203,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -3.272216796875,
      "rewards/margins": 15.560937881469727,
      "rewards/rejected": -18.831249237060547,
      "step": 5560
    },
    {
      "epoch": 2.3896577620426993,
      "grad_norm": 0.11846212570441392,
      "learning_rate": 4.024678111587982e-07,
      "logits/chosen": -1.095330834388733,
      "logits/rejected": -1.135986328125,
      "logps/chosen": -414.2250061035156,
      "logps/rejected": -567.7999877929688,
      "loss": 0.0015,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.0911865234375,
      "rewards/margins": 17.964061737060547,
      "rewards/rejected": -21.049999237060547,
      "step": 5570
    },
    {
      "epoch": 2.3939491470872225,
      "grad_norm": 30.694543881692244,
      "learning_rate": 4.0139484978540774e-07,
      "logits/chosen": -1.136865258216858,
      "logits/rejected": -1.18017578125,
      "logps/chosen": -381.5874938964844,
      "logps/rejected": -525.2999877929688,
      "loss": 0.0255,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -3.421826124191284,
      "rewards/margins": 15.760937690734863,
      "rewards/rejected": -19.178905487060547,
      "step": 5580
    },
    {
      "epoch": 2.3982405321317457,
      "grad_norm": 111.97407297625111,
      "learning_rate": 4.0032188841201715e-07,
      "logits/chosen": -1.190527319908142,
      "logits/rejected": -1.2721679210662842,
      "logps/chosen": -386.67498779296875,
      "logps/rejected": -502.04998779296875,
      "loss": 0.0166,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -3.891406297683716,
      "rewards/margins": 14.465624809265137,
      "rewards/rejected": -18.353124618530273,
      "step": 5590
    },
    {
      "epoch": 2.4025319171762685,
      "grad_norm": 0.6633418000195235,
      "learning_rate": 3.992489270386266e-07,
      "logits/chosen": -1.0438964366912842,
      "logits/rejected": -1.097387671470642,
      "logps/chosen": -406.5,
      "logps/rejected": -586.9000244140625,
      "loss": 0.018,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -4.134375095367432,
      "rewards/margins": 17.787500381469727,
      "rewards/rejected": -21.918750762939453,
      "step": 5600
    },
    {
      "epoch": 2.4068233022207917,
      "grad_norm": 0.13014608405177375,
      "learning_rate": 3.98175965665236e-07,
      "logits/chosen": -1.1730468273162842,
      "logits/rejected": -1.21875,
      "logps/chosen": -426.07501220703125,
      "logps/rejected": -563.5999755859375,
      "loss": 0.0012,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.1102051734924316,
      "rewards/margins": 17.415624618530273,
      "rewards/rejected": -20.528125762939453,
      "step": 5610
    },
    {
      "epoch": 2.411114687265315,
      "grad_norm": 0.06941121138354146,
      "learning_rate": 3.9710300429184543e-07,
      "logits/chosen": -1.0751464366912842,
      "logits/rejected": -1.101831078529358,
      "logps/chosen": -367.45001220703125,
      "logps/rejected": -513.0,
      "loss": 0.0062,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -3.8021483421325684,
      "rewards/margins": 16.232812881469727,
      "rewards/rejected": -20.046875,
      "step": 5620
    },
    {
      "epoch": 2.415406072309838,
      "grad_norm": 0.05770604341823292,
      "learning_rate": 3.9603004291845495e-07,
      "logits/chosen": -1.1044433116912842,
      "logits/rejected": -1.139794945716858,
      "logps/chosen": -387.3999938964844,
      "logps/rejected": -555.0999755859375,
      "loss": 0.0192,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -3.205883741378784,
      "rewards/margins": 16.59375,
      "rewards/rejected": -19.796875,
      "step": 5630
    },
    {
      "epoch": 2.4196974573543613,
      "grad_norm": 0.3871990584566747,
      "learning_rate": 3.9495708154506436e-07,
      "logits/chosen": -1.102270483970642,
      "logits/rejected": -1.134667992591858,
      "logps/chosen": -395.82501220703125,
      "logps/rejected": -564.4749755859375,
      "loss": 0.0087,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -3.7181639671325684,
      "rewards/margins": 17.6640625,
      "rewards/rejected": -21.373437881469727,
      "step": 5640
    },
    {
      "epoch": 2.4239888423988845,
      "grad_norm": 0.22896203342676788,
      "learning_rate": 3.938841201716738e-07,
      "logits/chosen": -1.145117163658142,
      "logits/rejected": -1.1711914539337158,
      "logps/chosen": -411.9750061035156,
      "logps/rejected": -519.4500122070312,
      "loss": 0.0139,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -3.2725586891174316,
      "rewards/margins": 15.540624618530273,
      "rewards/rejected": -18.815624237060547,
      "step": 5650
    },
    {
      "epoch": 2.428280227443407,
      "grad_norm": 0.04906188875799851,
      "learning_rate": 3.9281115879828324e-07,
      "logits/chosen": -1.1149413585662842,
      "logits/rejected": -1.1267578601837158,
      "logps/chosen": -402.1499938964844,
      "logps/rejected": -534.2999877929688,
      "loss": 0.0137,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -3.295117139816284,
      "rewards/margins": 16.167186737060547,
      "rewards/rejected": -19.457813262939453,
      "step": 5660
    },
    {
      "epoch": 2.4325716124879304,
      "grad_norm": 0.04723832920646726,
      "learning_rate": 3.9173819742489265e-07,
      "logits/chosen": -1.178320288658142,
      "logits/rejected": -1.22265625,
      "logps/chosen": -393.5625,
      "logps/rejected": -527.0750122070312,
      "loss": 0.0122,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.374462842941284,
      "rewards/margins": 15.359375,
      "rewards/rejected": -17.740625381469727,
      "step": 5670
    },
    {
      "epoch": 2.4368629975324536,
      "grad_norm": 4.204694029302814,
      "learning_rate": 3.9066523605150216e-07,
      "logits/chosen": -1.1519043445587158,
      "logits/rejected": -1.1873047351837158,
      "logps/chosen": -426.1625061035156,
      "logps/rejected": -559.4500122070312,
      "loss": 0.0006,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.138378858566284,
      "rewards/margins": 16.940624237060547,
      "rewards/rejected": -20.089061737060547,
      "step": 5680
    },
    {
      "epoch": 2.441154382576977,
      "grad_norm": 8.874464532835788,
      "learning_rate": 3.895922746781116e-07,
      "logits/chosen": -1.156835913658142,
      "logits/rejected": -1.219751000404358,
      "logps/chosen": -408.32501220703125,
      "logps/rejected": -513.7000122070312,
      "loss": 0.0072,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.2947754859924316,
      "rewards/margins": 16.020313262939453,
      "rewards/rejected": -18.296875,
      "step": 5690
    },
    {
      "epoch": 2.4454457676215,
      "grad_norm": 1.0893991899078694,
      "learning_rate": 3.88519313304721e-07,
      "logits/chosen": -1.19921875,
      "logits/rejected": -1.2463867664337158,
      "logps/chosen": -406.5625,
      "logps/rejected": -536.4000244140625,
      "loss": 0.0051,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.3580565452575684,
      "rewards/margins": 15.4296875,
      "rewards/rejected": -17.798437118530273,
      "step": 5700
    },
    {
      "epoch": 2.4497371526660228,
      "grad_norm": 2.7537734287907516,
      "learning_rate": 3.8744635193133045e-07,
      "logits/chosen": -1.1779296398162842,
      "logits/rejected": -1.244042992591858,
      "logps/chosen": -394.20001220703125,
      "logps/rejected": -514.25,
      "loss": 0.01,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.367358446121216,
      "rewards/margins": 15.1484375,
      "rewards/rejected": -17.509374618530273,
      "step": 5710
    },
    {
      "epoch": 2.454028537710546,
      "grad_norm": 0.05462432706493854,
      "learning_rate": 3.8637339055793986e-07,
      "logits/chosen": -1.158593773841858,
      "logits/rejected": -1.1767578125,
      "logps/chosen": -398.20001220703125,
      "logps/rejected": -530.2000122070312,
      "loss": 0.0178,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -2.8426756858825684,
      "rewards/margins": 15.193750381469727,
      "rewards/rejected": -18.034374237060547,
      "step": 5720
    },
    {
      "epoch": 2.458319922755069,
      "grad_norm": 0.051379750848194,
      "learning_rate": 3.853004291845494e-07,
      "logits/chosen": -1.146826148033142,
      "logits/rejected": -1.184228539466858,
      "logps/chosen": -404.5,
      "logps/rejected": -525.5250244140625,
      "loss": 0.0089,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.1160826683044434,
      "rewards/margins": 16.334375381469727,
      "rewards/rejected": -18.456249237060547,
      "step": 5730
    },
    {
      "epoch": 2.4626113077995924,
      "grad_norm": 0.484295764086103,
      "learning_rate": 3.842274678111588e-07,
      "logits/chosen": -1.123388648033142,
      "logits/rejected": -1.179101586341858,
      "logps/chosen": -415.625,
      "logps/rejected": -545.6500244140625,
      "loss": 0.0024,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.1312804222106934,
      "rewards/margins": 16.411718368530273,
      "rewards/rejected": -18.551563262939453,
      "step": 5740
    },
    {
      "epoch": 2.4669026928441156,
      "grad_norm": 0.6610913120056842,
      "learning_rate": 3.831545064377682e-07,
      "logits/chosen": -1.1625487804412842,
      "logits/rejected": -1.1875,
      "logps/chosen": -399.125,
      "logps/rejected": -557.0999755859375,
      "loss": 0.005,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.4585938453674316,
      "rewards/margins": 17.785938262939453,
      "rewards/rejected": -20.253124237060547,
      "step": 5750
    },
    {
      "epoch": 2.4711940778886383,
      "grad_norm": 0.7450581301080623,
      "learning_rate": 3.8208154506437766e-07,
      "logits/chosen": -1.1408202648162842,
      "logits/rejected": -1.20166015625,
      "logps/chosen": -384.7749938964844,
      "logps/rejected": -546.2999877929688,
      "loss": 0.0015,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.016162157058716,
      "rewards/margins": 17.685937881469727,
      "rewards/rejected": -20.698436737060547,
      "step": 5760
    },
    {
      "epoch": 2.4754854629331615,
      "grad_norm": 1.311313141668686,
      "learning_rate": 3.810085836909871e-07,
      "logits/chosen": -1.0562012195587158,
      "logits/rejected": -1.0993163585662842,
      "logps/chosen": -395.2250061035156,
      "logps/rejected": -524.3499755859375,
      "loss": 0.0314,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -3.298877000808716,
      "rewards/margins": 16.353124618530273,
      "rewards/rejected": -19.662500381469727,
      "step": 5770
    },
    {
      "epoch": 2.4797768479776847,
      "grad_norm": 0.3396052960460103,
      "learning_rate": 3.799356223175966e-07,
      "logits/chosen": -1.0969727039337158,
      "logits/rejected": -1.123632788658142,
      "logps/chosen": -416.54998779296875,
      "logps/rejected": -547.2999877929688,
      "loss": 0.0153,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -3.7972412109375,
      "rewards/margins": 17.232812881469727,
      "rewards/rejected": -21.021875381469727,
      "step": 5780
    },
    {
      "epoch": 2.484068233022208,
      "grad_norm": 0.04814824063700462,
      "learning_rate": 3.78862660944206e-07,
      "logits/chosen": -1.0678222179412842,
      "logits/rejected": -1.1090819835662842,
      "logps/chosen": -417.7749938964844,
      "logps/rejected": -588.4000244140625,
      "loss": 0.009,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -3.4354491233825684,
      "rewards/margins": 17.424999237060547,
      "rewards/rejected": -20.862499237060547,
      "step": 5790
    },
    {
      "epoch": 2.488359618066731,
      "grad_norm": 0.06811507493157097,
      "learning_rate": 3.777896995708154e-07,
      "logits/chosen": -1.046179175376892,
      "logits/rejected": -1.060937523841858,
      "logps/chosen": -410.20001220703125,
      "logps/rejected": -531.9749755859375,
      "loss": 0.0168,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -3.8173828125,
      "rewards/margins": 16.645313262939453,
      "rewards/rejected": -20.456249237060547,
      "step": 5800
    },
    {
      "epoch": 2.4926510031112543,
      "grad_norm": 2.1486622037504417,
      "learning_rate": 3.767167381974249e-07,
      "logits/chosen": -1.106298804283142,
      "logits/rejected": -1.143896460533142,
      "logps/chosen": -423.82501220703125,
      "logps/rejected": -566.75,
      "loss": 0.002,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.128515720367432,
      "rewards/margins": 15.717187881469727,
      "rewards/rejected": -19.853124618530273,
      "step": 5810
    },
    {
      "epoch": 2.4969423881557775,
      "grad_norm": 0.04581458119624618,
      "learning_rate": 3.756437768240343e-07,
      "logits/chosen": -1.1095702648162842,
      "logits/rejected": -1.171972632408142,
      "logps/chosen": -369.45001220703125,
      "logps/rejected": -522.0999755859375,
      "loss": 0.0137,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -3.3989500999450684,
      "rewards/margins": 16.640625,
      "rewards/rejected": -20.043750762939453,
      "step": 5820
    },
    {
      "epoch": 2.5012337732003003,
      "grad_norm": 1.52969525532562,
      "learning_rate": 3.745708154506438e-07,
      "logits/chosen": -1.089501976966858,
      "logits/rejected": -1.127587914466858,
      "logps/chosen": -372.3500061035156,
      "logps/rejected": -539.375,
      "loss": 0.0138,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -3.6758055686950684,
      "rewards/margins": 15.704687118530273,
      "rewards/rejected": -19.375,
      "step": 5830
    },
    {
      "epoch": 2.5055251582448235,
      "grad_norm": 0.2909350294143109,
      "learning_rate": 3.734978540772532e-07,
      "logits/chosen": -1.067968726158142,
      "logits/rejected": -1.1033203601837158,
      "logps/chosen": -397.125,
      "logps/rejected": -535.75,
      "loss": 0.0049,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -3.7779784202575684,
      "rewards/margins": 15.651562690734863,
      "rewards/rejected": -19.4296875,
      "step": 5840
    },
    {
      "epoch": 2.5098165432893467,
      "grad_norm": 0.13216010408953385,
      "learning_rate": 3.7242489270386263e-07,
      "logits/chosen": -1.064306616783142,
      "logits/rejected": -1.1462891101837158,
      "logps/chosen": -414.8500061035156,
      "logps/rejected": -565.6500244140625,
      "loss": 0.0199,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -3.442578077316284,
      "rewards/margins": 17.60546875,
      "rewards/rejected": -21.042186737060547,
      "step": 5850
    },
    {
      "epoch": 2.51410792833387,
      "grad_norm": 0.1458522240029636,
      "learning_rate": 3.713519313304721e-07,
      "logits/chosen": -1.0082519054412842,
      "logits/rejected": -1.050390601158142,
      "logps/chosen": -366.45001220703125,
      "logps/rejected": -476.07501220703125,
      "loss": 0.0141,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -3.1866698265075684,
      "rewards/margins": 15.307812690734863,
      "rewards/rejected": -18.509374618530273,
      "step": 5860
    },
    {
      "epoch": 2.518399313378393,
      "grad_norm": 1.0790216027876043,
      "learning_rate": 3.702789699570815e-07,
      "logits/chosen": -1.1498534679412842,
      "logits/rejected": -1.192968726158142,
      "logps/chosen": -400.2875061035156,
      "logps/rejected": -555.5999755859375,
      "loss": 0.0046,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -3.5619874000549316,
      "rewards/margins": 16.957813262939453,
      "rewards/rejected": -20.5234375,
      "step": 5870
    },
    {
      "epoch": 2.522690698422916,
      "grad_norm": 21.411581447229803,
      "learning_rate": 3.69206008583691e-07,
      "logits/chosen": -1.150976538658142,
      "logits/rejected": -1.16259765625,
      "logps/chosen": -413.1499938964844,
      "logps/rejected": -543.8499755859375,
      "loss": 0.0163,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -3.542773485183716,
      "rewards/margins": 16.506250381469727,
      "rewards/rejected": -20.057811737060547,
      "step": 5880
    },
    {
      "epoch": 2.526982083467439,
      "grad_norm": 0.063624364626722,
      "learning_rate": 3.6813304721030043e-07,
      "logits/chosen": -1.1453857421875,
      "logits/rejected": -1.1608397960662842,
      "logps/chosen": -393.3999938964844,
      "logps/rejected": -531.9000244140625,
      "loss": 0.0105,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -3.4307618141174316,
      "rewards/margins": 15.641406059265137,
      "rewards/rejected": -19.064062118530273,
      "step": 5890
    },
    {
      "epoch": 2.531273468511962,
      "grad_norm": 0.01016691276465561,
      "learning_rate": 3.6706008583690984e-07,
      "logits/chosen": -1.1398437023162842,
      "logits/rejected": -1.1657226085662842,
      "logps/chosen": -379.38751220703125,
      "logps/rejected": -525.5,
      "loss": 0.0327,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.431103467941284,
      "rewards/margins": 15.796875,
      "rewards/rejected": -18.235937118530273,
      "step": 5900
    },
    {
      "epoch": 2.5355648535564854,
      "grad_norm": 1.5617445529583414,
      "learning_rate": 3.659871244635193e-07,
      "logits/chosen": -1.2175781726837158,
      "logits/rejected": -1.2086913585662842,
      "logps/chosen": -388.79998779296875,
      "logps/rejected": -511.45001220703125,
      "loss": 0.0087,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.7445921897888184,
      "rewards/margins": 15.125,
      "rewards/rejected": -17.871875762939453,
      "step": 5910
    },
    {
      "epoch": 2.5398562386010086,
      "grad_norm": 3.0247896748281113,
      "learning_rate": 3.649141630901287e-07,
      "logits/chosen": -1.1659667491912842,
      "logits/rejected": -1.229248046875,
      "logps/chosen": -395.67498779296875,
      "logps/rejected": -553.2999877929688,
      "loss": 0.0022,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.319506883621216,
      "rewards/margins": 16.403125762939453,
      "rewards/rejected": -18.735937118530273,
      "step": 5920
    },
    {
      "epoch": 2.5441476236455314,
      "grad_norm": 0.12037995513584396,
      "learning_rate": 3.638412017167382e-07,
      "logits/chosen": -1.159570336341858,
      "logits/rejected": -1.2067382335662842,
      "logps/chosen": -396.48748779296875,
      "logps/rejected": -513.7000122070312,
      "loss": 0.009,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -1.9020111560821533,
      "rewards/margins": 15.9453125,
      "rewards/rejected": -17.84375,
      "step": 5930
    },
    {
      "epoch": 2.5484390086900546,
      "grad_norm": 0.0533418216161194,
      "learning_rate": 3.6276824034334764e-07,
      "logits/chosen": -1.101953148841858,
      "logits/rejected": -1.157373070716858,
      "logps/chosen": -376.75,
      "logps/rejected": -520.3499755859375,
      "loss": 0.0087,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.12359619140625,
      "rewards/margins": 15.671875,
      "rewards/rejected": -17.790624618530273,
      "step": 5940
    },
    {
      "epoch": 2.5527303937345778,
      "grad_norm": 3.651463623931559,
      "learning_rate": 3.6169527896995706e-07,
      "logits/chosen": -1.1693847179412842,
      "logits/rejected": -1.190332055091858,
      "logps/chosen": -383.67498779296875,
      "logps/rejected": -497.3999938964844,
      "loss": 0.0175,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -1.9588744640350342,
      "rewards/margins": 15.667187690734863,
      "rewards/rejected": -17.629688262939453,
      "step": 5950
    },
    {
      "epoch": 2.557021778779101,
      "grad_norm": 0.08250108047939421,
      "learning_rate": 3.606223175965665e-07,
      "logits/chosen": -1.133642554283142,
      "logits/rejected": -1.194921851158142,
      "logps/chosen": -425.61248779296875,
      "logps/rejected": -512.7999877929688,
      "loss": 0.0137,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.364971876144409,
      "rewards/margins": 16.376562118530273,
      "rewards/rejected": -18.743749618530273,
      "step": 5960
    },
    {
      "epoch": 2.561313163823624,
      "grad_norm": 25.782703464304635,
      "learning_rate": 3.5954935622317593e-07,
      "logits/chosen": -1.117089867591858,
      "logits/rejected": -1.174707055091858,
      "logps/chosen": -388.82501220703125,
      "logps/rejected": -517.5499877929688,
      "loss": 0.0099,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.824267625808716,
      "rewards/margins": 15.1953125,
      "rewards/rejected": -18.0078125,
      "step": 5970
    },
    {
      "epoch": 2.565604548868147,
      "grad_norm": 0.03578850988641143,
      "learning_rate": 3.584763948497854e-07,
      "logits/chosen": -1.1417236328125,
      "logits/rejected": -1.17041015625,
      "logps/chosen": -419.3999938964844,
      "logps/rejected": -534.0,
      "loss": 0.0047,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -3.2840819358825684,
      "rewards/margins": 15.675000190734863,
      "rewards/rejected": -18.9609375,
      "step": 5980
    },
    {
      "epoch": 2.5698959339126706,
      "grad_norm": 0.08619582933257304,
      "learning_rate": 3.5740343347639486e-07,
      "logits/chosen": -1.071533203125,
      "logits/rejected": -1.09228515625,
      "logps/chosen": -419.375,
      "logps/rejected": -550.125,
      "loss": 0.0046,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -3.695141553878784,
      "rewards/margins": 16.954687118530273,
      "rewards/rejected": -20.651561737060547,
      "step": 5990
    },
    {
      "epoch": 2.5741873189571933,
      "grad_norm": 0.027925863872517394,
      "learning_rate": 3.5633047210300427e-07,
      "logits/chosen": -1.094335913658142,
      "logits/rejected": -1.1645996570587158,
      "logps/chosen": -368.45001220703125,
      "logps/rejected": -517.25,
      "loss": 0.005,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -3.3781371116638184,
      "rewards/margins": 16.2265625,
      "rewards/rejected": -19.618749618530273,
      "step": 6000
    },
    {
      "epoch": 2.5784787040017165,
      "grad_norm": 0.4479928190697049,
      "learning_rate": 3.5525751072961373e-07,
      "logits/chosen": -1.1359374523162842,
      "logits/rejected": -1.155029296875,
      "logps/chosen": -409.07501220703125,
      "logps/rejected": -543.4500122070312,
      "loss": 0.0055,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.545483350753784,
      "rewards/margins": 17.326562881469727,
      "rewards/rejected": -19.885936737060547,
      "step": 6010
    },
    {
      "epoch": 2.5827700890462397,
      "grad_norm": 0.06470103911127432,
      "learning_rate": 3.5418454935622314e-07,
      "logits/chosen": -1.0671875476837158,
      "logits/rejected": -1.1250731945037842,
      "logps/chosen": -362.9750061035156,
      "logps/rejected": -529.4000244140625,
      "loss": 0.0007,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.0658936500549316,
      "rewards/margins": 16.126562118530273,
      "rewards/rejected": -19.201562881469727,
      "step": 6020
    },
    {
      "epoch": 2.587061474090763,
      "grad_norm": 0.6485464310767913,
      "learning_rate": 3.531115879828326e-07,
      "logits/chosen": -1.1291992664337158,
      "logits/rejected": -1.1618163585662842,
      "logps/chosen": -389.0,
      "logps/rejected": -536.7999877929688,
      "loss": 0.0114,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -3.468554735183716,
      "rewards/margins": 17.6875,
      "rewards/rejected": -21.157812118530273,
      "step": 6030
    },
    {
      "epoch": 2.591352859135286,
      "grad_norm": 0.0688533411850023,
      "learning_rate": 3.5203862660944207e-07,
      "logits/chosen": -1.049523949623108,
      "logits/rejected": -1.121801733970642,
      "logps/chosen": -378.7250061035156,
      "logps/rejected": -501.2250061035156,
      "loss": 0.0482,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.294482469558716,
      "rewards/margins": 15.535937309265137,
      "rewards/rejected": -17.837499618530273,
      "step": 6040
    },
    {
      "epoch": 2.595644244179809,
      "grad_norm": 16.365263159961124,
      "learning_rate": 3.509656652360515e-07,
      "logits/chosen": -1.085058569908142,
      "logits/rejected": -1.0940430164337158,
      "logps/chosen": -354.8500061035156,
      "logps/rejected": -525.0,
      "loss": 0.0093,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -3.345043897628784,
      "rewards/margins": 16.053125381469727,
      "rewards/rejected": -19.403125762939453,
      "step": 6050
    },
    {
      "epoch": 2.599935629224332,
      "grad_norm": 4.047277602416829,
      "learning_rate": 3.4989270386266095e-07,
      "logits/chosen": -1.113183617591858,
      "logits/rejected": -1.154687523841858,
      "logps/chosen": -431.7124938964844,
      "logps/rejected": -551.25,
      "loss": 0.0117,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -3.344970703125,
      "rewards/margins": 17.181249618530273,
      "rewards/rejected": -20.509374618530273,
      "step": 6060
    },
    {
      "epoch": 2.6042270142688553,
      "grad_norm": 0.3130990368515212,
      "learning_rate": 3.4881974248927036e-07,
      "logits/chosen": -0.9510742425918579,
      "logits/rejected": -1.0143554210662842,
      "logps/chosen": -369.4750061035156,
      "logps/rejected": -523.0999755859375,
      "loss": 0.0235,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -2.5815186500549316,
      "rewards/margins": 17.015625,
      "rewards/rejected": -19.610937118530273,
      "step": 6070
    },
    {
      "epoch": 2.6085183993133785,
      "grad_norm": 0.3784010252488101,
      "learning_rate": 3.4774678111587977e-07,
      "logits/chosen": -1.13525390625,
      "logits/rejected": -1.1865234375,
      "logps/chosen": -395.125,
      "logps/rejected": -550.6500244140625,
      "loss": 0.0004,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.036816358566284,
      "rewards/margins": 16.868749618530273,
      "rewards/rejected": -19.907812118530273,
      "step": 6080
    },
    {
      "epoch": 2.6128097843579017,
      "grad_norm": 1.4778778603259732,
      "learning_rate": 3.466738197424893e-07,
      "logits/chosen": -1.1120116710662842,
      "logits/rejected": -1.148046851158142,
      "logps/chosen": -405.6499938964844,
      "logps/rejected": -531.2999877929688,
      "loss": 0.0054,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.806347608566284,
      "rewards/margins": 17.1015625,
      "rewards/rejected": -19.924999237060547,
      "step": 6090
    },
    {
      "epoch": 2.6171011694024244,
      "grad_norm": 3.4659834168960564,
      "learning_rate": 3.456008583690987e-07,
      "logits/chosen": -1.1062500476837158,
      "logits/rejected": -1.144677758216858,
      "logps/chosen": -398.0249938964844,
      "logps/rejected": -533.3499755859375,
      "loss": 0.0069,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -3.404296875,
      "rewards/margins": 17.1328125,
      "rewards/rejected": -20.5390625,
      "step": 6100
    },
    {
      "epoch": 2.6213925544469476,
      "grad_norm": 0.3397205704093927,
      "learning_rate": 3.445278969957081e-07,
      "logits/chosen": -1.118554711341858,
      "logits/rejected": -1.148339867591858,
      "logps/chosen": -431.04998779296875,
      "logps/rejected": -563.25,
      "loss": 0.0058,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -3.5126953125,
      "rewards/margins": 16.701562881469727,
      "rewards/rejected": -20.214061737060547,
      "step": 6110
    },
    {
      "epoch": 2.625683939491471,
      "grad_norm": 0.09021591985807836,
      "learning_rate": 3.4345493562231757e-07,
      "logits/chosen": -1.115014672279358,
      "logits/rejected": -1.1656372547149658,
      "logps/chosen": -404.17498779296875,
      "logps/rejected": -566.75,
      "loss": 0.0058,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.7041993141174316,
      "rewards/margins": 17.203125,
      "rewards/rejected": -19.912500381469727,
      "step": 6120
    },
    {
      "epoch": 2.629975324535994,
      "grad_norm": 0.033456552404593026,
      "learning_rate": 3.42381974248927e-07,
      "logits/chosen": -1.1680176258087158,
      "logits/rejected": -1.219140648841858,
      "logps/chosen": -401.20001220703125,
      "logps/rejected": -536.9000244140625,
      "loss": 0.0162,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.124267578125,
      "rewards/margins": 16.042186737060547,
      "rewards/rejected": -18.170312881469727,
      "step": 6130
    },
    {
      "epoch": 2.6342667095805172,
      "grad_norm": 0.012679231099117165,
      "learning_rate": 3.413090128755365e-07,
      "logits/chosen": -1.0952637195587158,
      "logits/rejected": -1.1824219226837158,
      "logps/chosen": -381.67498779296875,
      "logps/rejected": -514.7999877929688,
      "loss": 0.0057,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.836718797683716,
      "rewards/margins": 15.743749618530273,
      "rewards/rejected": -18.587499618530273,
      "step": 6140
    },
    {
      "epoch": 2.63855809462504,
      "grad_norm": 0.5625709514809482,
      "learning_rate": 3.402360515021459e-07,
      "logits/chosen": -1.11376953125,
      "logits/rejected": -1.169531226158142,
      "logps/chosen": -427.375,
      "logps/rejected": -571.2000122070312,
      "loss": 0.0004,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.221057176589966,
      "rewards/margins": 16.440624237060547,
      "rewards/rejected": -19.667186737060547,
      "step": 6150
    },
    {
      "epoch": 2.6428494796695636,
      "grad_norm": 0.03841319460093605,
      "learning_rate": 3.391630901287553e-07,
      "logits/chosen": -1.087915062904358,
      "logits/rejected": -1.084863305091858,
      "logps/chosen": -367.1000061035156,
      "logps/rejected": -536.0,
      "loss": 0.0143,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -3.4153809547424316,
      "rewards/margins": 18.2265625,
      "rewards/rejected": -21.640625,
      "step": 6160
    },
    {
      "epoch": 2.6471408647140864,
      "grad_norm": 0.04919708597642205,
      "learning_rate": 3.380901287553648e-07,
      "logits/chosen": -1.2096679210662842,
      "logits/rejected": -1.2317383289337158,
      "logps/chosen": -422.20001220703125,
      "logps/rejected": -544.6500244140625,
      "loss": 0.0055,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -4.257714748382568,
      "rewards/margins": 16.329687118530273,
      "rewards/rejected": -20.595312118530273,
      "step": 6170
    },
    {
      "epoch": 2.6514322497586096,
      "grad_norm": 0.2859959319544514,
      "learning_rate": 3.370171673819742e-07,
      "logits/chosen": -1.0998046398162842,
      "logits/rejected": -1.1304199695587158,
      "logps/chosen": -375.70001220703125,
      "logps/rejected": -502.04998779296875,
      "loss": 0.0095,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -3.2306151390075684,
      "rewards/margins": 16.634374618530273,
      "rewards/rejected": -19.871875762939453,
      "step": 6180
    },
    {
      "epoch": 2.655723634803133,
      "grad_norm": 1.5862905027322383,
      "learning_rate": 3.359442060085837e-07,
      "logits/chosen": -1.22998046875,
      "logits/rejected": -1.2902343273162842,
      "logps/chosen": -394.7250061035156,
      "logps/rejected": -556.4000244140625,
      "loss": 0.0046,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -4.377172946929932,
      "rewards/margins": 17.848438262939453,
      "rewards/rejected": -22.225000381469727,
      "step": 6190
    },
    {
      "epoch": 2.660015019847656,
      "grad_norm": 0.12713119445498408,
      "learning_rate": 3.348712446351931e-07,
      "logits/chosen": -1.2356445789337158,
      "logits/rejected": -1.3152344226837158,
      "logps/chosen": -454.70001220703125,
      "logps/rejected": -576.2999877929688,
      "loss": 0.0044,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -4.132843017578125,
      "rewards/margins": 17.606250762939453,
      "rewards/rejected": -21.746875762939453,
      "step": 6200
    },
    {
      "epoch": 2.664306404892179,
      "grad_norm": 1.9771307577383137,
      "learning_rate": 3.3379828326180253e-07,
      "logits/chosen": -1.0989258289337158,
      "logits/rejected": -1.1527831554412842,
      "logps/chosen": -405.8999938964844,
      "logps/rejected": -577.25,
      "loss": 0.033,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -4.186327934265137,
      "rewards/margins": 17.9609375,
      "rewards/rejected": -22.149999618530273,
      "step": 6210
    },
    {
      "epoch": 2.668597789936702,
      "grad_norm": 0.3739970249160375,
      "learning_rate": 3.32725321888412e-07,
      "logits/chosen": -1.1715087890625,
      "logits/rejected": -1.2533690929412842,
      "logps/chosen": -415.875,
      "logps/rejected": -545.9000244140625,
      "loss": 0.0189,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -4.358788967132568,
      "rewards/margins": 15.454687118530273,
      "rewards/rejected": -19.818750381469727,
      "step": 6220
    },
    {
      "epoch": 2.672889174981225,
      "grad_norm": 0.028976249331164185,
      "learning_rate": 3.316523605150214e-07,
      "logits/chosen": -1.175683617591858,
      "logits/rejected": -1.2211425304412842,
      "logps/chosen": -392.75,
      "logps/rejected": -565.0499877929688,
      "loss": 0.0133,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -4.058007717132568,
      "rewards/margins": 17.690624237060547,
      "rewards/rejected": -21.740625381469727,
      "step": 6230
    },
    {
      "epoch": 2.6771805600257483,
      "grad_norm": 6.042038606435633,
      "learning_rate": 3.305793991416309e-07,
      "logits/chosen": -1.2159926891326904,
      "logits/rejected": -1.249017357826233,
      "logps/chosen": -383.70001220703125,
      "logps/rejected": -551.0999755859375,
      "loss": 0.0026,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.5869140625,
      "rewards/margins": 18.411718368530273,
      "rewards/rejected": -22.004688262939453,
      "step": 6240
    },
    {
      "epoch": 2.6814719450702715,
      "grad_norm": 2.442412286141214,
      "learning_rate": 3.2950643776824034e-07,
      "logits/chosen": -1.1765625476837158,
      "logits/rejected": -1.2177734375,
      "logps/chosen": -409.0562438964844,
      "logps/rejected": -518.375,
      "loss": 0.0051,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -3.406445264816284,
      "rewards/margins": 15.860937118530273,
      "rewards/rejected": -19.267187118530273,
      "step": 6250
    },
    {
      "epoch": 2.6857633301147947,
      "grad_norm": 0.18297352062757927,
      "learning_rate": 3.2843347639484975e-07,
      "logits/chosen": -1.121972680091858,
      "logits/rejected": -1.1828124523162842,
      "logps/chosen": -432.2250061035156,
      "logps/rejected": -546.9500122070312,
      "loss": 0.0189,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -5.041259765625,
      "rewards/margins": 16.928125381469727,
      "rewards/rejected": -21.96875,
      "step": 6260
    },
    {
      "epoch": 2.6900547151593175,
      "grad_norm": 0.14983124242754417,
      "learning_rate": 3.273605150214592e-07,
      "logits/chosen": -1.139794945716858,
      "logits/rejected": -1.1677734851837158,
      "logps/chosen": -416.82501220703125,
      "logps/rejected": -573.5499877929688,
      "loss": 0.0089,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -3.692065477371216,
      "rewards/margins": 17.765625,
      "rewards/rejected": -21.462499618530273,
      "step": 6270
    },
    {
      "epoch": 2.6943461002038407,
      "grad_norm": 0.2511333057878584,
      "learning_rate": 3.262875536480686e-07,
      "logits/chosen": -1.1207275390625,
      "logits/rejected": -1.1764647960662842,
      "logps/chosen": -416.20001220703125,
      "logps/rejected": -573.9500122070312,
      "loss": 0.0073,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -5.306445121765137,
      "rewards/margins": 16.901561737060547,
      "rewards/rejected": -22.223438262939453,
      "step": 6280
    },
    {
      "epoch": 2.698637485248364,
      "grad_norm": 0.017253096173398912,
      "learning_rate": 3.2521459227467814e-07,
      "logits/chosen": -1.1545898914337158,
      "logits/rejected": -1.2039062976837158,
      "logps/chosen": -420.07501220703125,
      "logps/rejected": -558.5,
      "loss": 0.0007,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.107079982757568,
      "rewards/margins": 18.3984375,
      "rewards/rejected": -22.493749618530273,
      "step": 6290
    },
    {
      "epoch": 2.702928870292887,
      "grad_norm": 0.16407107375088972,
      "learning_rate": 3.2414163090128755e-07,
      "logits/chosen": -1.1139647960662842,
      "logits/rejected": -1.1533691883087158,
      "logps/chosen": -381.95001220703125,
      "logps/rejected": -531.4000244140625,
      "loss": 0.0163,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -3.0899415016174316,
      "rewards/margins": 17.757030487060547,
      "rewards/rejected": -20.839061737060547,
      "step": 6300
    },
    {
      "epoch": 2.7072202553374103,
      "grad_norm": 1.558264793866727,
      "learning_rate": 3.2306866952789696e-07,
      "logits/chosen": -1.1912841796875,
      "logits/rejected": -1.2086181640625,
      "logps/chosen": -424.6875,
      "logps/rejected": -578.4000244140625,
      "loss": 0.0098,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -3.8907227516174316,
      "rewards/margins": 17.623437881469727,
      "rewards/rejected": -21.514062881469727,
      "step": 6310
    },
    {
      "epoch": 2.711511640381933,
      "grad_norm": 0.12449114444918907,
      "learning_rate": 3.219957081545064e-07,
      "logits/chosen": -1.0435791015625,
      "logits/rejected": -1.1114501953125,
      "logps/chosen": -389.0,
      "logps/rejected": -517.4500122070312,
      "loss": 0.0175,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -4.762793064117432,
      "rewards/margins": 17.571874618530273,
      "rewards/rejected": -22.345312118530273,
      "step": 6320
    },
    {
      "epoch": 2.7158030254264562,
      "grad_norm": 0.341440573494519,
      "learning_rate": 3.2092274678111584e-07,
      "logits/chosen": -1.094970703125,
      "logits/rejected": -1.14208984375,
      "logps/chosen": -343.95001220703125,
      "logps/rejected": -492.1499938964844,
      "loss": 0.0166,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -4.232324123382568,
      "rewards/margins": 16.479686737060547,
      "rewards/rejected": -20.714061737060547,
      "step": 6330
    },
    {
      "epoch": 2.7200944104709794,
      "grad_norm": 0.7303841002564108,
      "learning_rate": 3.198497854077253e-07,
      "logits/chosen": -1.0439453125,
      "logits/rejected": -1.110693335533142,
      "logps/chosen": -396.25,
      "logps/rejected": -521.5499877929688,
      "loss": 0.0161,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -3.961669921875,
      "rewards/margins": 16.139062881469727,
      "rewards/rejected": -20.104686737060547,
      "step": 6340
    },
    {
      "epoch": 2.7243857955155026,
      "grad_norm": 5.695949385887621,
      "learning_rate": 3.1877682403433476e-07,
      "logits/chosen": -1.0993835926055908,
      "logits/rejected": -1.168359398841858,
      "logps/chosen": -389.125,
      "logps/rejected": -515.8499755859375,
      "loss": 0.0053,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.613037109375,
      "rewards/margins": 15.879687309265137,
      "rewards/rejected": -18.4921875,
      "step": 6350
    },
    {
      "epoch": 2.728677180560026,
      "grad_norm": 0.30767965365017286,
      "learning_rate": 3.177038626609442e-07,
      "logits/chosen": -1.146875023841858,
      "logits/rejected": -1.176049828529358,
      "logps/chosen": -418.5,
      "logps/rejected": -559.2000122070312,
      "loss": 0.0046,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -3.7760252952575684,
      "rewards/margins": 17.5546875,
      "rewards/rejected": -21.331249237060547,
      "step": 6360
    },
    {
      "epoch": 2.7329685656045486,
      "grad_norm": 0.2511619576486326,
      "learning_rate": 3.1663090128755364e-07,
      "logits/chosen": -1.062890648841858,
      "logits/rejected": -1.1300780773162842,
      "logps/chosen": -379.7250061035156,
      "logps/rejected": -504.75,
      "loss": 0.0196,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.916796922683716,
      "rewards/margins": 16.706249237060547,
      "rewards/rejected": -19.635936737060547,
      "step": 6370
    },
    {
      "epoch": 2.7372599506490722,
      "grad_norm": 2.5712852046523103,
      "learning_rate": 3.1555793991416305e-07,
      "logits/chosen": -1.114648461341858,
      "logits/rejected": -1.174951195716858,
      "logps/chosen": -399.9750061035156,
      "logps/rejected": -506.67498779296875,
      "loss": 0.0186,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -3.8543944358825684,
      "rewards/margins": 14.822656631469727,
      "rewards/rejected": -18.676563262939453,
      "step": 6380
    },
    {
      "epoch": 2.741551335693595,
      "grad_norm": 58.23191231248602,
      "learning_rate": 3.144849785407725e-07,
      "logits/chosen": -1.143310546875,
      "logits/rejected": -1.168359398841858,
      "logps/chosen": -407.125,
      "logps/rejected": -522.0,
      "loss": 0.0179,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -3.79833984375,
      "rewards/margins": 17.501562118530273,
      "rewards/rejected": -21.295312881469727,
      "step": 6390
    },
    {
      "epoch": 2.745842720738118,
      "grad_norm": 0.665111643510761,
      "learning_rate": 3.13412017167382e-07,
      "logits/chosen": -1.10693359375,
      "logits/rejected": -1.1408202648162842,
      "logps/chosen": -384.3500061035156,
      "logps/rejected": -530.0250244140625,
      "loss": 0.0005,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.323779106140137,
      "rewards/margins": 16.717187881469727,
      "rewards/rejected": -21.037500381469727,
      "step": 6400
    },
    {
      "epoch": 2.7501341057826414,
      "grad_norm": 0.017159448941161805,
      "learning_rate": 3.123390557939914e-07,
      "logits/chosen": -1.1697266101837158,
      "logits/rejected": -1.212988257408142,
      "logps/chosen": -472.4750061035156,
      "logps/rejected": -624.2000122070312,
      "loss": 0.0046,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -3.854199171066284,
      "rewards/margins": 17.709375381469727,
      "rewards/rejected": -21.565624237060547,
      "step": 6410
    },
    {
      "epoch": 2.7544254908271646,
      "grad_norm": 0.2467001773901379,
      "learning_rate": 3.1126609442060085e-07,
      "logits/chosen": -1.136132836341858,
      "logits/rejected": -1.1658446788787842,
      "logps/chosen": -444.8999938964844,
      "logps/rejected": -580.0499877929688,
      "loss": 0.0087,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -3.6280274391174316,
      "rewards/margins": 18.415624618530273,
      "rewards/rejected": -22.043750762939453,
      "step": 6420
    },
    {
      "epoch": 2.758716875871688,
      "grad_norm": 1.7631911531526012,
      "learning_rate": 3.1019313304721026e-07,
      "logits/chosen": -1.027856469154358,
      "logits/rejected": -1.086035132408142,
      "logps/chosen": -378.0249938964844,
      "logps/rejected": -539.9000244140625,
      "loss": 0.0133,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -3.9754881858825684,
      "rewards/margins": 17.1484375,
      "rewards/rejected": -21.1328125,
      "step": 6430
    },
    {
      "epoch": 2.7630082609162105,
      "grad_norm": 70.40410095025409,
      "learning_rate": 3.0912017167381973e-07,
      "logits/chosen": -1.1338379383087158,
      "logits/rejected": -1.1833984851837158,
      "logps/chosen": -415.32501220703125,
      "logps/rejected": -554.0,
      "loss": 0.0044,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.824316501617432,
      "rewards/margins": 17.120311737060547,
      "rewards/rejected": -21.9453125,
      "step": 6440
    },
    {
      "epoch": 2.7672996459607337,
      "grad_norm": 0.6695117155853282,
      "learning_rate": 3.080472103004292e-07,
      "logits/chosen": -1.067724585533142,
      "logits/rejected": -1.0922362804412842,
      "logps/chosen": -398.29998779296875,
      "logps/rejected": -550.5250244140625,
      "loss": 0.0317,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -4.178808689117432,
      "rewards/margins": 17.3671875,
      "rewards/rejected": -21.548437118530273,
      "step": 6450
    },
    {
      "epoch": 2.771591031005257,
      "grad_norm": 0.13602896704869735,
      "learning_rate": 3.069742489270386e-07,
      "logits/chosen": -1.153222680091858,
      "logits/rejected": -1.1833007335662842,
      "logps/chosen": -396.5,
      "logps/rejected": -548.0499877929688,
      "loss": 0.0103,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -4.59136962890625,
      "rewards/margins": 17.310937881469727,
      "rewards/rejected": -21.896875381469727,
      "step": 6460
    },
    {
      "epoch": 2.77588241604978,
      "grad_norm": 0.2400847824783146,
      "learning_rate": 3.0590128755364807e-07,
      "logits/chosen": -1.1359131336212158,
      "logits/rejected": -1.168115258216858,
      "logps/chosen": -408.7250061035156,
      "logps/rejected": -523.75,
      "loss": 0.0176,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -2.640869140625,
      "rewards/margins": 17.017187118530273,
      "rewards/rejected": -19.659374237060547,
      "step": 6470
    },
    {
      "epoch": 2.7801738010943033,
      "grad_norm": 4.830861868678675,
      "learning_rate": 3.048283261802575e-07,
      "logits/chosen": -1.1575195789337158,
      "logits/rejected": -1.202050805091858,
      "logps/chosen": -391.75,
      "logps/rejected": -537.7999877929688,
      "loss": 0.0056,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -3.327099561691284,
      "rewards/margins": 16.549999237060547,
      "rewards/rejected": -19.879688262939453,
      "step": 6480
    },
    {
      "epoch": 2.784465186138826,
      "grad_norm": 0.04358226587651279,
      "learning_rate": 3.0375536480686694e-07,
      "logits/chosen": -1.1005859375,
      "logits/rejected": -1.18115234375,
      "logps/chosen": -406.7250061035156,
      "logps/rejected": -519.2750244140625,
      "loss": 0.0156,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.890429735183716,
      "rewards/margins": 16.075780868530273,
      "rewards/rejected": -18.954687118530273,
      "step": 6490
    },
    {
      "epoch": 2.7887565711833493,
      "grad_norm": 0.4869708208903308,
      "learning_rate": 3.026824034334764e-07,
      "logits/chosen": -1.068457007408142,
      "logits/rejected": -1.147851586341858,
      "logps/chosen": -372.125,
      "logps/rejected": -527.2999877929688,
      "loss": 0.0096,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -3.4208984375,
      "rewards/margins": 15.40625,
      "rewards/rejected": -18.818750381469727,
      "step": 6500
    },
    {
      "epoch": 2.7930479562278725,
      "grad_norm": 0.06855429348751023,
      "learning_rate": 3.016094420600858e-07,
      "logits/chosen": -1.082910180091858,
      "logits/rejected": -1.116601586341858,
      "logps/chosen": -391.95001220703125,
      "logps/rejected": -525.2000122070312,
      "loss": 0.0102,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -3.1814942359924316,
      "rewards/margins": 15.925000190734863,
      "rewards/rejected": -19.09375,
      "step": 6510
    },
    {
      "epoch": 2.7973393412723957,
      "grad_norm": 4.136598653759272,
      "learning_rate": 3.005364806866953e-07,
      "logits/chosen": -1.0823974609375,
      "logits/rejected": -1.117285132408142,
      "logps/chosen": -394.1499938964844,
      "logps/rejected": -549.5499877929688,
      "loss": 0.0092,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -3.1749024391174316,
      "rewards/margins": 16.801563262939453,
      "rewards/rejected": -19.975000381469727,
      "step": 6520
    },
    {
      "epoch": 2.801630726316919,
      "grad_norm": 0.43694777915440336,
      "learning_rate": 2.994635193133047e-07,
      "logits/chosen": -1.023950219154358,
      "logits/rejected": -1.0615723133087158,
      "logps/chosen": -383.7250061035156,
      "logps/rejected": -529.2999877929688,
      "loss": 0.0105,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.732421875,
      "rewards/margins": 16.376562118530273,
      "rewards/rejected": -19.104686737060547,
      "step": 6530
    },
    {
      "epoch": 2.8059221113614417,
      "grad_norm": 2.0686457624070194,
      "learning_rate": 2.9839055793991416e-07,
      "logits/chosen": -1.1383788585662842,
      "logits/rejected": -1.188720703125,
      "logps/chosen": -429.5874938964844,
      "logps/rejected": -560.9500122070312,
      "loss": 0.0098,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -3.555889844894409,
      "rewards/margins": 17.006250381469727,
      "rewards/rejected": -20.556249618530273,
      "step": 6540
    },
    {
      "epoch": 2.8102134964059653,
      "grad_norm": 5.025792322520393,
      "learning_rate": 2.973175965665236e-07,
      "logits/chosen": -1.0822021961212158,
      "logits/rejected": -1.1461913585662842,
      "logps/chosen": -429.1499938964844,
      "logps/rejected": -612.1500244140625,
      "loss": 0.0092,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -3.9110350608825684,
      "rewards/margins": 18.051563262939453,
      "rewards/rejected": -21.959375381469727,
      "step": 6550
    },
    {
      "epoch": 2.814504881450488,
      "grad_norm": 0.26008114539563354,
      "learning_rate": 2.9624463519313303e-07,
      "logits/chosen": -1.085302710533142,
      "logits/rejected": -1.118408203125,
      "logps/chosen": -372.1000061035156,
      "logps/rejected": -501.20001220703125,
      "loss": 0.0132,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -3.851733446121216,
      "rewards/margins": 16.317188262939453,
      "rewards/rejected": -20.1796875,
      "step": 6560
    },
    {
      "epoch": 2.8187962664950112,
      "grad_norm": 0.03320099176764248,
      "learning_rate": 2.9517167381974244e-07,
      "logits/chosen": -1.1650390625,
      "logits/rejected": -1.193945288658142,
      "logps/chosen": -429.0249938964844,
      "logps/rejected": -555.6500244140625,
      "loss": 0.0052,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -3.8432250022888184,
      "rewards/margins": 16.301563262939453,
      "rewards/rejected": -20.145313262939453,
      "step": 6570
    },
    {
      "epoch": 2.8230876515395344,
      "grad_norm": 34.29849607949799,
      "learning_rate": 2.940987124463519e-07,
      "logits/chosen": -1.01708984375,
      "logits/rejected": -1.067968726158142,
      "logps/chosen": -396.17498779296875,
      "logps/rejected": -539.25,
      "loss": 0.0121,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -3.326709032058716,
      "rewards/margins": 16.581249237060547,
      "rewards/rejected": -19.909374237060547,
      "step": 6580
    },
    {
      "epoch": 2.8273790365840576,
      "grad_norm": 0.31172731006123194,
      "learning_rate": 2.9302575107296137e-07,
      "logits/chosen": -0.9941161870956421,
      "logits/rejected": -1.0595214366912842,
      "logps/chosen": -410.8500061035156,
      "logps/rejected": -577.9500122070312,
      "loss": 0.0119,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -4.6103515625,
      "rewards/margins": 17.4765625,
      "rewards/rejected": -22.076562881469727,
      "step": 6590
    },
    {
      "epoch": 2.831670421628581,
      "grad_norm": 0.04453226691016711,
      "learning_rate": 2.9195278969957083e-07,
      "logits/chosen": -1.1447265148162842,
      "logits/rejected": -1.1728026866912842,
      "logps/chosen": -422.9750061035156,
      "logps/rejected": -582.5499877929688,
      "loss": 0.005,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -4.27734375,
      "rewards/margins": 17.201562881469727,
      "rewards/rejected": -21.481250762939453,
      "step": 6600
    },
    {
      "epoch": 2.8359618066731036,
      "grad_norm": 1.2485162350807397,
      "learning_rate": 2.9087982832618024e-07,
      "logits/chosen": -1.0518798828125,
      "logits/rejected": -1.110742211341858,
      "logps/chosen": -387.8500061035156,
      "logps/rejected": -543.2000122070312,
      "loss": 0.0258,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -4.219042778015137,
      "rewards/margins": 17.857812881469727,
      "rewards/rejected": -22.087499618530273,
      "step": 6610
    },
    {
      "epoch": 2.840253191717627,
      "grad_norm": 0.008505312707248821,
      "learning_rate": 2.8980686695278966e-07,
      "logits/chosen": -1.045263648033142,
      "logits/rejected": -1.122460961341858,
      "logps/chosen": -376.45001220703125,
      "logps/rejected": -545.6500244140625,
      "loss": 0.0119,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -4.165429592132568,
      "rewards/margins": 17.6640625,
      "rewards/rejected": -21.828125,
      "step": 6620
    },
    {
      "epoch": 2.84454457676215,
      "grad_norm": 2.1274236494928322,
      "learning_rate": 2.887339055793991e-07,
      "logits/chosen": -1.119287133216858,
      "logits/rejected": -1.1335937976837158,
      "logps/chosen": -409.0,
      "logps/rejected": -553.9000244140625,
      "loss": 0.0027,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.054858446121216,
      "rewards/margins": 17.3359375,
      "rewards/rejected": -20.395313262939453,
      "step": 6630
    },
    {
      "epoch": 2.848835961806673,
      "grad_norm": 0.012997654048413913,
      "learning_rate": 2.876609442060086e-07,
      "logits/chosen": -1.1417968273162842,
      "logits/rejected": -1.188378930091858,
      "logps/chosen": -426.95001220703125,
      "logps/rejected": -586.5499877929688,
      "loss": 0.0056,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -3.531787157058716,
      "rewards/margins": 16.921875,
      "rewards/rejected": -20.456249237060547,
      "step": 6640
    },
    {
      "epoch": 2.8531273468511964,
      "grad_norm": 1.8249718185823771,
      "learning_rate": 2.8658798283261805e-07,
      "logits/chosen": -1.0469238758087158,
      "logits/rejected": -1.085107445716858,
      "logps/chosen": -404.4750061035156,
      "logps/rejected": -527.3499755859375,
      "loss": 0.0092,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -3.287029981613159,
      "rewards/margins": 16.4453125,
      "rewards/rejected": -19.740625381469727,
      "step": 6650
    },
    {
      "epoch": 2.857418731895719,
      "grad_norm": 0.8605487912386847,
      "learning_rate": 2.8551502145922746e-07,
      "logits/chosen": -1.1165282726287842,
      "logits/rejected": -1.1824219226837158,
      "logps/chosen": -423.54998779296875,
      "logps/rejected": -592.25,
      "loss": 0.0003,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.170703172683716,
      "rewards/margins": 18.498437881469727,
      "rewards/rejected": -21.654687881469727,
      "step": 6660
    },
    {
      "epoch": 2.8617101169402424,
      "grad_norm": 0.17847628219851697,
      "learning_rate": 2.8444206008583687e-07,
      "logits/chosen": -1.138281226158142,
      "logits/rejected": -1.20849609375,
      "logps/chosen": -421.8999938964844,
      "logps/rejected": -537.1500244140625,
      "loss": 0.0151,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -3.1214842796325684,
      "rewards/margins": 16.09765625,
      "rewards/rejected": -19.217187881469727,
      "step": 6670
    },
    {
      "epoch": 2.8660015019847656,
      "grad_norm": 5.383701900903623,
      "learning_rate": 2.8336909871244633e-07,
      "logits/chosen": -1.065155029296875,
      "logits/rejected": -1.1383056640625,
      "logps/chosen": -370.625,
      "logps/rejected": -514.4500122070312,
      "loss": 0.014,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -3.181835889816284,
      "rewards/margins": 16.375,
      "rewards/rejected": -19.559375762939453,
      "step": 6680
    },
    {
      "epoch": 2.8702928870292888,
      "grad_norm": 2.317468571147531,
      "learning_rate": 2.822961373390558e-07,
      "logits/chosen": -1.126074194908142,
      "logits/rejected": -1.165429711341858,
      "logps/chosen": -385.875,
      "logps/rejected": -554.5499877929688,
      "loss": 0.0282,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -3.4695801734924316,
      "rewards/margins": 16.42578125,
      "rewards/rejected": -19.892187118530273,
      "step": 6690
    },
    {
      "epoch": 2.874584272073812,
      "grad_norm": 0.03680714788548126,
      "learning_rate": 2.8122317596566526e-07,
      "logits/chosen": -1.092382788658142,
      "logits/rejected": -1.110815405845642,
      "logps/chosen": -386.875,
      "logps/rejected": -531.75,
      "loss": 0.0131,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -3.378002882003784,
      "rewards/margins": 16.982812881469727,
      "rewards/rejected": -20.353124618530273,
      "step": 6700
    },
    {
      "epoch": 2.8788756571183347,
      "grad_norm": 33.476581892084894,
      "learning_rate": 2.8015021459227467e-07,
      "logits/chosen": -1.149804711341858,
      "logits/rejected": -1.1837890148162842,
      "logps/chosen": -392.2250061035156,
      "logps/rejected": -551.6500244140625,
      "loss": 0.0198,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -3.8650574684143066,
      "rewards/margins": 17.654687881469727,
      "rewards/rejected": -21.5234375,
      "step": 6710
    },
    {
      "epoch": 2.883167042162858,
      "grad_norm": 0.06874988179677158,
      "learning_rate": 2.790772532188841e-07,
      "logits/chosen": -1.149999976158142,
      "logits/rejected": -1.2208740711212158,
      "logps/chosen": -392.7875061035156,
      "logps/rejected": -532.7249755859375,
      "loss": 0.0004,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.7861328125,
      "rewards/margins": 16.9921875,
      "rewards/rejected": -20.779687881469727,
      "step": 6720
    },
    {
      "epoch": 2.887458427207381,
      "grad_norm": 0.08190278384087993,
      "learning_rate": 2.7800429184549355e-07,
      "logits/chosen": -1.1873047351837158,
      "logits/rejected": -1.23095703125,
      "logps/chosen": -390.75,
      "logps/rejected": -523.2999877929688,
      "loss": 0.0151,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.994982957839966,
      "rewards/margins": 17.225000381469727,
      "rewards/rejected": -20.220312118530273,
      "step": 6730
    },
    {
      "epoch": 2.8917498122519043,
      "grad_norm": 4.113618173725715,
      "learning_rate": 2.76931330472103e-07,
      "logits/chosen": -1.0952637195587158,
      "logits/rejected": -1.1465332508087158,
      "logps/chosen": -382.73748779296875,
      "logps/rejected": -543.7000122070312,
      "loss": 0.0132,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -4.748681545257568,
      "rewards/margins": 17.623437881469727,
      "rewards/rejected": -22.375,
      "step": 6740
    },
    {
      "epoch": 2.8960411972964275,
      "grad_norm": 0.004131997125234821,
      "learning_rate": 2.758583690987124e-07,
      "logits/chosen": -1.1780273914337158,
      "logits/rejected": -1.210546851158142,
      "logps/chosen": -387.4750061035156,
      "logps/rejected": -551.5499877929688,
      "loss": 0.0092,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -6.230761528015137,
      "rewards/margins": 17.21875,
      "rewards/rejected": -23.462499618530273,
      "step": 6750
    },
    {
      "epoch": 2.9003325823409507,
      "grad_norm": 0.10287402224650052,
      "learning_rate": 2.747854077253219e-07,
      "logits/chosen": -1.159570336341858,
      "logits/rejected": -1.193750023841858,
      "logps/chosen": -402.1625061035156,
      "logps/rejected": -575.8499755859375,
      "loss": 0.0046,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -5.483838081359863,
      "rewards/margins": 17.2890625,
      "rewards/rejected": -22.762500762939453,
      "step": 6760
    },
    {
      "epoch": 2.904623967385474,
      "grad_norm": 0.2387039663589461,
      "learning_rate": 2.737124463519313e-07,
      "logits/chosen": -1.1859619617462158,
      "logits/rejected": -1.1893310546875,
      "logps/chosen": -410.1499938964844,
      "logps/rejected": -551.4500122070312,
      "loss": 0.0063,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -6.254492282867432,
      "rewards/margins": 17.832813262939453,
      "rewards/rejected": -24.0703125,
      "step": 6770
    },
    {
      "epoch": 2.9089153524299967,
      "grad_norm": 0.0007330988858735252,
      "learning_rate": 2.7263948497854076e-07,
      "logits/chosen": -1.0857422351837158,
      "logits/rejected": -1.140869140625,
      "logps/chosen": -403.8500061035156,
      "logps/rejected": -552.75,
      "loss": 0.0223,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -5.592577934265137,
      "rewards/margins": 18.423437118530273,
      "rewards/rejected": -24.018749237060547,
      "step": 6780
    },
    {
      "epoch": 2.91320673747452,
      "grad_norm": 0.12279139603402796,
      "learning_rate": 2.715665236051502e-07,
      "logits/chosen": -1.043798804283142,
      "logits/rejected": -1.1020996570587158,
      "logps/chosen": -390.2124938964844,
      "logps/rejected": -557.5750122070312,
      "loss": 0.0191,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -5.569140434265137,
      "rewards/margins": 16.880468368530273,
      "rewards/rejected": -22.459375381469727,
      "step": 6790
    },
    {
      "epoch": 2.917498122519043,
      "grad_norm": 1.833745751437381,
      "learning_rate": 2.7049356223175964e-07,
      "logits/chosen": -1.106835961341858,
      "logits/rejected": -1.203466773033142,
      "logps/chosen": -435.5,
      "logps/rejected": -554.0999755859375,
      "loss": 0.0135,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -5.113671779632568,
      "rewards/margins": 17.121875762939453,
      "rewards/rejected": -22.2421875,
      "step": 6800
    },
    {
      "epoch": 2.9217895075635663,
      "grad_norm": 0.04453613121689767,
      "learning_rate": 2.694206008583691e-07,
      "logits/chosen": -1.107666015625,
      "logits/rejected": -1.173486351966858,
      "logps/chosen": -428.92498779296875,
      "logps/rejected": -598.2999877929688,
      "loss": 0.0089,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -5.703222751617432,
      "rewards/margins": 18.464061737060547,
      "rewards/rejected": -24.176563262939453,
      "step": 6810
    },
    {
      "epoch": 2.9260808926080895,
      "grad_norm": 0.016320720718881864,
      "learning_rate": 2.683476394849785e-07,
      "logits/chosen": -1.0830078125,
      "logits/rejected": -1.141015648841858,
      "logps/chosen": -398.8999938964844,
      "logps/rejected": -576.25,
      "loss": 0.0176,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -5.218457221984863,
      "rewards/margins": 20.004688262939453,
      "rewards/rejected": -25.228124618530273,
      "step": 6820
    },
    {
      "epoch": 2.930372277652612,
      "grad_norm": 0.11404054228480617,
      "learning_rate": 2.67274678111588e-07,
      "logits/chosen": -1.0846068859100342,
      "logits/rejected": -1.1232178211212158,
      "logps/chosen": -445.7749938964844,
      "logps/rejected": -590.5,
      "loss": 0.0344,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -4.5947265625,
      "rewards/margins": 18.526561737060547,
      "rewards/rejected": -23.1328125,
      "step": 6830
    },
    {
      "epoch": 2.9346636626971354,
      "grad_norm": 0.23925849581562467,
      "learning_rate": 2.6620171673819744e-07,
      "logits/chosen": -1.126953125,
      "logits/rejected": -1.169189453125,
      "logps/chosen": -401.625,
      "logps/rejected": -567.0999755859375,
      "loss": 0.0223,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -4.591064453125,
      "rewards/margins": 18.154687881469727,
      "rewards/rejected": -22.732812881469727,
      "step": 6840
    },
    {
      "epoch": 2.9389550477416586,
      "grad_norm": 0.07021392546832039,
      "learning_rate": 2.6512875536480685e-07,
      "logits/chosen": -1.1106445789337158,
      "logits/rejected": -1.0964844226837158,
      "logps/chosen": -381.5,
      "logps/rejected": -550.0499877929688,
      "loss": 0.0134,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -4.954174995422363,
      "rewards/margins": 17.379688262939453,
      "rewards/rejected": -22.326562881469727,
      "step": 6850
    },
    {
      "epoch": 2.943246432786182,
      "grad_norm": 0.5872394867884444,
      "learning_rate": 2.640557939914163e-07,
      "logits/chosen": -0.970440685749054,
      "logits/rejected": -1.053491234779358,
      "logps/chosen": -389.79998779296875,
      "logps/rejected": -540.0,
      "loss": 0.0225,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -4.573828220367432,
      "rewards/margins": 16.6640625,
      "rewards/rejected": -21.235937118530273,
      "step": 6860
    },
    {
      "epoch": 2.947537817830705,
      "grad_norm": 1.8932138624054262,
      "learning_rate": 2.629828326180257e-07,
      "logits/chosen": -1.070654273033142,
      "logits/rejected": -1.08984375,
      "logps/chosen": -367.7875061035156,
      "logps/rejected": -549.9000244140625,
      "loss": 0.0027,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.888842821121216,
      "rewards/margins": 17.149999618530273,
      "rewards/rejected": -21.037500381469727,
      "step": 6870
    },
    {
      "epoch": 2.9518292028752278,
      "grad_norm": 0.27225482651235966,
      "learning_rate": 2.619098712446352e-07,
      "logits/chosen": -0.9581543207168579,
      "logits/rejected": -1.0140380859375,
      "logps/chosen": -372.8125,
      "logps/rejected": -519.5499877929688,
      "loss": 0.0133,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.935961961746216,
      "rewards/margins": 17.8671875,
      "rewards/rejected": -20.803125381469727,
      "step": 6880
    },
    {
      "epoch": 2.956120587919751,
      "grad_norm": 0.07832943803901476,
      "learning_rate": 2.6083690987124465e-07,
      "logits/chosen": -1.066748023033142,
      "logits/rejected": -1.119726538658142,
      "logps/chosen": -376.4750061035156,
      "logps/rejected": -542.9000244140625,
      "loss": 0.003,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.11328125,
      "rewards/margins": 17.442188262939453,
      "rewards/rejected": -20.5546875,
      "step": 6890
    },
    {
      "epoch": 2.960411972964274,
      "grad_norm": 1.316772707856467,
      "learning_rate": 2.5976394849785406e-07,
      "logits/chosen": -0.922607421875,
      "logits/rejected": -0.977832019329071,
      "logps/chosen": -425.1875,
      "logps/rejected": -580.75,
      "loss": 0.0248,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -3.806640625,
      "rewards/margins": 18.154687881469727,
      "rewards/rejected": -21.953125,
      "step": 6900
    },
    {
      "epoch": 2.9647033580087974,
      "grad_norm": 0.626242401926231,
      "learning_rate": 2.5869098712446353e-07,
      "logits/chosen": -1.0491454601287842,
      "logits/rejected": -1.070898413658142,
      "logps/chosen": -418.54998779296875,
      "logps/rejected": -586.75,
      "loss": 0.0061,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -4.499072074890137,
      "rewards/margins": 18.095312118530273,
      "rewards/rejected": -22.59375,
      "step": 6910
    },
    {
      "epoch": 2.9689947430533206,
      "grad_norm": 0.44346695692015714,
      "learning_rate": 2.5761802575107294e-07,
      "logits/chosen": -1.0969970226287842,
      "logits/rejected": -1.1124999523162842,
      "logps/chosen": -389.17498779296875,
      "logps/rejected": -517.5499877929688,
      "loss": 0.0217,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -4.229687690734863,
      "rewards/margins": 16.91796875,
      "rewards/rejected": -21.1328125,
      "step": 6920
    },
    {
      "epoch": 2.9732861280978433,
      "grad_norm": 0.0032791312752799245,
      "learning_rate": 2.565450643776824e-07,
      "logits/chosen": -1.0750000476837158,
      "logits/rejected": -1.0897216796875,
      "logps/chosen": -411.8500061035156,
      "logps/rejected": -553.7000122070312,
      "loss": 0.0087,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -4.692675590515137,
      "rewards/margins": 18.251562118530273,
      "rewards/rejected": -22.9453125,
      "step": 6930
    },
    {
      "epoch": 2.977577513142367,
      "grad_norm": 0.12538616889953105,
      "learning_rate": 2.554721030042918e-07,
      "logits/chosen": -1.127343773841858,
      "logits/rejected": -1.1511719226837158,
      "logps/chosen": -413.67498779296875,
      "logps/rejected": -540.5499877929688,
      "loss": 0.005,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -4.697070121765137,
      "rewards/margins": 18.431249618530273,
      "rewards/rejected": -23.121875762939453,
      "step": 6940
    },
    {
      "epoch": 2.9818688981868897,
      "grad_norm": 12.361835458938065,
      "learning_rate": 2.543991416309013e-07,
      "logits/chosen": -0.992388904094696,
      "logits/rejected": -1.0128662586212158,
      "logps/chosen": -429.32501220703125,
      "logps/rejected": -545.3499755859375,
      "loss": 0.0461,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -4.654394626617432,
      "rewards/margins": 16.424219131469727,
      "rewards/rejected": -21.089061737060547,
      "step": 6950
    },
    {
      "epoch": 2.986160283231413,
      "grad_norm": 0.44444414662920156,
      "learning_rate": 2.5332618025751074e-07,
      "logits/chosen": -0.9417968988418579,
      "logits/rejected": -0.9969238042831421,
      "logps/chosen": -436.625,
      "logps/rejected": -599.7000122070312,
      "loss": 0.016,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -4.56640625,
      "rewards/margins": 18.0234375,
      "rewards/rejected": -22.590625762939453,
      "step": 6960
    },
    {
      "epoch": 2.990451668275936,
      "grad_norm": 0.6637141361142552,
      "learning_rate": 2.5225321888412015e-07,
      "logits/chosen": -1.0317871570587158,
      "logits/rejected": -1.1255371570587158,
      "logps/chosen": -413.6000061035156,
      "logps/rejected": -568.1500244140625,
      "loss": 0.0061,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -4.749218940734863,
      "rewards/margins": 17.642187118530273,
      "rewards/rejected": -22.378124237060547,
      "step": 6970
    },
    {
      "epoch": 2.9947430533204593,
      "grad_norm": 0.4310314072784117,
      "learning_rate": 2.5118025751072956e-07,
      "logits/chosen": -0.8998168706893921,
      "logits/rejected": -0.9791625738143921,
      "logps/chosen": -395.17498779296875,
      "logps/rejected": -519.2000122070312,
      "loss": 0.0273,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -3.4146971702575684,
      "rewards/margins": 15.4453125,
      "rewards/rejected": -18.862499237060547,
      "step": 6980
    },
    {
      "epoch": 2.9990344383649825,
      "grad_norm": 0.169618820490877,
      "learning_rate": 2.50107296137339e-07,
      "logits/chosen": -1.025292992591858,
      "logits/rejected": -1.0778319835662842,
      "logps/chosen": -407.70001220703125,
      "logps/rejected": -556.2999877929688,
      "loss": 0.0089,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -3.965087890625,
      "rewards/margins": 16.475780487060547,
      "rewards/rejected": -20.453125,
      "step": 6990
    },
    {
      "epoch": 3.003003969531166,
      "grad_norm": 0.012852492794570246,
      "learning_rate": 2.490343347639485e-07,
      "logits/chosen": -0.9653980135917664,
      "logits/rejected": -0.9903663396835327,
      "logps/chosen": -365.8513488769531,
      "logps/rejected": -534.9189453125,
      "loss": 0.0246,
      "rewards/accuracies": 0.9662162065505981,
      "rewards/chosen": -3.5451066493988037,
      "rewards/margins": 16.9577693939209,
      "rewards/rejected": -20.508445739746094,
      "step": 7000
    },
    {
      "epoch": 3.0072953545756893,
      "grad_norm": 0.1883159473286805,
      "learning_rate": 2.479613733905579e-07,
      "logits/chosen": -0.990039050579071,
      "logits/rejected": -1.045800805091858,
      "logps/chosen": -409.54998779296875,
      "logps/rejected": -574.5,
      "loss": 0.0087,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -3.3230957984924316,
      "rewards/margins": 18.131250381469727,
      "rewards/rejected": -21.464061737060547,
      "step": 7010
    },
    {
      "epoch": 3.0115867396202125,
      "grad_norm": 0.011470425094908726,
      "learning_rate": 2.4688841201716737e-07,
      "logits/chosen": -1.089941382408142,
      "logits/rejected": -1.15966796875,
      "logps/chosen": -394.17498779296875,
      "logps/rejected": -538.7999877929688,
      "loss": 0.0044,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -4.173593044281006,
      "rewards/margins": 16.142187118530273,
      "rewards/rejected": -20.309375762939453,
      "step": 7020
    },
    {
      "epoch": 3.0158781246647357,
      "grad_norm": 0.004629222547386246,
      "learning_rate": 2.4581545064377683e-07,
      "logits/chosen": -1.105126976966858,
      "logits/rejected": -1.142480492591858,
      "logps/chosen": -385.92498779296875,
      "logps/rejected": -559.7999877929688,
      "loss": 0.0087,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -3.8199219703674316,
      "rewards/margins": 18.246875762939453,
      "rewards/rejected": -22.064062118530273,
      "step": 7030
    },
    {
      "epoch": 3.0201695097092585,
      "grad_norm": 0.028089961491226577,
      "learning_rate": 2.4474248927038624e-07,
      "logits/chosen": -0.9541260004043579,
      "logits/rejected": -1.01806640625,
      "logps/chosen": -386.20001220703125,
      "logps/rejected": -528.0999755859375,
      "loss": 0.013,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -3.63623046875,
      "rewards/margins": 17.573436737060547,
      "rewards/rejected": -21.212499618530273,
      "step": 7040
    },
    {
      "epoch": 3.0244608947537817,
      "grad_norm": 0.04387816172302924,
      "learning_rate": 2.436695278969957e-07,
      "logits/chosen": -1.0131347179412842,
      "logits/rejected": -1.015051245689392,
      "logps/chosen": -414.0625,
      "logps/rejected": -565.5499877929688,
      "loss": 0.0262,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -4.078320503234863,
      "rewards/margins": 17.7578125,
      "rewards/rejected": -21.831249237060547,
      "step": 7050
    },
    {
      "epoch": 3.028752279798305,
      "grad_norm": 0.11957022050492318,
      "learning_rate": 2.425965665236051e-07,
      "logits/chosen": -1.130273461341858,
      "logits/rejected": -1.1540038585662842,
      "logps/chosen": -373.07501220703125,
      "logps/rejected": -533.5,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.1893310546875,
      "rewards/margins": 18.528125762939453,
      "rewards/rejected": -21.712499618530273,
      "step": 7060
    },
    {
      "epoch": 3.033043664842828,
      "grad_norm": 0.018614914745001016,
      "learning_rate": 2.415236051502146e-07,
      "logits/chosen": -0.995800793170929,
      "logits/rejected": -1.003808617591858,
      "logps/chosen": -402.6499938964844,
      "logps/rejected": -591.4000244140625,
      "loss": 0.0087,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -4.662353515625,
      "rewards/margins": 19.057811737060547,
      "rewards/rejected": -23.728124618530273,
      "step": 7070
    },
    {
      "epoch": 3.0373350498873513,
      "grad_norm": 0.023932764096977484,
      "learning_rate": 2.4045064377682404e-07,
      "logits/chosen": -1.041259765625,
      "logits/rejected": -1.0592772960662842,
      "logps/chosen": -403.2250061035156,
      "logps/rejected": -553.5999755859375,
      "loss": 0.0116,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -3.921142578125,
      "rewards/margins": 18.253124237060547,
      "rewards/rejected": -22.173437118530273,
      "step": 7080
    },
    {
      "epoch": 3.0416264349318745,
      "grad_norm": 0.1572713549294151,
      "learning_rate": 2.3937768240343345e-07,
      "logits/chosen": -1.0118286609649658,
      "logits/rejected": -1.040075659751892,
      "logps/chosen": -434.17498779296875,
      "logps/rejected": -539.5,
      "loss": 0.0045,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -4.736425876617432,
      "rewards/margins": 18.651561737060547,
      "rewards/rejected": -23.381250381469727,
      "step": 7090
    },
    {
      "epoch": 3.045917819976397,
      "grad_norm": 0.003381987283044917,
      "learning_rate": 2.383047210300429e-07,
      "logits/chosen": -0.9781738519668579,
      "logits/rejected": -1.025183081626892,
      "logps/chosen": -386.8125,
      "logps/rejected": -561.7000122070312,
      "loss": 0.0146,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -4.576025485992432,
      "rewards/margins": 18.471874237060547,
      "rewards/rejected": -23.048437118530273,
      "step": 7100
    },
    {
      "epoch": 3.0502092050209204,
      "grad_norm": 0.23883892279216642,
      "learning_rate": 2.3723175965665236e-07,
      "logits/chosen": -1.079126000404358,
      "logits/rejected": -1.1520507335662842,
      "logps/chosen": -392.0249938964844,
      "logps/rejected": -562.2999877929688,
      "loss": 0.0087,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -4.73046875,
      "rewards/margins": 17.6484375,
      "rewards/rejected": -22.375,
      "step": 7110
    },
    {
      "epoch": 3.0545005900654436,
      "grad_norm": 0.03626367810707846,
      "learning_rate": 2.361587982832618e-07,
      "logits/chosen": -1.0204346179962158,
      "logits/rejected": -1.0711181163787842,
      "logps/chosen": -397.70001220703125,
      "logps/rejected": -575.0499877929688,
      "loss": 0.0173,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -4.585888862609863,
      "rewards/margins": 18.375,
      "rewards/rejected": -22.956249237060547,
      "step": 7120
    },
    {
      "epoch": 3.058791975109967,
      "grad_norm": 0.8931073983603135,
      "learning_rate": 2.3508583690987126e-07,
      "logits/chosen": -1.0258300304412842,
      "logits/rejected": -1.06005859375,
      "logps/chosen": -394.9750061035156,
      "logps/rejected": -550.7000122070312,
      "loss": 0.0131,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -3.0218262672424316,
      "rewards/margins": 19.142187118530273,
      "rewards/rejected": -22.153125762939453,
      "step": 7130
    },
    {
      "epoch": 3.06308336015449,
      "grad_norm": 381.41618174962065,
      "learning_rate": 2.3401287553648067e-07,
      "logits/chosen": -1.052636742591858,
      "logits/rejected": -1.1193358898162842,
      "logps/chosen": -412.57501220703125,
      "logps/rejected": -579.4500122070312,
      "loss": 0.018,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -3.893359422683716,
      "rewards/margins": 18.348438262939453,
      "rewards/rejected": -22.243749618530273,
      "step": 7140
    },
    {
      "epoch": 3.0673747451990128,
      "grad_norm": 0.007556774507986806,
      "learning_rate": 2.329399141630901e-07,
      "logits/chosen": -1.105859398841858,
      "logits/rejected": -1.1364257335662842,
      "logps/chosen": -450.79998779296875,
      "logps/rejected": -578.0,
      "loss": 0.0087,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -4.035119533538818,
      "rewards/margins": 17.451562881469727,
      "rewards/rejected": -21.473438262939453,
      "step": 7150
    },
    {
      "epoch": 3.071666130243536,
      "grad_norm": 0.0031212743755631893,
      "learning_rate": 2.3186695278969957e-07,
      "logits/chosen": -0.997607409954071,
      "logits/rejected": -1.0497314929962158,
      "logps/chosen": -345.7875061035156,
      "logps/rejected": -530.5499877929688,
      "loss": 0.0173,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -3.4971680641174316,
      "rewards/margins": 18.112499237060547,
      "rewards/rejected": -21.604686737060547,
      "step": 7160
    },
    {
      "epoch": 3.075957515288059,
      "grad_norm": 0.0035072063326313675,
      "learning_rate": 2.30793991416309e-07,
      "logits/chosen": -1.1287109851837158,
      "logits/rejected": -1.1774413585662842,
      "logps/chosen": -402.2749938964844,
      "logps/rejected": -592.4500122070312,
      "loss": 0.0043,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -4.153515815734863,
      "rewards/margins": 19.020313262939453,
      "rewards/rejected": -23.181249618530273,
      "step": 7170
    },
    {
      "epoch": 3.0802489003325824,
      "grad_norm": 0.012461367374791187,
      "learning_rate": 2.2972103004291844e-07,
      "logits/chosen": -1.0167968273162842,
      "logits/rejected": -1.0549805164337158,
      "logps/chosen": -381.42498779296875,
      "logps/rejected": -546.5,
      "loss": 0.0146,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -3.7808837890625,
      "rewards/margins": 18.212499618530273,
      "rewards/rejected": -22.0,
      "step": 7180
    },
    {
      "epoch": 3.0845402853771056,
      "grad_norm": 1.9117107794391355,
      "learning_rate": 2.2864806866952788e-07,
      "logits/chosen": -0.992950439453125,
      "logits/rejected": -1.0628173351287842,
      "logps/chosen": -380.95001220703125,
      "logps/rejected": -541.7000122070312,
      "loss": 0.0216,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -3.6735596656799316,
      "rewards/margins": 17.440624237060547,
      "rewards/rejected": -21.120311737060547,
      "step": 7190
    },
    {
      "epoch": 3.0888316704216288,
      "grad_norm": 0.028919992689791785,
      "learning_rate": 2.2757510729613732e-07,
      "logits/chosen": -1.079736351966858,
      "logits/rejected": -1.138769507408142,
      "logps/chosen": -372.0249938964844,
      "logps/rejected": -532.25,
      "loss": 0.0045,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -3.756640672683716,
      "rewards/margins": 17.485937118530273,
      "rewards/rejected": -21.217187881469727,
      "step": 7200
    },
    {
      "epoch": 3.0931230554661515,
      "grad_norm": 0.07471126535731881,
      "learning_rate": 2.2650214592274678e-07,
      "logits/chosen": -1.1583983898162842,
      "logits/rejected": -1.188085913658142,
      "logps/chosen": -407.32501220703125,
      "logps/rejected": -533.3499755859375,
      "loss": 0.0087,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -3.2856383323669434,
      "rewards/margins": 18.310937881469727,
      "rewards/rejected": -21.600000381469727,
      "step": 7210
    },
    {
      "epoch": 3.0974144405106747,
      "grad_norm": 0.003378535872630436,
      "learning_rate": 2.2542918454935622e-07,
      "logits/chosen": -1.05596923828125,
      "logits/rejected": -1.1089355945587158,
      "logps/chosen": -427.1499938964844,
      "logps/rejected": -603.5499877929688,
      "loss": 0.0087,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -3.6424317359924316,
      "rewards/margins": 18.918750762939453,
      "rewards/rejected": -22.556249618530273,
      "step": 7220
    },
    {
      "epoch": 3.101705825555198,
      "grad_norm": 0.19273915535532818,
      "learning_rate": 2.2435622317596563e-07,
      "logits/chosen": -0.9581543207168579,
      "logits/rejected": -0.992053210735321,
      "logps/chosen": -387.17498779296875,
      "logps/rejected": -549.3499755859375,
      "loss": 0.026,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -3.709033250808716,
      "rewards/margins": 18.013280868530273,
      "rewards/rejected": -21.723438262939453,
      "step": 7230
    },
    {
      "epoch": 3.105997210599721,
      "grad_norm": 0.04131345163024332,
      "learning_rate": 2.232832618025751e-07,
      "logits/chosen": -1.099707007408142,
      "logits/rejected": -1.1299316883087158,
      "logps/chosen": -392.0,
      "logps/rejected": -563.9500122070312,
      "loss": 0.0187,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -4.202416896820068,
      "rewards/margins": 18.307811737060547,
      "rewards/rejected": -22.506250381469727,
      "step": 7240
    },
    {
      "epoch": 3.1102885956442443,
      "grad_norm": 0.14079651654500058,
      "learning_rate": 2.2221030042918453e-07,
      "logits/chosen": -1.146093726158142,
      "logits/rejected": -1.2034180164337158,
      "logps/chosen": -396.7749938964844,
      "logps/rejected": -578.4500122070312,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.9156250953674316,
      "rewards/margins": 19.290624618530273,
      "rewards/rejected": -23.193750381469727,
      "step": 7250
    },
    {
      "epoch": 3.1145799806887675,
      "grad_norm": 1.209043460456609,
      "learning_rate": 2.21137339055794e-07,
      "logits/chosen": -1.071557641029358,
      "logits/rejected": -1.10400390625,
      "logps/chosen": -410.29998779296875,
      "logps/rejected": -569.4000244140625,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.0126953125,
      "rewards/margins": 18.615625381469727,
      "rewards/rejected": -22.640625,
      "step": 7260
    },
    {
      "epoch": 3.1188713657332903,
      "grad_norm": 0.012056094666844732,
      "learning_rate": 2.2006437768240343e-07,
      "logits/chosen": -1.0742676258087158,
      "logits/rejected": -1.121923804283142,
      "logps/chosen": -393.2749938964844,
      "logps/rejected": -562.1500244140625,
      "loss": 0.0046,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -3.611523389816284,
      "rewards/margins": 19.310937881469727,
      "rewards/rejected": -22.917186737060547,
      "step": 7270
    },
    {
      "epoch": 3.1231627507778135,
      "grad_norm": 0.010540804035967381,
      "learning_rate": 2.1899141630901284e-07,
      "logits/chosen": -1.112939476966858,
      "logits/rejected": -1.143652319908142,
      "logps/chosen": -413.7250061035156,
      "logps/rejected": -578.9000244140625,
      "loss": 0.0052,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -4.840966701507568,
      "rewards/margins": 18.317188262939453,
      "rewards/rejected": -23.162500381469727,
      "step": 7280
    },
    {
      "epoch": 3.1274541358223367,
      "grad_norm": 0.003982637050952266,
      "learning_rate": 2.179184549356223e-07,
      "logits/chosen": -0.9849609136581421,
      "logits/rejected": -1.0363280773162842,
      "logps/chosen": -404.1875,
      "logps/rejected": -567.2000122070312,
      "loss": 0.0216,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -4.919921875,
      "rewards/margins": 18.524999618530273,
      "rewards/rejected": -23.434375762939453,
      "step": 7290
    },
    {
      "epoch": 3.13174552086686,
      "grad_norm": 0.04562213197660063,
      "learning_rate": 2.1684549356223175e-07,
      "logits/chosen": -1.0841064453125,
      "logits/rejected": -1.1157715320587158,
      "logps/chosen": -414.82501220703125,
      "logps/rejected": -572.7999877929688,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.109326124191284,
      "rewards/margins": 18.7265625,
      "rewards/rejected": -21.848438262939453,
      "step": 7300
    },
    {
      "epoch": 3.136036905911383,
      "grad_norm": 0.004849745674894276,
      "learning_rate": 2.157725321888412e-07,
      "logits/chosen": -1.102197289466858,
      "logits/rejected": -1.153173804283142,
      "logps/chosen": -383.7250061035156,
      "logps/rejected": -556.1500244140625,
      "loss": 0.0003,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.674267530441284,
      "rewards/margins": 19.375,
      "rewards/rejected": -23.045312881469727,
      "step": 7310
    },
    {
      "epoch": 3.140328290955906,
      "grad_norm": 0.0065743927772143025,
      "learning_rate": 2.1469957081545062e-07,
      "logits/chosen": -1.049658179283142,
      "logits/rejected": -1.080322265625,
      "logps/chosen": -398.98748779296875,
      "logps/rejected": -563.8499755859375,
      "loss": 0.0173,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -4.922070503234863,
      "rewards/margins": 19.034374237060547,
      "rewards/rejected": -23.942188262939453,
      "step": 7320
    },
    {
      "epoch": 3.144619676000429,
      "grad_norm": 0.0031089621853393,
      "learning_rate": 2.1362660944206006e-07,
      "logits/chosen": -1.13623046875,
      "logits/rejected": -1.19580078125,
      "logps/chosen": -415.3999938964844,
      "logps/rejected": -586.5999755859375,
      "loss": 0.0087,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -3.1915526390075684,
      "rewards/margins": 19.740625381469727,
      "rewards/rejected": -22.942188262939453,
      "step": 7330
    },
    {
      "epoch": 3.148911061044952,
      "grad_norm": 0.30038112449218907,
      "learning_rate": 2.1255364806866952e-07,
      "logits/chosen": -1.0879395008087158,
      "logits/rejected": -1.1259644031524658,
      "logps/chosen": -417.1000061035156,
      "logps/rejected": -538.4749755859375,
      "loss": 0.0044,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -4.221093654632568,
      "rewards/margins": 17.765625,
      "rewards/rejected": -21.985937118530273,
      "step": 7340
    },
    {
      "epoch": 3.1532024460894754,
      "grad_norm": 0.05788266136678705,
      "learning_rate": 2.1148068669527896e-07,
      "logits/chosen": -1.0761229991912842,
      "logits/rejected": -1.1498534679412842,
      "logps/chosen": -431.3999938964844,
      "logps/rejected": -586.9500122070312,
      "loss": 0.013,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -4.797558784484863,
      "rewards/margins": 19.076562881469727,
      "rewards/rejected": -23.864063262939453,
      "step": 7350
    },
    {
      "epoch": 3.1574938311339986,
      "grad_norm": 8.441814448929113e-05,
      "learning_rate": 2.1040772532188842e-07,
      "logits/chosen": -1.105615258216858,
      "logits/rejected": -1.178320288658142,
      "logps/chosen": -446.17498779296875,
      "logps/rejected": -558.1500244140625,
      "loss": 0.0044,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -4.286523342132568,
      "rewards/margins": 18.715625762939453,
      "rewards/rejected": -22.995311737060547,
      "step": 7360
    },
    {
      "epoch": 3.161785216178522,
      "grad_norm": 0.009512097378561528,
      "learning_rate": 2.0933476394849783e-07,
      "logits/chosen": -1.0654785633087158,
      "logits/rejected": -1.091210961341858,
      "logps/chosen": -413.79998779296875,
      "logps/rejected": -584.9500122070312,
      "loss": 0.013,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -4.941504001617432,
      "rewards/margins": 18.870311737060547,
      "rewards/rejected": -23.803125381469727,
      "step": 7370
    },
    {
      "epoch": 3.1660766012230446,
      "grad_norm": 0.012706208621092691,
      "learning_rate": 2.0826180257510727e-07,
      "logits/chosen": -1.134033203125,
      "logits/rejected": -1.168701171875,
      "logps/chosen": -371.8999938964844,
      "logps/rejected": -546.7249755859375,
      "loss": 0.0043,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -4.395215034484863,
      "rewards/margins": 18.8359375,
      "rewards/rejected": -23.223438262939453,
      "step": 7380
    },
    {
      "epoch": 3.1703679862675678,
      "grad_norm": 0.002616459486340226,
      "learning_rate": 2.0718884120171674e-07,
      "logits/chosen": -1.1020019054412842,
      "logits/rejected": -1.154150366783142,
      "logps/chosen": -419.3500061035156,
      "logps/rejected": -570.4000244140625,
      "loss": 0.0044,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -4.803515434265137,
      "rewards/margins": 18.598438262939453,
      "rewards/rejected": -23.395313262939453,
      "step": 7390
    },
    {
      "epoch": 3.174659371312091,
      "grad_norm": 0.0015572982583173006,
      "learning_rate": 2.0611587982832617e-07,
      "logits/chosen": -1.1628906726837158,
      "logits/rejected": -1.1892578601837158,
      "logps/chosen": -397.20001220703125,
      "logps/rejected": -569.0750122070312,
      "loss": 0.0087,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -4.714745998382568,
      "rewards/margins": 19.996875762939453,
      "rewards/rejected": -24.709375381469727,
      "step": 7400
    },
    {
      "epoch": 3.178950756356614,
      "grad_norm": 0.013222345952245489,
      "learning_rate": 2.0504291845493564e-07,
      "logits/chosen": -1.067285180091858,
      "logits/rejected": -1.106689453125,
      "logps/chosen": -409.04998779296875,
      "logps/rejected": -583.8499755859375,
      "loss": 0.0024,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.060351371765137,
      "rewards/margins": 19.479686737060547,
      "rewards/rejected": -24.553125381469727,
      "step": 7410
    },
    {
      "epoch": 3.1832421414011374,
      "grad_norm": 0.1557683369587535,
      "learning_rate": 2.0396995708154505e-07,
      "logits/chosen": -1.09765625,
      "logits/rejected": -1.124414086341858,
      "logps/chosen": -416.70001220703125,
      "logps/rejected": -608.0499877929688,
      "loss": 0.013,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -3.558154344558716,
      "rewards/margins": 20.721874237060547,
      "rewards/rejected": -24.28125,
      "step": 7420
    },
    {
      "epoch": 3.1875335264456606,
      "grad_norm": 0.015822904250508526,
      "learning_rate": 2.0289699570815449e-07,
      "logits/chosen": -1.0731201171875,
      "logits/rejected": -1.1497070789337158,
      "logps/chosen": -421.9375,
      "logps/rejected": -557.4500122070312,
      "loss": 0.0048,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -4.479150295257568,
      "rewards/margins": 19.7265625,
      "rewards/rejected": -24.200000762939453,
      "step": 7430
    },
    {
      "epoch": 3.1918249114901833,
      "grad_norm": 0.0007927777741120745,
      "learning_rate": 2.0182403433476395e-07,
      "logits/chosen": -1.1225097179412842,
      "logits/rejected": -1.154882788658142,
      "logps/chosen": -424.875,
      "logps/rejected": -601.0,
      "loss": 0.0107,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -5.0126953125,
      "rewards/margins": 19.643749237060547,
      "rewards/rejected": -24.654687881469727,
      "step": 7440
    },
    {
      "epoch": 3.1961162965347065,
      "grad_norm": 3.878737566950894,
      "learning_rate": 2.007510729613734e-07,
      "logits/chosen": -1.180810570716858,
      "logits/rejected": -1.2202637195587158,
      "logps/chosen": -442.75,
      "logps/rejected": -605.7000122070312,
      "loss": 0.0131,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -4.404687404632568,
      "rewards/margins": 19.315624237060547,
      "rewards/rejected": -23.712499618530273,
      "step": 7450
    },
    {
      "epoch": 3.2004076815792297,
      "grad_norm": 0.0059448786132387495,
      "learning_rate": 1.996781115879828e-07,
      "logits/chosen": -1.172753930091858,
      "logits/rejected": -1.2078125476837158,
      "logps/chosen": -412.2250061035156,
      "logps/rejected": -580.5499877929688,
      "loss": 0.013,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -5.168530464172363,
      "rewards/margins": 19.796875,
      "rewards/rejected": -24.973438262939453,
      "step": 7460
    },
    {
      "epoch": 3.204699066623753,
      "grad_norm": 0.026566817279985014,
      "learning_rate": 1.9860515021459226e-07,
      "logits/chosen": -1.0465819835662842,
      "logits/rejected": -1.122460961341858,
      "logps/chosen": -398.0249938964844,
      "logps/rejected": -569.4000244140625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.805920362472534,
      "rewards/margins": 18.8125,
      "rewards/rejected": -22.615625381469727,
      "step": 7470
    },
    {
      "epoch": 3.208990451668276,
      "grad_norm": 0.0570582178524798,
      "learning_rate": 1.975321888412017e-07,
      "logits/chosen": -1.108007788658142,
      "logits/rejected": -1.130468726158142,
      "logps/chosen": -403.5874938964844,
      "logps/rejected": -521.5499877929688,
      "loss": 0.0087,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -3.728283643722534,
      "rewards/margins": 17.185937881469727,
      "rewards/rejected": -20.90625,
      "step": 7480
    },
    {
      "epoch": 3.213281836712799,
      "grad_norm": 0.10074334390774173,
      "learning_rate": 1.9645922746781116e-07,
      "logits/chosen": -1.1065247058868408,
      "logits/rejected": -1.1827392578125,
      "logps/chosen": -422.1499938964844,
      "logps/rejected": -591.0999755859375,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.498730659484863,
      "rewards/margins": 19.071874618530273,
      "rewards/rejected": -23.564062118530273,
      "step": 7490
    },
    {
      "epoch": 3.217573221757322,
      "grad_norm": 0.005671957823497198,
      "learning_rate": 1.953862660944206e-07,
      "logits/chosen": -1.144433617591858,
      "logits/rejected": -1.178320288658142,
      "logps/chosen": -423.57501220703125,
      "logps/rejected": -575.0,
      "loss": 0.0087,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -5.390625,
      "rewards/margins": 18.6796875,
      "rewards/rejected": -24.060937881469727,
      "step": 7500
    },
    {
      "epoch": 3.2218646068018453,
      "grad_norm": 0.009949736203342541,
      "learning_rate": 1.9431330472103e-07,
      "logits/chosen": -1.1103515625,
      "logits/rejected": -1.1331055164337158,
      "logps/chosen": -389.57501220703125,
      "logps/rejected": -552.0499877929688,
      "loss": 0.0087,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -4.111572265625,
      "rewards/margins": 18.084375381469727,
      "rewards/rejected": -22.185937881469727,
      "step": 7510
    },
    {
      "epoch": 3.2261559918463685,
      "grad_norm": 0.01987592164099171,
      "learning_rate": 1.9324034334763948e-07,
      "logits/chosen": -1.0822021961212158,
      "logits/rejected": -1.1204102039337158,
      "logps/chosen": -419.875,
      "logps/rejected": -578.2999877929688,
      "loss": 0.0087,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -4.7591552734375,
      "rewards/margins": 19.084375381469727,
      "rewards/rejected": -23.837499618530273,
      "step": 7520
    },
    {
      "epoch": 3.2304473768908917,
      "grad_norm": 0.00017210986266684387,
      "learning_rate": 1.9216738197424891e-07,
      "logits/chosen": -1.0680663585662842,
      "logits/rejected": -1.1095702648162842,
      "logps/chosen": -452.17498779296875,
      "logps/rejected": -638.7000122070312,
      "loss": 0.0087,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -4.590478420257568,
      "rewards/margins": 20.362499237060547,
      "rewards/rejected": -24.942188262939453,
      "step": 7530
    },
    {
      "epoch": 3.2347387619354144,
      "grad_norm": 0.0015576399921244165,
      "learning_rate": 1.9109442060085838e-07,
      "logits/chosen": -1.194921851158142,
      "logits/rejected": -1.219824194908142,
      "logps/chosen": -406.61248779296875,
      "logps/rejected": -553.9000244140625,
      "loss": 0.0047,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -5.10986328125,
      "rewards/margins": 19.25,
      "rewards/rejected": -24.370311737060547,
      "step": 7540
    },
    {
      "epoch": 3.2390301469799376,
      "grad_norm": 0.0016778297394876907,
      "learning_rate": 1.900214592274678e-07,
      "logits/chosen": -0.9443359375,
      "logits/rejected": -1.007104516029358,
      "logps/chosen": -429.88751220703125,
      "logps/rejected": -594.1500244140625,
      "loss": 0.0044,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -4.302148342132568,
      "rewards/margins": 19.889062881469727,
      "rewards/rejected": -24.182811737060547,
      "step": 7550
    },
    {
      "epoch": 3.243321532024461,
      "grad_norm": 0.011251358614406274,
      "learning_rate": 1.8894849785407723e-07,
      "logits/chosen": -0.925488293170929,
      "logits/rejected": -0.976611316204071,
      "logps/chosen": -360.54998779296875,
      "logps/rejected": -566.5,
      "loss": 0.0243,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -4.0889892578125,
      "rewards/margins": 19.979686737060547,
      "rewards/rejected": -24.060937881469727,
      "step": 7560
    },
    {
      "epoch": 3.247612917068984,
      "grad_norm": 0.8187483739243558,
      "learning_rate": 1.878755364806867e-07,
      "logits/chosen": -1.0807616710662842,
      "logits/rejected": -1.1400878429412842,
      "logps/chosen": -416.4375,
      "logps/rejected": -547.5999755859375,
      "loss": 0.0131,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -4.018945217132568,
      "rewards/margins": 18.854686737060547,
      "rewards/rejected": -22.876562118530273,
      "step": 7570
    },
    {
      "epoch": 3.2519043021135072,
      "grad_norm": 0.009726602724960827,
      "learning_rate": 1.8680257510729613e-07,
      "logits/chosen": -1.021997094154358,
      "logits/rejected": -1.089990258216858,
      "logps/chosen": -403.125,
      "logps/rejected": -568.0999755859375,
      "loss": 0.0014,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.712011814117432,
      "rewards/margins": 18.395313262939453,
      "rewards/rejected": -23.106250762939453,
      "step": 7580
    },
    {
      "epoch": 3.2561956871580304,
      "grad_norm": 0.014207458339286431,
      "learning_rate": 1.857296137339056e-07,
      "logits/chosen": -1.108789086341858,
      "logits/rejected": -1.1352050304412842,
      "logps/chosen": -411.5249938964844,
      "logps/rejected": -559.5499877929688,
      "loss": 0.0043,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -3.5798583030700684,
      "rewards/margins": 19.768749237060547,
      "rewards/rejected": -23.334375381469727,
      "step": 7590
    },
    {
      "epoch": 3.260487072202553,
      "grad_norm": 0.006328679161499758,
      "learning_rate": 1.84656652360515e-07,
      "logits/chosen": -1.1180908679962158,
      "logits/rejected": -1.165136694908142,
      "logps/chosen": -446.42498779296875,
      "logps/rejected": -599.8499755859375,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.420214653015137,
      "rewards/margins": 18.715625762939453,
      "rewards/rejected": -23.134374618530273,
      "step": 7600
    },
    {
      "epoch": 3.2647784572470764,
      "grad_norm": 0.03741625752538844,
      "learning_rate": 1.8358369098712444e-07,
      "logits/chosen": -1.1806640625,
      "logits/rejected": -1.2287108898162842,
      "logps/chosen": -409.4750061035156,
      "logps/rejected": -563.5,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.049829006195068,
      "rewards/margins": 19.295312881469727,
      "rewards/rejected": -23.348438262939453,
      "step": 7610
    },
    {
      "epoch": 3.2690698422915996,
      "grad_norm": 6.541874933149319,
      "learning_rate": 1.825107296137339e-07,
      "logits/chosen": -1.063818335533142,
      "logits/rejected": -1.106054663658142,
      "logps/chosen": -437.5,
      "logps/rejected": -580.0,
      "loss": 0.0056,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -4.458349704742432,
      "rewards/margins": 17.975000381469727,
      "rewards/rejected": -22.432811737060547,
      "step": 7620
    },
    {
      "epoch": 3.273361227336123,
      "grad_norm": 0.006254891814542349,
      "learning_rate": 1.8143776824034334e-07,
      "logits/chosen": -1.151953101158142,
      "logits/rejected": -1.208251953125,
      "logps/chosen": -421.6000061035156,
      "logps/rejected": -543.7999877929688,
      "loss": 0.0043,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -4.941992282867432,
      "rewards/margins": 18.354686737060547,
      "rewards/rejected": -23.287500381469727,
      "step": 7630
    },
    {
      "epoch": 3.277652612380646,
      "grad_norm": 0.032208732610917536,
      "learning_rate": 1.8036480686695278e-07,
      "logits/chosen": -1.007836937904358,
      "logits/rejected": -1.048681616783142,
      "logps/chosen": -381.625,
      "logps/rejected": -563.5250244140625,
      "loss": 0.0043,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -3.9909911155700684,
      "rewards/margins": 19.082813262939453,
      "rewards/rejected": -23.084375381469727,
      "step": 7640
    },
    {
      "epoch": 3.281943997425169,
      "grad_norm": 0.003997162204992917,
      "learning_rate": 1.7929184549356222e-07,
      "logits/chosen": -1.1282470226287842,
      "logits/rejected": -1.153710961341858,
      "logps/chosen": -383.3125,
      "logps/rejected": -555.9000244140625,
      "loss": 0.0043,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -4.053027153015137,
      "rewards/margins": 18.149999618530273,
      "rewards/rejected": -22.196874618530273,
      "step": 7650
    },
    {
      "epoch": 3.286235382469692,
      "grad_norm": 0.0016323746838290136,
      "learning_rate": 1.7821888412017165e-07,
      "logits/chosen": -1.15185546875,
      "logits/rejected": -1.191796898841858,
      "logps/chosen": -415.0249938964844,
      "logps/rejected": -565.0999755859375,
      "loss": 0.0044,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -4.0958251953125,
      "rewards/margins": 19.348438262939453,
      "rewards/rejected": -23.439062118530273,
      "step": 7660
    },
    {
      "epoch": 3.290526767514215,
      "grad_norm": 15.325049227365643,
      "learning_rate": 1.7714592274678112e-07,
      "logits/chosen": -1.088964819908142,
      "logits/rejected": -1.140527367591858,
      "logps/chosen": -391.3500061035156,
      "logps/rejected": -536.0999755859375,
      "loss": 0.0136,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -4.533056735992432,
      "rewards/margins": 18.339061737060547,
      "rewards/rejected": -22.864063262939453,
      "step": 7670
    },
    {
      "epoch": 3.2948181525587383,
      "grad_norm": 2.616642428342158,
      "learning_rate": 1.7607296137339055e-07,
      "logits/chosen": -1.1190917491912842,
      "logits/rejected": -1.156494140625,
      "logps/chosen": -436.45001220703125,
      "logps/rejected": -606.6500244140625,
      "loss": 0.0132,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -3.8875732421875,
      "rewards/margins": 19.284374237060547,
      "rewards/rejected": -23.174999237060547,
      "step": 7680
    },
    {
      "epoch": 3.2991095376032615,
      "grad_norm": 0.014484864568080628,
      "learning_rate": 1.75e-07,
      "logits/chosen": -1.0398681163787842,
      "logits/rejected": -1.076904296875,
      "logps/chosen": -429.3374938964844,
      "logps/rejected": -590.1500244140625,
      "loss": 0.0051,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -5.8251953125,
      "rewards/margins": 17.607812881469727,
      "rewards/rejected": -23.428125381469727,
      "step": 7690
    },
    {
      "epoch": 3.3034009226477847,
      "grad_norm": 0.008089391321756274,
      "learning_rate": 1.7392703862660943e-07,
      "logits/chosen": -1.09716796875,
      "logits/rejected": -1.154394507408142,
      "logps/chosen": -394.5,
      "logps/rejected": -598.4000244140625,
      "loss": 0.0044,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -5.268945217132568,
      "rewards/margins": 20.6015625,
      "rewards/rejected": -25.856250762939453,
      "step": 7700
    },
    {
      "epoch": 3.3076923076923075,
      "grad_norm": 0.0075349949953169,
      "learning_rate": 1.7285407725321887e-07,
      "logits/chosen": -1.15869140625,
      "logits/rejected": -1.2034180164337158,
      "logps/chosen": -418.4750061035156,
      "logps/rejected": -576.9000244140625,
      "loss": 0.0049,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -5.557421684265137,
      "rewards/margins": 18.826562881469727,
      "rewards/rejected": -24.390625,
      "step": 7710
    },
    {
      "epoch": 3.3119836927368307,
      "grad_norm": 0.03239436523639938,
      "learning_rate": 1.7178111587982833e-07,
      "logits/chosen": -1.123437523841858,
      "logits/rejected": -1.1945312023162842,
      "logps/chosen": -388.54998779296875,
      "logps/rejected": -546.0999755859375,
      "loss": 0.0087,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -4.828125,
      "rewards/margins": 18.75,
      "rewards/rejected": -23.595312118530273,
      "step": 7720
    },
    {
      "epoch": 3.316275077781354,
      "grad_norm": 0.012913075946874353,
      "learning_rate": 1.7070815450643777e-07,
      "logits/chosen": -1.129150390625,
      "logits/rejected": -1.186621069908142,
      "logps/chosen": -409.375,
      "logps/rejected": -576.0499877929688,
      "loss": 0.0087,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -5.027783393859863,
      "rewards/margins": 19.018749237060547,
      "rewards/rejected": -24.049999237060547,
      "step": 7730
    },
    {
      "epoch": 3.320566462825877,
      "grad_norm": 0.3081401482911639,
      "learning_rate": 1.696351931330472e-07,
      "logits/chosen": -1.031457543373108,
      "logits/rejected": -1.0567505359649658,
      "logps/chosen": -432.29998779296875,
      "logps/rejected": -571.7000122070312,
      "loss": 0.0131,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -4.695898532867432,
      "rewards/margins": 19.145313262939453,
      "rewards/rejected": -23.831249237060547,
      "step": 7740
    },
    {
      "epoch": 3.3248578478704003,
      "grad_norm": 0.06500816883734759,
      "learning_rate": 1.6856223175965664e-07,
      "logits/chosen": -1.1437499523162842,
      "logits/rejected": -1.174218773841858,
      "logps/chosen": -421.92498779296875,
      "logps/rejected": -589.1500244140625,
      "loss": 0.013,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -5.138085842132568,
      "rewards/margins": 18.364063262939453,
      "rewards/rejected": -23.503124237060547,
      "step": 7750
    },
    {
      "epoch": 3.3291492329149235,
      "grad_norm": 0.010808352624494583,
      "learning_rate": 1.6748927038626608e-07,
      "logits/chosen": -1.135009765625,
      "logits/rejected": -1.1886107921600342,
      "logps/chosen": -453.75,
      "logps/rejected": -600.1500244140625,
      "loss": 0.0006,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.870019435882568,
      "rewards/margins": 18.845312118530273,
      "rewards/rejected": -23.728124618530273,
      "step": 7760
    },
    {
      "epoch": 3.3334406179594462,
      "grad_norm": 0.054381280214576826,
      "learning_rate": 1.6641630901287554e-07,
      "logits/chosen": -1.214746117591858,
      "logits/rejected": -1.224511742591858,
      "logps/chosen": -443.5249938964844,
      "logps/rejected": -586.5499877929688,
      "loss": 0.0087,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -5.556542873382568,
      "rewards/margins": 18.353124618530273,
      "rewards/rejected": -23.924999237060547,
      "step": 7770
    },
    {
      "epoch": 3.3377320030039694,
      "grad_norm": 0.13625770026161219,
      "learning_rate": 1.6534334763948496e-07,
      "logits/chosen": -1.093237280845642,
      "logits/rejected": -1.158935546875,
      "logps/chosen": -430.67498779296875,
      "logps/rejected": -594.6749877929688,
      "loss": 0.0087,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -4.848876953125,
      "rewards/margins": 20.254688262939453,
      "rewards/rejected": -25.100000381469727,
      "step": 7780
    },
    {
      "epoch": 3.3420233880484926,
      "grad_norm": 0.026210135428165512,
      "learning_rate": 1.6427038626609442e-07,
      "logits/chosen": -1.149023413658142,
      "logits/rejected": -1.197851538658142,
      "logps/chosen": -437.23748779296875,
      "logps/rejected": -610.0999755859375,
      "loss": 0.0084,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -4.936718940734863,
      "rewards/margins": 19.004688262939453,
      "rewards/rejected": -23.943750381469727,
      "step": 7790
    },
    {
      "epoch": 3.346314773093016,
      "grad_norm": 0.009880691659967484,
      "learning_rate": 1.6319742489270386e-07,
      "logits/chosen": -1.033203125,
      "logits/rejected": -1.090966820716858,
      "logps/chosen": -414.9125061035156,
      "logps/rejected": -589.3499755859375,
      "loss": 0.0173,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -4.807104587554932,
      "rewards/margins": 19.185937881469727,
      "rewards/rejected": -24.009374618530273,
      "step": 7800
    },
    {
      "epoch": 3.350606158137539,
      "grad_norm": 0.019381656753884636,
      "learning_rate": 1.621244635193133e-07,
      "logits/chosen": -1.123022437095642,
      "logits/rejected": -1.153924584388733,
      "logps/chosen": -402.6000061035156,
      "logps/rejected": -547.7000122070312,
      "loss": 0.026,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -5.301855564117432,
      "rewards/margins": 18.357812881469727,
      "rewards/rejected": -23.659374237060547,
      "step": 7810
    },
    {
      "epoch": 3.3548975431820622,
      "grad_norm": 19.218437911000994,
      "learning_rate": 1.6105150214592276e-07,
      "logits/chosen": -1.194921851158142,
      "logits/rejected": -1.1987793445587158,
      "logps/chosen": -384.5,
      "logps/rejected": -568.8499755859375,
      "loss": 0.0048,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -5.235449314117432,
      "rewards/margins": 18.671875,
      "rewards/rejected": -23.90625,
      "step": 7820
    },
    {
      "epoch": 3.359188928226585,
      "grad_norm": 0.03159748574516458,
      "learning_rate": 1.5997854077253217e-07,
      "logits/chosen": -1.0241210460662842,
      "logits/rejected": -1.0974609851837158,
      "logps/chosen": -426.9750061035156,
      "logps/rejected": -602.0,
      "loss": 0.0141,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -5.150830268859863,
      "rewards/margins": 19.799999237060547,
      "rewards/rejected": -24.940624237060547,
      "step": 7830
    },
    {
      "epoch": 3.363480313271108,
      "grad_norm": 0.008474964749175029,
      "learning_rate": 1.5890557939914163e-07,
      "logits/chosen": -1.1995117664337158,
      "logits/rejected": -1.245214819908142,
      "logps/chosen": -419.7250061035156,
      "logps/rejected": -633.1500244140625,
      "loss": 0.0043,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -4.628204345703125,
      "rewards/margins": 20.345312118530273,
      "rewards/rejected": -24.975000381469727,
      "step": 7840
    },
    {
      "epoch": 3.3677716983156314,
      "grad_norm": 0.1128953239722341,
      "learning_rate": 1.5783261802575107e-07,
      "logits/chosen": -1.0924804210662842,
      "logits/rejected": -1.138574242591858,
      "logps/chosen": -379.7250061035156,
      "logps/rejected": -577.8499755859375,
      "loss": 0.0087,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -5.306542873382568,
      "rewards/margins": 19.215625762939453,
      "rewards/rejected": -24.524999618530273,
      "step": 7850
    },
    {
      "epoch": 3.3720630833601546,
      "grad_norm": 0.010289820953143607,
      "learning_rate": 1.567596566523605e-07,
      "logits/chosen": -1.1528809070587158,
      "logits/rejected": -1.223242163658142,
      "logps/chosen": -408.29998779296875,
      "logps/rejected": -566.9000244140625,
      "loss": 0.0043,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -5.283154487609863,
      "rewards/margins": 19.600000381469727,
      "rewards/rejected": -24.889062881469727,
      "step": 7860
    },
    {
      "epoch": 3.376354468404678,
      "grad_norm": 0.028458235935922418,
      "learning_rate": 1.5568669527896995e-07,
      "logits/chosen": -1.1438477039337158,
      "logits/rejected": -1.193945288658142,
      "logps/chosen": -415.42498779296875,
      "logps/rejected": -568.2000122070312,
      "loss": 0.0181,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -4.93316650390625,
      "rewards/margins": 19.53125,
      "rewards/rejected": -24.453125,
      "step": 7870
    },
    {
      "epoch": 3.3806458534492005,
      "grad_norm": 0.03620517556992207,
      "learning_rate": 1.5461373390557938e-07,
      "logits/chosen": -1.1495361328125,
      "logits/rejected": -1.1992676258087158,
      "logps/chosen": -446.9750061035156,
      "logps/rejected": -568.3499755859375,
      "loss": 0.013,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -5.263952732086182,
      "rewards/margins": 18.0703125,
      "rewards/rejected": -23.339061737060547,
      "step": 7880
    },
    {
      "epoch": 3.3849372384937237,
      "grad_norm": 0.006374012863317528,
      "learning_rate": 1.5354077253218882e-07,
      "logits/chosen": -1.0954711437225342,
      "logits/rejected": -1.147705078125,
      "logps/chosen": -412.7250061035156,
      "logps/rejected": -591.7999877929688,
      "loss": 0.0087,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -5.102929592132568,
      "rewards/margins": 20.232812881469727,
      "rewards/rejected": -25.340625762939453,
      "step": 7890
    },
    {
      "epoch": 3.389228623538247,
      "grad_norm": 0.01200608581709591,
      "learning_rate": 1.5246781115879828e-07,
      "logits/chosen": -1.158593773841858,
      "logits/rejected": -1.17626953125,
      "logps/chosen": -404.9750061035156,
      "logps/rejected": -591.2000122070312,
      "loss": 0.013,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -4.934814453125,
      "rewards/margins": 20.407812118530273,
      "rewards/rejected": -25.337499618530273,
      "step": 7900
    },
    {
      "epoch": 3.39352000858277,
      "grad_norm": 0.003774743701495429,
      "learning_rate": 1.5139484978540772e-07,
      "logits/chosen": -1.17041015625,
      "logits/rejected": -1.1916015148162842,
      "logps/chosen": -429.1000061035156,
      "logps/rejected": -584.7000122070312,
      "loss": 0.0043,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -5.457421779632568,
      "rewards/margins": 19.032812118530273,
      "rewards/rejected": -24.481250762939453,
      "step": 7910
    },
    {
      "epoch": 3.3978113936272933,
      "grad_norm": 0.01691640942005166,
      "learning_rate": 1.5032188841201716e-07,
      "logits/chosen": -1.115966796875,
      "logits/rejected": -1.16015625,
      "logps/chosen": -407.2250061035156,
      "logps/rejected": -586.9000244140625,
      "loss": 0.0043,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -4.930468559265137,
      "rewards/margins": 20.854686737060547,
      "rewards/rejected": -25.784374237060547,
      "step": 7920
    },
    {
      "epoch": 3.402102778671816,
      "grad_norm": 0.0006230714969723587,
      "learning_rate": 1.492489270386266e-07,
      "logits/chosen": -1.102783203125,
      "logits/rejected": -1.1399657726287842,
      "logps/chosen": -416.1000061035156,
      "logps/rejected": -595.5999755859375,
      "loss": 0.0043,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -4.092761039733887,
      "rewards/margins": 20.723438262939453,
      "rewards/rejected": -24.799999237060547,
      "step": 7930
    },
    {
      "epoch": 3.4063941637163393,
      "grad_norm": 0.34343849654144964,
      "learning_rate": 1.4817596566523603e-07,
      "logits/chosen": -1.085107445716858,
      "logits/rejected": -1.1638672351837158,
      "logps/chosen": -441.29998779296875,
      "logps/rejected": -627.6500244140625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.952160835266113,
      "rewards/margins": 21.689062118530273,
      "rewards/rejected": -26.6328125,
      "step": 7940
    },
    {
      "epoch": 3.4106855487608625,
      "grad_norm": 0.01324665773238559,
      "learning_rate": 1.471030042918455e-07,
      "logits/chosen": -1.107421875,
      "logits/rejected": -1.1540038585662842,
      "logps/chosen": -393.67498779296875,
      "logps/rejected": -558.7000122070312,
      "loss": 0.0043,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -4.610058784484863,
      "rewards/margins": 19.417186737060547,
      "rewards/rejected": -24.015625,
      "step": 7950
    },
    {
      "epoch": 3.4149769338053857,
      "grad_norm": 0.0030413582330390187,
      "learning_rate": 1.460300429184549e-07,
      "logits/chosen": -1.148290991783142,
      "logits/rejected": -1.20556640625,
      "logps/chosen": -412.07501220703125,
      "logps/rejected": -580.5499877929688,
      "loss": 0.0086,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -4.329687595367432,
      "rewards/margins": 20.0234375,
      "rewards/rejected": -24.356250762939453,
      "step": 7960
    },
    {
      "epoch": 3.419268318849909,
      "grad_norm": 0.07526660026754835,
      "learning_rate": 1.4495708154506437e-07,
      "logits/chosen": -1.134057641029358,
      "logits/rejected": -1.143310546875,
      "logps/chosen": -423.2250061035156,
      "logps/rejected": -587.3499755859375,
      "loss": 0.0131,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -5.134228706359863,
      "rewards/margins": 19.142187118530273,
      "rewards/rejected": -24.270313262939453,
      "step": 7970
    },
    {
      "epoch": 3.423559703894432,
      "grad_norm": 0.0010662944733789812,
      "learning_rate": 1.438841201716738e-07,
      "logits/chosen": -1.068212866783142,
      "logits/rejected": -1.1121094226837158,
      "logps/chosen": -397.1000061035156,
      "logps/rejected": -593.3499755859375,
      "loss": 0.0298,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -4.976513862609863,
      "rewards/margins": 20.193750381469727,
      "rewards/rejected": -25.184375762939453,
      "step": 7980
    },
    {
      "epoch": 3.4278510889389553,
      "grad_norm": 0.012003322880143812,
      "learning_rate": 1.4281115879828325e-07,
      "logits/chosen": -1.142480492591858,
      "logits/rejected": -1.201171875,
      "logps/chosen": -426.75,
      "logps/rejected": -592.7000122070312,
      "loss": 0.0087,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -4.889746189117432,
      "rewards/margins": 20.056249618530273,
      "rewards/rejected": -24.951562881469727,
      "step": 7990
    },
    {
      "epoch": 3.432142473983478,
      "grad_norm": 82.2444202819973,
      "learning_rate": 1.417381974248927e-07,
      "logits/chosen": -1.158105492591858,
      "logits/rejected": -1.186621069908142,
      "logps/chosen": -403.1000061035156,
      "logps/rejected": -577.1500244140625,
      "loss": 0.0178,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -5.324609279632568,
      "rewards/margins": 20.339061737060547,
      "rewards/rejected": -25.649999618530273,
      "step": 8000
    },
    {
      "epoch": 3.4364338590280012,
      "grad_norm": 0.030842181041924824,
      "learning_rate": 1.4066523605150212e-07,
      "logits/chosen": -1.1731445789337158,
      "logits/rejected": -1.223388671875,
      "logps/chosen": -434.2749938964844,
      "logps/rejected": -620.8499755859375,
      "loss": 0.0044,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -4.7216796875,
      "rewards/margins": 20.114063262939453,
      "rewards/rejected": -24.834375381469727,
      "step": 8010
    },
    {
      "epoch": 3.4407252440725244,
      "grad_norm": 0.001207826784579018,
      "learning_rate": 1.395922746781116e-07,
      "logits/chosen": -1.15625,
      "logits/rejected": -1.19287109375,
      "logps/chosen": -441.7749938964844,
      "logps/rejected": -597.25,
      "loss": 0.013,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -5.583788871765137,
      "rewards/margins": 18.717187881469727,
      "rewards/rejected": -24.301563262939453,
      "step": 8020
    },
    {
      "epoch": 3.4450166291170476,
      "grad_norm": 0.0019996286224288145,
      "learning_rate": 1.3851931330472102e-07,
      "logits/chosen": -1.1330077648162842,
      "logits/rejected": -1.163964867591858,
      "logps/chosen": -382.3999938964844,
      "logps/rejected": -603.5,
      "loss": 0.013,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -5.21051025390625,
      "rewards/margins": 19.8046875,
      "rewards/rejected": -25.009374618530273,
      "step": 8030
    },
    {
      "epoch": 3.449308014161571,
      "grad_norm": 1.4631629423211996,
      "learning_rate": 1.3744635193133046e-07,
      "logits/chosen": -1.207128882408142,
      "logits/rejected": -1.254492163658142,
      "logps/chosen": -437.9624938964844,
      "logps/rejected": -642.5,
      "loss": 0.0048,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -4.7852783203125,
      "rewards/margins": 20.143749237060547,
      "rewards/rejected": -24.954687118530273,
      "step": 8040
    },
    {
      "epoch": 3.4535993992060936,
      "grad_norm": 0.013598695279643053,
      "learning_rate": 1.3637339055793993e-07,
      "logits/chosen": -1.19677734375,
      "logits/rejected": -1.249414086341858,
      "logps/chosen": -420.70001220703125,
      "logps/rejected": -577.2000122070312,
      "loss": 0.0089,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -4.064160346984863,
      "rewards/margins": 19.543750762939453,
      "rewards/rejected": -23.603124618530273,
      "step": 8050
    },
    {
      "epoch": 3.457890784250617,
      "grad_norm": 0.02450884297939579,
      "learning_rate": 1.3530042918454934e-07,
      "logits/chosen": -1.124548316001892,
      "logits/rejected": -1.1672852039337158,
      "logps/chosen": -417.625,
      "logps/rejected": -574.1500244140625,
      "loss": 0.013,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -4.458691596984863,
      "rewards/margins": 19.470312118530273,
      "rewards/rejected": -23.931249618530273,
      "step": 8060
    },
    {
      "epoch": 3.46218216929514,
      "grad_norm": 0.0032686712163600604,
      "learning_rate": 1.342274678111588e-07,
      "logits/chosen": -1.049658179283142,
      "logits/rejected": -1.137939453125,
      "logps/chosen": -390.20001220703125,
      "logps/rejected": -571.9000244140625,
      "loss": 0.0175,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -5.260351657867432,
      "rewards/margins": 19.470312118530273,
      "rewards/rejected": -24.723438262939453,
      "step": 8070
    },
    {
      "epoch": 3.466473554339663,
      "grad_norm": 0.015332844797687078,
      "learning_rate": 1.3315450643776824e-07,
      "logits/chosen": -1.1382324695587158,
      "logits/rejected": -1.178955078125,
      "logps/chosen": -381.2250061035156,
      "logps/rejected": -591.0499877929688,
      "loss": 0.0044,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -5.236132621765137,
      "rewards/margins": 20.931249618530273,
      "rewards/rejected": -26.159374237060547,
      "step": 8080
    },
    {
      "epoch": 3.4707649393841864,
      "grad_norm": 0.7092839372780634,
      "learning_rate": 1.3208154506437768e-07,
      "logits/chosen": -1.175073266029358,
      "logits/rejected": -1.2043945789337158,
      "logps/chosen": -399.98748779296875,
      "logps/rejected": -568.7999877929688,
      "loss": 0.0087,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -5.623486518859863,
      "rewards/margins": 19.123437881469727,
      "rewards/rejected": -24.753124237060547,
      "step": 8090
    },
    {
      "epoch": 3.475056324428709,
      "grad_norm": 0.006423304329476774,
      "learning_rate": 1.310085836909871e-07,
      "logits/chosen": -1.1774413585662842,
      "logits/rejected": -1.2195312976837158,
      "logps/chosen": -423.3999938964844,
      "logps/rejected": -630.4500122070312,
      "loss": 0.0002,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.335741996765137,
      "rewards/margins": 20.939062118530273,
      "rewards/rejected": -25.264062881469727,
      "step": 8100
    },
    {
      "epoch": 3.4793477094732324,
      "grad_norm": 0.007657271842911379,
      "learning_rate": 1.2993562231759655e-07,
      "logits/chosen": -1.140771508216858,
      "logits/rejected": -1.171044945716858,
      "logps/chosen": -436.5,
      "logps/rejected": -606.1500244140625,
      "loss": 0.013,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -4.464306831359863,
      "rewards/margins": 19.760936737060547,
      "rewards/rejected": -24.228124618530273,
      "step": 8110
    },
    {
      "epoch": 3.4836390945177556,
      "grad_norm": 0.008409242658158721,
      "learning_rate": 1.2886266094420601e-07,
      "logits/chosen": -1.159570336341858,
      "logits/rejected": -1.2205078601837158,
      "logps/chosen": -424.45001220703125,
      "logps/rejected": -583.6500244140625,
      "loss": 0.0087,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -5.229467868804932,
      "rewards/margins": 19.120311737060547,
      "rewards/rejected": -24.350000381469727,
      "step": 8120
    },
    {
      "epoch": 3.4879304795622788,
      "grad_norm": 0.03381226688135793,
      "learning_rate": 1.2778969957081545e-07,
      "logits/chosen": -1.171875,
      "logits/rejected": -1.204199194908142,
      "logps/chosen": -395.1000061035156,
      "logps/rejected": -569.0999755859375,
      "loss": 0.0087,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -4.168774604797363,
      "rewards/margins": 19.012500762939453,
      "rewards/rejected": -23.182811737060547,
      "step": 8130
    },
    {
      "epoch": 3.492221864606802,
      "grad_norm": 0.08610832246223908,
      "learning_rate": 1.267167381974249e-07,
      "logits/chosen": -1.195214867591858,
      "logits/rejected": -1.246679663658142,
      "logps/chosen": -428.07501220703125,
      "logps/rejected": -584.9000244140625,
      "loss": 0.0043,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -4.810595512390137,
      "rewards/margins": 18.720312118530273,
      "rewards/rejected": -23.524999618530273,
      "step": 8140
    },
    {
      "epoch": 3.496513249651325,
      "grad_norm": 0.010742797782946869,
      "learning_rate": 1.2564377682403433e-07,
      "logits/chosen": -1.132714867591858,
      "logits/rejected": -1.18798828125,
      "logps/chosen": -445.125,
      "logps/rejected": -640.75,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.706835746765137,
      "rewards/margins": 21.009374618530273,
      "rewards/rejected": -25.721874237060547,
      "step": 8150
    },
    {
      "epoch": 3.5008046346958483,
      "grad_norm": 0.0057678395562779546,
      "learning_rate": 1.2457081545064376e-07,
      "logits/chosen": -1.196997046470642,
      "logits/rejected": -1.2295410633087158,
      "logps/chosen": -454.07501220703125,
      "logps/rejected": -605.5,
      "loss": 0.0087,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -4.809887886047363,
      "rewards/margins": 20.048437118530273,
      "rewards/rejected": -24.862499237060547,
      "step": 8160
    },
    {
      "epoch": 3.505096019740371,
      "grad_norm": 0.015205031767764289,
      "learning_rate": 1.2349785407725323e-07,
      "logits/chosen": -1.1923828125,
      "logits/rejected": -1.2169921398162842,
      "logps/chosen": -385.5249938964844,
      "logps/rejected": -577.25,
      "loss": 0.0173,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -4.830883979797363,
      "rewards/margins": 19.621875762939453,
      "rewards/rejected": -24.454687118530273,
      "step": 8170
    },
    {
      "epoch": 3.5093874047848943,
      "grad_norm": 0.005920411784687027,
      "learning_rate": 1.2242489270386264e-07,
      "logits/chosen": -1.035009741783142,
      "logits/rejected": -1.063562035560608,
      "logps/chosen": -399.25,
      "logps/rejected": -602.9000244140625,
      "loss": 0.0043,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -5.027441501617432,
      "rewards/margins": 20.659374237060547,
      "rewards/rejected": -25.681249618530273,
      "step": 8180
    },
    {
      "epoch": 3.5136787898294175,
      "grad_norm": 0.02422476760611223,
      "learning_rate": 1.213519313304721e-07,
      "logits/chosen": -1.177832007408142,
      "logits/rejected": -1.165429711341858,
      "logps/chosen": -418.32501220703125,
      "logps/rejected": -613.0999755859375,
      "loss": 0.0044,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -5.979101657867432,
      "rewards/margins": 20.059375762939453,
      "rewards/rejected": -26.046875,
      "step": 8190
    },
    {
      "epoch": 3.5179701748739407,
      "grad_norm": 0.0035385363225243207,
      "learning_rate": 1.2027896995708154e-07,
      "logits/chosen": -1.1566040515899658,
      "logits/rejected": -1.198144555091858,
      "logps/chosen": -441.82501220703125,
      "logps/rejected": -585.1500244140625,
      "loss": 0.0043,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -5.383935451507568,
      "rewards/margins": 19.307811737060547,
      "rewards/rejected": -24.696874618530273,
      "step": 8200
    },
    {
      "epoch": 3.522261559918464,
      "grad_norm": 0.015340100058724833,
      "learning_rate": 1.1920600858369098e-07,
      "logits/chosen": -1.1886718273162842,
      "logits/rejected": -1.2389647960662842,
      "logps/chosen": -432.54998779296875,
      "logps/rejected": -567.1500244140625,
      "loss": 0.0087,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -5.60498046875,
      "rewards/margins": 19.3984375,
      "rewards/rejected": -25.0078125,
      "step": 8210
    },
    {
      "epoch": 3.5265529449629867,
      "grad_norm": 0.0046605012507767735,
      "learning_rate": 1.1813304721030043e-07,
      "logits/chosen": -1.330078125,
      "logits/rejected": -1.330468773841858,
      "logps/chosen": -458.8999938964844,
      "logps/rejected": -621.5,
      "loss": 0.0005,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.110888481140137,
      "rewards/margins": 19.424999237060547,
      "rewards/rejected": -24.53125,
      "step": 8220
    },
    {
      "epoch": 3.53084433000751,
      "grad_norm": 0.05525991035519728,
      "learning_rate": 1.1706008583690987e-07,
      "logits/chosen": -1.1921875476837158,
      "logits/rejected": -1.2214844226837158,
      "logps/chosen": -421.4750061035156,
      "logps/rejected": -572.1500244140625,
      "loss": 0.0043,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -5.306542873382568,
      "rewards/margins": 19.339061737060547,
      "rewards/rejected": -24.65625,
      "step": 8230
    },
    {
      "epoch": 3.535135715052033,
      "grad_norm": 0.007116013633970248,
      "learning_rate": 1.1598712446351932e-07,
      "logits/chosen": -1.2008788585662842,
      "logits/rejected": -1.27734375,
      "logps/chosen": -426.125,
      "logps/rejected": -556.5,
      "loss": 0.0043,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -4.968554496765137,
      "rewards/margins": 19.420312881469727,
      "rewards/rejected": -24.392187118530273,
      "step": 8240
    },
    {
      "epoch": 3.5394271000965563,
      "grad_norm": 0.3805637531011174,
      "learning_rate": 1.1491416309012874e-07,
      "logits/chosen": -1.2132079601287842,
      "logits/rejected": -1.25390625,
      "logps/chosen": -417.0,
      "logps/rejected": -588.25,
      "loss": 0.0044,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -5.109375,
      "rewards/margins": 19.826562881469727,
      "rewards/rejected": -24.934375762939453,
      "step": 8250
    },
    {
      "epoch": 3.5437184851410795,
      "grad_norm": 0.029586090663247917,
      "learning_rate": 1.1384120171673819e-07,
      "logits/chosen": -1.114111304283142,
      "logits/rejected": -1.1476562023162842,
      "logps/chosen": -391.1499938964844,
      "logps/rejected": -568.6500244140625,
      "loss": 0.009,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -4.879443168640137,
      "rewards/margins": 19.456249237060547,
      "rewards/rejected": -24.3359375,
      "step": 8260
    },
    {
      "epoch": 3.548009870185602,
      "grad_norm": 150.64189207084723,
      "learning_rate": 1.1276824034334763e-07,
      "logits/chosen": -1.1459472179412842,
      "logits/rejected": -1.168212890625,
      "logps/chosen": -414.7749938964844,
      "logps/rejected": -588.5,
      "loss": 0.0126,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -4.650586128234863,
      "rewards/margins": 20.784374237060547,
      "rewards/rejected": -25.424999237060547,
      "step": 8270
    },
    {
      "epoch": 3.5523012552301254,
      "grad_norm": 8.545537961573473,
      "learning_rate": 1.1169527896995708e-07,
      "logits/chosen": -1.1368896961212158,
      "logits/rejected": -1.181640625,
      "logps/chosen": -362.375,
      "logps/rejected": -557.7999877929688,
      "loss": 0.0054,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -4.553124904632568,
      "rewards/margins": 19.496875762939453,
      "rewards/rejected": -24.0546875,
      "step": 8280
    },
    {
      "epoch": 3.5565926402746486,
      "grad_norm": 0.03396961142928566,
      "learning_rate": 1.1062231759656653e-07,
      "logits/chosen": -1.143798828125,
      "logits/rejected": -1.1921265125274658,
      "logps/chosen": -410.125,
      "logps/rejected": -590.0499877929688,
      "loss": 0.013,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -5.22509765625,
      "rewards/margins": 19.774999618530273,
      "rewards/rejected": -25.012500762939453,
      "step": 8290
    },
    {
      "epoch": 3.560884025319172,
      "grad_norm": 0.03836962606758187,
      "learning_rate": 1.0954935622317595e-07,
      "logits/chosen": -1.1392333507537842,
      "logits/rejected": -1.1644287109375,
      "logps/chosen": -409.75,
      "logps/rejected": -575.5,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.59912109375,
      "rewards/margins": 19.339061737060547,
      "rewards/rejected": -24.950000762939453,
      "step": 8300
    },
    {
      "epoch": 3.565175410363695,
      "grad_norm": 0.1319076339282744,
      "learning_rate": 1.084763948497854e-07,
      "logits/chosen": -1.100488305091858,
      "logits/rejected": -1.1226074695587158,
      "logps/chosen": -398.7250061035156,
      "logps/rejected": -591.5,
      "loss": 0.0173,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -5.386523246765137,
      "rewards/margins": 19.967967987060547,
      "rewards/rejected": -25.342187881469727,
      "step": 8310
    },
    {
      "epoch": 3.5694667954082178,
      "grad_norm": 0.013255963897236104,
      "learning_rate": 1.0740343347639484e-07,
      "logits/chosen": -1.27001953125,
      "logits/rejected": -1.300390601158142,
      "logps/chosen": -441.79998779296875,
      "logps/rejected": -585.2000122070312,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.059618949890137,
      "rewards/margins": 20.268749237060547,
      "rewards/rejected": -25.326562881469727,
      "step": 8320
    },
    {
      "epoch": 3.573758180452741,
      "grad_norm": 0.017935620569501418,
      "learning_rate": 1.0633047210300429e-07,
      "logits/chosen": -1.1029052734375,
      "logits/rejected": -1.1689453125,
      "logps/chosen": -394.20001220703125,
      "logps/rejected": -607.5999755859375,
      "loss": 0.0074,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -4.909960746765137,
      "rewards/margins": 20.596874237060547,
      "rewards/rejected": -25.503124237060547,
      "step": 8330
    },
    {
      "epoch": 3.578049565497264,
      "grad_norm": 0.010512842233830077,
      "learning_rate": 1.0525751072961373e-07,
      "logits/chosen": -1.196679711341858,
      "logits/rejected": -1.2370116710662842,
      "logps/chosen": -474.42498779296875,
      "logps/rejected": -621.1500244140625,
      "loss": 0.0043,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -5.546093940734863,
      "rewards/margins": 19.149999618530273,
      "rewards/rejected": -24.700000762939453,
      "step": 8340
    },
    {
      "epoch": 3.5823409505417874,
      "grad_norm": 1.1612297571321948,
      "learning_rate": 1.0418454935622317e-07,
      "logits/chosen": -1.201171875,
      "logits/rejected": -1.254785180091858,
      "logps/chosen": -449.5,
      "logps/rejected": -580.7000122070312,
      "loss": 0.0133,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -5.069091796875,
      "rewards/margins": 18.453125,
      "rewards/rejected": -23.526561737060547,
      "step": 8350
    },
    {
      "epoch": 3.5866323355863106,
      "grad_norm": 0.6031746746918992,
      "learning_rate": 1.031115879828326e-07,
      "logits/chosen": -1.0625,
      "logits/rejected": -1.134130835533142,
      "logps/chosen": -399.625,
      "logps/rejected": -579.2999877929688,
      "loss": 0.013,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -4.983544826507568,
      "rewards/margins": 19.201562881469727,
      "rewards/rejected": -24.189062118530273,
      "step": 8360
    },
    {
      "epoch": 3.5909237206308338,
      "grad_norm": 0.034385041454836895,
      "learning_rate": 1.0203862660944206e-07,
      "logits/chosen": -1.203027367591858,
      "logits/rejected": -1.19384765625,
      "logps/chosen": -429.875,
      "logps/rejected": -607.2999877929688,
      "loss": 0.0043,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -5.474609375,
      "rewards/margins": 19.285938262939453,
      "rewards/rejected": -24.753124237060547,
      "step": 8370
    },
    {
      "epoch": 3.595215105675357,
      "grad_norm": 0.006919599887907935,
      "learning_rate": 1.0096566523605151e-07,
      "logits/chosen": -1.13134765625,
      "logits/rejected": -1.174560546875,
      "logps/chosen": -407.29998779296875,
      "logps/rejected": -548.125,
      "loss": 0.0243,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -5.267187595367432,
      "rewards/margins": 18.051563262939453,
      "rewards/rejected": -23.314062118530273,
      "step": 8380
    },
    {
      "epoch": 3.5995064907198797,
      "grad_norm": 0.426549414736646,
      "learning_rate": 9.989270386266093e-08,
      "logits/chosen": -1.186669945716858,
      "logits/rejected": -1.2199218273162842,
      "logps/chosen": -385.375,
      "logps/rejected": -589.4000244140625,
      "loss": 0.0088,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -4.575488090515137,
      "rewards/margins": 20.6171875,
      "rewards/rejected": -25.196874618530273,
      "step": 8390
    },
    {
      "epoch": 3.603797875764403,
      "grad_norm": 0.014428582321994409,
      "learning_rate": 9.881974248927038e-08,
      "logits/chosen": -1.153564453125,
      "logits/rejected": -1.216699242591858,
      "logps/chosen": -425.67498779296875,
      "logps/rejected": -595.1500244140625,
      "loss": 0.0173,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -4.942578315734863,
      "rewards/margins": 18.684375762939453,
      "rewards/rejected": -23.623437881469727,
      "step": 8400
    },
    {
      "epoch": 3.608089260808926,
      "grad_norm": 0.007049499544214462,
      "learning_rate": 9.774678111587982e-08,
      "logits/chosen": -1.1842041015625,
      "logits/rejected": -1.2451171875,
      "logps/chosen": -437.95001220703125,
      "logps/rejected": -596.4500122070312,
      "loss": 0.0043,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -5.171093940734863,
      "rewards/margins": 19.451562881469727,
      "rewards/rejected": -24.625,
      "step": 8410
    },
    {
      "epoch": 3.6123806458534493,
      "grad_norm": 0.023125194554903444,
      "learning_rate": 9.667381974248927e-08,
      "logits/chosen": -1.2317383289337158,
      "logits/rejected": -1.305029273033142,
      "logps/chosen": -401.7250061035156,
      "logps/rejected": -572.9500122070312,
      "loss": 0.0044,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -5.224707126617432,
      "rewards/margins": 19.629688262939453,
      "rewards/rejected": -24.860937118530273,
      "step": 8420
    },
    {
      "epoch": 3.6166720308979725,
      "grad_norm": 0.0010436821012540236,
      "learning_rate": 9.560085836909871e-08,
      "logits/chosen": -1.1028320789337158,
      "logits/rejected": -1.158300757408142,
      "logps/chosen": -378.20001220703125,
      "logps/rejected": -565.4000244140625,
      "loss": 0.0188,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -4.609570503234863,
      "rewards/margins": 20.212499618530273,
      "rewards/rejected": -24.814062118530273,
      "step": 8430
    },
    {
      "epoch": 3.6209634159424953,
      "grad_norm": 0.02702237318003246,
      "learning_rate": 9.452789699570815e-08,
      "logits/chosen": -1.1737792491912842,
      "logits/rejected": -1.2073242664337158,
      "logps/chosen": -430.3500061035156,
      "logps/rejected": -611.25,
      "loss": 0.013,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -5.385644435882568,
      "rewards/margins": 19.059375762939453,
      "rewards/rejected": -24.435937881469727,
      "step": 8440
    },
    {
      "epoch": 3.6252548009870185,
      "grad_norm": 0.052244393047937335,
      "learning_rate": 9.34549356223176e-08,
      "logits/chosen": -1.1113770008087158,
      "logits/rejected": -1.147680640220642,
      "logps/chosen": -408.92498779296875,
      "logps/rejected": -577.1500244140625,
      "loss": 0.0087,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -4.613085746765137,
      "rewards/margins": 20.084375381469727,
      "rewards/rejected": -24.690624237060547,
      "step": 8450
    },
    {
      "epoch": 3.6295461860315417,
      "grad_norm": 0.01939988567590373,
      "learning_rate": 9.238197424892703e-08,
      "logits/chosen": -1.1155273914337158,
      "logits/rejected": -1.158056616783142,
      "logps/chosen": -398.5,
      "logps/rejected": -593.8499755859375,
      "loss": 0.0087,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -4.90771484375,
      "rewards/margins": 18.248437881469727,
      "rewards/rejected": -23.167186737060547,
      "step": 8460
    },
    {
      "epoch": 3.633837571076065,
      "grad_norm": 0.11734794410730569,
      "learning_rate": 9.130901287553648e-08,
      "logits/chosen": -1.0884277820587158,
      "logits/rejected": -1.158544898033142,
      "logps/chosen": -376.9125061035156,
      "logps/rejected": -567.9500122070312,
      "loss": 0.0089,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -5.1920166015625,
      "rewards/margins": 19.7734375,
      "rewards/rejected": -24.971874237060547,
      "step": 8470
    },
    {
      "epoch": 3.638128956120588,
      "grad_norm": 63.50828196672935,
      "learning_rate": 9.023605150214592e-08,
      "logits/chosen": -1.1023437976837158,
      "logits/rejected": -1.1388671398162842,
      "logps/chosen": -423.4750061035156,
      "logps/rejected": -574.5,
      "loss": 0.0417,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -5.053515434265137,
      "rewards/margins": 19.235937118530273,
      "rewards/rejected": -24.299999237060547,
      "step": 8480
    },
    {
      "epoch": 3.642420341165111,
      "grad_norm": 0.5946343830684734,
      "learning_rate": 8.916309012875536e-08,
      "logits/chosen": -1.1142089366912842,
      "logits/rejected": -1.1716797351837158,
      "logps/chosen": -426.1499938964844,
      "logps/rejected": -591.9000244140625,
      "loss": 0.0173,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -5.137402534484863,
      "rewards/margins": 19.606250762939453,
      "rewards/rejected": -24.732812881469727,
      "step": 8490
    },
    {
      "epoch": 3.646711726209634,
      "grad_norm": 11.20458934232315,
      "learning_rate": 8.80901287553648e-08,
      "logits/chosen": -1.1569335460662842,
      "logits/rejected": -1.1858642101287842,
      "logps/chosen": -430.82501220703125,
      "logps/rejected": -553.2999877929688,
      "loss": 0.0024,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.823876857757568,
      "rewards/margins": 18.964061737060547,
      "rewards/rejected": -23.771875381469727,
      "step": 8500
    },
    {
      "epoch": 3.651003111254157,
      "grad_norm": 0.016355504391497244,
      "learning_rate": 8.701716738197425e-08,
      "logits/chosen": -1.1650879383087158,
      "logits/rejected": -1.203466773033142,
      "logps/chosen": -430.79998779296875,
      "logps/rejected": -581.75,
      "loss": 0.013,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -4.818799018859863,
      "rewards/margins": 19.073436737060547,
      "rewards/rejected": -23.887500762939453,
      "step": 8510
    },
    {
      "epoch": 3.6552944962986804,
      "grad_norm": 0.011776871793671703,
      "learning_rate": 8.594420600858368e-08,
      "logits/chosen": -1.256738305091858,
      "logits/rejected": -1.2671387195587158,
      "logps/chosen": -394.6000061035156,
      "logps/rejected": -591.25,
      "loss": 0.0087,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -5.537060737609863,
      "rewards/margins": 19.8671875,
      "rewards/rejected": -25.415624618530273,
      "step": 8520
    },
    {
      "epoch": 3.6595858813432036,
      "grad_norm": 0.047168494203315085,
      "learning_rate": 8.487124463519314e-08,
      "logits/chosen": -1.1472656726837158,
      "logits/rejected": -1.207983374595642,
      "logps/chosen": -428.5,
      "logps/rejected": -604.0,
      "loss": 0.0043,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -5.100488185882568,
      "rewards/margins": 20.3125,
      "rewards/rejected": -25.407812118530273,
      "step": 8530
    },
    {
      "epoch": 3.6638772663877264,
      "grad_norm": 0.008613503592405865,
      "learning_rate": 8.379828326180257e-08,
      "logits/chosen": -1.1666991710662842,
      "logits/rejected": -1.2208983898162842,
      "logps/chosen": -403.54998779296875,
      "logps/rejected": -592.7999877929688,
      "loss": 0.0087,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -5.676465034484863,
      "rewards/margins": 20.146875381469727,
      "rewards/rejected": -25.837499618530273,
      "step": 8540
    },
    {
      "epoch": 3.66816865143225,
      "grad_norm": 0.7005315117328474,
      "learning_rate": 8.272532188841201e-08,
      "logits/chosen": -1.1213867664337158,
      "logits/rejected": -1.1579101085662842,
      "logps/chosen": -422.57501220703125,
      "logps/rejected": -569.5,
      "loss": 0.0173,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -5.765429496765137,
      "rewards/margins": 19.556249618530273,
      "rewards/rejected": -25.325000762939453,
      "step": 8550
    },
    {
      "epoch": 3.6724600364767728,
      "grad_norm": 0.009648771564342145,
      "learning_rate": 8.165236051502146e-08,
      "logits/chosen": -1.0265991687774658,
      "logits/rejected": -1.0940673351287842,
      "logps/chosen": -419.04998779296875,
      "logps/rejected": -574.5999755859375,
      "loss": 0.0177,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -5.427783012390137,
      "rewards/margins": 18.901561737060547,
      "rewards/rejected": -24.318750381469727,
      "step": 8560
    },
    {
      "epoch": 3.676751421521296,
      "grad_norm": 0.009627641295976356,
      "learning_rate": 8.05793991416309e-08,
      "logits/chosen": -1.0927002429962158,
      "logits/rejected": -1.1087524890899658,
      "logps/chosen": -387.875,
      "logps/rejected": -571.0250244140625,
      "loss": 0.0176,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -5.486718654632568,
      "rewards/margins": 19.465625762939453,
      "rewards/rejected": -24.956249237060547,
      "step": 8570
    },
    {
      "epoch": 3.681042806565819,
      "grad_norm": 0.02063193695921579,
      "learning_rate": 7.950643776824034e-08,
      "logits/chosen": -1.140649437904358,
      "logits/rejected": -1.1853516101837158,
      "logps/chosen": -426.1000061035156,
      "logps/rejected": -594.8499755859375,
      "loss": 0.0087,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -5.322265625,
      "rewards/margins": 19.876562118530273,
      "rewards/rejected": -25.1875,
      "step": 8580
    },
    {
      "epoch": 3.6853341916103424,
      "grad_norm": 0.008542792450343666,
      "learning_rate": 7.843347639484977e-08,
      "logits/chosen": -1.1705443859100342,
      "logits/rejected": -1.227148413658142,
      "logps/chosen": -430.8500061035156,
      "logps/rejected": -624.3499755859375,
      "loss": 0.0043,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -5.431054592132568,
      "rewards/margins": 19.567188262939453,
      "rewards/rejected": -24.987499237060547,
      "step": 8590
    },
    {
      "epoch": 3.6896255766548656,
      "grad_norm": 0.03321597572821151,
      "learning_rate": 7.736051502145922e-08,
      "logits/chosen": -1.0995361804962158,
      "logits/rejected": -1.1615478992462158,
      "logps/chosen": -407.625,
      "logps/rejected": -612.0999755859375,
      "loss": 0.0087,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -6.202050685882568,
      "rewards/margins": 20.03125,
      "rewards/rejected": -26.237499237060547,
      "step": 8600
    },
    {
      "epoch": 3.6939169616993883,
      "grad_norm": 0.7687063951855772,
      "learning_rate": 7.628755364806867e-08,
      "logits/chosen": -1.105932593345642,
      "logits/rejected": -1.1331787109375,
      "logps/chosen": -403.8125,
      "logps/rejected": -590.6124877929688,
      "loss": 0.0173,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -4.930078029632568,
      "rewards/margins": 19.387500762939453,
      "rewards/rejected": -24.321874618530273,
      "step": 8610
    },
    {
      "epoch": 3.6982083467439115,
      "grad_norm": 0.013251509903096039,
      "learning_rate": 7.521459227467811e-08,
      "logits/chosen": -1.134667992591858,
      "logits/rejected": -1.1613280773162842,
      "logps/chosen": -373.82501220703125,
      "logps/rejected": -526.8499755859375,
      "loss": 0.0389,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -5.658789157867432,
      "rewards/margins": 18.024999618530273,
      "rewards/rejected": -23.681249618530273,
      "step": 8620
    },
    {
      "epoch": 3.7024997317884347,
      "grad_norm": 0.009906141003661512,
      "learning_rate": 7.414163090128755e-08,
      "logits/chosen": -1.14990234375,
      "logits/rejected": -1.189843773841858,
      "logps/chosen": -413.3999938964844,
      "logps/rejected": -600.0499877929688,
      "loss": 0.0044,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -6.3974609375,
      "rewards/margins": 20.021875381469727,
      "rewards/rejected": -26.421875,
      "step": 8630
    },
    {
      "epoch": 3.706791116832958,
      "grad_norm": 2.345995632282609,
      "learning_rate": 7.306866952789699e-08,
      "logits/chosen": -1.185546875,
      "logits/rejected": -1.227441430091858,
      "logps/chosen": -413.1499938964844,
      "logps/rejected": -609.8499755859375,
      "loss": 0.0043,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -5.889452934265137,
      "rewards/margins": 20.482812881469727,
      "rewards/rejected": -26.365625381469727,
      "step": 8640
    },
    {
      "epoch": 3.711082501877481,
      "grad_norm": 0.05875854584665483,
      "learning_rate": 7.199570815450644e-08,
      "logits/chosen": -1.144628882408142,
      "logits/rejected": -1.1782715320587158,
      "logps/chosen": -426.79998779296875,
      "logps/rejected": -587.25,
      "loss": 0.0043,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -5.925000190734863,
      "rewards/margins": 19.9140625,
      "rewards/rejected": -25.840625762939453,
      "step": 8650
    },
    {
      "epoch": 3.715373886922004,
      "grad_norm": 0.0009066449772924871,
      "learning_rate": 7.092274678111587e-08,
      "logits/chosen": -1.1748046875,
      "logits/rejected": -1.2217285633087158,
      "logps/chosen": -434.70001220703125,
      "logps/rejected": -581.0499877929688,
      "loss": 0.0088,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -4.889843940734863,
      "rewards/margins": 19.135936737060547,
      "rewards/rejected": -24.028125762939453,
      "step": 8660
    },
    {
      "epoch": 3.719665271966527,
      "grad_norm": 0.018555419012686265,
      "learning_rate": 6.984978540772533e-08,
      "logits/chosen": -1.1774413585662842,
      "logits/rejected": -1.1995117664337158,
      "logps/chosen": -414.1000061035156,
      "logps/rejected": -586.5499877929688,
      "loss": 0.0056,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -5.23974609375,
      "rewards/margins": 19.862499237060547,
      "rewards/rejected": -25.104686737060547,
      "step": 8670
    },
    {
      "epoch": 3.7239566570110503,
      "grad_norm": 0.07186117976952897,
      "learning_rate": 6.877682403433475e-08,
      "logits/chosen": -1.076025366783142,
      "logits/rejected": -1.126611351966858,
      "logps/chosen": -417.5,
      "logps/rejected": -574.7000122070312,
      "loss": 0.0049,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -5.630566596984863,
      "rewards/margins": 20.09375,
      "rewards/rejected": -25.71875,
      "step": 8680
    },
    {
      "epoch": 3.7282480420555735,
      "grad_norm": 0.3500929697138476,
      "learning_rate": 6.77038626609442e-08,
      "logits/chosen": -1.169531226158142,
      "logits/rejected": -1.218359351158142,
      "logps/chosen": -430.42498779296875,
      "logps/rejected": -622.4500122070312,
      "loss": 0.013,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -5.948290824890137,
      "rewards/margins": 20.340625762939453,
      "rewards/rejected": -26.306249618530273,
      "step": 8690
    },
    {
      "epoch": 3.7325394271000967,
      "grad_norm": 0.04287049011548471,
      "learning_rate": 6.663090128755365e-08,
      "logits/chosen": -1.178808569908142,
      "logits/rejected": -1.217871069908142,
      "logps/chosen": -413.5375061035156,
      "logps/rejected": -552.3499755859375,
      "loss": 0.0098,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -5.390527248382568,
      "rewards/margins": 19.048437118530273,
      "rewards/rejected": -24.451562881469727,
      "step": 8700
    },
    {
      "epoch": 3.7368308121446194,
      "grad_norm": 0.7638228099649663,
      "learning_rate": 6.555793991416309e-08,
      "logits/chosen": -1.1450684070587158,
      "logits/rejected": -1.1697266101837158,
      "logps/chosen": -440.8500061035156,
      "logps/rejected": -608.3499755859375,
      "loss": 0.0043,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -5.300195217132568,
      "rewards/margins": 20.012500762939453,
      "rewards/rejected": -25.318750381469727,
      "step": 8710
    },
    {
      "epoch": 3.741122197189143,
      "grad_norm": 0.05362729115251585,
      "learning_rate": 6.448497854077254e-08,
      "logits/chosen": -1.2199218273162842,
      "logits/rejected": -1.2786133289337158,
      "logps/chosen": -424.7749938964844,
      "logps/rejected": -570.4249877929688,
      "loss": 0.0087,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -5.472265720367432,
      "rewards/margins": 18.778125762939453,
      "rewards/rejected": -24.248437881469727,
      "step": 8720
    },
    {
      "epoch": 3.745413582233666,
      "grad_norm": 0.0015457125940328896,
      "learning_rate": 6.341201716738196e-08,
      "logits/chosen": -1.105859398841858,
      "logits/rejected": -1.1628906726837158,
      "logps/chosen": -440.0249938964844,
      "logps/rejected": -604.9000244140625,
      "loss": 0.0087,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -5.180371284484863,
      "rewards/margins": 20.107812881469727,
      "rewards/rejected": -25.296875,
      "step": 8730
    },
    {
      "epoch": 3.749704967278189,
      "grad_norm": 0.00897334687468335,
      "learning_rate": 6.233905579399141e-08,
      "logits/chosen": -1.1243896484375,
      "logits/rejected": -1.203271508216858,
      "logps/chosen": -404.0,
      "logps/rejected": -584.3499755859375,
      "loss": 0.0044,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -6.476171970367432,
      "rewards/margins": 18.776561737060547,
      "rewards/rejected": -25.256250381469727,
      "step": 8740
    },
    {
      "epoch": 3.7539963523227122,
      "grad_norm": 0.8382933068986903,
      "learning_rate": 6.126609442060085e-08,
      "logits/chosen": -1.149169921875,
      "logits/rejected": -1.165283203125,
      "logps/chosen": -408.1875,
      "logps/rejected": -603.4500122070312,
      "loss": 0.013,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -5.2392578125,
      "rewards/margins": 20.681249618530273,
      "rewards/rejected": -25.921875,
      "step": 8750
    },
    {
      "epoch": 3.7582877373672354,
      "grad_norm": 0.019262215984209605,
      "learning_rate": 6.01931330472103e-08,
      "logits/chosen": -1.12060546875,
      "logits/rejected": -1.1537353992462158,
      "logps/chosen": -411.61248779296875,
      "logps/rejected": -578.25,
      "loss": 0.0173,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -4.814013481140137,
      "rewards/margins": 19.459375381469727,
      "rewards/rejected": -24.253124237060547,
      "step": 8760
    },
    {
      "epoch": 3.7625791224117586,
      "grad_norm": 0.003031795927683085,
      "learning_rate": 5.912017167381974e-08,
      "logits/chosen": -1.1413085460662842,
      "logits/rejected": -1.15234375,
      "logps/chosen": -414.0249938964844,
      "logps/rejected": -589.9000244140625,
      "loss": 0.013,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -5.6142578125,
      "rewards/margins": 19.428125381469727,
      "rewards/rejected": -25.042186737060547,
      "step": 8770
    },
    {
      "epoch": 3.7668705074562814,
      "grad_norm": 0.004916795604384672,
      "learning_rate": 5.8047210300429184e-08,
      "logits/chosen": -1.1911132335662842,
      "logits/rejected": -1.215917944908142,
      "logps/chosen": -438.0249938964844,
      "logps/rejected": -594.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.287841796875,
      "rewards/margins": 19.8984375,
      "rewards/rejected": -25.212499618530273,
      "step": 8780
    },
    {
      "epoch": 3.7711618925008046,
      "grad_norm": 0.2725158571382876,
      "learning_rate": 5.697424892703862e-08,
      "logits/chosen": -1.133886694908142,
      "logits/rejected": -1.1789062023162842,
      "logps/chosen": -446.2250061035156,
      "logps/rejected": -621.5,
      "loss": 0.0044,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -5.412695407867432,
      "rewards/margins": 20.174999237060547,
      "rewards/rejected": -25.590625762939453,
      "step": 8790
    },
    {
      "epoch": 3.775453277545328,
      "grad_norm": 0.0003881513738795367,
      "learning_rate": 5.5901287553648065e-08,
      "logits/chosen": -1.129003882408142,
      "logits/rejected": -1.1956055164337158,
      "logps/chosen": -433.82501220703125,
      "logps/rejected": -624.4000244140625,
      "loss": 0.0044,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -5.087450981140137,
      "rewards/margins": 19.653125762939453,
      "rewards/rejected": -24.745311737060547,
      "step": 8800
    },
    {
      "epoch": 3.779744662589851,
      "grad_norm": 0.016658203031237096,
      "learning_rate": 5.482832618025751e-08,
      "logits/chosen": -1.269433617591858,
      "logits/rejected": -1.296972632408142,
      "logps/chosen": -416.7250061035156,
      "logps/rejected": -603.25,
      "loss": 0.0087,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -4.615283012390137,
      "rewards/margins": 20.009374618530273,
      "rewards/rejected": -24.635936737060547,
      "step": 8810
    },
    {
      "epoch": 3.784036047634374,
      "grad_norm": 0.11604748123006396,
      "learning_rate": 5.3755364806866953e-08,
      "logits/chosen": -1.096435546875,
      "logits/rejected": -1.1467773914337158,
      "logps/chosen": -365.92498779296875,
      "logps/rejected": -561.9000244140625,
      "loss": 0.0044,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -5.550488471984863,
      "rewards/margins": 19.528125762939453,
      "rewards/rejected": -25.0703125,
      "step": 8820
    },
    {
      "epoch": 3.788327432678897,
      "grad_norm": 0.001816564124599535,
      "learning_rate": 5.268240343347639e-08,
      "logits/chosen": -1.1362793445587158,
      "logits/rejected": -1.186132788658142,
      "logps/chosen": -390.875,
      "logps/rejected": -574.2999877929688,
      "loss": 0.0173,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -5.0009765625,
      "rewards/margins": 19.954687118530273,
      "rewards/rejected": -24.962499618530273,
      "step": 8830
    },
    {
      "epoch": 3.79261881772342,
      "grad_norm": 0.009763879262715845,
      "learning_rate": 5.1609442060085835e-08,
      "logits/chosen": -1.130029320716858,
      "logits/rejected": -1.1472656726837158,
      "logps/chosen": -416.57501220703125,
      "logps/rejected": -603.2000122070312,
      "loss": 0.0044,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -5.990234375,
      "rewards/margins": 18.829687118530273,
      "rewards/rejected": -24.815624237060547,
      "step": 8840
    },
    {
      "epoch": 3.7969102027679433,
      "grad_norm": 0.034224130519936925,
      "learning_rate": 5.053648068669528e-08,
      "logits/chosen": -1.200659155845642,
      "logits/rejected": -1.220312476158142,
      "logps/chosen": -416.29998779296875,
      "logps/rejected": -588.9500122070312,
      "loss": 0.0043,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -4.940087795257568,
      "rewards/margins": 19.4140625,
      "rewards/rejected": -24.340625762939453,
      "step": 8850
    },
    {
      "epoch": 3.8012015878124665,
      "grad_norm": 0.004546701605052007,
      "learning_rate": 4.9463519313304716e-08,
      "logits/chosen": -1.0882568359375,
      "logits/rejected": -1.15478515625,
      "logps/chosen": -398.73748779296875,
      "logps/rejected": -563.2999877929688,
      "loss": 0.0043,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -5.063867092132568,
      "rewards/margins": 19.649999618530273,
      "rewards/rejected": -24.703125,
      "step": 8860
    },
    {
      "epoch": 3.8054929728569897,
      "grad_norm": 0.0036128855282883234,
      "learning_rate": 4.839055793991416e-08,
      "logits/chosen": -1.1306641101837158,
      "logits/rejected": -1.1904296875,
      "logps/chosen": -398.1499938964844,
      "logps/rejected": -548.5,
      "loss": 0.014,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -4.91455078125,
      "rewards/margins": 19.639062881469727,
      "rewards/rejected": -24.553125381469727,
      "step": 8870
    },
    {
      "epoch": 3.8097843579015125,
      "grad_norm": 0.3817359344560652,
      "learning_rate": 4.73175965665236e-08,
      "logits/chosen": -1.0784423351287842,
      "logits/rejected": -1.135498046875,
      "logps/chosen": -421.7250061035156,
      "logps/rejected": -567.2000122070312,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.0078125,
      "rewards/margins": 19.745311737060547,
      "rewards/rejected": -24.762500762939453,
      "step": 8880
    },
    {
      "epoch": 3.8140757429460357,
      "grad_norm": 0.006240846256120957,
      "learning_rate": 4.624463519313305e-08,
      "logits/chosen": -1.186914086341858,
      "logits/rejected": -1.21044921875,
      "logps/chosen": -409.375,
      "logps/rejected": -592.7000122070312,
      "loss": 0.013,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -5.807470798492432,
      "rewards/margins": 19.232812881469727,
      "rewards/rejected": -25.059375762939453,
      "step": 8890
    },
    {
      "epoch": 3.818367127990559,
      "grad_norm": 0.007952048563600921,
      "learning_rate": 4.517167381974249e-08,
      "logits/chosen": -1.1313965320587158,
      "logits/rejected": -1.179589867591858,
      "logps/chosen": -421.8500061035156,
      "logps/rejected": -565.8499755859375,
      "loss": 0.0087,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -5.587695121765137,
      "rewards/margins": 18.723438262939453,
      "rewards/rejected": -24.303125381469727,
      "step": 8900
    },
    {
      "epoch": 3.822658513035082,
      "grad_norm": 0.01724901381153147,
      "learning_rate": 4.409871244635193e-08,
      "logits/chosen": -1.06494140625,
      "logits/rejected": -1.092626929283142,
      "logps/chosen": -414.125,
      "logps/rejected": -609.3499755859375,
      "loss": 0.0174,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -5.836181640625,
      "rewards/margins": 20.185937881469727,
      "rewards/rejected": -26.0078125,
      "step": 8910
    },
    {
      "epoch": 3.8269498980796053,
      "grad_norm": 0.008326764096645971,
      "learning_rate": 4.3025751072961374e-08,
      "logits/chosen": -1.177734375,
      "logits/rejected": -1.1969726085662842,
      "logps/chosen": -388.3500061035156,
      "logps/rejected": -570.0999755859375,
      "loss": 0.0043,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -4.16650390625,
      "rewards/margins": 19.923437118530273,
      "rewards/rejected": -24.079687118530273,
      "step": 8920
    },
    {
      "epoch": 3.831241283124128,
      "grad_norm": 0.33242534611412916,
      "learning_rate": 4.195278969957081e-08,
      "logits/chosen": -1.22900390625,
      "logits/rejected": -1.289648413658142,
      "logps/chosen": -411.20001220703125,
      "logps/rejected": -577.8499755859375,
      "loss": 0.0043,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -6.445068359375,
      "rewards/margins": 19.932811737060547,
      "rewards/rejected": -26.368749618530273,
      "step": 8930
    },
    {
      "epoch": 3.8355326681686517,
      "grad_norm": 0.006276648363819293,
      "learning_rate": 4.0879828326180256e-08,
      "logits/chosen": -1.109375,
      "logits/rejected": -1.175878882408142,
      "logps/chosen": -416.6499938964844,
      "logps/rejected": -578.6500244140625,
      "loss": 0.0044,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -5.922656059265137,
      "rewards/margins": 19.681249618530273,
      "rewards/rejected": -25.612499237060547,
      "step": 8940
    },
    {
      "epoch": 3.8398240532131744,
      "grad_norm": 0.013605303107122827,
      "learning_rate": 3.980686695278969e-08,
      "logits/chosen": -1.2322266101837158,
      "logits/rejected": -1.2883789539337158,
      "logps/chosen": -406.75,
      "logps/rejected": -600.2999877929688,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.696215629577637,
      "rewards/margins": 19.8515625,
      "rewards/rejected": -25.565624237060547,
      "step": 8950
    },
    {
      "epoch": 3.8441154382576976,
      "grad_norm": 0.009679535285228419,
      "learning_rate": 3.873390557939914e-08,
      "logits/chosen": -1.1540405750274658,
      "logits/rejected": -1.1765625476837158,
      "logps/chosen": -404.875,
      "logps/rejected": -574.75,
      "loss": 0.0087,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -4.732251167297363,
      "rewards/margins": 20.024999618530273,
      "rewards/rejected": -24.762500762939453,
      "step": 8960
    },
    {
      "epoch": 3.848406823302221,
      "grad_norm": 0.003163681431655965,
      "learning_rate": 3.766094420600859e-08,
      "logits/chosen": -1.166406273841858,
      "logits/rejected": -1.20166015625,
      "logps/chosen": -406.9750061035156,
      "logps/rejected": -614.75,
      "loss": 0.0173,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -4.773535251617432,
      "rewards/margins": 21.017187118530273,
      "rewards/rejected": -25.787500381469727,
      "step": 8970
    },
    {
      "epoch": 3.852698208346744,
      "grad_norm": 0.24376124547916087,
      "learning_rate": 3.6587982832618025e-08,
      "logits/chosen": -1.1422851085662842,
      "logits/rejected": -1.1750977039337158,
      "logps/chosen": -422.67498779296875,
      "logps/rejected": -582.9500122070312,
      "loss": 0.0044,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -5.2293701171875,
      "rewards/margins": 19.390625,
      "rewards/rejected": -24.618749618530273,
      "step": 8980
    },
    {
      "epoch": 3.8569895933912672,
      "grad_norm": 0.18061318769593468,
      "learning_rate": 3.551502145922747e-08,
      "logits/chosen": -1.19287109375,
      "logits/rejected": -1.2336914539337158,
      "logps/chosen": -421.875,
      "logps/rejected": -567.9500122070312,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.7822265625,
      "rewards/margins": 19.9453125,
      "rewards/rejected": -24.725000381469727,
      "step": 8990
    },
    {
      "epoch": 3.86128097843579,
      "grad_norm": 7.442839648940899,
      "learning_rate": 3.444206008583691e-08,
      "logits/chosen": -1.174658179283142,
      "logits/rejected": -1.2516601085662842,
      "logps/chosen": -433.29998779296875,
      "logps/rejected": -603.4000244140625,
      "loss": 0.006,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -6.202343940734863,
      "rewards/margins": 19.2578125,
      "rewards/rejected": -25.450000762939453,
      "step": 9000
    },
    {
      "epoch": 3.865572363480313,
      "grad_norm": 0.0038657539091527873,
      "learning_rate": 3.336909871244635e-08,
      "logits/chosen": -1.1528809070587158,
      "logits/rejected": -1.1877930164337158,
      "logps/chosen": -418.3500061035156,
      "logps/rejected": -598.7000122070312,
      "loss": 0.0002,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.153857231140137,
      "rewards/margins": 20.620311737060547,
      "rewards/rejected": -25.787500381469727,
      "step": 9010
    },
    {
      "epoch": 3.8698637485248364,
      "grad_norm": 0.01779707500865911,
      "learning_rate": 3.229613733905579e-08,
      "logits/chosen": -1.1757323741912842,
      "logits/rejected": -1.226171851158142,
      "logps/chosen": -412.1000061035156,
      "logps/rejected": -600.75,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.138916015625,
      "rewards/margins": 20.018749237060547,
      "rewards/rejected": -25.162500381469727,
      "step": 9020
    },
    {
      "epoch": 3.8741551335693596,
      "grad_norm": 0.00692639799944452,
      "learning_rate": 3.122317596566524e-08,
      "logits/chosen": -1.174414038658142,
      "logits/rejected": -1.226171851158142,
      "logps/chosen": -412.75,
      "logps/rejected": -568.5999755859375,
      "loss": 0.005,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -6.194726467132568,
      "rewards/margins": 19.270313262939453,
      "rewards/rejected": -25.475000381469727,
      "step": 9030
    },
    {
      "epoch": 3.878446518613883,
      "grad_norm": 0.13162765013878067,
      "learning_rate": 3.0150214592274677e-08,
      "logits/chosen": -1.137182593345642,
      "logits/rejected": -1.1811034679412842,
      "logps/chosen": -395.7749938964844,
      "logps/rejected": -565.5499877929688,
      "loss": 0.013,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -5.177380561828613,
      "rewards/margins": 20.206249237060547,
      "rewards/rejected": -25.387500762939453,
      "step": 9040
    },
    {
      "epoch": 3.8827379036584055,
      "grad_norm": 1.5255453705841067,
      "learning_rate": 2.9077253218884117e-08,
      "logits/chosen": -1.1381347179412842,
      "logits/rejected": -1.1803710460662842,
      "logps/chosen": -406.23748779296875,
      "logps/rejected": -585.0750122070312,
      "loss": 0.013,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -5.33056640625,
      "rewards/margins": 19.440624237060547,
      "rewards/rejected": -24.756250381469727,
      "step": 9050
    },
    {
      "epoch": 3.8870292887029287,
      "grad_norm": 0.006947421828447531,
      "learning_rate": 2.800429184549356e-08,
      "logits/chosen": -1.19189453125,
      "logits/rejected": -1.2062499523162842,
      "logps/chosen": -422.8999938964844,
      "logps/rejected": -578.5,
      "loss": 0.0106,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -5.400341987609863,
      "rewards/margins": 18.881250381469727,
      "rewards/rejected": -24.28125,
      "step": 9060
    },
    {
      "epoch": 3.891320673747452,
      "grad_norm": 0.16422902027485328,
      "learning_rate": 2.6931330472103006e-08,
      "logits/chosen": -1.1759765148162842,
      "logits/rejected": -1.220678687095642,
      "logps/chosen": -408.82501220703125,
      "logps/rejected": -594.0499877929688,
      "loss": 0.0087,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -5.097021579742432,
      "rewards/margins": 20.837499618530273,
      "rewards/rejected": -25.924999237060547,
      "step": 9070
    },
    {
      "epoch": 3.895612058791975,
      "grad_norm": 0.0018380816437804641,
      "learning_rate": 2.5858369098712446e-08,
      "logits/chosen": -1.1725585460662842,
      "logits/rejected": -1.2155272960662842,
      "logps/chosen": -436.6000061035156,
      "logps/rejected": -597.8499755859375,
      "loss": 0.0045,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -5.912011623382568,
      "rewards/margins": 19.528125762939453,
      "rewards/rejected": -25.434375762939453,
      "step": 9080
    },
    {
      "epoch": 3.8999034438364983,
      "grad_norm": 0.18461594911843315,
      "learning_rate": 2.4785407725321887e-08,
      "logits/chosen": -1.102294921875,
      "logits/rejected": -1.150976538658142,
      "logps/chosen": -397.6499938964844,
      "logps/rejected": -586.5999755859375,
      "loss": 0.0191,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -6.325097560882568,
      "rewards/margins": 18.228124618530273,
      "rewards/rejected": -24.556249618530273,
      "step": 9090
    },
    {
      "epoch": 3.904194828881021,
      "grad_norm": 0.010766425612135111,
      "learning_rate": 2.3712446351931328e-08,
      "logits/chosen": -1.1015136241912842,
      "logits/rejected": -1.1376953125,
      "logps/chosen": -424.70001220703125,
      "logps/rejected": -594.9500122070312,
      "loss": 0.001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.347729682922363,
      "rewards/margins": 20.268749237060547,
      "rewards/rejected": -25.612499237060547,
      "step": 9100
    },
    {
      "epoch": 3.9084862139255447,
      "grad_norm": 0.0038996436424056013,
      "learning_rate": 2.2639484978540772e-08,
      "logits/chosen": -1.143652319908142,
      "logits/rejected": -1.1872069835662842,
      "logps/chosen": -444.1499938964844,
      "logps/rejected": -628.25,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -6.237011909484863,
      "rewards/margins": 20.728124618530273,
      "rewards/rejected": -26.962499618530273,
      "step": 9110
    },
    {
      "epoch": 3.9127775989700675,
      "grad_norm": 0.5447118938993043,
      "learning_rate": 2.1566523605150216e-08,
      "logits/chosen": -1.068750023841858,
      "logits/rejected": -1.108642578125,
      "logps/chosen": -394.36248779296875,
      "logps/rejected": -565.0250244140625,
      "loss": 0.026,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -4.714257717132568,
      "rewards/margins": 18.384374618530273,
      "rewards/rejected": -23.100000381469727,
      "step": 9120
    },
    {
      "epoch": 3.9170689840145907,
      "grad_norm": 0.0057429841399033515,
      "learning_rate": 2.0493562231759657e-08,
      "logits/chosen": -1.143457055091858,
      "logits/rejected": -1.150976538658142,
      "logps/chosen": -421.1000061035156,
      "logps/rejected": -595.0,
      "loss": 0.0087,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -5.350878715515137,
      "rewards/margins": 18.831249237060547,
      "rewards/rejected": -24.190624237060547,
      "step": 9130
    },
    {
      "epoch": 3.921360369059114,
      "grad_norm": 0.0017402942739525417,
      "learning_rate": 1.9420600858369097e-08,
      "logits/chosen": -1.19970703125,
      "logits/rejected": -1.252539038658142,
      "logps/chosen": -429.04998779296875,
      "logps/rejected": -625.0499877929688,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -6.002539157867432,
      "rewards/margins": 20.151561737060547,
      "rewards/rejected": -26.153125762939453,
      "step": 9140
    },
    {
      "epoch": 3.925651754103637,
      "grad_norm": 0.00236581055839428,
      "learning_rate": 1.834763948497854e-08,
      "logits/chosen": -1.138085961341858,
      "logits/rejected": -1.1799805164337158,
      "logps/chosen": -401.79998779296875,
      "logps/rejected": -605.5999755859375,
      "loss": 0.0043,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -5.485497951507568,
      "rewards/margins": 20.420312881469727,
      "rewards/rejected": -25.915624618530273,
      "step": 9150
    },
    {
      "epoch": 3.9299431391481603,
      "grad_norm": 6.580424826946325,
      "learning_rate": 1.7274678111587982e-08,
      "logits/chosen": -1.1097900867462158,
      "logits/rejected": -1.149999976158142,
      "logps/chosen": -383.375,
      "logps/rejected": -568.0499877929688,
      "loss": 0.0089,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -5.230664253234863,
      "rewards/margins": 19.6640625,
      "rewards/rejected": -24.895313262939453,
      "step": 9160
    },
    {
      "epoch": 3.934234524192683,
      "grad_norm": 1.8991061018483988,
      "learning_rate": 1.6201716738197423e-08,
      "logits/chosen": -1.115026831626892,
      "logits/rejected": -1.196874976158142,
      "logps/chosen": -403.57501220703125,
      "logps/rejected": -599.8499755859375,
      "loss": 0.0747,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -5.606640815734863,
      "rewards/margins": 19.954687118530273,
      "rewards/rejected": -25.551563262939453,
      "step": 9170
    },
    {
      "epoch": 3.9385259092372062,
      "grad_norm": 0.09548242071216996,
      "learning_rate": 1.5128755364806867e-08,
      "logits/chosen": -1.131445288658142,
      "logits/rejected": -1.160546898841858,
      "logps/chosen": -399.7250061035156,
      "logps/rejected": -576.5,
      "loss": 0.0043,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -5.366747856140137,
      "rewards/margins": 19.971874237060547,
      "rewards/rejected": -25.340625762939453,
      "step": 9180
    },
    {
      "epoch": 3.9428172942817294,
      "grad_norm": 0.10064597849730939,
      "learning_rate": 1.4055793991416308e-08,
      "logits/chosen": -1.2252929210662842,
      "logits/rejected": -1.27001953125,
      "logps/chosen": -433.7749938964844,
      "logps/rejected": -619.7000122070312,
      "loss": 0.0043,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -5.671582221984863,
      "rewards/margins": 19.793750762939453,
      "rewards/rejected": -25.434375762939453,
      "step": 9190
    },
    {
      "epoch": 3.9471086793262526,
      "grad_norm": 0.5725614041457924,
      "learning_rate": 1.2982832618025752e-08,
      "logits/chosen": -1.1168944835662842,
      "logits/rejected": -1.1872069835662842,
      "logps/chosen": -434.7250061035156,
      "logps/rejected": -609.25,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.802099704742432,
      "rewards/margins": 20.440624237060547,
      "rewards/rejected": -26.240625381469727,
      "step": 9200
    },
    {
      "epoch": 3.951400064370776,
      "grad_norm": 0.010639907743597816,
      "learning_rate": 1.1909871244635193e-08,
      "logits/chosen": -1.1735351085662842,
      "logits/rejected": -1.197119116783142,
      "logps/chosen": -376.70001220703125,
      "logps/rejected": -564.0499877929688,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.043798923492432,
      "rewards/margins": 20.931249618530273,
      "rewards/rejected": -24.978124618530273,
      "step": 9210
    },
    {
      "epoch": 3.9556914494152986,
      "grad_norm": 0.00221853467668428,
      "learning_rate": 1.0836909871244635e-08,
      "logits/chosen": -1.120849609375,
      "logits/rejected": -1.188134789466858,
      "logps/chosen": -408.42498779296875,
      "logps/rejected": -593.9000244140625,
      "loss": 0.0087,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -5.590112209320068,
      "rewards/margins": 19.431249618530273,
      "rewards/rejected": -25.032812118530273,
      "step": 9220
    },
    {
      "epoch": 3.959982834459822,
      "grad_norm": 0.026782085902069052,
      "learning_rate": 9.763948497854078e-09,
      "logits/chosen": -1.1114013195037842,
      "logits/rejected": -1.1640136241912842,
      "logps/chosen": -405.6499938964844,
      "logps/rejected": -546.8499755859375,
      "loss": 0.0043,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -4.98828125,
      "rewards/margins": 18.870311737060547,
      "rewards/rejected": -23.865625381469727,
      "step": 9230
    },
    {
      "epoch": 3.964274219504345,
      "grad_norm": 0.007751701127611905,
      "learning_rate": 8.69098712446352e-09,
      "logits/chosen": -1.10498046875,
      "logits/rejected": -1.1596190929412842,
      "logps/chosen": -410.92498779296875,
      "logps/rejected": -585.5499877929688,
      "loss": 0.0216,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -5.873486518859863,
      "rewards/margins": 18.854686737060547,
      "rewards/rejected": -24.717187881469727,
      "step": 9240
    },
    {
      "epoch": 3.968565604548868,
      "grad_norm": 0.021010697026064194,
      "learning_rate": 7.61802575107296e-09,
      "logits/chosen": -1.1213867664337158,
      "logits/rejected": -1.1759765148162842,
      "logps/chosen": -414.07501220703125,
      "logps/rejected": -579.6500244140625,
      "loss": 0.0088,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -5.167773246765137,
      "rewards/margins": 18.953125,
      "rewards/rejected": -24.131250381469727,
      "step": 9250
    },
    {
      "epoch": 3.9728569895933914,
      "grad_norm": 0.4471115466545836,
      "learning_rate": 6.545064377682403e-09,
      "logits/chosen": -1.13018798828125,
      "logits/rejected": -1.1712524890899658,
      "logps/chosen": -453.7250061035156,
      "logps/rejected": -580.2000122070312,
      "loss": 0.0053,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -5.656396389007568,
      "rewards/margins": 19.264062881469727,
      "rewards/rejected": -24.928125381469727,
      "step": 9260
    },
    {
      "epoch": 3.977148374637914,
      "grad_norm": 0.006739333990448072,
      "learning_rate": 5.4721030042918455e-09,
      "logits/chosen": -1.0823974609375,
      "logits/rejected": -1.1277344226837158,
      "logps/chosen": -428.2749938964844,
      "logps/rejected": -596.4000244140625,
      "loss": 0.0087,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -4.913866996765137,
      "rewards/margins": 19.3046875,
      "rewards/rejected": -24.223438262939453,
      "step": 9270
    },
    {
      "epoch": 3.9814397596824374,
      "grad_norm": 0.0057730693487199666,
      "learning_rate": 4.399141630901287e-09,
      "logits/chosen": -1.19189453125,
      "logits/rejected": -1.2080078125,
      "logps/chosen": -414.2124938964844,
      "logps/rejected": -621.0499877929688,
      "loss": 0.0048,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -5.322363376617432,
      "rewards/margins": 19.765625,
      "rewards/rejected": -25.095312118530273,
      "step": 9280
    },
    {
      "epoch": 3.9857311447269606,
      "grad_norm": 0.038885102751617884,
      "learning_rate": 3.3261802575107295e-09,
      "logits/chosen": -1.110742211341858,
      "logits/rejected": -1.1547362804412842,
      "logps/chosen": -426.38751220703125,
      "logps/rejected": -619.0999755859375,
      "loss": 0.0087,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -5.202246189117432,
      "rewards/margins": 20.028125762939453,
      "rewards/rejected": -25.223438262939453,
      "step": 9290
    },
    {
      "epoch": 3.9900225297714838,
      "grad_norm": 0.0033490480018467236,
      "learning_rate": 2.2532188841201715e-09,
      "logits/chosen": -1.171484351158142,
      "logits/rejected": -1.1748046875,
      "logps/chosen": -415.54998779296875,
      "logps/rejected": -554.0,
      "loss": 0.013,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -4.995409965515137,
      "rewards/margins": 18.40625,
      "rewards/rejected": -23.401561737060547,
      "step": 9300
    },
    {
      "epoch": 3.994313914816007,
      "grad_norm": 0.005656690901700299,
      "learning_rate": 1.1802575107296135e-09,
      "logits/chosen": -1.192480444908142,
      "logits/rejected": -1.225976586341858,
      "logps/chosen": -417.6000061035156,
      "logps/rejected": -604.9500122070312,
      "loss": 0.0043,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -5.453711032867432,
      "rewards/margins": 19.671875,
      "rewards/rejected": -25.118749618530273,
      "step": 9310
    },
    {
      "epoch": 3.99860529986053,
      "grad_norm": 2.1753181183889985,
      "learning_rate": 1.0729613733905578e-10,
      "logits/chosen": -1.1315429210662842,
      "logits/rejected": -1.206640601158142,
      "logps/chosen": -410.875,
      "logps/rejected": -560.0,
      "loss": 0.0177,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -5.658984184265137,
      "rewards/margins": 19.8203125,
      "rewards/rejected": -25.481250762939453,
      "step": 9320
    }
  ],
  "logging_steps": 10,
  "max_steps": 9320,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 4,
  "save_steps": 500,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 1,
  "trial_name": null,
  "trial_params": null
}