{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.6640392362935716,
  "eval_steps": 10000,
  "global_step": 9500,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0,
      "learning_rate": 6.666666666666666e-10,
      "logits/chosen": -2.8252084255218506,
      "logits/rejected": -2.962496042251587,
      "logps/chosen": -145.02584838867188,
      "logps/rejected": -154.2101593017578,
      "loss": 0.6931,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.0,
      "learning_rate": 6.666666666666666e-09,
      "logits/chosen": -2.758862018585205,
      "logits/rejected": -2.733859062194824,
      "logps/chosen": -232.04269409179688,
      "logps/rejected": -191.10357666015625,
      "loss": 0.693,
      "rewards/accuracies": 0.4444444477558136,
      "rewards/chosen": -0.0011513985227793455,
      "rewards/margins": -0.001275173737667501,
      "rewards/rejected": 0.00012377536040730774,
      "step": 10
    },
    {
      "epoch": 0.0,
      "learning_rate": 1.3333333333333332e-08,
      "logits/chosen": -2.7594728469848633,
      "logits/rejected": -2.7667908668518066,
      "logps/chosen": -246.5968780517578,
      "logps/rejected": -213.8126678466797,
      "loss": 0.6931,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.0009503777837380767,
      "rewards/margins": 0.00045707420213148,
      "rewards/rejected": -0.0014074521604925394,
      "step": 20
    },
    {
      "epoch": 0.01,
      "learning_rate": 2e-08,
      "logits/chosen": -2.7216076850891113,
      "logits/rejected": -2.7468366622924805,
      "logps/chosen": -258.4103698730469,
      "logps/rejected": -267.4147033691406,
      "loss": 0.6937,
      "rewards/accuracies": 0.42500001192092896,
      "rewards/chosen": -0.00010261923307552934,
      "rewards/margins": -0.0023887469433248043,
      "rewards/rejected": 0.002286128234118223,
      "step": 30
    },
    {
      "epoch": 0.01,
      "learning_rate": 2.6666666666666664e-08,
      "logits/chosen": -2.871617078781128,
      "logits/rejected": -2.801084518432617,
      "logps/chosen": -242.60867309570312,
      "logps/rejected": -232.33499145507812,
      "loss": 0.6928,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.0018373109633103013,
      "rewards/margins": 0.00041042696102522314,
      "rewards/rejected": 0.0014268836239352822,
      "step": 40
    },
    {
      "epoch": 0.01,
      "learning_rate": 3.333333333333333e-08,
      "logits/chosen": -2.680046796798706,
      "logits/rejected": -2.714757204055786,
      "logps/chosen": -240.3429718017578,
      "logps/rejected": -184.65933227539062,
      "loss": 0.6932,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -2.7602236514212564e-05,
      "rewards/margins": 0.00031870920793153346,
      "rewards/rejected": -0.0003463116008788347,
      "step": 50
    },
    {
      "epoch": 0.01,
      "learning_rate": 4e-08,
      "logits/chosen": -2.796433448791504,
      "logits/rejected": -2.823643684387207,
      "logps/chosen": -217.6321258544922,
      "logps/rejected": -171.95498657226562,
      "loss": 0.6931,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -0.00015609370893798769,
      "rewards/margins": 0.00023442933161277324,
      "rewards/rejected": -0.00039052290958352387,
      "step": 60
    },
    {
      "epoch": 0.01,
      "learning_rate": 4.666666666666667e-08,
      "logits/chosen": -2.7592403888702393,
      "logits/rejected": -2.7623109817504883,
      "logps/chosen": -216.8944854736328,
      "logps/rejected": -218.29275512695312,
      "loss": 0.6928,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.001479600090533495,
      "rewards/margins": 0.00030887164757587016,
      "rewards/rejected": -0.0017884715925902128,
      "step": 70
    },
    {
      "epoch": 0.01,
      "learning_rate": 5.333333333333333e-08,
      "logits/chosen": -2.6831583976745605,
      "logits/rejected": -2.708047866821289,
      "logps/chosen": -271.333984375,
      "logps/rejected": -211.88742065429688,
      "loss": 0.6931,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.001955702668055892,
      "rewards/margins": 0.003820312675088644,
      "rewards/rejected": -0.0018646096577867866,
      "step": 80
    },
    {
      "epoch": 0.02,
      "learning_rate": 6e-08,
      "logits/chosen": -2.7753825187683105,
      "logits/rejected": -2.754791259765625,
      "logps/chosen": -197.63711547851562,
      "logps/rejected": -200.37274169921875,
      "loss": 0.6933,
      "rewards/accuracies": 0.32499998807907104,
      "rewards/chosen": -0.001247849315404892,
      "rewards/margins": -0.002036663005128503,
      "rewards/rejected": 0.000788813573308289,
      "step": 90
    },
    {
      "epoch": 0.02,
      "learning_rate": 6.666666666666665e-08,
      "logits/chosen": -2.799436569213867,
      "logits/rejected": -2.7348275184631348,
      "logps/chosen": -273.90789794921875,
      "logps/rejected": -223.2024383544922,
      "loss": 0.6931,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0007920874049887061,
      "rewards/margins": -1.7427420971216634e-05,
      "rewards/rejected": -0.000774660031311214,
      "step": 100
    },
    {
      "epoch": 0.02,
      "learning_rate": 7.333333333333333e-08,
      "logits/chosen": -2.702688217163086,
      "logits/rejected": -2.7402188777923584,
      "logps/chosen": -237.420654296875,
      "logps/rejected": -176.0985870361328,
      "loss": 0.693,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.0006930113304406404,
      "rewards/margins": 0.0016551170265302062,
      "rewards/rejected": -0.0009621057542972267,
      "step": 110
    },
    {
      "epoch": 0.02,
      "learning_rate": 8e-08,
      "logits/chosen": -2.7931458950042725,
      "logits/rejected": -2.732015371322632,
      "logps/chosen": -234.2708282470703,
      "logps/rejected": -191.72213745117188,
      "loss": 0.6933,
      "rewards/accuracies": 0.30000001192092896,
      "rewards/chosen": -0.001032972359098494,
      "rewards/margins": -0.0020988043397665024,
      "rewards/rejected": 0.001065832213498652,
      "step": 120
    },
    {
      "epoch": 0.02,
      "learning_rate": 8.666666666666666e-08,
      "logits/chosen": -2.7383856773376465,
      "logits/rejected": -2.778128147125244,
      "logps/chosen": -236.73348999023438,
      "logps/rejected": -212.9311065673828,
      "loss": 0.6931,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.0009975699940696359,
      "rewards/margins": -0.0017474631313234568,
      "rewards/rejected": 0.0007498931954614818,
      "step": 130
    },
    {
      "epoch": 0.02,
      "learning_rate": 9.333333333333334e-08,
      "logits/chosen": -2.770118236541748,
      "logits/rejected": -2.8189587593078613,
      "logps/chosen": -242.80325317382812,
      "logps/rejected": -248.46408081054688,
      "loss": 0.6932,
      "rewards/accuracies": 0.42500001192092896,
      "rewards/chosen": -0.001506878761574626,
      "rewards/margins": -0.0005925501463934779,
      "rewards/rejected": -0.0009143284405581653,
      "step": 140
    },
    {
      "epoch": 0.03,
      "learning_rate": 1e-07,
      "logits/chosen": -2.7896523475646973,
      "logits/rejected": -2.8789305686950684,
      "logps/chosen": -279.1468505859375,
      "logps/rejected": -286.92449951171875,
      "loss": 0.6932,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.001270245062187314,
      "rewards/margins": 0.0007720521534793079,
      "rewards/rejected": 0.0004981928505003452,
      "step": 150
    },
    {
      "epoch": 0.03,
      "learning_rate": 9.994962216624685e-08,
      "logits/chosen": -2.7570908069610596,
      "logits/rejected": -2.7672576904296875,
      "logps/chosen": -237.09164428710938,
      "logps/rejected": -215.8179473876953,
      "loss": 0.6935,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.0016938255866989493,
      "rewards/margins": -0.00298044690862298,
      "rewards/rejected": 0.0012866210890933871,
      "step": 160
    },
    {
      "epoch": 0.03,
      "learning_rate": 9.989924433249369e-08,
      "logits/chosen": -2.8499808311462402,
      "logits/rejected": -2.8154406547546387,
      "logps/chosen": -264.3780822753906,
      "logps/rejected": -214.450439453125,
      "loss": 0.6932,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.001735908561386168,
      "rewards/margins": 0.001296285423450172,
      "rewards/rejected": 0.00043962281779386103,
      "step": 170
    },
    {
      "epoch": 0.03,
      "learning_rate": 9.984886649874054e-08,
      "logits/chosen": -2.6970016956329346,
      "logits/rejected": -2.7668747901916504,
      "logps/chosen": -222.513671875,
      "logps/rejected": -189.9994354248047,
      "loss": 0.6933,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.0020346306264400482,
      "rewards/margins": 0.0011342220241203904,
      "rewards/rejected": 0.0009004086023196578,
      "step": 180
    },
    {
      "epoch": 0.03,
      "learning_rate": 9.97984886649874e-08,
      "logits/chosen": -2.730062484741211,
      "logits/rejected": -2.8065617084503174,
      "logps/chosen": -278.373046875,
      "logps/rejected": -261.8786315917969,
      "loss": 0.6933,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.0013691270723938942,
      "rewards/margins": -0.0007420595502480865,
      "rewards/rejected": -0.0006270675803534687,
      "step": 190
    },
    {
      "epoch": 0.04,
      "learning_rate": 9.974811083123425e-08,
      "logits/chosen": -2.773843765258789,
      "logits/rejected": -2.774496555328369,
      "logps/chosen": -243.3163604736328,
      "logps/rejected": -234.8438720703125,
      "loss": 0.6931,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.0005632409593090415,
      "rewards/margins": -4.069525675731711e-05,
      "rewards/rejected": 0.0006039362633600831,
      "step": 200
    },
    {
      "epoch": 0.04,
      "learning_rate": 9.96977329974811e-08,
      "logits/chosen": -2.797464370727539,
      "logits/rejected": -2.779179811477661,
      "logps/chosen": -213.2550506591797,
      "logps/rejected": -183.4717254638672,
      "loss": 0.6929,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.001099438639357686,
      "rewards/margins": -0.0010665215086191893,
      "rewards/rejected": 0.0021659601479768753,
      "step": 210
    },
    {
      "epoch": 0.04,
      "learning_rate": 9.964735516372796e-08,
      "logits/chosen": -2.672825336456299,
      "logits/rejected": -2.753871440887451,
      "logps/chosen": -187.55667114257812,
      "logps/rejected": -199.6050262451172,
      "loss": 0.6932,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.00040147211984731257,
      "rewards/margins": -0.0006626142421737313,
      "rewards/rejected": 0.0002611424424685538,
      "step": 220
    },
    {
      "epoch": 0.04,
      "learning_rate": 9.959697732997481e-08,
      "logits/chosen": -2.73313570022583,
      "logits/rejected": -2.7167673110961914,
      "logps/chosen": -207.70272827148438,
      "logps/rejected": -201.5740203857422,
      "loss": 0.6933,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.001323978416621685,
      "rewards/margins": -0.0013272857759147882,
      "rewards/rejected": 3.307475708425045e-06,
      "step": 230
    },
    {
      "epoch": 0.04,
      "learning_rate": 9.954659949622166e-08,
      "logits/chosen": -2.7659072875976562,
      "logits/rejected": -2.854525566101074,
      "logps/chosen": -247.24267578125,
      "logps/rejected": -216.25,
      "loss": 0.6931,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -0.0007479648338630795,
      "rewards/margins": 0.00017742058844305575,
      "rewards/rejected": -0.0009253855096176267,
      "step": 240
    },
    {
      "epoch": 0.04,
      "learning_rate": 9.94962216624685e-08,
      "logits/chosen": -2.681570053100586,
      "logits/rejected": -2.746302843093872,
      "logps/chosen": -305.2917785644531,
      "logps/rejected": -295.3072509765625,
      "loss": 0.693,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.0004612560442183167,
      "rewards/margins": 0.0005774974706582725,
      "rewards/rejected": -0.0001162414118880406,
      "step": 250
    },
    {
      "epoch": 0.05,
      "learning_rate": 9.944584382871536e-08,
      "logits/chosen": -2.7773802280426025,
      "logits/rejected": -2.7356131076812744,
      "logps/chosen": -260.58135986328125,
      "logps/rejected": -247.7050323486328,
      "loss": 0.6932,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.0014422254171222448,
      "rewards/margins": 0.0032965210266411304,
      "rewards/rejected": -0.0018542958423495293,
      "step": 260
    },
    {
      "epoch": 0.05,
      "learning_rate": 9.939546599496221e-08,
      "logits/chosen": -2.6985695362091064,
      "logits/rejected": -2.768378257751465,
      "logps/chosen": -209.56796264648438,
      "logps/rejected": -199.95033264160156,
      "loss": 0.6935,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -0.000545819231774658,
      "rewards/margins": 0.0005106240278109908,
      "rewards/rejected": -0.0010564432013779879,
      "step": 270
    },
    {
      "epoch": 0.05,
      "learning_rate": 9.934508816120907e-08,
      "logits/chosen": -2.7142832279205322,
      "logits/rejected": -2.727756977081299,
      "logps/chosen": -311.3549499511719,
      "logps/rejected": -293.3232727050781,
      "loss": 0.693,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.001072610029950738,
      "rewards/margins": 0.0006248473073355854,
      "rewards/rejected": 0.0004477625770960003,
      "step": 280
    },
    {
      "epoch": 0.05,
      "learning_rate": 9.929471032745591e-08,
      "logits/chosen": -2.6512770652770996,
      "logits/rejected": -2.677826404571533,
      "logps/chosen": -217.1204833984375,
      "logps/rejected": -199.79481506347656,
      "loss": 0.6928,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 5.9052497817901894e-05,
      "rewards/margins": 0.00203351816162467,
      "rewards/rejected": -0.0019744650926440954,
      "step": 290
    },
    {
      "epoch": 0.05,
      "learning_rate": 9.924433249370276e-08,
      "logits/chosen": -2.6881537437438965,
      "logits/rejected": -2.7492778301239014,
      "logps/chosen": -277.6586608886719,
      "logps/rejected": -240.2971954345703,
      "loss": 0.6931,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.0018934666877612472,
      "rewards/margins": 0.001739538973197341,
      "rewards/rejected": 0.00015392780187539756,
      "step": 300
    },
    {
      "epoch": 0.05,
      "learning_rate": 9.919395465994961e-08,
      "logits/chosen": -2.7815330028533936,
      "logits/rejected": -2.7532501220703125,
      "logps/chosen": -257.75836181640625,
      "logps/rejected": -200.2654571533203,
      "loss": 0.6934,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.0008091029012575746,
      "rewards/margins": -0.001112941768951714,
      "rewards/rejected": 0.00030383875127881765,
      "step": 310
    },
    {
      "epoch": 0.06,
      "learning_rate": 9.914357682619647e-08,
      "logits/chosen": -2.826770782470703,
      "logits/rejected": -2.845146417617798,
      "logps/chosen": -250.01553344726562,
      "logps/rejected": -218.0773162841797,
      "loss": 0.6933,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.00093313108664006,
      "rewards/margins": 0.0006354941288009286,
      "rewards/rejected": 0.00029763689963147044,
      "step": 320
    },
    {
      "epoch": 0.06,
      "learning_rate": 9.909319899244332e-08,
      "logits/chosen": -2.6958234310150146,
      "logits/rejected": -2.7005648612976074,
      "logps/chosen": -236.06967163085938,
      "logps/rejected": -190.2051239013672,
      "loss": 0.6931,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.0011252507101744413,
      "rewards/margins": -0.00028516483143903315,
      "rewards/rejected": 0.0014104156289249659,
      "step": 330
    },
    {
      "epoch": 0.06,
      "learning_rate": 9.904282115869017e-08,
      "logits/chosen": -2.7159180641174316,
      "logits/rejected": -2.755709648132324,
      "logps/chosen": -232.99978637695312,
      "logps/rejected": -195.90316772460938,
      "loss": 0.6931,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.0008891105535440147,
      "rewards/margins": 0.0006613129517063498,
      "rewards/rejected": 0.00022779754363000393,
      "step": 340
    },
    {
      "epoch": 0.06,
      "learning_rate": 9.899244332493703e-08,
      "logits/chosen": -2.8205747604370117,
      "logits/rejected": -2.790215253829956,
      "logps/chosen": -235.89877319335938,
      "logps/rejected": -194.9135284423828,
      "loss": 0.6931,
      "rewards/accuracies": 0.42500001192092896,
      "rewards/chosen": -0.0006920454325154424,
      "rewards/margins": -0.0019427335355430841,
      "rewards/rejected": 0.001250688568688929,
      "step": 350
    },
    {
      "epoch": 0.06,
      "learning_rate": 9.894206549118388e-08,
      "logits/chosen": -2.706644058227539,
      "logits/rejected": -2.7300548553466797,
      "logps/chosen": -224.51806640625,
      "logps/rejected": -253.60177612304688,
      "loss": 0.6932,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.0006736659561283886,
      "rewards/margins": -0.002054039854556322,
      "rewards/rejected": 0.0013803739566355944,
      "step": 360
    },
    {
      "epoch": 0.06,
      "learning_rate": 9.889168765743072e-08,
      "logits/chosen": -2.7433345317840576,
      "logits/rejected": -2.810014247894287,
      "logps/chosen": -223.27615356445312,
      "logps/rejected": -212.83901977539062,
      "loss": 0.693,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -0.00019951626018155366,
      "rewards/margins": -0.000935495481826365,
      "rewards/rejected": 0.0007359791197814047,
      "step": 370
    },
    {
      "epoch": 0.07,
      "learning_rate": 9.884130982367758e-08,
      "logits/chosen": -2.8322505950927734,
      "logits/rejected": -2.8160457611083984,
      "logps/chosen": -257.120361328125,
      "logps/rejected": -204.89254760742188,
      "loss": 0.693,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.001022899174131453,
      "rewards/margins": 0.000910973350983113,
      "rewards/rejected": 0.0001119256266974844,
      "step": 380
    },
    {
      "epoch": 0.07,
      "learning_rate": 9.879093198992443e-08,
      "logits/chosen": -2.675583839416504,
      "logits/rejected": -2.6759538650512695,
      "logps/chosen": -195.3834686279297,
      "logps/rejected": -161.30630493164062,
      "loss": 0.6931,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.0015894779935479164,
      "rewards/margins": 0.0003656911721918732,
      "rewards/rejected": 0.0012237867340445518,
      "step": 390
    },
    {
      "epoch": 0.07,
      "learning_rate": 9.874055415617128e-08,
      "logits/chosen": -2.775231122970581,
      "logits/rejected": -2.8162126541137695,
      "logps/chosen": -293.67376708984375,
      "logps/rejected": -233.018798828125,
      "loss": 0.693,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0006875410908833146,
      "rewards/margins": -0.0008610993390902877,
      "rewards/rejected": 0.00017355827731080353,
      "step": 400
    },
    {
      "epoch": 0.07,
      "learning_rate": 9.869017632241812e-08,
      "logits/chosen": -2.7781994342803955,
      "logits/rejected": -2.8431591987609863,
      "logps/chosen": -244.99267578125,
      "logps/rejected": -192.35743713378906,
      "loss": 0.6932,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.00142839050386101,
      "rewards/margins": -0.0006640168721787632,
      "rewards/rejected": 0.0020924073178321123,
      "step": 410
    },
    {
      "epoch": 0.07,
      "learning_rate": 9.863979848866498e-08,
      "logits/chosen": -2.7019505500793457,
      "logits/rejected": -2.7378628253936768,
      "logps/chosen": -237.5064697265625,
      "logps/rejected": -203.72897338867188,
      "loss": 0.6937,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0007829790702089667,
      "rewards/margins": -0.0010059999767690897,
      "rewards/rejected": 0.0017889788141474128,
      "step": 420
    },
    {
      "epoch": 0.08,
      "learning_rate": 9.858942065491183e-08,
      "logits/chosen": -2.741657257080078,
      "logits/rejected": -2.771892547607422,
      "logps/chosen": -246.7175750732422,
      "logps/rejected": -202.51193237304688,
      "loss": 0.693,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.00037730118492618203,
      "rewards/margins": 0.00044115298078395426,
      "rewards/rejected": -6.385180313372985e-05,
      "step": 430
    },
    {
      "epoch": 0.08,
      "learning_rate": 9.853904282115868e-08,
      "logits/chosen": -2.741157054901123,
      "logits/rejected": -2.7170910835266113,
      "logps/chosen": -230.9762725830078,
      "logps/rejected": -197.1507110595703,
      "loss": 0.6933,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -5.774973033112474e-05,
      "rewards/margins": -0.0021546499338001013,
      "rewards/rejected": 0.002096900250762701,
      "step": 440
    },
    {
      "epoch": 0.08,
      "learning_rate": 9.848866498740554e-08,
      "logits/chosen": -2.797440767288208,
      "logits/rejected": -2.752779245376587,
      "logps/chosen": -245.9729766845703,
      "logps/rejected": -222.7683868408203,
      "loss": 0.6933,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": 0.0005045041325502098,
      "rewards/margins": -0.0016038778703659773,
      "rewards/rejected": 0.0021083818282932043,
      "step": 450
    },
    {
      "epoch": 0.08,
      "learning_rate": 9.843828715365239e-08,
      "logits/chosen": -2.728309154510498,
      "logits/rejected": -2.7414863109588623,
      "logps/chosen": -225.4750518798828,
      "logps/rejected": -217.5722198486328,
      "loss": 0.6929,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.0019893483258783817,
      "rewards/margins": 0.0019943092484027147,
      "rewards/rejected": -4.960992555425037e-06,
      "step": 460
    },
    {
      "epoch": 0.08,
      "learning_rate": 9.838790931989925e-08,
      "logits/chosen": -2.6399004459381104,
      "logits/rejected": -2.6418204307556152,
      "logps/chosen": -259.00262451171875,
      "logps/rejected": -232.1974639892578,
      "loss": 0.693,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.00039179419400170445,
      "rewards/margins": -0.001004178891889751,
      "rewards/rejected": 0.0006123848725110292,
      "step": 470
    },
    {
      "epoch": 0.08,
      "learning_rate": 9.833753148614609e-08,
      "logits/chosen": -2.7012977600097656,
      "logits/rejected": -2.734711170196533,
      "logps/chosen": -259.11505126953125,
      "logps/rejected": -212.99191284179688,
      "loss": 0.693,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 3.07522714138031e-05,
      "rewards/margins": -0.0009926442289724946,
      "rewards/rejected": 0.0010233965003862977,
      "step": 480
    },
    {
      "epoch": 0.09,
      "learning_rate": 9.828715365239294e-08,
      "logits/chosen": -2.779294967651367,
      "logits/rejected": -2.742389678955078,
      "logps/chosen": -219.0150909423828,
      "logps/rejected": -213.0858612060547,
      "loss": 0.6936,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0007726555922999978,
      "rewards/margins": -0.0002816848282236606,
      "rewards/rejected": 0.0010543405078351498,
      "step": 490
    },
    {
      "epoch": 0.09,
      "learning_rate": 9.823677581863979e-08,
      "logits/chosen": -2.7667160034179688,
      "logits/rejected": -2.746671199798584,
      "logps/chosen": -241.52261352539062,
      "logps/rejected": -244.81637573242188,
      "loss": 0.6932,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0008761234348639846,
      "rewards/margins": 0.000312745978590101,
      "rewards/rejected": 0.0005633773980662227,
      "step": 500
    },
    {
      "epoch": 0.09,
      "learning_rate": 9.818639798488665e-08,
      "logits/chosen": -2.773653507232666,
      "logits/rejected": -2.7031195163726807,
      "logps/chosen": -188.9192352294922,
      "logps/rejected": -198.82492065429688,
      "loss": 0.6927,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.0020457953214645386,
      "rewards/margins": 0.0025221453979611397,
      "rewards/rejected": -0.0004763497563544661,
      "step": 510
    },
    {
      "epoch": 0.09,
      "learning_rate": 9.813602015113349e-08,
      "logits/chosen": -2.767975330352783,
      "logits/rejected": -2.745241641998291,
      "logps/chosen": -255.3336639404297,
      "logps/rejected": -210.2073516845703,
      "loss": 0.6932,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.0016272428911179304,
      "rewards/margins": -3.840450517600402e-05,
      "rewards/rejected": 0.0016656474908813834,
      "step": 520
    },
    {
      "epoch": 0.09,
      "learning_rate": 9.808564231738034e-08,
      "logits/chosen": -2.701375961303711,
      "logits/rejected": -2.740355968475342,
      "logps/chosen": -194.14346313476562,
      "logps/rejected": -195.3939971923828,
      "loss": 0.6931,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.001990049611777067,
      "rewards/margins": -0.0021429904736578465,
      "rewards/rejected": 0.004133040551096201,
      "step": 530
    },
    {
      "epoch": 0.09,
      "learning_rate": 9.80352644836272e-08,
      "logits/chosen": -2.7491579055786133,
      "logits/rejected": -2.8547916412353516,
      "logps/chosen": -257.10809326171875,
      "logps/rejected": -218.61026000976562,
      "loss": 0.6932,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0014056519139558077,
      "rewards/margins": -7.3385540417802986e-06,
      "rewards/rejected": 0.001412990503013134,
      "step": 540
    },
    {
      "epoch": 0.1,
      "learning_rate": 9.798488664987405e-08,
      "logits/chosen": -2.762587308883667,
      "logits/rejected": -2.7138381004333496,
      "logps/chosen": -250.11111450195312,
      "logps/rejected": -210.0104217529297,
      "loss": 0.6931,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.0026454334147274494,
      "rewards/margins": 0.0003759107203222811,
      "rewards/rejected": 0.0022695225197821856,
      "step": 550
    },
    {
      "epoch": 0.1,
      "learning_rate": 9.79345088161209e-08,
      "logits/chosen": -2.7964959144592285,
      "logits/rejected": -2.812797784805298,
      "logps/chosen": -262.03253173828125,
      "logps/rejected": -205.50057983398438,
      "loss": 0.6931,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.0009436160326004028,
      "rewards/margins": 0.0005760550848208368,
      "rewards/rejected": 0.00036756088957190514,
      "step": 560
    },
    {
      "epoch": 0.1,
      "learning_rate": 9.788413098236776e-08,
      "logits/chosen": -2.8215677738189697,
      "logits/rejected": -2.803280830383301,
      "logps/chosen": -243.072998046875,
      "logps/rejected": -198.3536376953125,
      "loss": 0.6928,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.0028980812057852745,
      "rewards/margins": 0.002744231838732958,
      "rewards/rejected": 0.00015384898870252073,
      "step": 570
    },
    {
      "epoch": 0.1,
      "learning_rate": 9.783375314861461e-08,
      "logits/chosen": -2.8663644790649414,
      "logits/rejected": -2.8302829265594482,
      "logps/chosen": -231.59359741210938,
      "logps/rejected": -258.3666076660156,
      "loss": 0.6932,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.0001844273356255144,
      "rewards/margins": -0.002044522203505039,
      "rewards/rejected": 0.0022289494518190622,
      "step": 580
    },
    {
      "epoch": 0.1,
      "learning_rate": 9.778337531486146e-08,
      "logits/chosen": -2.7864925861358643,
      "logits/rejected": -2.7147817611694336,
      "logps/chosen": -229.01431274414062,
      "logps/rejected": -186.32522583007812,
      "loss": 0.6928,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 2.3794593744241865e-06,
      "rewards/margins": 0.0006320485845208168,
      "rewards/rejected": -0.000629669229965657,
      "step": 590
    },
    {
      "epoch": 0.11,
      "learning_rate": 9.77329974811083e-08,
      "logits/chosen": -2.754164218902588,
      "logits/rejected": -2.7349648475646973,
      "logps/chosen": -216.50918579101562,
      "logps/rejected": -186.99282836914062,
      "loss": 0.6928,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.0005909929750487208,
      "rewards/margins": 5.385982876759954e-05,
      "rewards/rejected": 0.0005371329025365412,
      "step": 600
    },
    {
      "epoch": 0.11,
      "learning_rate": 9.768261964735516e-08,
      "logits/chosen": -2.754659652709961,
      "logits/rejected": -2.727217197418213,
      "logps/chosen": -236.4612274169922,
      "logps/rejected": -213.8798828125,
      "loss": 0.693,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.0012212401488795877,
      "rewards/margins": 0.00041647389298304915,
      "rewards/rejected": 0.0008047660812735558,
      "step": 610
    },
    {
      "epoch": 0.11,
      "learning_rate": 9.763224181360201e-08,
      "logits/chosen": -2.767381191253662,
      "logits/rejected": -2.773684501647949,
      "logps/chosen": -276.1978759765625,
      "logps/rejected": -241.90969848632812,
      "loss": 0.6926,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": 0.0022741584107279778,
      "rewards/margins": 0.0036654185969382524,
      "rewards/rejected": -0.0013912600697949529,
      "step": 620
    },
    {
      "epoch": 0.11,
      "learning_rate": 9.758186397984886e-08,
      "logits/chosen": -2.7231290340423584,
      "logits/rejected": -2.799967050552368,
      "logps/chosen": -288.275634765625,
      "logps/rejected": -215.36392211914062,
      "loss": 0.6932,
      "rewards/accuracies": 0.3499999940395355,
      "rewards/chosen": -8.152765076374635e-05,
      "rewards/margins": -0.003193531883880496,
      "rewards/rejected": 0.0031120043713599443,
      "step": 630
    },
    {
      "epoch": 0.11,
      "learning_rate": 9.75314861460957e-08,
      "logits/chosen": -2.8058621883392334,
      "logits/rejected": -2.811411142349243,
      "logps/chosen": -263.43182373046875,
      "logps/rejected": -255.8742218017578,
      "loss": 0.6928,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": 0.0022689211182296276,
      "rewards/margins": 0.0014448559377342463,
      "rewards/rejected": 0.0008240645984187722,
      "step": 640
    },
    {
      "epoch": 0.11,
      "learning_rate": 9.748110831234256e-08,
      "logits/chosen": -2.8055179119110107,
      "logits/rejected": -2.751683235168457,
      "logps/chosen": -190.2101287841797,
      "logps/rejected": -180.5990447998047,
      "loss": 0.6926,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": 0.00034066467196680605,
      "rewards/margins": 0.001460460713133216,
      "rewards/rejected": -0.0011197957210242748,
      "step": 650
    },
    {
      "epoch": 0.12,
      "learning_rate": 9.743073047858941e-08,
      "logits/chosen": -2.6653835773468018,
      "logits/rejected": -2.6121246814727783,
      "logps/chosen": -209.80648803710938,
      "logps/rejected": -229.5626983642578,
      "loss": 0.6933,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.0011562244035303593,
      "rewards/margins": 0.00043965905206277966,
      "rewards/rejected": 0.000716565118636936,
      "step": 660
    },
    {
      "epoch": 0.12,
      "learning_rate": 9.738035264483628e-08,
      "logits/chosen": -2.756425380706787,
      "logits/rejected": -2.6906371116638184,
      "logps/chosen": -208.0061798095703,
      "logps/rejected": -227.17526245117188,
      "loss": 0.6929,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.003370731370523572,
      "rewards/margins": 0.0021752621978521347,
      "rewards/rejected": 0.0011954689398407936,
      "step": 670
    },
    {
      "epoch": 0.12,
      "learning_rate": 9.732997481108312e-08,
      "logits/chosen": -2.748220920562744,
      "logits/rejected": -2.739067554473877,
      "logps/chosen": -204.1517791748047,
      "logps/rejected": -174.79559326171875,
      "loss": 0.6928,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.0015211543068289757,
      "rewards/margins": 0.001060990383848548,
      "rewards/rejected": 0.0004601640102919191,
      "step": 680
    },
    {
      "epoch": 0.12,
      "learning_rate": 9.727959697732997e-08,
      "logits/chosen": -2.7568039894104004,
      "logits/rejected": -2.7389419078826904,
      "logps/chosen": -223.5104522705078,
      "logps/rejected": -172.09033203125,
      "loss": 0.6937,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -4.150771928834729e-05,
      "rewards/margins": -0.0012603879440575838,
      "rewards/rejected": 0.001218880177475512,
      "step": 690
    },
    {
      "epoch": 0.12,
      "learning_rate": 9.722921914357683e-08,
      "logits/chosen": -2.762721300125122,
      "logits/rejected": -2.750661611557007,
      "logps/chosen": -230.8249969482422,
      "logps/rejected": -231.24960327148438,
      "loss": 0.6929,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.0016020803013816476,
      "rewards/margins": 0.0007085275137796998,
      "rewards/rejected": 0.0008935527876019478,
      "step": 700
    },
    {
      "epoch": 0.12,
      "learning_rate": 9.717884130982368e-08,
      "logits/chosen": -2.6903281211853027,
      "logits/rejected": -2.668412685394287,
      "logps/chosen": -173.5829315185547,
      "logps/rejected": -174.61099243164062,
      "loss": 0.6931,
      "rewards/accuracies": 0.42500001192092896,
      "rewards/chosen": -0.0005260295001789927,
      "rewards/margins": -0.0008247641962952912,
      "rewards/rejected": 0.00029873469611629844,
      "step": 710
    },
    {
      "epoch": 0.13,
      "learning_rate": 9.712846347607052e-08,
      "logits/chosen": -2.7260491847991943,
      "logits/rejected": -2.723248243331909,
      "logps/chosen": -228.8632049560547,
      "logps/rejected": -191.02725219726562,
      "loss": 0.6935,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.002371068811044097,
      "rewards/margins": -0.0012089888332411647,
      "rewards/rejected": 0.00358005752786994,
      "step": 720
    },
    {
      "epoch": 0.13,
      "learning_rate": 9.707808564231737e-08,
      "logits/chosen": -2.7456154823303223,
      "logits/rejected": -2.681551456451416,
      "logps/chosen": -257.8016052246094,
      "logps/rejected": -246.4755096435547,
      "loss": 0.6933,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.000685253064148128,
      "rewards/margins": -0.0014969928888604045,
      "rewards/rejected": 0.002182246185839176,
      "step": 730
    },
    {
      "epoch": 0.13,
      "learning_rate": 9.702770780856423e-08,
      "logits/chosen": -2.7827036380767822,
      "logits/rejected": -2.7841458320617676,
      "logps/chosen": -221.7444610595703,
      "logps/rejected": -198.41885375976562,
      "loss": 0.6932,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.0009337845258414745,
      "rewards/margins": 8.86632114998065e-05,
      "rewards/rejected": 0.0008451213943772018,
      "step": 740
    },
    {
      "epoch": 0.13,
      "learning_rate": 9.697732997481108e-08,
      "logits/chosen": -2.763047456741333,
      "logits/rejected": -2.8110077381134033,
      "logps/chosen": -236.8753204345703,
      "logps/rejected": -191.76792907714844,
      "loss": 0.6931,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.003436017083004117,
      "rewards/margins": 0.000934536918066442,
      "rewards/rejected": 0.0025014798156917095,
      "step": 750
    },
    {
      "epoch": 0.13,
      "learning_rate": 9.692695214105792e-08,
      "logits/chosen": -2.726303815841675,
      "logits/rejected": -2.7912750244140625,
      "logps/chosen": -315.33441162109375,
      "logps/rejected": -216.3941650390625,
      "loss": 0.6924,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.0015838576946407557,
      "rewards/margins": 0.0014837884809821844,
      "rewards/rejected": 0.0001000691408989951,
      "step": 760
    },
    {
      "epoch": 0.13,
      "learning_rate": 9.687657430730478e-08,
      "logits/chosen": -2.702937602996826,
      "logits/rejected": -2.7665514945983887,
      "logps/chosen": -221.53292846679688,
      "logps/rejected": -171.7909393310547,
      "loss": 0.6927,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.0022713050711899996,
      "rewards/margins": -0.00044591521145775914,
      "rewards/rejected": 0.0027172204572707415,
      "step": 770
    },
    {
      "epoch": 0.14,
      "learning_rate": 9.682619647355164e-08,
      "logits/chosen": -2.83231520652771,
      "logits/rejected": -2.841844081878662,
      "logps/chosen": -228.69546508789062,
      "logps/rejected": -169.39346313476562,
      "loss": 0.6928,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.001805447624064982,
      "rewards/margins": -0.0005494118086062372,
      "rewards/rejected": 0.002354859607294202,
      "step": 780
    },
    {
      "epoch": 0.14,
      "learning_rate": 9.677581863979848e-08,
      "logits/chosen": -2.7778494358062744,
      "logits/rejected": -2.7811062335968018,
      "logps/chosen": -215.41586303710938,
      "logps/rejected": -208.3562469482422,
      "loss": 0.6927,
      "rewards/accuracies": 0.3499999940395355,
      "rewards/chosen": 0.00014900062524247915,
      "rewards/margins": -0.002759304828941822,
      "rewards/rejected": 0.0029083057306706905,
      "step": 790
    },
    {
      "epoch": 0.14,
      "learning_rate": 9.672544080604534e-08,
      "logits/chosen": -2.706941604614258,
      "logits/rejected": -2.7043490409851074,
      "logps/chosen": -252.07980346679688,
      "logps/rejected": -224.51388549804688,
      "loss": 0.693,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": 0.0011368256527930498,
      "rewards/margins": 0.001408792450092733,
      "rewards/rejected": -0.0002719667972996831,
      "step": 800
    },
    {
      "epoch": 0.14,
      "learning_rate": 9.667506297229219e-08,
      "logits/chosen": -2.7257018089294434,
      "logits/rejected": -2.688807725906372,
      "logps/chosen": -230.65444946289062,
      "logps/rejected": -203.5872802734375,
      "loss": 0.6928,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.0041201431304216385,
      "rewards/margins": 0.0013161466922610998,
      "rewards/rejected": 0.0028039959724992514,
      "step": 810
    },
    {
      "epoch": 0.14,
      "learning_rate": 9.662468513853904e-08,
      "logits/chosen": -2.7131431102752686,
      "logits/rejected": -2.7395095825195312,
      "logps/chosen": -246.7789764404297,
      "logps/rejected": -224.97299194335938,
      "loss": 0.6929,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0022838334552943707,
      "rewards/margins": 0.0005551796639338136,
      "rewards/rejected": 0.0017286535585299134,
      "step": 820
    },
    {
      "epoch": 0.15,
      "learning_rate": 9.657430730478588e-08,
      "logits/chosen": -2.801877498626709,
      "logits/rejected": -2.7862987518310547,
      "logps/chosen": -242.6244659423828,
      "logps/rejected": -223.6712646484375,
      "loss": 0.693,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0037963681388646364,
      "rewards/margins": 0.0008951274794526398,
      "rewards/rejected": 0.0029012407176196575,
      "step": 830
    },
    {
      "epoch": 0.15,
      "learning_rate": 9.652392947103274e-08,
      "logits/chosen": -2.763369560241699,
      "logits/rejected": -2.7350258827209473,
      "logps/chosen": -231.41952514648438,
      "logps/rejected": -207.5250244140625,
      "loss": 0.6931,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0018902644515037537,
      "rewards/margins": 0.000653119117487222,
      "rewards/rejected": 0.0012371453922241926,
      "step": 840
    },
    {
      "epoch": 0.15,
      "learning_rate": 9.647355163727959e-08,
      "logits/chosen": -2.709563732147217,
      "logits/rejected": -2.7220940589904785,
      "logps/chosen": -229.4707489013672,
      "logps/rejected": -200.20779418945312,
      "loss": 0.6927,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.0027879923582077026,
      "rewards/margins": 0.0006196832982823253,
      "rewards/rejected": 0.002168309409171343,
      "step": 850
    },
    {
      "epoch": 0.15,
      "learning_rate": 9.642317380352644e-08,
      "logits/chosen": -2.7222867012023926,
      "logits/rejected": -2.686190128326416,
      "logps/chosen": -250.36752319335938,
      "logps/rejected": -195.4890899658203,
      "loss": 0.6932,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.0026868334971368313,
      "rewards/margins": -0.00014689173258375376,
      "rewards/rejected": 0.0028337249532341957,
      "step": 860
    },
    {
      "epoch": 0.15,
      "learning_rate": 9.637279596977329e-08,
      "logits/chosen": -2.6420130729675293,
      "logits/rejected": -2.7638611793518066,
      "logps/chosen": -272.90576171875,
      "logps/rejected": -225.89804077148438,
      "loss": 0.6931,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.003827697830274701,
      "rewards/margins": 0.001123771769925952,
      "rewards/rejected": 0.002703926060348749,
      "step": 870
    },
    {
      "epoch": 0.15,
      "learning_rate": 9.632241813602014e-08,
      "logits/chosen": -2.804816722869873,
      "logits/rejected": -2.793016195297241,
      "logps/chosen": -236.7283477783203,
      "logps/rejected": -191.2239990234375,
      "loss": 0.693,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.0009727630531415343,
      "rewards/margins": -0.0009720485541038215,
      "rewards/rejected": 0.0019448116654530168,
      "step": 880
    },
    {
      "epoch": 0.16,
      "learning_rate": 9.6272040302267e-08,
      "logits/chosen": -2.7938032150268555,
      "logits/rejected": -2.7507739067077637,
      "logps/chosen": -206.41567993164062,
      "logps/rejected": -178.12667846679688,
      "loss": 0.693,
      "rewards/accuracies": 0.42500001192092896,
      "rewards/chosen": 0.0008494978537783027,
      "rewards/margins": -0.0007513789460062981,
      "rewards/rejected": 0.0016008767997846007,
      "step": 890
    },
    {
      "epoch": 0.16,
      "learning_rate": 9.622166246851386e-08,
      "logits/chosen": -2.7419960498809814,
      "logits/rejected": -2.8216521739959717,
      "logps/chosen": -292.3438720703125,
      "logps/rejected": -209.7271270751953,
      "loss": 0.6927,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.00268277944996953,
      "rewards/margins": -0.00072367989923805,
      "rewards/rejected": 0.0034064589999616146,
      "step": 900
    },
    {
      "epoch": 0.16,
      "learning_rate": 9.61712846347607e-08,
      "logits/chosen": -2.716244697570801,
      "logits/rejected": -2.7290053367614746,
      "logps/chosen": -218.2609100341797,
      "logps/rejected": -196.04891967773438,
      "loss": 0.6932,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.0014504556311294436,
      "rewards/margins": -0.0006075268611311913,
      "rewards/rejected": 0.0020579826086759567,
      "step": 910
    },
    {
      "epoch": 0.16,
      "learning_rate": 9.612090680100755e-08,
      "logits/chosen": -2.7427549362182617,
      "logits/rejected": -2.7771146297454834,
      "logps/chosen": -205.1763916015625,
      "logps/rejected": -197.89157104492188,
      "loss": 0.6931,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": 0.001757201156578958,
      "rewards/margins": 0.0007729234057478607,
      "rewards/rejected": 0.0009842776926234365,
      "step": 920
    },
    {
      "epoch": 0.16,
      "learning_rate": 9.607052896725441e-08,
      "logits/chosen": -2.810089349746704,
      "logits/rejected": -2.7477164268493652,
      "logps/chosen": -234.19094848632812,
      "logps/rejected": -238.1735076904297,
      "loss": 0.6928,
      "rewards/accuracies": 0.42500001192092896,
      "rewards/chosen": 0.0020914555061608553,
      "rewards/margins": -0.0015822440618649125,
      "rewards/rejected": 0.003673699451610446,
      "step": 930
    },
    {
      "epoch": 0.16,
      "learning_rate": 9.602015113350126e-08,
      "logits/chosen": -2.7463126182556152,
      "logits/rejected": -2.7717044353485107,
      "logps/chosen": -318.3886413574219,
      "logps/rejected": -217.30325317382812,
      "loss": 0.6932,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": 0.0019021205371245742,
      "rewards/margins": -0.0013187190052121878,
      "rewards/rejected": 0.003220838960260153,
      "step": 940
    },
    {
      "epoch": 0.17,
      "learning_rate": 9.59697732997481e-08,
      "logits/chosen": -2.7562918663024902,
      "logits/rejected": -2.737678050994873,
      "logps/chosen": -212.1610870361328,
      "logps/rejected": -169.73379516601562,
      "loss": 0.6928,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.003521175589412451,
      "rewards/margins": 0.00361448572948575,
      "rewards/rejected": -9.331032924819738e-05,
      "step": 950
    },
    {
      "epoch": 0.17,
      "learning_rate": 9.591939546599495e-08,
      "logits/chosen": -2.785679340362549,
      "logits/rejected": -2.775289535522461,
      "logps/chosen": -266.58660888671875,
      "logps/rejected": -248.35580444335938,
      "loss": 0.6934,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.002367380540817976,
      "rewards/margins": -0.001244062907062471,
      "rewards/rejected": 0.0036114430986344814,
      "step": 960
    },
    {
      "epoch": 0.17,
      "learning_rate": 9.586901763224181e-08,
      "logits/chosen": -2.775019645690918,
      "logits/rejected": -2.751147747039795,
      "logps/chosen": -272.42840576171875,
      "logps/rejected": -262.1443176269531,
      "loss": 0.6928,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.0036885985173285007,
      "rewards/margins": 0.002885077614337206,
      "rewards/rejected": 0.0008035210776142776,
      "step": 970
    },
    {
      "epoch": 0.17,
      "learning_rate": 9.581863979848866e-08,
      "logits/chosen": -2.726874351501465,
      "logits/rejected": -2.768751621246338,
      "logps/chosen": -236.5287322998047,
      "logps/rejected": -211.50131225585938,
      "loss": 0.6928,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.0035156819503754377,
      "rewards/margins": -0.0017192515078932047,
      "rewards/rejected": 0.005234933458268642,
      "step": 980
    },
    {
      "epoch": 0.17,
      "learning_rate": 9.57682619647355e-08,
      "logits/chosen": -2.7328333854675293,
      "logits/rejected": -2.7810556888580322,
      "logps/chosen": -253.5846405029297,
      "logps/rejected": -208.56729125976562,
      "loss": 0.693,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.004323097877204418,
      "rewards/margins": 0.001049490412697196,
      "rewards/rejected": 0.003273607464507222,
      "step": 990
    },
    {
      "epoch": 0.18,
      "learning_rate": 9.571788413098237e-08,
      "logits/chosen": -2.775546073913574,
      "logits/rejected": -2.7548234462738037,
      "logps/chosen": -274.9341735839844,
      "logps/rejected": -214.09378051757812,
      "loss": 0.693,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.004553532227873802,
      "rewards/margins": 0.0013717750553041697,
      "rewards/rejected": 0.003181757405400276,
      "step": 1000
    },
    {
      "epoch": 0.18,
      "learning_rate": 9.566750629722922e-08,
      "logits/chosen": -2.7104849815368652,
      "logits/rejected": -2.698439121246338,
      "logps/chosen": -255.5958251953125,
      "logps/rejected": -174.08676147460938,
      "loss": 0.6928,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.003329707309603691,
      "rewards/margins": 0.0009661338990554214,
      "rewards/rejected": 0.002363573294132948,
      "step": 1010
    },
    {
      "epoch": 0.18,
      "learning_rate": 9.561712846347608e-08,
      "logits/chosen": -2.667184591293335,
      "logits/rejected": -2.633574962615967,
      "logps/chosen": -197.6795196533203,
      "logps/rejected": -169.07440185546875,
      "loss": 0.6929,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.000696418690495193,
      "rewards/margins": 0.0006790427723899484,
      "rewards/rejected": 1.7376034520566463e-05,
      "step": 1020
    },
    {
      "epoch": 0.18,
      "learning_rate": 9.556675062972292e-08,
      "logits/chosen": -2.7279274463653564,
      "logits/rejected": -2.725475311279297,
      "logps/chosen": -224.8273468017578,
      "logps/rejected": -199.46923828125,
      "loss": 0.693,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.0031109179835766554,
      "rewards/margins": -3.088197991019115e-05,
      "rewards/rejected": 0.0031417999416589737,
      "step": 1030
    },
    {
      "epoch": 0.18,
      "learning_rate": 9.551637279596977e-08,
      "logits/chosen": -2.7452080249786377,
      "logits/rejected": -2.748317003250122,
      "logps/chosen": -251.00180053710938,
      "logps/rejected": -221.8520050048828,
      "loss": 0.6929,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": 0.002598334103822708,
      "rewards/margins": -0.00022798255668021739,
      "rewards/rejected": 0.002826316747814417,
      "step": 1040
    },
    {
      "epoch": 0.18,
      "learning_rate": 9.546599496221662e-08,
      "logits/chosen": -2.7736282348632812,
      "logits/rejected": -2.7318246364593506,
      "logps/chosen": -261.21356201171875,
      "logps/rejected": -215.4080352783203,
      "loss": 0.6927,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.0046277581714093685,
      "rewards/margins": 0.0004109583387617022,
      "rewards/rejected": 0.004216799978166819,
      "step": 1050
    },
    {
      "epoch": 0.19,
      "learning_rate": 9.541561712846348e-08,
      "logits/chosen": -2.831284999847412,
      "logits/rejected": -2.7485132217407227,
      "logps/chosen": -263.9079284667969,
      "logps/rejected": -222.78909301757812,
      "loss": 0.6932,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.004149717278778553,
      "rewards/margins": -0.0001199671532958746,
      "rewards/rejected": 0.004269684199243784,
      "step": 1060
    },
    {
      "epoch": 0.19,
      "learning_rate": 9.536523929471032e-08,
      "logits/chosen": -2.7859325408935547,
      "logits/rejected": -2.7743453979492188,
      "logps/chosen": -205.18862915039062,
      "logps/rejected": -163.95297241210938,
      "loss": 0.6926,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.001009289757348597,
      "rewards/margins": -0.0014739797916263342,
      "rewards/rejected": 0.002483269665390253,
      "step": 1070
    },
    {
      "epoch": 0.19,
      "learning_rate": 9.531486146095717e-08,
      "logits/chosen": -2.7141454219818115,
      "logits/rejected": -2.742264747619629,
      "logps/chosen": -212.10800170898438,
      "logps/rejected": -211.097900390625,
      "loss": 0.6927,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": 0.002731418702751398,
      "rewards/margins": -0.0012434081872925162,
      "rewards/rejected": 0.003974826540797949,
      "step": 1080
    },
    {
      "epoch": 0.19,
      "learning_rate": 9.526448362720403e-08,
      "logits/chosen": -2.646359920501709,
      "logits/rejected": -2.671069622039795,
      "logps/chosen": -197.65640258789062,
      "logps/rejected": -180.06640625,
      "loss": 0.6929,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.0025646693538874388,
      "rewards/margins": 0.0011420946102589369,
      "rewards/rejected": 0.0014225749764591455,
      "step": 1090
    },
    {
      "epoch": 0.19,
      "learning_rate": 9.521410579345087e-08,
      "logits/chosen": -2.7953693866729736,
      "logits/rejected": -2.7932047843933105,
      "logps/chosen": -197.06411743164062,
      "logps/rejected": -199.5815887451172,
      "loss": 0.6929,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.004566690884530544,
      "rewards/margins": 4.555946361506358e-05,
      "rewards/rejected": 0.00452113151550293,
      "step": 1100
    },
    {
      "epoch": 0.19,
      "learning_rate": 9.516372795969773e-08,
      "logits/chosen": -2.75532865524292,
      "logits/rejected": -2.768794536590576,
      "logps/chosen": -247.06533813476562,
      "logps/rejected": -203.4183807373047,
      "loss": 0.6934,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": 0.0037621301598846912,
      "rewards/margins": -0.00023272607359103858,
      "rewards/rejected": 0.003994855564087629,
      "step": 1110
    },
    {
      "epoch": 0.2,
      "learning_rate": 9.511335012594459e-08,
      "logits/chosen": -2.7900328636169434,
      "logits/rejected": -2.747842311859131,
      "logps/chosen": -233.60934448242188,
      "logps/rejected": -209.6956329345703,
      "loss": 0.6927,
      "rewards/accuracies": 0.42500001192092896,
      "rewards/chosen": 0.002573530189692974,
      "rewards/margins": -0.0018167212838307023,
      "rewards/rejected": 0.004390250891447067,
      "step": 1120
    },
    {
      "epoch": 0.2,
      "learning_rate": 9.506297229219144e-08,
      "logits/chosen": -2.8266825675964355,
      "logits/rejected": -2.7168092727661133,
      "logps/chosen": -266.6517028808594,
      "logps/rejected": -240.08877563476562,
      "loss": 0.6932,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.005716273095458746,
      "rewards/margins": 0.0027064047753810883,
      "rewards/rejected": 0.003009868785738945,
      "step": 1130
    },
    {
      "epoch": 0.2,
      "learning_rate": 9.501259445843828e-08,
      "logits/chosen": -2.7689754962921143,
      "logits/rejected": -2.7763471603393555,
      "logps/chosen": -289.73931884765625,
      "logps/rejected": -222.8748016357422,
      "loss": 0.693,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.0030738338828086853,
      "rewards/margins": -0.001451755058951676,
      "rewards/rejected": 0.004525588359683752,
      "step": 1140
    },
    {
      "epoch": 0.2,
      "learning_rate": 9.496221662468513e-08,
      "logits/chosen": -2.768038272857666,
      "logits/rejected": -2.754030704498291,
      "logps/chosen": -234.2838592529297,
      "logps/rejected": -202.77279663085938,
      "loss": 0.6925,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.004740605130791664,
      "rewards/margins": 0.0016939297784119844,
      "rewards/rejected": 0.0030466755852103233,
      "step": 1150
    },
    {
      "epoch": 0.2,
      "learning_rate": 9.491183879093199e-08,
      "logits/chosen": -2.8069653511047363,
      "logits/rejected": -2.7526118755340576,
      "logps/chosen": -209.11172485351562,
      "logps/rejected": -200.98428344726562,
      "loss": 0.6925,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.005559694021940231,
      "rewards/margins": 0.002170537132769823,
      "rewards/rejected": 0.0033891566563397646,
      "step": 1160
    },
    {
      "epoch": 0.2,
      "learning_rate": 9.486146095717884e-08,
      "logits/chosen": -2.7833895683288574,
      "logits/rejected": -2.755847930908203,
      "logps/chosen": -228.0541229248047,
      "logps/rejected": -222.4268035888672,
      "loss": 0.6926,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.00539207411929965,
      "rewards/margins": 0.0042550102807581425,
      "rewards/rejected": 0.001137065002694726,
      "step": 1170
    },
    {
      "epoch": 0.21,
      "learning_rate": 9.481108312342568e-08,
      "logits/chosen": -2.8028454780578613,
      "logits/rejected": -2.798689365386963,
      "logps/chosen": -251.14501953125,
      "logps/rejected": -244.967041015625,
      "loss": 0.6929,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": 0.0028695925138890743,
      "rewards/margins": 0.0012147616362199187,
      "rewards/rejected": 0.0016548307612538338,
      "step": 1180
    },
    {
      "epoch": 0.21,
      "learning_rate": 9.476070528967254e-08,
      "logits/chosen": -2.7660348415374756,
      "logits/rejected": -2.8378560543060303,
      "logps/chosen": -250.03372192382812,
      "logps/rejected": -201.1326446533203,
      "loss": 0.6933,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": 0.005208217538893223,
      "rewards/margins": 0.0004181812109891325,
      "rewards/rejected": 0.004790036473423243,
      "step": 1190
    },
    {
      "epoch": 0.21,
      "learning_rate": 9.471032745591939e-08,
      "logits/chosen": -2.738584280014038,
      "logits/rejected": -2.799468517303467,
      "logps/chosen": -294.7505187988281,
      "logps/rejected": -218.1544647216797,
      "loss": 0.6929,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.002887497888877988,
      "rewards/margins": 0.0018897324334830046,
      "rewards/rejected": 0.000997765688225627,
      "step": 1200
    },
    {
      "epoch": 0.21,
      "learning_rate": 9.465994962216624e-08,
      "logits/chosen": -2.744781970977783,
      "logits/rejected": -2.764321804046631,
      "logps/chosen": -203.63406372070312,
      "logps/rejected": -165.10145568847656,
      "loss": 0.6934,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.0023968457244336605,
      "rewards/margins": 0.00017872030730359256,
      "rewards/rejected": 0.002218125155195594,
      "step": 1210
    },
    {
      "epoch": 0.21,
      "learning_rate": 9.460957178841308e-08,
      "logits/chosen": -2.7360353469848633,
      "logits/rejected": -2.746555805206299,
      "logps/chosen": -211.74185180664062,
      "logps/rejected": -180.95497131347656,
      "loss": 0.6921,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.004236987791955471,
      "rewards/margins": 0.0014550769701600075,
      "rewards/rejected": 0.002781911287456751,
      "step": 1220
    },
    {
      "epoch": 0.22,
      "learning_rate": 9.455919395465995e-08,
      "logits/chosen": -2.7374870777130127,
      "logits/rejected": -2.7698867321014404,
      "logps/chosen": -215.5163116455078,
      "logps/rejected": -210.37368774414062,
      "loss": 0.693,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.004980830010026693,
      "rewards/margins": 0.0006271142628975213,
      "rewards/rejected": 0.004353716038167477,
      "step": 1230
    },
    {
      "epoch": 0.22,
      "learning_rate": 9.45088161209068e-08,
      "logits/chosen": -2.807363271713257,
      "logits/rejected": -2.7921762466430664,
      "logps/chosen": -199.8319091796875,
      "logps/rejected": -177.23934936523438,
      "loss": 0.6926,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.004381081089377403,
      "rewards/margins": 0.0005592384259216487,
      "rewards/rejected": 0.00382184237241745,
      "step": 1240
    },
    {
      "epoch": 0.22,
      "learning_rate": 9.445843828715366e-08,
      "logits/chosen": -2.8015992641448975,
      "logits/rejected": -2.7855136394500732,
      "logps/chosen": -253.15487670898438,
      "logps/rejected": -222.3345947265625,
      "loss": 0.6928,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": 0.004838630557060242,
      "rewards/margins": -9.69449756667018e-05,
      "rewards/rejected": 0.004935575183480978,
      "step": 1250
    },
    {
      "epoch": 0.22,
      "learning_rate": 9.44080604534005e-08,
      "logits/chosen": -2.6941018104553223,
      "logits/rejected": -2.6818172931671143,
      "logps/chosen": -231.94412231445312,
      "logps/rejected": -228.63485717773438,
      "loss": 0.6922,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": 0.00432557100430131,
      "rewards/margins": -0.0011508779134601355,
      "rewards/rejected": 0.005476449616253376,
      "step": 1260
    },
    {
      "epoch": 0.22,
      "learning_rate": 9.435768261964735e-08,
      "logits/chosen": -2.788661479949951,
      "logits/rejected": -2.713569402694702,
      "logps/chosen": -228.61123657226562,
      "logps/rejected": -230.6959686279297,
      "loss": 0.6933,
      "rewards/accuracies": 0.42500001192092896,
      "rewards/chosen": 0.0030228032264858484,
      "rewards/margins": -0.002682460006326437,
      "rewards/rejected": 0.005705262999981642,
      "step": 1270
    },
    {
      "epoch": 0.22,
      "learning_rate": 9.43073047858942e-08,
      "logits/chosen": -2.7489657402038574,
      "logits/rejected": -2.7751262187957764,
      "logps/chosen": -260.9920349121094,
      "logps/rejected": -214.2970733642578,
      "loss": 0.6932,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": 0.00232306937687099,
      "rewards/margins": -0.002894314704462886,
      "rewards/rejected": 0.005217384081333876,
      "step": 1280
    },
    {
      "epoch": 0.23,
      "learning_rate": 9.425692695214106e-08,
      "logits/chosen": -2.7719390392303467,
      "logits/rejected": -2.8384242057800293,
      "logps/chosen": -253.56301879882812,
      "logps/rejected": -174.05593872070312,
      "loss": 0.6929,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.00489515345543623,
      "rewards/margins": 0.0007793476688675582,
      "rewards/rejected": 0.00411580502986908,
      "step": 1290
    },
    {
      "epoch": 0.23,
      "learning_rate": 9.42065491183879e-08,
      "logits/chosen": -2.787092447280884,
      "logits/rejected": -2.73928165435791,
      "logps/chosen": -230.39700317382812,
      "logps/rejected": -205.6045684814453,
      "loss": 0.693,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.004592005163431168,
      "rewards/margins": 0.0004286689218133688,
      "rewards/rejected": 0.004163335543125868,
      "step": 1300
    },
    {
      "epoch": 0.23,
      "learning_rate": 9.415617128463475e-08,
      "logits/chosen": -2.7811431884765625,
      "logits/rejected": -2.778087615966797,
      "logps/chosen": -242.17929077148438,
      "logps/rejected": -206.31338500976562,
      "loss": 0.6927,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.0039786649867892265,
      "rewards/margins": -0.0004098299832548946,
      "rewards/rejected": 0.00438849488273263,
      "step": 1310
    },
    {
      "epoch": 0.23,
      "learning_rate": 9.41057934508816e-08,
      "logits/chosen": -2.6959187984466553,
      "logits/rejected": -2.788710832595825,
      "logps/chosen": -239.3393096923828,
      "logps/rejected": -177.31842041015625,
      "loss": 0.6925,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.0030113484244793653,
      "rewards/margins": 0.0008161927689798176,
      "rewards/rejected": 0.0021951559465378523,
      "step": 1320
    },
    {
      "epoch": 0.23,
      "learning_rate": 9.405541561712846e-08,
      "logits/chosen": -2.8158926963806152,
      "logits/rejected": -2.7934420108795166,
      "logps/chosen": -246.5731964111328,
      "logps/rejected": -215.53076171875,
      "loss": 0.6925,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": 0.0015319122467190027,
      "rewards/margins": -0.0007288408814929426,
      "rewards/rejected": 0.0022607529535889626,
      "step": 1330
    },
    {
      "epoch": 0.23,
      "learning_rate": 9.400503778337531e-08,
      "logits/chosen": -2.648024320602417,
      "logits/rejected": -2.6812193393707275,
      "logps/chosen": -253.42678833007812,
      "logps/rejected": -231.6710205078125,
      "loss": 0.6928,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.006225033663213253,
      "rewards/margins": 0.002081912010908127,
      "rewards/rejected": 0.004143121186643839,
      "step": 1340
    },
    {
      "epoch": 0.24,
      "learning_rate": 9.395465994962217e-08,
      "logits/chosen": -2.7687926292419434,
      "logits/rejected": -2.7247531414031982,
      "logps/chosen": -198.0591278076172,
      "logps/rejected": -180.5609893798828,
      "loss": 0.6924,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": 0.004522668197751045,
      "rewards/margins": 0.0002378983626840636,
      "rewards/rejected": 0.0042847697623074055,
      "step": 1350
    },
    {
      "epoch": 0.24,
      "learning_rate": 9.390428211586902e-08,
      "logits/chosen": -2.747763156890869,
      "logits/rejected": -2.706991672515869,
      "logps/chosen": -255.20736694335938,
      "logps/rejected": -249.22946166992188,
      "loss": 0.6927,
      "rewards/accuracies": 0.42500001192092896,
      "rewards/chosen": 0.003953252919018269,
      "rewards/margins": -0.000359726749593392,
      "rewards/rejected": 0.004312979988753796,
      "step": 1360
    },
    {
      "epoch": 0.24,
      "learning_rate": 9.385390428211587e-08,
      "logits/chosen": -2.7608680725097656,
      "logits/rejected": -2.807486057281494,
      "logps/chosen": -219.6439971923828,
      "logps/rejected": -175.39483642578125,
      "loss": 0.6927,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.002831274876371026,
      "rewards/margins": 0.00012180556223029271,
      "rewards/rejected": 0.0027094685938209295,
      "step": 1370
    },
    {
      "epoch": 0.24,
      "learning_rate": 9.380352644836271e-08,
      "logits/chosen": -2.713686466217041,
      "logits/rejected": -2.7140908241271973,
      "logps/chosen": -242.8909912109375,
      "logps/rejected": -245.1560821533203,
      "loss": 0.6929,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0032959680538624525,
      "rewards/margins": -0.0015910521615296602,
      "rewards/rejected": 0.004887019749730825,
      "step": 1380
    },
    {
      "epoch": 0.24,
      "learning_rate": 9.375314861460957e-08,
      "logits/chosen": -2.7107300758361816,
      "logits/rejected": -2.703914165496826,
      "logps/chosen": -230.3936767578125,
      "logps/rejected": -253.17105102539062,
      "loss": 0.6931,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": 0.002635772805660963,
      "rewards/margins": -0.0005686979275196791,
      "rewards/rejected": 0.0032044709660112858,
      "step": 1390
    },
    {
      "epoch": 0.25,
      "learning_rate": 9.370277078085642e-08,
      "logits/chosen": -2.7594103813171387,
      "logits/rejected": -2.766557455062866,
      "logps/chosen": -224.17626953125,
      "logps/rejected": -215.5638427734375,
      "loss": 0.6927,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": 0.005116119980812073,
      "rewards/margins": 0.001255401293747127,
      "rewards/rejected": 0.0038607188034802675,
      "step": 1400
    },
    {
      "epoch": 0.25,
      "learning_rate": 9.365239294710328e-08,
      "logits/chosen": -2.756953477859497,
      "logits/rejected": -2.758610486984253,
      "logps/chosen": -235.735107421875,
      "logps/rejected": -188.66738891601562,
      "loss": 0.6923,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.005391458515077829,
      "rewards/margins": 0.0025972898583859205,
      "rewards/rejected": 0.002794168423861265,
      "step": 1410
    },
    {
      "epoch": 0.25,
      "learning_rate": 9.360201511335012e-08,
      "logits/chosen": -2.655500888824463,
      "logits/rejected": -2.6824567317962646,
      "logps/chosen": -214.3753662109375,
      "logps/rejected": -198.02159118652344,
      "loss": 0.6925,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.0019323080778121948,
      "rewards/margins": 5.818065255880356e-05,
      "rewards/rejected": 0.0018741274252533913,
      "step": 1420
    },
    {
      "epoch": 0.25,
      "learning_rate": 9.355163727959697e-08,
      "logits/chosen": -2.748979091644287,
      "logits/rejected": -2.7408335208892822,
      "logps/chosen": -218.5498046875,
      "logps/rejected": -226.24478149414062,
      "loss": 0.693,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.003222791478037834,
      "rewards/margins": -0.0020806575194001198,
      "rewards/rejected": 0.005303448997437954,
      "step": 1430
    },
    {
      "epoch": 0.25,
      "learning_rate": 9.350125944584382e-08,
      "logits/chosen": -2.83701229095459,
      "logits/rejected": -2.812077283859253,
      "logps/chosen": -219.4445343017578,
      "logps/rejected": -199.6473846435547,
      "loss": 0.6925,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.005975143052637577,
      "rewards/margins": 0.0015650525456294417,
      "rewards/rejected": 0.004410090856254101,
      "step": 1440
    },
    {
      "epoch": 0.25,
      "learning_rate": 9.345088161209068e-08,
      "logits/chosen": -2.7849693298339844,
      "logits/rejected": -2.7290408611297607,
      "logps/chosen": -208.5734405517578,
      "logps/rejected": -195.73760986328125,
      "loss": 0.6927,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": 0.0017629098147153854,
      "rewards/margins": -0.0009751931065693498,
      "rewards/rejected": 0.0027381028048694134,
      "step": 1450
    },
    {
      "epoch": 0.26,
      "learning_rate": 9.340050377833753e-08,
      "logits/chosen": -2.7637295722961426,
      "logits/rejected": -2.76435923576355,
      "logps/chosen": -247.49789428710938,
      "logps/rejected": -220.59597778320312,
      "loss": 0.6925,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.007490481249988079,
      "rewards/margins": 0.0033264905214309692,
      "rewards/rejected": 0.00416399072855711,
      "step": 1460
    },
    {
      "epoch": 0.26,
      "learning_rate": 9.335012594458438e-08,
      "logits/chosen": -2.8603591918945312,
      "logits/rejected": -2.8554975986480713,
      "logps/chosen": -260.99822998046875,
      "logps/rejected": -210.4870147705078,
      "loss": 0.6927,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.0044870320707559586,
      "rewards/margins": 0.0010521438671275973,
      "rewards/rejected": 0.003434887621551752,
      "step": 1470
    },
    {
      "epoch": 0.26,
      "learning_rate": 9.329974811083124e-08,
      "logits/chosen": -2.7806200981140137,
      "logits/rejected": -2.780532121658325,
      "logps/chosen": -220.0015106201172,
      "logps/rejected": -210.1516571044922,
      "loss": 0.6933,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.0038058434147387743,
      "rewards/margins": -0.0014907947042956948,
      "rewards/rejected": 0.005296637769788504,
      "step": 1480
    },
    {
      "epoch": 0.26,
      "learning_rate": 9.324937027707808e-08,
      "logits/chosen": -2.716339111328125,
      "logits/rejected": -2.7717671394348145,
      "logps/chosen": -222.77645874023438,
      "logps/rejected": -188.9876708984375,
      "loss": 0.6925,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.006295431405305862,
      "rewards/margins": 0.001769342226907611,
      "rewards/rejected": 0.004526089411228895,
      "step": 1490
    },
    {
      "epoch": 0.26,
      "learning_rate": 9.319899244332493e-08,
      "logits/chosen": -2.7848076820373535,
      "logits/rejected": -2.763967990875244,
      "logps/chosen": -258.8097839355469,
      "logps/rejected": -231.8975372314453,
      "loss": 0.6931,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.00430386234074831,
      "rewards/margins": 0.000930764595977962,
      "rewards/rejected": 0.0033730976283550262,
      "step": 1500
    },
    {
      "epoch": 0.26,
      "learning_rate": 9.314861460957179e-08,
      "logits/chosen": -2.7079567909240723,
      "logits/rejected": -2.73983097076416,
      "logps/chosen": -266.87939453125,
      "logps/rejected": -215.9412078857422,
      "loss": 0.6923,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.0085039883852005,
      "rewards/margins": 0.0026515196077525616,
      "rewards/rejected": 0.005852467846125364,
      "step": 1510
    },
    {
      "epoch": 0.27,
      "learning_rate": 9.309823677581864e-08,
      "logits/chosen": -2.796665668487549,
      "logits/rejected": -2.7821192741394043,
      "logps/chosen": -223.2317352294922,
      "logps/rejected": -186.51785278320312,
      "loss": 0.6927,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.00340632232837379,
      "rewards/margins": 0.001166484085842967,
      "rewards/rejected": 0.0022398384753614664,
      "step": 1520
    },
    {
      "epoch": 0.27,
      "learning_rate": 9.304785894206548e-08,
      "logits/chosen": -2.7554478645324707,
      "logits/rejected": -2.720654010772705,
      "logps/chosen": -242.2590789794922,
      "logps/rejected": -228.3076171875,
      "loss": 0.6928,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.006177330389618874,
      "rewards/margins": 0.003083443734794855,
      "rewards/rejected": 0.003093887586146593,
      "step": 1530
    },
    {
      "epoch": 0.27,
      "learning_rate": 9.299748110831233e-08,
      "logits/chosen": -2.735020875930786,
      "logits/rejected": -2.69610595703125,
      "logps/chosen": -235.02078247070312,
      "logps/rejected": -209.3193817138672,
      "loss": 0.6924,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": 0.003275824710726738,
      "rewards/margins": -0.002175877569243312,
      "rewards/rejected": 0.005451702047139406,
      "step": 1540
    },
    {
      "epoch": 0.27,
      "learning_rate": 9.294710327455919e-08,
      "logits/chosen": -2.7271084785461426,
      "logits/rejected": -2.754586696624756,
      "logps/chosen": -241.9888916015625,
      "logps/rejected": -214.42196655273438,
      "loss": 0.6921,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.006132029928267002,
      "rewards/margins": 0.002107930602505803,
      "rewards/rejected": 0.004024098627269268,
      "step": 1550
    },
    {
      "epoch": 0.27,
      "learning_rate": 9.289672544080604e-08,
      "logits/chosen": -2.7789578437805176,
      "logits/rejected": -2.804396390914917,
      "logps/chosen": -242.86337280273438,
      "logps/rejected": -193.51954650878906,
      "loss": 0.6928,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.005735357757657766,
      "rewards/margins": 0.0003629502607509494,
      "rewards/rejected": 0.0053724078461527824,
      "step": 1560
    },
    {
      "epoch": 0.28,
      "learning_rate": 9.28463476070529e-08,
      "logits/chosen": -2.7950856685638428,
      "logits/rejected": -2.756988048553467,
      "logps/chosen": -223.4980010986328,
      "logps/rejected": -219.8987579345703,
      "loss": 0.6921,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.004248038865625858,
      "rewards/margins": 0.0013514070305973291,
      "rewards/rejected": 0.002896632067859173,
      "step": 1570
    },
    {
      "epoch": 0.28,
      "learning_rate": 9.279596977329975e-08,
      "logits/chosen": -2.705752372741699,
      "logits/rejected": -2.7348334789276123,
      "logps/chosen": -230.68222045898438,
      "logps/rejected": -227.8123321533203,
      "loss": 0.6923,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.003918840549886227,
      "rewards/margins": 0.0002412328467471525,
      "rewards/rejected": 0.0036776072811335325,
      "step": 1580
    },
    {
      "epoch": 0.28,
      "learning_rate": 9.27455919395466e-08,
      "logits/chosen": -2.7165350914001465,
      "logits/rejected": -2.7095649242401123,
      "logps/chosen": -202.18948364257812,
      "logps/rejected": -204.86062622070312,
      "loss": 0.6927,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.0033042896538972855,
      "rewards/margins": -0.0008645458146929741,
      "rewards/rejected": 0.0041688354685902596,
      "step": 1590
    },
    {
      "epoch": 0.28,
      "learning_rate": 9.269521410579345e-08,
      "logits/chosen": -2.708650588989258,
      "logits/rejected": -2.6804311275482178,
      "logps/chosen": -204.07569885253906,
      "logps/rejected": -151.32139587402344,
      "loss": 0.6925,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.004048886708915234,
      "rewards/margins": 0.0010695180390030146,
      "rewards/rejected": 0.0029793691355735064,
      "step": 1600
    },
    {
      "epoch": 0.28,
      "learning_rate": 9.26448362720403e-08,
      "logits/chosen": -2.7520134449005127,
      "logits/rejected": -2.7817509174346924,
      "logps/chosen": -231.3147430419922,
      "logps/rejected": -186.743896484375,
      "loss": 0.6921,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.006745592691004276,
      "rewards/margins": 0.0036534499377012253,
      "rewards/rejected": 0.0030921432189643383,
      "step": 1610
    },
    {
      "epoch": 0.28,
      "learning_rate": 9.259445843828715e-08,
      "logits/chosen": -2.8089406490325928,
      "logits/rejected": -2.804744243621826,
      "logps/chosen": -294.253662109375,
      "logps/rejected": -250.2303924560547,
      "loss": 0.6923,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.008810626342892647,
      "rewards/margins": 0.00140268518589437,
      "rewards/rejected": 0.007407941855490208,
      "step": 1620
    },
    {
      "epoch": 0.29,
      "learning_rate": 9.2544080604534e-08,
      "logits/chosen": -2.788504123687744,
      "logits/rejected": -2.818932294845581,
      "logps/chosen": -204.68771362304688,
      "logps/rejected": -171.62033081054688,
      "loss": 0.6927,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": 0.003130528377369046,
      "rewards/margins": -0.0011326493695378304,
      "rewards/rejected": 0.004263177514076233,
      "step": 1630
    },
    {
      "epoch": 0.29,
      "learning_rate": 9.249370277078086e-08,
      "logits/chosen": -2.75852632522583,
      "logits/rejected": -2.810488700866699,
      "logps/chosen": -293.2353820800781,
      "logps/rejected": -233.1583709716797,
      "loss": 0.693,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.005525670945644379,
      "rewards/margins": 0.0027149529196321964,
      "rewards/rejected": 0.002810717560350895,
      "step": 1640
    },
    {
      "epoch": 0.29,
      "learning_rate": 9.24433249370277e-08,
      "logits/chosen": -2.7465367317199707,
      "logits/rejected": -2.7176971435546875,
      "logps/chosen": -248.1861114501953,
      "logps/rejected": -205.9281005859375,
      "loss": 0.6924,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.007562885992228985,
      "rewards/margins": 0.005088142119348049,
      "rewards/rejected": 0.0024747434072196484,
      "step": 1650
    },
    {
      "epoch": 0.29,
      "learning_rate": 9.239294710327455e-08,
      "logits/chosen": -2.765758752822876,
      "logits/rejected": -2.76737904548645,
      "logps/chosen": -236.78482055664062,
      "logps/rejected": -216.59408569335938,
      "loss": 0.6925,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.00701189273968339,
      "rewards/margins": 0.0009126948425546288,
      "rewards/rejected": 0.006099197082221508,
      "step": 1660
    },
    {
      "epoch": 0.29,
      "learning_rate": 9.23425692695214e-08,
      "logits/chosen": -2.7198102474212646,
      "logits/rejected": -2.676805257797241,
      "logps/chosen": -190.41085815429688,
      "logps/rejected": -199.9924774169922,
      "loss": 0.6927,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.005398167297244072,
      "rewards/margins": 0.0025181311648339033,
      "rewards/rejected": 0.0028800363652408123,
      "step": 1670
    },
    {
      "epoch": 0.29,
      "learning_rate": 9.229219143576826e-08,
      "logits/chosen": -2.7450883388519287,
      "logits/rejected": -2.7034592628479004,
      "logps/chosen": -285.408935546875,
      "logps/rejected": -261.0296936035156,
      "loss": 0.6922,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.004709193948656321,
      "rewards/margins": -0.00033829378662630916,
      "rewards/rejected": 0.005047488491982222,
      "step": 1680
    },
    {
      "epoch": 0.3,
      "learning_rate": 9.224181360201511e-08,
      "logits/chosen": -2.6765735149383545,
      "logits/rejected": -2.5730743408203125,
      "logps/chosen": -218.64990234375,
      "logps/rejected": -193.49989318847656,
      "loss": 0.6929,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.005248272325843573,
      "rewards/margins": 0.0031143042724579573,
      "rewards/rejected": 0.002133968286216259,
      "step": 1690
    },
    {
      "epoch": 0.3,
      "learning_rate": 9.219143576826196e-08,
      "logits/chosen": -2.7516093254089355,
      "logits/rejected": -2.796564817428589,
      "logps/chosen": -259.0831298828125,
      "logps/rejected": -185.15670776367188,
      "loss": 0.6922,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.006098913960158825,
      "rewards/margins": 0.0033921864815056324,
      "rewards/rejected": 0.002706727711483836,
      "step": 1700
    },
    {
      "epoch": 0.3,
      "learning_rate": 9.214105793450882e-08,
      "logits/chosen": -2.8071539402008057,
      "logits/rejected": -2.7718913555145264,
      "logps/chosen": -237.6824188232422,
      "logps/rejected": -192.46731567382812,
      "loss": 0.6926,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.006562414113432169,
      "rewards/margins": 0.0027120665181428194,
      "rewards/rejected": 0.0038503475952893496,
      "step": 1710
    },
    {
      "epoch": 0.3,
      "learning_rate": 9.209068010075567e-08,
      "logits/chosen": -2.801098346710205,
      "logits/rejected": -2.8045449256896973,
      "logps/chosen": -306.48004150390625,
      "logps/rejected": -233.9066162109375,
      "loss": 0.6923,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.007683928124606609,
      "rewards/margins": 0.003144733142107725,
      "rewards/rejected": 0.004539194516837597,
      "step": 1720
    },
    {
      "epoch": 0.3,
      "learning_rate": 9.204030226700251e-08,
      "logits/chosen": -2.7108566761016846,
      "logits/rejected": -2.805584669113159,
      "logps/chosen": -190.70640563964844,
      "logps/rejected": -164.53231811523438,
      "loss": 0.6922,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.005603504832834005,
      "rewards/margins": 0.0018408369505777955,
      "rewards/rejected": 0.0037626686971634626,
      "step": 1730
    },
    {
      "epoch": 0.3,
      "learning_rate": 9.198992443324937e-08,
      "logits/chosen": -2.7416088581085205,
      "logits/rejected": -2.7634804248809814,
      "logps/chosen": -187.78787231445312,
      "logps/rejected": -176.73611450195312,
      "loss": 0.6928,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": 0.003316228510811925,
      "rewards/margins": -0.0012748127337545156,
      "rewards/rejected": 0.004591041244566441,
      "step": 1740
    },
    {
      "epoch": 0.31,
      "learning_rate": 9.193954659949622e-08,
      "logits/chosen": -2.7573604583740234,
      "logits/rejected": -2.685169219970703,
      "logps/chosen": -220.46957397460938,
      "logps/rejected": -186.6195831298828,
      "loss": 0.6923,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.00505624245852232,
      "rewards/margins": 0.0036626472137868404,
      "rewards/rejected": 0.0013935946626588702,
      "step": 1750
    },
    {
      "epoch": 0.31,
      "learning_rate": 9.188916876574306e-08,
      "logits/chosen": -2.8284783363342285,
      "logits/rejected": -2.7774031162261963,
      "logps/chosen": -270.4326171875,
      "logps/rejected": -247.10494995117188,
      "loss": 0.6922,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.007807993330061436,
      "rewards/margins": 0.003054631408303976,
      "rewards/rejected": 0.004753361456096172,
      "step": 1760
    },
    {
      "epoch": 0.31,
      "learning_rate": 9.183879093198991e-08,
      "logits/chosen": -2.7399754524230957,
      "logits/rejected": -2.783146619796753,
      "logps/chosen": -237.53921508789062,
      "logps/rejected": -206.88241577148438,
      "loss": 0.6923,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.00769572239369154,
      "rewards/margins": 0.0022291478235274553,
      "rewards/rejected": 0.005466574802994728,
      "step": 1770
    },
    {
      "epoch": 0.31,
      "learning_rate": 9.178841309823677e-08,
      "logits/chosen": -2.716817855834961,
      "logits/rejected": -2.719804286956787,
      "logps/chosen": -235.63916015625,
      "logps/rejected": -200.00222778320312,
      "loss": 0.6924,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.006353571079671383,
      "rewards/margins": 0.0007235562661662698,
      "rewards/rejected": 0.00563001586124301,
      "step": 1780
    },
    {
      "epoch": 0.31,
      "learning_rate": 9.173803526448362e-08,
      "logits/chosen": -2.7352333068847656,
      "logits/rejected": -2.7781901359558105,
      "logps/chosen": -175.88961791992188,
      "logps/rejected": -172.944091796875,
      "loss": 0.6924,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.005051255226135254,
      "rewards/margins": 0.00013072407455183566,
      "rewards/rejected": 0.004920531529933214,
      "step": 1790
    },
    {
      "epoch": 0.32,
      "learning_rate": 9.168765743073047e-08,
      "logits/chosen": -2.737471342086792,
      "logits/rejected": -2.71102237701416,
      "logps/chosen": -250.81787109375,
      "logps/rejected": -211.5341796875,
      "loss": 0.6923,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.007261678576469421,
      "rewards/margins": 0.0013637794181704521,
      "rewards/rejected": 0.005897899158298969,
      "step": 1800
    },
    {
      "epoch": 0.32,
      "learning_rate": 9.163727959697733e-08,
      "logits/chosen": -2.7573001384735107,
      "logits/rejected": -2.777223587036133,
      "logps/chosen": -223.52700805664062,
      "logps/rejected": -224.9475860595703,
      "loss": 0.6925,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.005940387956798077,
      "rewards/margins": 0.002223807852715254,
      "rewards/rejected": 0.003716580104082823,
      "step": 1810
    },
    {
      "epoch": 0.32,
      "learning_rate": 9.158690176322418e-08,
      "logits/chosen": -2.7546377182006836,
      "logits/rejected": -2.8038830757141113,
      "logps/chosen": -210.6802215576172,
      "logps/rejected": -191.94479370117188,
      "loss": 0.6928,
      "rewards/accuracies": 0.42500001192092896,
      "rewards/chosen": 0.005436464212834835,
      "rewards/margins": -0.0017134780064225197,
      "rewards/rejected": 0.007149942219257355,
      "step": 1820
    },
    {
      "epoch": 0.32,
      "learning_rate": 9.153652392947104e-08,
      "logits/chosen": -2.793966293334961,
      "logits/rejected": -2.84097957611084,
      "logps/chosen": -275.74078369140625,
      "logps/rejected": -247.7950439453125,
      "loss": 0.6932,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.006203091703355312,
      "rewards/margins": 0.0017260663444176316,
      "rewards/rejected": 0.004477024544030428,
      "step": 1830
    },
    {
      "epoch": 0.32,
      "learning_rate": 9.148614609571788e-08,
      "logits/chosen": -2.7271361351013184,
      "logits/rejected": -2.8027615547180176,
      "logps/chosen": -197.15615844726562,
      "logps/rejected": -185.2477264404297,
      "loss": 0.6925,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.004695483483374119,
      "rewards/margins": -0.0009570989641360939,
      "rewards/rejected": 0.00565258227288723,
      "step": 1840
    },
    {
      "epoch": 0.32,
      "learning_rate": 9.143576826196473e-08,
      "logits/chosen": -2.6608946323394775,
      "logits/rejected": -2.6794865131378174,
      "logps/chosen": -248.6991729736328,
      "logps/rejected": -243.78225708007812,
      "loss": 0.6925,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": 0.006408816669136286,
      "rewards/margins": -0.0003214823664166033,
      "rewards/rejected": 0.0067302994430065155,
      "step": 1850
    },
    {
      "epoch": 0.33,
      "learning_rate": 9.138539042821158e-08,
      "logits/chosen": -2.7787938117980957,
      "logits/rejected": -2.773528575897217,
      "logps/chosen": -276.1280212402344,
      "logps/rejected": -227.1744842529297,
      "loss": 0.6927,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.009555719792842865,
      "rewards/margins": 0.003407822223380208,
      "rewards/rejected": 0.006147897802293301,
      "step": 1860
    },
    {
      "epoch": 0.33,
      "learning_rate": 9.133501259445844e-08,
      "logits/chosen": -2.759768009185791,
      "logits/rejected": -2.7878026962280273,
      "logps/chosen": -282.8551330566406,
      "logps/rejected": -236.4971160888672,
      "loss": 0.6929,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.007030332926660776,
      "rewards/margins": 0.0005596103728748858,
      "rewards/rejected": 0.006470722146332264,
      "step": 1870
    },
    {
      "epoch": 0.33,
      "learning_rate": 9.128463476070528e-08,
      "logits/chosen": -2.7400457859039307,
      "logits/rejected": -2.7290937900543213,
      "logps/chosen": -228.9811248779297,
      "logps/rejected": -190.52259826660156,
      "loss": 0.6921,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.008042901754379272,
      "rewards/margins": 0.004830832127481699,
      "rewards/rejected": 0.00321206939406693,
      "step": 1880
    },
    {
      "epoch": 0.33,
      "learning_rate": 9.123425692695213e-08,
      "logits/chosen": -2.7177040576934814,
      "logits/rejected": -2.746002197265625,
      "logps/chosen": -208.7105255126953,
      "logps/rejected": -178.6358642578125,
      "loss": 0.6924,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.008034145459532738,
      "rewards/margins": 0.0038532826583832502,
      "rewards/rejected": 0.0041808634996414185,
      "step": 1890
    },
    {
      "epoch": 0.33,
      "learning_rate": 9.118387909319898e-08,
      "logits/chosen": -2.7194418907165527,
      "logits/rejected": -2.7867624759674072,
      "logps/chosen": -260.50518798828125,
      "logps/rejected": -232.93331909179688,
      "loss": 0.6923,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.009059436619281769,
      "rewards/margins": 0.0009833091171458364,
      "rewards/rejected": 0.008076125755906105,
      "step": 1900
    },
    {
      "epoch": 0.33,
      "learning_rate": 9.113350125944584e-08,
      "logits/chosen": -2.6974074840545654,
      "logits/rejected": -2.720205545425415,
      "logps/chosen": -229.29800415039062,
      "logps/rejected": -222.5083465576172,
      "loss": 0.6927,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.006830728612840176,
      "rewards/margins": 9.487769420957193e-05,
      "rewards/rejected": 0.006735851056873798,
      "step": 1910
    },
    {
      "epoch": 0.34,
      "learning_rate": 9.108312342569269e-08,
      "logits/chosen": -2.682278633117676,
      "logits/rejected": -2.689655303955078,
      "logps/chosen": -262.4538879394531,
      "logps/rejected": -208.20834350585938,
      "loss": 0.6922,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.008090605027973652,
      "rewards/margins": 0.005382605362683535,
      "rewards/rejected": 0.0027079996652901173,
      "step": 1920
    },
    {
      "epoch": 0.34,
      "learning_rate": 9.103274559193955e-08,
      "logits/chosen": -2.655890703201294,
      "logits/rejected": -2.764970541000366,
      "logps/chosen": -277.8880310058594,
      "logps/rejected": -202.5020751953125,
      "loss": 0.6928,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.007924163714051247,
      "rewards/margins": 0.0037231750320643187,
      "rewards/rejected": 0.004200988449156284,
      "step": 1930
    },
    {
      "epoch": 0.34,
      "learning_rate": 9.09823677581864e-08,
      "logits/chosen": -2.7730414867401123,
      "logits/rejected": -2.708861827850342,
      "logps/chosen": -233.4267578125,
      "logps/rejected": -205.75045776367188,
      "loss": 0.6928,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.007474385201931,
      "rewards/margins": 0.0025435371790081263,
      "rewards/rejected": 0.00493084779009223,
      "step": 1940
    },
    {
      "epoch": 0.34,
      "learning_rate": 9.093198992443325e-08,
      "logits/chosen": -2.7270901203155518,
      "logits/rejected": -2.7830066680908203,
      "logps/chosen": -269.06103515625,
      "logps/rejected": -233.0624237060547,
      "loss": 0.6924,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.006805942859500647,
      "rewards/margins": 0.002863366389647126,
      "rewards/rejected": 0.003942577634006739,
      "step": 1950
    },
    {
      "epoch": 0.34,
      "learning_rate": 9.088161209068009e-08,
      "logits/chosen": -2.756842851638794,
      "logits/rejected": -2.7270755767822266,
      "logps/chosen": -205.07064819335938,
      "logps/rejected": -199.98719787597656,
      "loss": 0.6924,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.006842183880507946,
      "rewards/margins": 0.0016147240530699492,
      "rewards/rejected": 0.0052274600602686405,
      "step": 1960
    },
    {
      "epoch": 0.35,
      "learning_rate": 9.083123425692695e-08,
      "logits/chosen": -2.772458076477051,
      "logits/rejected": -2.720886707305908,
      "logps/chosen": -222.73904418945312,
      "logps/rejected": -192.80467224121094,
      "loss": 0.6925,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.008435122668743134,
      "rewards/margins": 0.0015853416407480836,
      "rewards/rejected": 0.006849782075732946,
      "step": 1970
    },
    {
      "epoch": 0.35,
      "learning_rate": 9.07808564231738e-08,
      "logits/chosen": -2.8323588371276855,
      "logits/rejected": -2.757342576980591,
      "logps/chosen": -221.9827117919922,
      "logps/rejected": -206.13693237304688,
      "loss": 0.6925,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.006544102914631367,
      "rewards/margins": 0.0011714848224073648,
      "rewards/rejected": 0.005372617393732071,
      "step": 1980
    },
    {
      "epoch": 0.35,
      "learning_rate": 9.073047858942065e-08,
      "logits/chosen": -2.839995861053467,
      "logits/rejected": -2.778444290161133,
      "logps/chosen": -198.0923309326172,
      "logps/rejected": -185.09725952148438,
      "loss": 0.692,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.00797742698341608,
      "rewards/margins": 0.0033867559395730495,
      "rewards/rejected": 0.004590670112520456,
      "step": 1990
    },
    {
      "epoch": 0.35,
      "learning_rate": 9.06801007556675e-08,
      "logits/chosen": -2.7065186500549316,
      "logits/rejected": -2.6786417961120605,
      "logps/chosen": -200.3252716064453,
      "logps/rejected": -174.55252075195312,
      "loss": 0.6927,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0027497578412294388,
      "rewards/margins": 0.00032762327464297414,
      "rewards/rejected": 0.002422134391963482,
      "step": 2000
    },
    {
      "epoch": 0.35,
      "learning_rate": 9.062972292191435e-08,
      "logits/chosen": -2.8198843002319336,
      "logits/rejected": -2.72586727142334,
      "logps/chosen": -250.848388671875,
      "logps/rejected": -223.4097900390625,
      "loss": 0.692,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.010807116515934467,
      "rewards/margins": 0.0031298950780183077,
      "rewards/rejected": 0.007677220739424229,
      "step": 2010
    },
    {
      "epoch": 0.35,
      "learning_rate": 9.05793450881612e-08,
      "logits/chosen": -2.7961342334747314,
      "logits/rejected": -2.793257713317871,
      "logps/chosen": -228.30099487304688,
      "logps/rejected": -172.77639770507812,
      "loss": 0.6922,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.00609872629866004,
      "rewards/margins": 0.0020499066449701786,
      "rewards/rejected": 0.004048819653689861,
      "step": 2020
    },
    {
      "epoch": 0.36,
      "learning_rate": 9.052896725440806e-08,
      "logits/chosen": -2.773071765899658,
      "logits/rejected": -2.726912260055542,
      "logps/chosen": -210.5772705078125,
      "logps/rejected": -182.91647338867188,
      "loss": 0.6918,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.008080719970166683,
      "rewards/margins": 0.002562993671745062,
      "rewards/rejected": 0.005517727229744196,
      "step": 2030
    },
    {
      "epoch": 0.36,
      "learning_rate": 9.047858942065491e-08,
      "logits/chosen": -2.694596767425537,
      "logits/rejected": -2.707979917526245,
      "logps/chosen": -219.9010772705078,
      "logps/rejected": -220.4337158203125,
      "loss": 0.6923,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.007597730960696936,
      "rewards/margins": -0.0006745406426489353,
      "rewards/rejected": 0.008272271603345871,
      "step": 2040
    },
    {
      "epoch": 0.36,
      "learning_rate": 9.042821158690176e-08,
      "logits/chosen": -2.823132276535034,
      "logits/rejected": -2.8142189979553223,
      "logps/chosen": -229.84036254882812,
      "logps/rejected": -180.3568572998047,
      "loss": 0.6922,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.00825478509068489,
      "rewards/margins": 0.00363583629950881,
      "rewards/rejected": 0.004618949256837368,
      "step": 2050
    },
    {
      "epoch": 0.36,
      "learning_rate": 9.037783375314862e-08,
      "logits/chosen": -2.7286908626556396,
      "logits/rejected": -2.7378149032592773,
      "logps/chosen": -174.80532836914062,
      "logps/rejected": -159.1217803955078,
      "loss": 0.6924,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": 0.005188413895666599,
      "rewards/margins": 0.001207556575536728,
      "rewards/rejected": 0.003980856854468584,
      "step": 2060
    },
    {
      "epoch": 0.36,
      "learning_rate": 9.032745591939546e-08,
      "logits/chosen": -2.7917840480804443,
      "logits/rejected": -2.7678382396698,
      "logps/chosen": -211.1044921875,
      "logps/rejected": -186.33775329589844,
      "loss": 0.6922,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.006869046948850155,
      "rewards/margins": 0.002652390394359827,
      "rewards/rejected": 0.004216657020151615,
      "step": 2070
    },
    {
      "epoch": 0.36,
      "learning_rate": 9.027707808564231e-08,
      "logits/chosen": -2.773383617401123,
      "logits/rejected": -2.7457468509674072,
      "logps/chosen": -196.5582733154297,
      "logps/rejected": -157.29763793945312,
      "loss": 0.6931,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.004717974923551083,
      "rewards/margins": -0.0003334479406476021,
      "rewards/rejected": 0.005051423329859972,
      "step": 2080
    },
    {
      "epoch": 0.37,
      "learning_rate": 9.022670025188916e-08,
      "logits/chosen": -2.829695224761963,
      "logits/rejected": -2.8286352157592773,
      "logps/chosen": -257.33575439453125,
      "logps/rejected": -222.24081420898438,
      "loss": 0.693,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": 0.008855994790792465,
      "rewards/margins": -0.0004033507429994643,
      "rewards/rejected": 0.009259345009922981,
      "step": 2090
    },
    {
      "epoch": 0.37,
      "learning_rate": 9.017632241813602e-08,
      "logits/chosen": -2.6822662353515625,
      "logits/rejected": -2.7216250896453857,
      "logps/chosen": -204.204345703125,
      "logps/rejected": -213.64120483398438,
      "loss": 0.6925,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": 0.005887551233172417,
      "rewards/margins": -0.001325045945122838,
      "rewards/rejected": 0.007212596945464611,
      "step": 2100
    },
    {
      "epoch": 0.37,
      "learning_rate": 9.012594458438286e-08,
      "logits/chosen": -2.717801332473755,
      "logits/rejected": -2.6509881019592285,
      "logps/chosen": -201.35829162597656,
      "logps/rejected": -204.86557006835938,
      "loss": 0.6931,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.005306928418576717,
      "rewards/margins": -0.0017324115615338087,
      "rewards/rejected": 0.007039339747279882,
      "step": 2110
    },
    {
      "epoch": 0.37,
      "learning_rate": 9.007556675062971e-08,
      "logits/chosen": -2.7330944538116455,
      "logits/rejected": -2.7502284049987793,
      "logps/chosen": -235.6743621826172,
      "logps/rejected": -227.54446411132812,
      "loss": 0.6923,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": 0.007282434497028589,
      "rewards/margins": 0.0004998206277377903,
      "rewards/rejected": 0.006782613694667816,
      "step": 2120
    },
    {
      "epoch": 0.37,
      "learning_rate": 9.002518891687657e-08,
      "logits/chosen": -2.7793288230895996,
      "logits/rejected": -2.7879912853240967,
      "logps/chosen": -244.0114288330078,
      "logps/rejected": -195.60763549804688,
      "loss": 0.6928,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.00913340412080288,
      "rewards/margins": 0.002795944456011057,
      "rewards/rejected": 0.006337459199130535,
      "step": 2130
    },
    {
      "epoch": 0.37,
      "learning_rate": 8.997481108312342e-08,
      "logits/chosen": -2.770501136779785,
      "logits/rejected": -2.75308895111084,
      "logps/chosen": -263.10198974609375,
      "logps/rejected": -227.291015625,
      "loss": 0.6928,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": 0.005692030768841505,
      "rewards/margins": -0.001495410455390811,
      "rewards/rejected": 0.007187440060079098,
      "step": 2140
    },
    {
      "epoch": 0.38,
      "learning_rate": 8.992443324937027e-08,
      "logits/chosen": -2.722250461578369,
      "logits/rejected": -2.782747745513916,
      "logps/chosen": -217.0636444091797,
      "logps/rejected": -190.8524627685547,
      "loss": 0.6923,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.0067198313772678375,
      "rewards/margins": 0.00041137990774586797,
      "rewards/rejected": 0.006308451294898987,
      "step": 2150
    },
    {
      "epoch": 0.38,
      "learning_rate": 8.987405541561713e-08,
      "logits/chosen": -2.792790651321411,
      "logits/rejected": -2.831502676010132,
      "logps/chosen": -235.6027069091797,
      "logps/rejected": -213.80899047851562,
      "loss": 0.6922,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.006459805183112621,
      "rewards/margins": 0.0006882447050884366,
      "rewards/rejected": 0.005771561060100794,
      "step": 2160
    },
    {
      "epoch": 0.38,
      "learning_rate": 8.982367758186398e-08,
      "logits/chosen": -2.6963181495666504,
      "logits/rejected": -2.763627529144287,
      "logps/chosen": -216.1712646484375,
      "logps/rejected": -189.81759643554688,
      "loss": 0.6921,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.008661460131406784,
      "rewards/margins": 0.00203773588873446,
      "rewards/rejected": 0.006623723544180393,
      "step": 2170
    },
    {
      "epoch": 0.38,
      "learning_rate": 8.977329974811083e-08,
      "logits/chosen": -2.735170364379883,
      "logits/rejected": -2.7084457874298096,
      "logps/chosen": -182.16720581054688,
      "logps/rejected": -163.56094360351562,
      "loss": 0.6927,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.008674652315676212,
      "rewards/margins": 0.003342932555824518,
      "rewards/rejected": 0.0053317188285291195,
      "step": 2180
    },
    {
      "epoch": 0.38,
      "learning_rate": 8.972292191435767e-08,
      "logits/chosen": -2.750704288482666,
      "logits/rejected": -2.779202938079834,
      "logps/chosen": -241.536865234375,
      "logps/rejected": -190.92633056640625,
      "loss": 0.6928,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.010668774135410786,
      "rewards/margins": 0.0013826647773385048,
      "rewards/rejected": 0.009286109358072281,
      "step": 2190
    },
    {
      "epoch": 0.39,
      "learning_rate": 8.967254408060453e-08,
      "logits/chosen": -2.7246861457824707,
      "logits/rejected": -2.675992488861084,
      "logps/chosen": -223.83837890625,
      "logps/rejected": -188.50747680664062,
      "loss": 0.6919,
      "rewards/accuracies": 0.42500001192092896,
      "rewards/chosen": 0.007452371530234814,
      "rewards/margins": -0.0014652060344815254,
      "rewards/rejected": 0.008917578496038914,
      "step": 2200
    },
    {
      "epoch": 0.39,
      "learning_rate": 8.962216624685138e-08,
      "logits/chosen": -2.7461745738983154,
      "logits/rejected": -2.690744400024414,
      "logps/chosen": -201.64651489257812,
      "logps/rejected": -210.37405395507812,
      "loss": 0.6927,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.005704388953745365,
      "rewards/margins": -0.00024541860329918563,
      "rewards/rejected": 0.005949807353317738,
      "step": 2210
    },
    {
      "epoch": 0.39,
      "learning_rate": 8.957178841309823e-08,
      "logits/chosen": -2.8038201332092285,
      "logits/rejected": -2.75268816947937,
      "logps/chosen": -254.98904418945312,
      "logps/rejected": -221.23080444335938,
      "loss": 0.6925,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.009008856490254402,
      "rewards/margins": 0.0005879864911548793,
      "rewards/rejected": 0.008420869708061218,
      "step": 2220
    },
    {
      "epoch": 0.39,
      "learning_rate": 8.952141057934508e-08,
      "logits/chosen": -2.804844617843628,
      "logits/rejected": -2.805863380432129,
      "logps/chosen": -242.4755859375,
      "logps/rejected": -208.1300048828125,
      "loss": 0.6921,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.008145569823682308,
      "rewards/margins": 0.0016121035441756248,
      "rewards/rejected": 0.006533467210829258,
      "step": 2230
    },
    {
      "epoch": 0.39,
      "learning_rate": 8.947103274559193e-08,
      "logits/chosen": -2.7161951065063477,
      "logits/rejected": -2.6106388568878174,
      "logps/chosen": -236.38534545898438,
      "logps/rejected": -237.8708038330078,
      "loss": 0.6921,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.01004200428724289,
      "rewards/margins": 0.004435664042830467,
      "rewards/rejected": 0.0056063407100737095,
      "step": 2240
    },
    {
      "epoch": 0.39,
      "learning_rate": 8.942065491183878e-08,
      "logits/chosen": -2.7104029655456543,
      "logits/rejected": -2.7299609184265137,
      "logps/chosen": -244.9352569580078,
      "logps/rejected": -201.58877563476562,
      "loss": 0.6926,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.008890941739082336,
      "rewards/margins": 0.0025217377115041018,
      "rewards/rejected": 0.006369204260408878,
      "step": 2250
    },
    {
      "epoch": 0.4,
      "learning_rate": 8.937027707808565e-08,
      "logits/chosen": -2.7342441082000732,
      "logits/rejected": -2.7766692638397217,
      "logps/chosen": -249.001708984375,
      "logps/rejected": -195.75601196289062,
      "loss": 0.6924,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.011680737137794495,
      "rewards/margins": 0.007096041925251484,
      "rewards/rejected": 0.004584694746881723,
      "step": 2260
    },
    {
      "epoch": 0.4,
      "learning_rate": 8.931989924433249e-08,
      "logits/chosen": -2.7314367294311523,
      "logits/rejected": -2.751647472381592,
      "logps/chosen": -257.7462158203125,
      "logps/rejected": -192.61325073242188,
      "loss": 0.6914,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.009932273998856544,
      "rewards/margins": 0.002873816527426243,
      "rewards/rejected": 0.007058457471430302,
      "step": 2270
    },
    {
      "epoch": 0.4,
      "learning_rate": 8.926952141057934e-08,
      "logits/chosen": -2.7829346656799316,
      "logits/rejected": -2.6927618980407715,
      "logps/chosen": -214.0055389404297,
      "logps/rejected": -167.7479705810547,
      "loss": 0.6918,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.01033037155866623,
      "rewards/margins": 0.006925082765519619,
      "rewards/rejected": 0.003405289025977254,
      "step": 2280
    },
    {
      "epoch": 0.4,
      "learning_rate": 8.92191435768262e-08,
      "logits/chosen": -2.7139039039611816,
      "logits/rejected": -2.74005389213562,
      "logps/chosen": -240.3031768798828,
      "logps/rejected": -191.8223419189453,
      "loss": 0.6925,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.006939934100955725,
      "rewards/margins": 0.0009533121483400464,
      "rewards/rejected": 0.0059866225346922874,
      "step": 2290
    },
    {
      "epoch": 0.4,
      "learning_rate": 8.916876574307305e-08,
      "logits/chosen": -2.7282981872558594,
      "logits/rejected": -2.712113380432129,
      "logps/chosen": -237.7392578125,
      "logps/rejected": -224.4556427001953,
      "loss": 0.692,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.010463756509125233,
      "rewards/margins": 0.004110955633223057,
      "rewards/rejected": 0.006352802272886038,
      "step": 2300
    },
    {
      "epoch": 0.4,
      "learning_rate": 8.911838790931989e-08,
      "logits/chosen": -2.7684943675994873,
      "logits/rejected": -2.7373266220092773,
      "logps/chosen": -206.5482635498047,
      "logps/rejected": -212.13406372070312,
      "loss": 0.6925,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": 0.006885471288114786,
      "rewards/margins": -0.002132121007889509,
      "rewards/rejected": 0.009017592296004295,
      "step": 2310
    },
    {
      "epoch": 0.41,
      "learning_rate": 8.906801007556674e-08,
      "logits/chosen": -2.805568218231201,
      "logits/rejected": -2.8355040550231934,
      "logps/chosen": -275.70587158203125,
      "logps/rejected": -209.14218139648438,
      "loss": 0.6926,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.007967405952513218,
      "rewards/margins": 0.002225789474323392,
      "rewards/rejected": 0.00574161671102047,
      "step": 2320
    },
    {
      "epoch": 0.41,
      "learning_rate": 8.90176322418136e-08,
      "logits/chosen": -2.7412164211273193,
      "logits/rejected": -2.7422406673431396,
      "logps/chosen": -244.7394256591797,
      "logps/rejected": -208.07717895507812,
      "loss": 0.6924,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.00783600378781557,
      "rewards/margins": 0.0005075406515970826,
      "rewards/rejected": 0.007328461855649948,
      "step": 2330
    },
    {
      "epoch": 0.41,
      "learning_rate": 8.896725440806045e-08,
      "logits/chosen": -2.8028624057769775,
      "logits/rejected": -2.796525716781616,
      "logps/chosen": -272.66632080078125,
      "logps/rejected": -222.811279296875,
      "loss": 0.6921,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.010351276025176048,
      "rewards/margins": 0.0018881557043641806,
      "rewards/rejected": 0.008463121019303799,
      "step": 2340
    },
    {
      "epoch": 0.41,
      "learning_rate": 8.891687657430729e-08,
      "logits/chosen": -2.6541576385498047,
      "logits/rejected": -2.686614751815796,
      "logps/chosen": -231.4371337890625,
      "logps/rejected": -216.7913818359375,
      "loss": 0.6924,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.0074166180565953255,
      "rewards/margins": -0.0016186184948310256,
      "rewards/rejected": 0.009035235270857811,
      "step": 2350
    },
    {
      "epoch": 0.41,
      "learning_rate": 8.886649874055415e-08,
      "logits/chosen": -2.631671190261841,
      "logits/rejected": -2.760439395904541,
      "logps/chosen": -225.0189666748047,
      "logps/rejected": -235.6808624267578,
      "loss": 0.6929,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.007628731429576874,
      "rewards/margins": -0.00035653618397191167,
      "rewards/rejected": 0.007985267788171768,
      "step": 2360
    },
    {
      "epoch": 0.42,
      "learning_rate": 8.881612090680101e-08,
      "logits/chosen": -2.6786155700683594,
      "logits/rejected": -2.7173733711242676,
      "logps/chosen": -220.6988983154297,
      "logps/rejected": -195.98582458496094,
      "loss": 0.6922,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.009033399634063244,
      "rewards/margins": 0.0017579678678885102,
      "rewards/rejected": 0.007275431416928768,
      "step": 2370
    },
    {
      "epoch": 0.42,
      "learning_rate": 8.876574307304785e-08,
      "logits/chosen": -2.774136543273926,
      "logits/rejected": -2.6647706031799316,
      "logps/chosen": -304.55181884765625,
      "logps/rejected": -307.3960876464844,
      "loss": 0.6923,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.012085049413144588,
      "rewards/margins": 0.0007154010236263275,
      "rewards/rejected": 0.01136965025216341,
      "step": 2380
    },
    {
      "epoch": 0.42,
      "learning_rate": 8.871536523929471e-08,
      "logits/chosen": -2.6545538902282715,
      "logits/rejected": -2.752094030380249,
      "logps/chosen": -250.7977752685547,
      "logps/rejected": -221.78964233398438,
      "loss": 0.6922,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": 0.008444622159004211,
      "rewards/margins": 0.0009982278570532799,
      "rewards/rejected": 0.0074463943019509315,
      "step": 2390
    },
    {
      "epoch": 0.42,
      "learning_rate": 8.866498740554156e-08,
      "logits/chosen": -2.7769970893859863,
      "logits/rejected": -2.749866485595703,
      "logps/chosen": -210.654541015625,
      "logps/rejected": -191.4696807861328,
      "loss": 0.6921,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.008832408115267754,
      "rewards/margins": -0.0005183859029784799,
      "rewards/rejected": 0.009350794367492199,
      "step": 2400
    },
    {
      "epoch": 0.42,
      "learning_rate": 8.861460957178841e-08,
      "logits/chosen": -2.697442054748535,
      "logits/rejected": -2.7253782749176025,
      "logps/chosen": -220.7960968017578,
      "logps/rejected": -217.39108276367188,
      "loss": 0.6921,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": 0.0069009833969175816,
      "rewards/margins": -0.0006977998418733478,
      "rewards/rejected": 0.007598782889544964,
      "step": 2410
    },
    {
      "epoch": 0.42,
      "learning_rate": 8.856423173803525e-08,
      "logits/chosen": -2.7221148014068604,
      "logits/rejected": -2.731081962585449,
      "logps/chosen": -213.35916137695312,
      "logps/rejected": -180.56887817382812,
      "loss": 0.6921,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.006449718959629536,
      "rewards/margins": 0.00094635970890522,
      "rewards/rejected": 0.005503358785063028,
      "step": 2420
    },
    {
      "epoch": 0.43,
      "learning_rate": 8.851385390428211e-08,
      "logits/chosen": -2.726142644882202,
      "logits/rejected": -2.7781872749328613,
      "logps/chosen": -233.19851684570312,
      "logps/rejected": -214.63565063476562,
      "loss": 0.6923,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.009057990275323391,
      "rewards/margins": 0.00277666375041008,
      "rewards/rejected": 0.006281326524913311,
      "step": 2430
    },
    {
      "epoch": 0.43,
      "learning_rate": 8.846347607052896e-08,
      "logits/chosen": -2.721498966217041,
      "logits/rejected": -2.74996018409729,
      "logps/chosen": -246.1979217529297,
      "logps/rejected": -196.91104125976562,
      "loss": 0.6918,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.009174585342407227,
      "rewards/margins": 0.002294916193932295,
      "rewards/rejected": 0.00687966775149107,
      "step": 2440
    },
    {
      "epoch": 0.43,
      "learning_rate": 8.841309823677582e-08,
      "logits/chosen": -2.7128663063049316,
      "logits/rejected": -2.6650452613830566,
      "logps/chosen": -206.5331573486328,
      "logps/rejected": -173.64071655273438,
      "loss": 0.6924,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": 0.007573836483061314,
      "rewards/margins": 0.0018665831303223968,
      "rewards/rejected": 0.005707253236323595,
      "step": 2450
    },
    {
      "epoch": 0.43,
      "learning_rate": 8.836272040302266e-08,
      "logits/chosen": -2.788109302520752,
      "logits/rejected": -2.7255702018737793,
      "logps/chosen": -243.81491088867188,
      "logps/rejected": -197.00755310058594,
      "loss": 0.6926,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.010449462570250034,
      "rewards/margins": 0.003042304888367653,
      "rewards/rejected": 0.007407157216221094,
      "step": 2460
    },
    {
      "epoch": 0.43,
      "learning_rate": 8.831234256926951e-08,
      "logits/chosen": -2.745732307434082,
      "logits/rejected": -2.720219135284424,
      "logps/chosen": -210.3990478515625,
      "logps/rejected": -215.95071411132812,
      "loss": 0.6925,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.007016859017312527,
      "rewards/margins": 0.0003235770855098963,
      "rewards/rejected": 0.006693282164633274,
      "step": 2470
    },
    {
      "epoch": 0.43,
      "learning_rate": 8.826196473551638e-08,
      "logits/chosen": -2.744358539581299,
      "logits/rejected": -2.735334873199463,
      "logps/chosen": -184.0402374267578,
      "logps/rejected": -155.08839416503906,
      "loss": 0.6924,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": 0.00902140885591507,
      "rewards/margins": 0.004157154820859432,
      "rewards/rejected": 0.004864254035055637,
      "step": 2480
    },
    {
      "epoch": 0.44,
      "learning_rate": 8.821158690176323e-08,
      "logits/chosen": -2.7525582313537598,
      "logits/rejected": -2.7596418857574463,
      "logps/chosen": -218.2732391357422,
      "logps/rejected": -201.02366638183594,
      "loss": 0.6921,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": 0.01294918917119503,
      "rewards/margins": 0.0058312080800533295,
      "rewards/rejected": 0.00711798295378685,
      "step": 2490
    },
    {
      "epoch": 0.44,
      "learning_rate": 8.816120906801007e-08,
      "logits/chosen": -2.754030466079712,
      "logits/rejected": -2.7371068000793457,
      "logps/chosen": -214.6430206298828,
      "logps/rejected": -185.87380981445312,
      "loss": 0.6921,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.01002354547381401,
      "rewards/margins": 0.0026681842282414436,
      "rewards/rejected": 0.00735536077991128,
      "step": 2500
    },
    {
      "epoch": 0.44,
      "learning_rate": 8.811083123425692e-08,
      "logits/chosen": -2.768505096435547,
      "logits/rejected": -2.7552056312561035,
      "logps/chosen": -227.3952178955078,
      "logps/rejected": -212.49172973632812,
      "loss": 0.6925,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.008513586595654488,
      "rewards/margins": 0.001957664964720607,
      "rewards/rejected": 0.006555921398103237,
      "step": 2510
    },
    {
      "epoch": 0.44,
      "learning_rate": 8.806045340050378e-08,
      "logits/chosen": -2.7537245750427246,
      "logits/rejected": -2.755566120147705,
      "logps/chosen": -294.26031494140625,
      "logps/rejected": -249.40060424804688,
      "loss": 0.6924,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.014117250218987465,
      "rewards/margins": 0.0032351291738450527,
      "rewards/rejected": 0.0108821215108037,
      "step": 2520
    },
    {
      "epoch": 0.44,
      "learning_rate": 8.801007556675063e-08,
      "logits/chosen": -2.787724018096924,
      "logits/rejected": -2.756760835647583,
      "logps/chosen": -284.3551330566406,
      "logps/rejected": -223.2763671875,
      "loss": 0.6927,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.009325772523880005,
      "rewards/margins": 0.0031861222814768553,
      "rewards/rejected": 0.006139650940895081,
      "step": 2530
    },
    {
      "epoch": 0.44,
      "learning_rate": 8.795969773299747e-08,
      "logits/chosen": -2.672646999359131,
      "logits/rejected": -2.7240772247314453,
      "logps/chosen": -194.29666137695312,
      "logps/rejected": -166.72015380859375,
      "loss": 0.6921,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.009259345009922981,
      "rewards/margins": 0.002904356922954321,
      "rewards/rejected": 0.006354988552629948,
      "step": 2540
    },
    {
      "epoch": 0.45,
      "learning_rate": 8.790931989924433e-08,
      "logits/chosen": -2.6944260597229004,
      "logits/rejected": -2.743349552154541,
      "logps/chosen": -232.59182739257812,
      "logps/rejected": -204.32626342773438,
      "loss": 0.6916,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.010777676478028297,
      "rewards/margins": 0.002251842524856329,
      "rewards/rejected": 0.00852583535015583,
      "step": 2550
    },
    {
      "epoch": 0.45,
      "learning_rate": 8.785894206549118e-08,
      "logits/chosen": -2.6557891368865967,
      "logits/rejected": -2.734790325164795,
      "logps/chosen": -276.6233825683594,
      "logps/rejected": -263.9059753417969,
      "loss": 0.6928,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.011154399253427982,
      "rewards/margins": 0.002435199683532119,
      "rewards/rejected": 0.008719200268387794,
      "step": 2560
    },
    {
      "epoch": 0.45,
      "learning_rate": 8.780856423173803e-08,
      "logits/chosen": -2.7870709896087646,
      "logits/rejected": -2.7531352043151855,
      "logps/chosen": -197.4013671875,
      "logps/rejected": -182.07659912109375,
      "loss": 0.693,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.008912255987524986,
      "rewards/margins": 0.00424787774682045,
      "rewards/rejected": 0.004664379172027111,
      "step": 2570
    },
    {
      "epoch": 0.45,
      "learning_rate": 8.775818639798487e-08,
      "logits/chosen": -2.7127013206481934,
      "logits/rejected": -2.784416437149048,
      "logps/chosen": -247.58114624023438,
      "logps/rejected": -234.8740997314453,
      "loss": 0.6924,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.00897514633834362,
      "rewards/margins": 0.0012818890390917659,
      "rewards/rejected": 0.0076932585798203945,
      "step": 2580
    },
    {
      "epoch": 0.45,
      "learning_rate": 8.770780856423174e-08,
      "logits/chosen": -2.750242233276367,
      "logits/rejected": -2.770826578140259,
      "logps/chosen": -231.67666625976562,
      "logps/rejected": -193.50149536132812,
      "loss": 0.6921,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.007797654718160629,
      "rewards/margins": 0.0008120322600007057,
      "rewards/rejected": 0.006985623389482498,
      "step": 2590
    },
    {
      "epoch": 0.46,
      "learning_rate": 8.765743073047859e-08,
      "logits/chosen": -2.7939043045043945,
      "logits/rejected": -2.732703685760498,
      "logps/chosen": -181.970458984375,
      "logps/rejected": -197.69735717773438,
      "loss": 0.6921,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.008163347840309143,
      "rewards/margins": 3.1356001272797585e-05,
      "rewards/rejected": 0.00813199020922184,
      "step": 2600
    },
    {
      "epoch": 0.46,
      "learning_rate": 8.760705289672545e-08,
      "logits/chosen": -2.7721047401428223,
      "logits/rejected": -2.727365016937256,
      "logps/chosen": -208.9429473876953,
      "logps/rejected": -212.2381134033203,
      "loss": 0.6921,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.008429192937910557,
      "rewards/margins": 0.002313527511432767,
      "rewards/rejected": 0.006115665193647146,
      "step": 2610
    },
    {
      "epoch": 0.46,
      "learning_rate": 8.755667506297229e-08,
      "logits/chosen": -2.7597005367279053,
      "logits/rejected": -2.754197597503662,
      "logps/chosen": -199.6351318359375,
      "logps/rejected": -197.25167846679688,
      "loss": 0.6919,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.008318079635500908,
      "rewards/margins": -0.001612034859135747,
      "rewards/rejected": 0.009930115193128586,
      "step": 2620
    },
    {
      "epoch": 0.46,
      "learning_rate": 8.750629722921914e-08,
      "logits/chosen": -2.7147693634033203,
      "logits/rejected": -2.7820794582366943,
      "logps/chosen": -217.3456573486328,
      "logps/rejected": -171.41885375976562,
      "loss": 0.6921,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": 0.007606199476867914,
      "rewards/margins": -0.00042509473860263824,
      "rewards/rejected": 0.008031293749809265,
      "step": 2630
    },
    {
      "epoch": 0.46,
      "learning_rate": 8.7455919395466e-08,
      "logits/chosen": -2.7982594966888428,
      "logits/rejected": -2.821972608566284,
      "logps/chosen": -213.735107421875,
      "logps/rejected": -194.77943420410156,
      "loss": 0.6923,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.007399165071547031,
      "rewards/margins": 0.0010075327008962631,
      "rewards/rejected": 0.006391631904989481,
      "step": 2640
    },
    {
      "epoch": 0.46,
      "learning_rate": 8.740554156171285e-08,
      "logits/chosen": -2.8446202278137207,
      "logits/rejected": -2.813603162765503,
      "logps/chosen": -252.73654174804688,
      "logps/rejected": -244.2529754638672,
      "loss": 0.6924,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.012703513726592064,
      "rewards/margins": 0.0044145723804831505,
      "rewards/rejected": 0.008288940414786339,
      "step": 2650
    },
    {
      "epoch": 0.47,
      "learning_rate": 8.735516372795969e-08,
      "logits/chosen": -2.6939728260040283,
      "logits/rejected": -2.7148683071136475,
      "logps/chosen": -245.3076629638672,
      "logps/rejected": -201.79263305664062,
      "loss": 0.6918,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.012571310624480247,
      "rewards/margins": 0.004034299403429031,
      "rewards/rejected": 0.008537010289728642,
      "step": 2660
    },
    {
      "epoch": 0.47,
      "learning_rate": 8.730478589420654e-08,
      "logits/chosen": -2.7818963527679443,
      "logits/rejected": -2.770908832550049,
      "logps/chosen": -262.9079895019531,
      "logps/rejected": -277.7655334472656,
      "loss": 0.6919,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.013273214921355247,
      "rewards/margins": 0.003978852182626724,
      "rewards/rejected": 0.009294363670051098,
      "step": 2670
    },
    {
      "epoch": 0.47,
      "learning_rate": 8.72544080604534e-08,
      "logits/chosen": -2.766702651977539,
      "logits/rejected": -2.7301106452941895,
      "logps/chosen": -262.9296875,
      "logps/rejected": -208.35519409179688,
      "loss": 0.6917,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.012347053736448288,
      "rewards/margins": 0.004885281436145306,
      "rewards/rejected": 0.007461773697286844,
      "step": 2680
    },
    {
      "epoch": 0.47,
      "learning_rate": 8.720403022670024e-08,
      "logits/chosen": -2.7742788791656494,
      "logits/rejected": -2.7576308250427246,
      "logps/chosen": -204.61465454101562,
      "logps/rejected": -202.59237670898438,
      "loss": 0.6927,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": 0.008512269705533981,
      "rewards/margins": -0.0016176451463252306,
      "rewards/rejected": 0.010129915550351143,
      "step": 2690
    },
    {
      "epoch": 0.47,
      "learning_rate": 8.71536523929471e-08,
      "logits/chosen": -2.7163846492767334,
      "logits/rejected": -2.8089888095855713,
      "logps/chosen": -248.59158325195312,
      "logps/rejected": -242.52670288085938,
      "loss": 0.692,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.012646988034248352,
      "rewards/margins": 0.0014709432143718004,
      "rewards/rejected": 0.011176046915352345,
      "step": 2700
    },
    {
      "epoch": 0.47,
      "learning_rate": 8.710327455919396e-08,
      "logits/chosen": -2.7283523082733154,
      "logits/rejected": -2.709995746612549,
      "logps/chosen": -247.4280242919922,
      "logps/rejected": -207.9947052001953,
      "loss": 0.6922,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.009775625541806221,
      "rewards/margins": -0.0002165736659662798,
      "rewards/rejected": 0.009992199949920177,
      "step": 2710
    },
    {
      "epoch": 0.48,
      "learning_rate": 8.705289672544081e-08,
      "logits/chosen": -2.8027420043945312,
      "logits/rejected": -2.8436694145202637,
      "logps/chosen": -202.6864471435547,
      "logps/rejected": -221.2021026611328,
      "loss": 0.6925,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.007427896372973919,
      "rewards/margins": -0.0015940333250910044,
      "rewards/rejected": 0.009021928533911705,
      "step": 2720
    },
    {
      "epoch": 0.48,
      "learning_rate": 8.700251889168765e-08,
      "logits/chosen": -2.680032253265381,
      "logits/rejected": -2.7500858306884766,
      "logps/chosen": -225.693359375,
      "logps/rejected": -206.8023223876953,
      "loss": 0.6926,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.00921904668211937,
      "rewards/margins": 0.0021597917657345533,
      "rewards/rejected": 0.007059253752231598,
      "step": 2730
    },
    {
      "epoch": 0.48,
      "learning_rate": 8.69521410579345e-08,
      "logits/chosen": -2.7341065406799316,
      "logits/rejected": -2.755167245864868,
      "logps/chosen": -193.05245971679688,
      "logps/rejected": -170.5665740966797,
      "loss": 0.6923,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": 0.005958110094070435,
      "rewards/margins": 0.0001765273918863386,
      "rewards/rejected": 0.005781582556664944,
      "step": 2740
    },
    {
      "epoch": 0.48,
      "learning_rate": 8.690176322418136e-08,
      "logits/chosen": -2.7305095195770264,
      "logits/rejected": -2.7078235149383545,
      "logps/chosen": -233.4241485595703,
      "logps/rejected": -181.7233123779297,
      "loss": 0.6922,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.010703453794121742,
      "rewards/margins": 0.002297390252351761,
      "rewards/rejected": 0.008406064473092556,
      "step": 2750
    },
    {
      "epoch": 0.48,
      "learning_rate": 8.685138539042821e-08,
      "logits/chosen": -2.761038303375244,
      "logits/rejected": -2.6922402381896973,
      "logps/chosen": -191.82229614257812,
      "logps/rejected": -203.77139282226562,
      "loss": 0.6924,
      "rewards/accuracies": 0.42500001192092896,
      "rewards/chosen": 0.008268560282886028,
      "rewards/margins": 6.912881508469582e-05,
      "rewards/rejected": 0.00819943193346262,
      "step": 2760
    },
    {
      "epoch": 0.49,
      "learning_rate": 8.680100755667505e-08,
      "logits/chosen": -2.7586519718170166,
      "logits/rejected": -2.695542097091675,
      "logps/chosen": -230.31625366210938,
      "logps/rejected": -218.05117797851562,
      "loss": 0.6927,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.00877605751156807,
      "rewards/margins": 0.0006737986695952713,
      "rewards/rejected": 0.008102258667349815,
      "step": 2770
    },
    {
      "epoch": 0.49,
      "learning_rate": 8.67506297229219e-08,
      "logits/chosen": -2.7693123817443848,
      "logits/rejected": -2.8033013343811035,
      "logps/chosen": -268.99603271484375,
      "logps/rejected": -246.95742797851562,
      "loss": 0.6915,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.015248882584273815,
      "rewards/margins": 0.0025188270956277847,
      "rewards/rejected": 0.012730054557323456,
      "step": 2780
    },
    {
      "epoch": 0.49,
      "learning_rate": 8.670025188916876e-08,
      "logits/chosen": -2.764688014984131,
      "logits/rejected": -2.758216619491577,
      "logps/chosen": -206.0214080810547,
      "logps/rejected": -178.74722290039062,
      "loss": 0.6922,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.01020827703177929,
      "rewards/margins": 0.0022834674455225468,
      "rewards/rejected": 0.00792481005191803,
      "step": 2790
    },
    {
      "epoch": 0.49,
      "learning_rate": 8.664987405541561e-08,
      "logits/chosen": -2.723081350326538,
      "logits/rejected": -2.7940492630004883,
      "logps/chosen": -203.63230895996094,
      "logps/rejected": -179.57211303710938,
      "loss": 0.6918,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.00934748724102974,
      "rewards/margins": 0.0031101994682103395,
      "rewards/rejected": 0.0062372866086661816,
      "step": 2800
    },
    {
      "epoch": 0.49,
      "learning_rate": 8.659949622166247e-08,
      "logits/chosen": -2.743274211883545,
      "logits/rejected": -2.7649941444396973,
      "logps/chosen": -209.7469024658203,
      "logps/rejected": -220.0643768310547,
      "loss": 0.6921,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.008425967767834663,
      "rewards/margins": 0.0009932437678799033,
      "rewards/rejected": 0.007432723883539438,
      "step": 2810
    },
    {
      "epoch": 0.49,
      "learning_rate": 8.654911838790932e-08,
      "logits/chosen": -2.7621662616729736,
      "logits/rejected": -2.7177906036376953,
      "logps/chosen": -215.0526580810547,
      "logps/rejected": -190.55581665039062,
      "loss": 0.6922,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.010186800733208656,
      "rewards/margins": 0.0018928442150354385,
      "rewards/rejected": 0.008293956518173218,
      "step": 2820
    },
    {
      "epoch": 0.5,
      "learning_rate": 8.649874055415617e-08,
      "logits/chosen": -2.8004374504089355,
      "logits/rejected": -2.7490596771240234,
      "logps/chosen": -252.2415008544922,
      "logps/rejected": -221.1431427001953,
      "loss": 0.6919,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.012386414222419262,
      "rewards/margins": 0.0008427410502918065,
      "rewards/rejected": 0.01154367346316576,
      "step": 2830
    },
    {
      "epoch": 0.5,
      "learning_rate": 8.644836272040303e-08,
      "logits/chosen": -2.810251235961914,
      "logits/rejected": -2.7899842262268066,
      "logps/chosen": -239.23046875,
      "logps/rejected": -231.6127471923828,
      "loss": 0.6926,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.012533411383628845,
      "rewards/margins": 0.0014264009660109878,
      "rewards/rejected": 0.011107010766863823,
      "step": 2840
    },
    {
      "epoch": 0.5,
      "learning_rate": 8.639798488664987e-08,
      "logits/chosen": -2.784536600112915,
      "logits/rejected": -2.7550041675567627,
      "logps/chosen": -248.12338256835938,
      "logps/rejected": -197.28073120117188,
      "loss": 0.6915,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.014371541328728199,
      "rewards/margins": 0.003601966891437769,
      "rewards/rejected": 0.010769573971629143,
      "step": 2850
    },
    {
      "epoch": 0.5,
      "learning_rate": 8.634760705289672e-08,
      "logits/chosen": -2.8218891620635986,
      "logits/rejected": -2.742927074432373,
      "logps/chosen": -196.79637145996094,
      "logps/rejected": -190.49765014648438,
      "loss": 0.6923,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.00799397099763155,
      "rewards/margins": -0.001486542634665966,
      "rewards/rejected": 0.009480513632297516,
      "step": 2860
    },
    {
      "epoch": 0.5,
      "learning_rate": 8.629722921914358e-08,
      "logits/chosen": -2.766782522201538,
      "logits/rejected": -2.7526514530181885,
      "logps/chosen": -237.14126586914062,
      "logps/rejected": -223.09487915039062,
      "loss": 0.6926,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.010379480198025703,
      "rewards/margins": 0.003819843288511038,
      "rewards/rejected": 0.006559636443853378,
      "step": 2870
    },
    {
      "epoch": 0.5,
      "learning_rate": 8.624685138539043e-08,
      "logits/chosen": -2.694092273712158,
      "logits/rejected": -2.7033393383026123,
      "logps/chosen": -186.11802673339844,
      "logps/rejected": -168.0809326171875,
      "loss": 0.6917,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.010750077664852142,
      "rewards/margins": 0.00394693948328495,
      "rewards/rejected": 0.006803138647228479,
      "step": 2880
    },
    {
      "epoch": 0.51,
      "learning_rate": 8.619647355163727e-08,
      "logits/chosen": -2.7816433906555176,
      "logits/rejected": -2.6702535152435303,
      "logps/chosen": -207.6709747314453,
      "logps/rejected": -180.0514678955078,
      "loss": 0.6912,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.011553024873137474,
      "rewards/margins": 0.00432405062019825,
      "rewards/rejected": 0.007228974252939224,
      "step": 2890
    },
    {
      "epoch": 0.51,
      "learning_rate": 8.614609571788412e-08,
      "logits/chosen": -2.7347779273986816,
      "logits/rejected": -2.7691762447357178,
      "logps/chosen": -230.37808227539062,
      "logps/rejected": -183.2679901123047,
      "loss": 0.6913,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.009555203840136528,
      "rewards/margins": 0.005391639657318592,
      "rewards/rejected": 0.004163564182817936,
      "step": 2900
    },
    {
      "epoch": 0.51,
      "learning_rate": 8.609571788413098e-08,
      "logits/chosen": -2.8352231979370117,
      "logits/rejected": -2.7687888145446777,
      "logps/chosen": -208.2872772216797,
      "logps/rejected": -177.54238891601562,
      "loss": 0.6922,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.007140007801353931,
      "rewards/margins": 0.00021615605510305613,
      "rewards/rejected": 0.006923851557075977,
      "step": 2910
    },
    {
      "epoch": 0.51,
      "learning_rate": 8.604534005037783e-08,
      "logits/chosen": -2.801035165786743,
      "logits/rejected": -2.7708606719970703,
      "logps/chosen": -267.33074951171875,
      "logps/rejected": -247.96484375,
      "loss": 0.6919,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.012757198885083199,
      "rewards/margins": 0.003161564003676176,
      "rewards/rejected": 0.00959563534706831,
      "step": 2920
    },
    {
      "epoch": 0.51,
      "learning_rate": 8.599496221662468e-08,
      "logits/chosen": -2.798997163772583,
      "logits/rejected": -2.7465548515319824,
      "logps/chosen": -221.67544555664062,
      "logps/rejected": -173.8419952392578,
      "loss": 0.6921,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0111413998529315,
      "rewards/margins": 0.0030490579083561897,
      "rewards/rejected": 0.008092342875897884,
      "step": 2930
    },
    {
      "epoch": 0.51,
      "learning_rate": 8.594458438287154e-08,
      "logits/chosen": -2.745358943939209,
      "logits/rejected": -2.7377078533172607,
      "logps/chosen": -241.1657257080078,
      "logps/rejected": -220.897705078125,
      "loss": 0.6917,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.007529490627348423,
      "rewards/margins": -6.388258043443784e-05,
      "rewards/rejected": 0.00759337330237031,
      "step": 2940
    },
    {
      "epoch": 0.52,
      "learning_rate": 8.589420654911839e-08,
      "logits/chosen": -2.70111346244812,
      "logits/rejected": -2.75768780708313,
      "logps/chosen": -242.2528076171875,
      "logps/rejected": -191.04638671875,
      "loss": 0.6916,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": 0.013275270350277424,
      "rewards/margins": 0.005159012973308563,
      "rewards/rejected": 0.00811625737696886,
      "step": 2950
    },
    {
      "epoch": 0.52,
      "learning_rate": 8.584382871536524e-08,
      "logits/chosen": -2.7602486610412598,
      "logits/rejected": -2.752366065979004,
      "logps/chosen": -266.2757263183594,
      "logps/rejected": -203.05038452148438,
      "loss": 0.6925,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.01176394708454609,
      "rewards/margins": 0.002135035116225481,
      "rewards/rejected": 0.009628912433981895,
      "step": 2960
    },
    {
      "epoch": 0.52,
      "learning_rate": 8.579345088161209e-08,
      "logits/chosen": -2.7365550994873047,
      "logits/rejected": -2.801192045211792,
      "logps/chosen": -237.6589813232422,
      "logps/rejected": -185.87973022460938,
      "loss": 0.6913,
      "rewards/accuracies": 0.42500001192092896,
      "rewards/chosen": 0.013516830280423164,
      "rewards/margins": 0.0016924828523769975,
      "rewards/rejected": 0.01182434894144535,
      "step": 2970
    },
    {
      "epoch": 0.52,
      "learning_rate": 8.574307304785894e-08,
      "logits/chosen": -2.7330987453460693,
      "logits/rejected": -2.7530648708343506,
      "logps/chosen": -234.9535675048828,
      "logps/rejected": -240.99813842773438,
      "loss": 0.6917,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.015216231346130371,
      "rewards/margins": 0.004241775255650282,
      "rewards/rejected": 0.010974457487463951,
      "step": 2980
    },
    {
      "epoch": 0.52,
      "learning_rate": 8.569269521410579e-08,
      "logits/chosen": -2.765040397644043,
      "logits/rejected": -2.7268776893615723,
      "logps/chosen": -204.8981475830078,
      "logps/rejected": -212.19705200195312,
      "loss": 0.6916,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.013496240600943565,
      "rewards/margins": 0.0013568944996222854,
      "rewards/rejected": 0.012139346450567245,
      "step": 2990
    },
    {
      "epoch": 0.53,
      "learning_rate": 8.564231738035263e-08,
      "logits/chosen": -2.8201658725738525,
      "logits/rejected": -2.7453672885894775,
      "logps/chosen": -232.99221801757812,
      "logps/rejected": -246.0234832763672,
      "loss": 0.6916,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.014087246730923653,
      "rewards/margins": 0.0024903384037315845,
      "rewards/rejected": 0.01159690786153078,
      "step": 3000
    },
    {
      "epoch": 0.53,
      "learning_rate": 8.559193954659949e-08,
      "logits/chosen": -2.8084664344787598,
      "logits/rejected": -2.7652931213378906,
      "logps/chosen": -276.5728454589844,
      "logps/rejected": -220.3472900390625,
      "loss": 0.6917,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.012886193580925465,
      "rewards/margins": 0.0032176401000469923,
      "rewards/rejected": 0.009668553248047829,
      "step": 3010
    },
    {
      "epoch": 0.53,
      "learning_rate": 8.554156171284634e-08,
      "logits/chosen": -2.7719342708587646,
      "logits/rejected": -2.784611701965332,
      "logps/chosen": -238.6121826171875,
      "logps/rejected": -219.5137939453125,
      "loss": 0.692,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.010869468562304974,
      "rewards/margins": 0.0018772600451484323,
      "rewards/rejected": 0.00899220909923315,
      "step": 3020
    },
    {
      "epoch": 0.53,
      "learning_rate": 8.54911838790932e-08,
      "logits/chosen": -2.8003268241882324,
      "logits/rejected": -2.795473098754883,
      "logps/chosen": -229.3064422607422,
      "logps/rejected": -202.93734741210938,
      "loss": 0.6921,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.011820869520306587,
      "rewards/margins": 0.003250813577324152,
      "rewards/rejected": 0.008570056408643723,
      "step": 3030
    },
    {
      "epoch": 0.53,
      "learning_rate": 8.544080604534005e-08,
      "logits/chosen": -2.7847442626953125,
      "logits/rejected": -2.734316349029541,
      "logps/chosen": -201.69662475585938,
      "logps/rejected": -186.3741455078125,
      "loss": 0.6918,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.010374282486736774,
      "rewards/margins": 0.0019776958506554365,
      "rewards/rejected": 0.008396586403250694,
      "step": 3040
    },
    {
      "epoch": 0.53,
      "learning_rate": 8.53904282115869e-08,
      "logits/chosen": -2.7927279472351074,
      "logits/rejected": -2.799988269805908,
      "logps/chosen": -238.63778686523438,
      "logps/rejected": -213.6460723876953,
      "loss": 0.692,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.015687134116888046,
      "rewards/margins": 0.0027517948765307665,
      "rewards/rejected": 0.012935337610542774,
      "step": 3050
    },
    {
      "epoch": 0.54,
      "learning_rate": 8.534005037783375e-08,
      "logits/chosen": -2.6211276054382324,
      "logits/rejected": -2.582188606262207,
      "logps/chosen": -215.00106811523438,
      "logps/rejected": -182.13609313964844,
      "loss": 0.6923,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.011294371448457241,
      "rewards/margins": 0.001207565190270543,
      "rewards/rejected": 0.010086806491017342,
      "step": 3060
    },
    {
      "epoch": 0.54,
      "learning_rate": 8.528967254408061e-08,
      "logits/chosen": -2.757004737854004,
      "logits/rejected": -2.819514036178589,
      "logps/chosen": -221.3273162841797,
      "logps/rejected": -190.31556701660156,
      "loss": 0.6921,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.011431792750954628,
      "rewards/margins": 0.001231744303368032,
      "rewards/rejected": 0.010200048796832561,
      "step": 3070
    },
    {
      "epoch": 0.54,
      "learning_rate": 8.523929471032745e-08,
      "logits/chosen": -2.7367422580718994,
      "logits/rejected": -2.7337427139282227,
      "logps/chosen": -263.4180603027344,
      "logps/rejected": -230.3423309326172,
      "loss": 0.6916,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.015658225864171982,
      "rewards/margins": 0.0033743069507181644,
      "rewards/rejected": 0.012283921241760254,
      "step": 3080
    },
    {
      "epoch": 0.54,
      "learning_rate": 8.51889168765743e-08,
      "logits/chosen": -2.800112247467041,
      "logits/rejected": -2.7367866039276123,
      "logps/chosen": -218.00650024414062,
      "logps/rejected": -169.1083526611328,
      "loss": 0.6923,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.012381301261484623,
      "rewards/margins": 0.005484296940267086,
      "rewards/rejected": 0.006897004786878824,
      "step": 3090
    },
    {
      "epoch": 0.54,
      "learning_rate": 8.513853904282116e-08,
      "logits/chosen": -2.7532482147216797,
      "logits/rejected": -2.7856948375701904,
      "logps/chosen": -257.03302001953125,
      "logps/rejected": -208.6194305419922,
      "loss": 0.6925,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.014161042869091034,
      "rewards/margins": 0.002776671200990677,
      "rewards/rejected": 0.011384371668100357,
      "step": 3100
    },
    {
      "epoch": 0.54,
      "learning_rate": 8.508816120906801e-08,
      "logits/chosen": -2.7750141620635986,
      "logits/rejected": -2.7644925117492676,
      "logps/chosen": -237.3404541015625,
      "logps/rejected": -179.40928649902344,
      "loss": 0.692,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.010464548133313656,
      "rewards/margins": -0.0005554911913350224,
      "rewards/rejected": 0.011020039208233356,
      "step": 3110
    },
    {
      "epoch": 0.55,
      "learning_rate": 8.503778337531485e-08,
      "logits/chosen": -2.740729808807373,
      "logits/rejected": -2.842524766921997,
      "logps/chosen": -237.0664520263672,
      "logps/rejected": -211.84603881835938,
      "loss": 0.6909,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.011694247834384441,
      "rewards/margins": -0.0019409225787967443,
      "rewards/rejected": 0.01363517064601183,
      "step": 3120
    },
    {
      "epoch": 0.55,
      "learning_rate": 8.49874055415617e-08,
      "logits/chosen": -2.7111763954162598,
      "logits/rejected": -2.7561068534851074,
      "logps/chosen": -263.2214660644531,
      "logps/rejected": -206.15432739257812,
      "loss": 0.6909,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.013094994239509106,
      "rewards/margins": 0.0035277139395475388,
      "rewards/rejected": 0.009567281231284142,
      "step": 3130
    },
    {
      "epoch": 0.55,
      "learning_rate": 8.493702770780856e-08,
      "logits/chosen": -2.720445394515991,
      "logits/rejected": -2.7390005588531494,
      "logps/chosen": -226.5557098388672,
      "logps/rejected": -216.19784545898438,
      "loss": 0.6922,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.012567183002829552,
      "rewards/margins": 0.0029570916667580605,
      "rewards/rejected": 0.009610090404748917,
      "step": 3140
    },
    {
      "epoch": 0.55,
      "learning_rate": 8.488664987405541e-08,
      "logits/chosen": -2.736865758895874,
      "logits/rejected": -2.7989611625671387,
      "logps/chosen": -270.09747314453125,
      "logps/rejected": -230.9444122314453,
      "loss": 0.6917,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.014618085697293282,
      "rewards/margins": 0.003351908642798662,
      "rewards/rejected": 0.011266176588833332,
      "step": 3150
    },
    {
      "epoch": 0.55,
      "learning_rate": 8.483627204030226e-08,
      "logits/chosen": -2.685662031173706,
      "logits/rejected": -2.750356912612915,
      "logps/chosen": -271.740234375,
      "logps/rejected": -247.66781616210938,
      "loss": 0.6918,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": 0.01378130353987217,
      "rewards/margins": 0.0003704793634824455,
      "rewards/rejected": 0.013410824351012707,
      "step": 3160
    },
    {
      "epoch": 0.56,
      "learning_rate": 8.478589420654912e-08,
      "logits/chosen": -2.7123258113861084,
      "logits/rejected": -2.8179118633270264,
      "logps/chosen": -233.6159210205078,
      "logps/rejected": -223.5303955078125,
      "loss": 0.6915,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.014396825805306435,
      "rewards/margins": 0.003176445607095957,
      "rewards/rejected": 0.01122037973254919,
      "step": 3170
    },
    {
      "epoch": 0.56,
      "learning_rate": 8.473551637279597e-08,
      "logits/chosen": -2.7784855365753174,
      "logits/rejected": -2.7305197715759277,
      "logps/chosen": -258.8409118652344,
      "logps/rejected": -214.46981811523438,
      "loss": 0.6914,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.01418989896774292,
      "rewards/margins": 0.0008127937326207757,
      "rewards/rejected": 0.013377104885876179,
      "step": 3180
    },
    {
      "epoch": 0.56,
      "learning_rate": 8.468513853904283e-08,
      "logits/chosen": -2.7307345867156982,
      "logits/rejected": -2.750396251678467,
      "logps/chosen": -214.94189453125,
      "logps/rejected": -179.4948272705078,
      "loss": 0.6911,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.012370200827717781,
      "rewards/margins": 0.0020483885891735554,
      "rewards/rejected": 0.010321812704205513,
      "step": 3190
    },
    {
      "epoch": 0.56,
      "learning_rate": 8.463476070528967e-08,
      "logits/chosen": -2.716055154800415,
      "logits/rejected": -2.805588722229004,
      "logps/chosen": -236.4894256591797,
      "logps/rejected": -175.95303344726562,
      "loss": 0.6919,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.012703245505690575,
      "rewards/margins": 0.0037045180797576904,
      "rewards/rejected": 0.008998729288578033,
      "step": 3200
    },
    {
      "epoch": 0.56,
      "learning_rate": 8.458438287153652e-08,
      "logits/chosen": -2.7577478885650635,
      "logits/rejected": -2.724205493927002,
      "logps/chosen": -223.320556640625,
      "logps/rejected": -222.6649932861328,
      "loss": 0.6917,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.010703946463763714,
      "rewards/margins": 0.0017586011672392488,
      "rewards/rejected": 0.0089453449472785,
      "step": 3210
    },
    {
      "epoch": 0.56,
      "learning_rate": 8.453400503778337e-08,
      "logits/chosen": -2.7715744972229004,
      "logits/rejected": -2.7677001953125,
      "logps/chosen": -298.09503173828125,
      "logps/rejected": -261.20379638671875,
      "loss": 0.692,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.01782422699034214,
      "rewards/margins": 0.004209473729133606,
      "rewards/rejected": 0.013614753261208534,
      "step": 3220
    },
    {
      "epoch": 0.57,
      "learning_rate": 8.448362720403023e-08,
      "logits/chosen": -2.7621097564697266,
      "logits/rejected": -2.7440028190612793,
      "logps/chosen": -214.5993194580078,
      "logps/rejected": -223.1764678955078,
      "loss": 0.6918,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.011158348992466927,
      "rewards/margins": 0.004007419571280479,
      "rewards/rejected": 0.007150929421186447,
      "step": 3230
    },
    {
      "epoch": 0.57,
      "learning_rate": 8.443324937027707e-08,
      "logits/chosen": -2.788442611694336,
      "logits/rejected": -2.7765183448791504,
      "logps/chosen": -225.32937622070312,
      "logps/rejected": -209.044677734375,
      "loss": 0.6916,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.014530852437019348,
      "rewards/margins": 0.00227823993191123,
      "rewards/rejected": 0.01225261203944683,
      "step": 3240
    },
    {
      "epoch": 0.57,
      "learning_rate": 8.438287153652392e-08,
      "logits/chosen": -2.7100307941436768,
      "logits/rejected": -2.7646098136901855,
      "logps/chosen": -236.94656372070312,
      "logps/rejected": -199.8778533935547,
      "loss": 0.6908,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.01314136665314436,
      "rewards/margins": 0.004657679237425327,
      "rewards/rejected": 0.008483686484396458,
      "step": 3250
    },
    {
      "epoch": 0.57,
      "learning_rate": 8.433249370277077e-08,
      "logits/chosen": -2.819563865661621,
      "logits/rejected": -2.7609450817108154,
      "logps/chosen": -253.8069305419922,
      "logps/rejected": -241.51123046875,
      "loss": 0.6923,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.012674269266426563,
      "rewards/margins": 0.0009184217196889222,
      "rewards/rejected": 0.011755848303437233,
      "step": 3260
    },
    {
      "epoch": 0.57,
      "learning_rate": 8.428211586901763e-08,
      "logits/chosen": -2.685859203338623,
      "logits/rejected": -2.7351815700531006,
      "logps/chosen": -227.080810546875,
      "logps/rejected": -203.94802856445312,
      "loss": 0.6917,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.014903845265507698,
      "rewards/margins": 0.0018497860291972756,
      "rewards/rejected": 0.013054059818387032,
      "step": 3270
    },
    {
      "epoch": 0.57,
      "learning_rate": 8.423173803526448e-08,
      "logits/chosen": -2.7697246074676514,
      "logits/rejected": -2.8137755393981934,
      "logps/chosen": -251.1637725830078,
      "logps/rejected": -195.75613403320312,
      "loss": 0.6918,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.012458008714020252,
      "rewards/margins": 0.004305009264498949,
      "rewards/rejected": 0.008152998983860016,
      "step": 3280
    },
    {
      "epoch": 0.58,
      "learning_rate": 8.418136020151134e-08,
      "logits/chosen": -2.773317813873291,
      "logits/rejected": -2.779855489730835,
      "logps/chosen": -223.1930389404297,
      "logps/rejected": -198.455322265625,
      "loss": 0.6925,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.011802679859101772,
      "rewards/margins": 0.0005634138360619545,
      "rewards/rejected": 0.011239266023039818,
      "step": 3290
    },
    {
      "epoch": 0.58,
      "learning_rate": 8.413098236775819e-08,
      "logits/chosen": -2.7221474647521973,
      "logits/rejected": -2.702664375305176,
      "logps/chosen": -274.08392333984375,
      "logps/rejected": -247.7399444580078,
      "loss": 0.6915,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.019806358963251114,
      "rewards/margins": 0.0065388367511332035,
      "rewards/rejected": 0.013267521746456623,
      "step": 3300
    },
    {
      "epoch": 0.58,
      "learning_rate": 8.408060453400503e-08,
      "logits/chosen": -2.8209447860717773,
      "logits/rejected": -2.834242582321167,
      "logps/chosen": -247.25802612304688,
      "logps/rejected": -201.07403564453125,
      "loss": 0.692,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.016469743102788925,
      "rewards/margins": 0.004864540882408619,
      "rewards/rejected": 0.011605201289057732,
      "step": 3310
    },
    {
      "epoch": 0.58,
      "learning_rate": 8.403022670025188e-08,
      "logits/chosen": -2.7461419105529785,
      "logits/rejected": -2.732264280319214,
      "logps/chosen": -227.6762237548828,
      "logps/rejected": -209.2851104736328,
      "loss": 0.691,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.012286007404327393,
      "rewards/margins": 0.0037416242994368076,
      "rewards/rejected": 0.008544383570551872,
      "step": 3320
    },
    {
      "epoch": 0.58,
      "learning_rate": 8.397984886649874e-08,
      "logits/chosen": -2.713840961456299,
      "logits/rejected": -2.692565441131592,
      "logps/chosen": -228.81851196289062,
      "logps/rejected": -199.67190551757812,
      "loss": 0.6916,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.017076753079891205,
      "rewards/margins": 0.006653298623859882,
      "rewards/rejected": 0.010423455387353897,
      "step": 3330
    },
    {
      "epoch": 0.59,
      "learning_rate": 8.392947103274559e-08,
      "logits/chosen": -2.811196804046631,
      "logits/rejected": -2.7759921550750732,
      "logps/chosen": -237.18215942382812,
      "logps/rejected": -184.60023498535156,
      "loss": 0.6912,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.015599017962813377,
      "rewards/margins": 0.004597905091941357,
      "rewards/rejected": 0.011001111008226871,
      "step": 3340
    },
    {
      "epoch": 0.59,
      "learning_rate": 8.387909319899243e-08,
      "logits/chosen": -2.712432861328125,
      "logits/rejected": -2.8248891830444336,
      "logps/chosen": -284.87091064453125,
      "logps/rejected": -221.70162963867188,
      "loss": 0.6911,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.015140047296881676,
      "rewards/margins": -0.00015327543951570988,
      "rewards/rejected": 0.015293324366211891,
      "step": 3350
    },
    {
      "epoch": 0.59,
      "learning_rate": 8.382871536523928e-08,
      "logits/chosen": -2.778764009475708,
      "logits/rejected": -2.8072402477264404,
      "logps/chosen": -249.6968536376953,
      "logps/rejected": -218.6016082763672,
      "loss": 0.6911,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.01566213183104992,
      "rewards/margins": 0.0012918005231767893,
      "rewards/rejected": 0.014370331540703773,
      "step": 3360
    },
    {
      "epoch": 0.59,
      "learning_rate": 8.377833753148614e-08,
      "logits/chosen": -2.770099639892578,
      "logits/rejected": -2.793405771255493,
      "logps/chosen": -242.748291015625,
      "logps/rejected": -207.92636108398438,
      "loss": 0.6917,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.0115726999938488,
      "rewards/margins": 0.004362519830465317,
      "rewards/rejected": 0.0072101810947060585,
      "step": 3370
    },
    {
      "epoch": 0.59,
      "learning_rate": 8.372795969773299e-08,
      "logits/chosen": -2.7587761878967285,
      "logits/rejected": -2.8187568187713623,
      "logps/chosen": -288.847412109375,
      "logps/rejected": -241.4041290283203,
      "loss": 0.6924,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.019069483503699303,
      "rewards/margins": 0.003561516525223851,
      "rewards/rejected": 0.01550796814262867,
      "step": 3380
    },
    {
      "epoch": 0.59,
      "learning_rate": 8.367758186397985e-08,
      "logits/chosen": -2.773113489151001,
      "logits/rejected": -2.843980312347412,
      "logps/chosen": -235.67578125,
      "logps/rejected": -217.6513214111328,
      "loss": 0.6924,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.011275621131062508,
      "rewards/margins": -0.0001427721290383488,
      "rewards/rejected": 0.01141839288175106,
      "step": 3390
    },
    {
      "epoch": 0.6,
      "learning_rate": 8.36272040302267e-08,
      "logits/chosen": -2.858863115310669,
      "logits/rejected": -2.758176326751709,
      "logps/chosen": -248.9114227294922,
      "logps/rejected": -233.37063598632812,
      "loss": 0.6912,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.019003715366125107,
      "rewards/margins": 0.0045727817341685295,
      "rewards/rejected": 0.014430937357246876,
      "step": 3400
    },
    {
      "epoch": 0.6,
      "learning_rate": 8.357682619647355e-08,
      "logits/chosen": -2.7978243827819824,
      "logits/rejected": -2.765048027038574,
      "logps/chosen": -215.2990264892578,
      "logps/rejected": -195.3312225341797,
      "loss": 0.692,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.012433426454663277,
      "rewards/margins": 0.0025961692444980145,
      "rewards/rejected": 0.009837256744503975,
      "step": 3410
    },
    {
      "epoch": 0.6,
      "learning_rate": 8.35264483627204e-08,
      "logits/chosen": -2.8078112602233887,
      "logits/rejected": -2.7712979316711426,
      "logps/chosen": -225.05715942382812,
      "logps/rejected": -191.1551971435547,
      "loss": 0.6917,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.011315623298287392,
      "rewards/margins": 0.004302586428821087,
      "rewards/rejected": 0.007013037800788879,
      "step": 3420
    },
    {
      "epoch": 0.6,
      "learning_rate": 8.347607052896725e-08,
      "logits/chosen": -2.701322078704834,
      "logits/rejected": -2.6965625286102295,
      "logps/chosen": -190.59228515625,
      "logps/rejected": -183.9395294189453,
      "loss": 0.6921,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.013294106349349022,
      "rewards/margins": 0.0023096068762242794,
      "rewards/rejected": 0.01098449993878603,
      "step": 3430
    },
    {
      "epoch": 0.6,
      "learning_rate": 8.34256926952141e-08,
      "logits/chosen": -2.743400812149048,
      "logits/rejected": -2.7452921867370605,
      "logps/chosen": -258.9476318359375,
      "logps/rejected": -218.33700561523438,
      "loss": 0.6913,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.014264727011322975,
      "rewards/margins": 0.0023330049589276314,
      "rewards/rejected": 0.011931722052395344,
      "step": 3440
    },
    {
      "epoch": 0.6,
      "learning_rate": 8.337531486146095e-08,
      "logits/chosen": -2.759187698364258,
      "logits/rejected": -2.7136118412017822,
      "logps/chosen": -221.3391876220703,
      "logps/rejected": -180.05746459960938,
      "loss": 0.692,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.013907767832279205,
      "rewards/margins": 0.004912000615149736,
      "rewards/rejected": 0.008995766751468182,
      "step": 3450
    },
    {
      "epoch": 0.61,
      "learning_rate": 8.332493702770781e-08,
      "logits/chosen": -2.671977996826172,
      "logits/rejected": -2.743464946746826,
      "logps/chosen": -240.4029083251953,
      "logps/rejected": -222.5838623046875,
      "loss": 0.6921,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.011776207946240902,
      "rewards/margins": 0.0016693586949259043,
      "rewards/rejected": 0.010106848552823067,
      "step": 3460
    },
    {
      "epoch": 0.61,
      "learning_rate": 8.327455919395465e-08,
      "logits/chosen": -2.760468006134033,
      "logits/rejected": -2.7636971473693848,
      "logps/chosen": -210.5015106201172,
      "logps/rejected": -191.4309539794922,
      "loss": 0.6913,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.012788991443812847,
      "rewards/margins": 0.0014230671804398298,
      "rewards/rejected": 0.011365924030542374,
      "step": 3470
    },
    {
      "epoch": 0.61,
      "learning_rate": 8.32241813602015e-08,
      "logits/chosen": -2.7345573902130127,
      "logits/rejected": -2.7453200817108154,
      "logps/chosen": -260.7548828125,
      "logps/rejected": -212.4744110107422,
      "loss": 0.6919,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.016130229458212852,
      "rewards/margins": 0.0023641791194677353,
      "rewards/rejected": 0.013766050338745117,
      "step": 3480
    },
    {
      "epoch": 0.61,
      "learning_rate": 8.317380352644836e-08,
      "logits/chosen": -2.7350945472717285,
      "logits/rejected": -2.7537972927093506,
      "logps/chosen": -261.77496337890625,
      "logps/rejected": -223.4450225830078,
      "loss": 0.6917,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.016082722693681717,
      "rewards/margins": 0.0003509976086206734,
      "rewards/rejected": 0.015731725841760635,
      "step": 3490
    },
    {
      "epoch": 0.61,
      "learning_rate": 8.312342569269521e-08,
      "logits/chosen": -2.717952013015747,
      "logits/rejected": -2.6623072624206543,
      "logps/chosen": -249.53726196289062,
      "logps/rejected": -235.1985321044922,
      "loss": 0.692,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.016532665118575096,
      "rewards/margins": 0.003554257797077298,
      "rewards/rejected": 0.012978407554328442,
      "step": 3500
    },
    {
      "epoch": 0.61,
      "learning_rate": 8.307304785894206e-08,
      "logits/chosen": -2.788193702697754,
      "logits/rejected": -2.796319007873535,
      "logps/chosen": -226.8190155029297,
      "logps/rejected": -194.38963317871094,
      "loss": 0.6909,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.016295744106173515,
      "rewards/margins": 0.00471791485324502,
      "rewards/rejected": 0.011577830649912357,
      "step": 3510
    },
    {
      "epoch": 0.62,
      "learning_rate": 8.302267002518892e-08,
      "logits/chosen": -2.719542980194092,
      "logits/rejected": -2.699769973754883,
      "logps/chosen": -219.9004669189453,
      "logps/rejected": -181.20230102539062,
      "loss": 0.6921,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.010805217549204826,
      "rewards/margins": 0.002578904153779149,
      "rewards/rejected": 0.008226314559578896,
      "step": 3520
    },
    {
      "epoch": 0.62,
      "learning_rate": 8.297229219143577e-08,
      "logits/chosen": -2.7711305618286133,
      "logits/rejected": -2.761723756790161,
      "logps/chosen": -265.3699951171875,
      "logps/rejected": -235.4482421875,
      "loss": 0.6915,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": 0.015871699899435043,
      "rewards/margins": 0.0017049426678568125,
      "rewards/rejected": 0.014166759327054024,
      "step": 3530
    },
    {
      "epoch": 0.62,
      "learning_rate": 8.292191435768262e-08,
      "logits/chosen": -2.7074053287506104,
      "logits/rejected": -2.709003210067749,
      "logps/chosen": -230.49472045898438,
      "logps/rejected": -194.09027099609375,
      "loss": 0.691,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.01634136773645878,
      "rewards/margins": 0.006125512532889843,
      "rewards/rejected": 0.01021585427224636,
      "step": 3540
    },
    {
      "epoch": 0.62,
      "learning_rate": 8.287153652392946e-08,
      "logits/chosen": -2.709663152694702,
      "logits/rejected": -2.7229886054992676,
      "logps/chosen": -214.0351104736328,
      "logps/rejected": -223.4528350830078,
      "loss": 0.6913,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": 0.01229914091527462,
      "rewards/margins": -0.001658583409152925,
      "rewards/rejected": 0.013957725837826729,
      "step": 3550
    },
    {
      "epoch": 0.62,
      "learning_rate": 8.282115869017632e-08,
      "logits/chosen": -2.6734941005706787,
      "logits/rejected": -2.6738343238830566,
      "logps/chosen": -186.98057556152344,
      "logps/rejected": -172.816650390625,
      "loss": 0.6913,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": 0.00906095840036869,
      "rewards/margins": 0.0004602237604558468,
      "rewards/rejected": 0.008600736036896706,
      "step": 3560
    },
    {
      "epoch": 0.63,
      "learning_rate": 8.277078085642317e-08,
      "logits/chosen": -2.7180447578430176,
      "logits/rejected": -2.687653064727783,
      "logps/chosen": -180.7360382080078,
      "logps/rejected": -178.57864379882812,
      "loss": 0.6921,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": 0.007827245630323887,
      "rewards/margins": -0.0009862823644652963,
      "rewards/rejected": 0.008813528344035149,
      "step": 3570
    },
    {
      "epoch": 0.63,
      "learning_rate": 8.272040302267002e-08,
      "logits/chosen": -2.7027173042297363,
      "logits/rejected": -2.6870369911193848,
      "logps/chosen": -208.6110382080078,
      "logps/rejected": -167.7442626953125,
      "loss": 0.692,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.01145018357783556,
      "rewards/margins": 0.0035087857395410538,
      "rewards/rejected": 0.007941396906971931,
      "step": 3580
    },
    {
      "epoch": 0.63,
      "learning_rate": 8.267002518891686e-08,
      "logits/chosen": -2.7568917274475098,
      "logits/rejected": -2.6639156341552734,
      "logps/chosen": -226.4164276123047,
      "logps/rejected": -186.1335906982422,
      "loss": 0.6917,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.013830587267875671,
      "rewards/margins": 0.0034837317653000355,
      "rewards/rejected": 0.010346856899559498,
      "step": 3590
    },
    {
      "epoch": 0.63,
      "learning_rate": 8.261964735516372e-08,
      "logits/chosen": -2.8411202430725098,
      "logits/rejected": -2.8351664543151855,
      "logps/chosen": -207.5603790283203,
      "logps/rejected": -206.82534790039062,
      "loss": 0.6918,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": 0.012709485366940498,
      "rewards/margins": -0.0002613008546177298,
      "rewards/rejected": 0.012970787473022938,
      "step": 3600
    },
    {
      "epoch": 0.63,
      "learning_rate": 8.256926952141057e-08,
      "logits/chosen": -2.85115909576416,
      "logits/rejected": -2.836247682571411,
      "logps/chosen": -217.4815673828125,
      "logps/rejected": -217.00668334960938,
      "loss": 0.6919,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.015054309740662575,
      "rewards/margins": 0.001863854588009417,
      "rewards/rejected": 0.013190452940762043,
      "step": 3610
    },
    {
      "epoch": 0.63,
      "learning_rate": 8.251889168765743e-08,
      "logits/chosen": -2.74369740486145,
      "logits/rejected": -2.7580039501190186,
      "logps/chosen": -280.5350341796875,
      "logps/rejected": -250.761962890625,
      "loss": 0.692,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.02001040428876877,
      "rewards/margins": 0.003991215955466032,
      "rewards/rejected": 0.01601918786764145,
      "step": 3620
    },
    {
      "epoch": 0.64,
      "learning_rate": 8.246851385390428e-08,
      "logits/chosen": -2.7671589851379395,
      "logits/rejected": -2.7030556201934814,
      "logps/chosen": -226.4773712158203,
      "logps/rejected": -183.44740295410156,
      "loss": 0.6919,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.014777980744838715,
      "rewards/margins": 0.006304244510829449,
      "rewards/rejected": 0.008473738096654415,
      "step": 3630
    },
    {
      "epoch": 0.64,
      "learning_rate": 8.241813602015113e-08,
      "logits/chosen": -2.7083332538604736,
      "logits/rejected": -2.695016860961914,
      "logps/chosen": -215.36013793945312,
      "logps/rejected": -189.6326904296875,
      "loss": 0.6912,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.013701984658837318,
      "rewards/margins": 0.0003051554667763412,
      "rewards/rejected": 0.01339682936668396,
      "step": 3640
    },
    {
      "epoch": 0.64,
      "learning_rate": 8.236775818639799e-08,
      "logits/chosen": -2.798619270324707,
      "logits/rejected": -2.793792247772217,
      "logps/chosen": -232.0058135986328,
      "logps/rejected": -157.7205810546875,
      "loss": 0.691,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.01850191317498684,
      "rewards/margins": 0.0096164895221591,
      "rewards/rejected": 0.008885422721505165,
      "step": 3650
    },
    {
      "epoch": 0.64,
      "learning_rate": 8.231738035264483e-08,
      "logits/chosen": -2.7713623046875,
      "logits/rejected": -2.6689579486846924,
      "logps/chosen": -201.49600219726562,
      "logps/rejected": -234.8392333984375,
      "loss": 0.692,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.012621419504284859,
      "rewards/margins": -0.00029890690348111093,
      "rewards/rejected": 0.012920325621962547,
      "step": 3660
    },
    {
      "epoch": 0.64,
      "learning_rate": 8.226700251889168e-08,
      "logits/chosen": -2.767369508743286,
      "logits/rejected": -2.744903326034546,
      "logps/chosen": -206.9405059814453,
      "logps/rejected": -207.5795440673828,
      "loss": 0.6919,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.011534372344613075,
      "rewards/margins": -0.0018367780139669776,
      "rewards/rejected": 0.013371150009334087,
      "step": 3670
    },
    {
      "epoch": 0.64,
      "learning_rate": 8.221662468513853e-08,
      "logits/chosen": -2.7466063499450684,
      "logits/rejected": -2.765381097793579,
      "logps/chosen": -222.0699462890625,
      "logps/rejected": -205.1193084716797,
      "loss": 0.6914,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.012425544671714306,
      "rewards/margins": -0.00028726962045766413,
      "rewards/rejected": 0.012712815776467323,
      "step": 3680
    },
    {
      "epoch": 0.65,
      "learning_rate": 8.216624685138539e-08,
      "logits/chosen": -2.7388596534729004,
      "logits/rejected": -2.7755088806152344,
      "logps/chosen": -275.1582336425781,
      "logps/rejected": -225.3055419921875,
      "loss": 0.6914,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": 0.025951793417334557,
      "rewards/margins": 0.010948443785309792,
      "rewards/rejected": 0.015003351494669914,
      "step": 3690
    },
    {
      "epoch": 0.65,
      "learning_rate": 8.211586901763223e-08,
      "logits/chosen": -2.7461769580841064,
      "logits/rejected": -2.708989143371582,
      "logps/chosen": -228.46145629882812,
      "logps/rejected": -185.53692626953125,
      "loss": 0.6913,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.017765622586011887,
      "rewards/margins": 0.00780740100890398,
      "rewards/rejected": 0.009958220645785332,
      "step": 3700
    },
    {
      "epoch": 0.65,
      "learning_rate": 8.206549118387908e-08,
      "logits/chosen": -2.733278751373291,
      "logits/rejected": -2.7251486778259277,
      "logps/chosen": -260.572509765625,
      "logps/rejected": -225.13363647460938,
      "loss": 0.6911,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.019569676369428635,
      "rewards/margins": 0.005702861584722996,
      "rewards/rejected": 0.013866816647350788,
      "step": 3710
    },
    {
      "epoch": 0.65,
      "learning_rate": 8.201511335012594e-08,
      "logits/chosen": -2.6957457065582275,
      "logits/rejected": -2.7972590923309326,
      "logps/chosen": -267.50677490234375,
      "logps/rejected": -215.5604705810547,
      "loss": 0.6915,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.020175551995635033,
      "rewards/margins": 0.005252575036138296,
      "rewards/rejected": 0.01492297649383545,
      "step": 3720
    },
    {
      "epoch": 0.65,
      "learning_rate": 8.19647355163728e-08,
      "logits/chosen": -2.713207960128784,
      "logits/rejected": -2.72871994972229,
      "logps/chosen": -233.4993896484375,
      "logps/rejected": -204.14529418945312,
      "loss": 0.6929,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.014108104631304741,
      "rewards/margins": -0.003692076075822115,
      "rewards/rejected": 0.01780017837882042,
      "step": 3730
    },
    {
      "epoch": 0.66,
      "learning_rate": 8.191435768261964e-08,
      "logits/chosen": -2.748497247695923,
      "logits/rejected": -2.683332681655884,
      "logps/chosen": -240.52297973632812,
      "logps/rejected": -233.1362762451172,
      "loss": 0.6911,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.016229938715696335,
      "rewards/margins": 0.0006633886368945241,
      "rewards/rejected": 0.015566547401249409,
      "step": 3740
    },
    {
      "epoch": 0.66,
      "learning_rate": 8.18639798488665e-08,
      "logits/chosen": -2.7239058017730713,
      "logits/rejected": -2.740793228149414,
      "logps/chosen": -247.53268432617188,
      "logps/rejected": -272.5812683105469,
      "loss": 0.6916,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.0161224864423275,
      "rewards/margins": 0.0009723890689201653,
      "rewards/rejected": 0.015150099992752075,
      "step": 3750
    },
    {
      "epoch": 0.66,
      "learning_rate": 8.181360201511335e-08,
      "logits/chosen": -2.7831082344055176,
      "logits/rejected": -2.815091609954834,
      "logps/chosen": -233.9501495361328,
      "logps/rejected": -208.8319854736328,
      "loss": 0.6909,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.017066344618797302,
      "rewards/margins": 0.0011864884290844202,
      "rewards/rejected": 0.015879856422543526,
      "step": 3760
    },
    {
      "epoch": 0.66,
      "learning_rate": 8.17632241813602e-08,
      "logits/chosen": -2.7610182762145996,
      "logits/rejected": -2.765444278717041,
      "logps/chosen": -269.3284912109375,
      "logps/rejected": -238.10537719726562,
      "loss": 0.6912,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": 0.019205298274755478,
      "rewards/margins": 0.007051876280456781,
      "rewards/rejected": 0.012153422459959984,
      "step": 3770
    },
    {
      "epoch": 0.66,
      "learning_rate": 8.171284634760704e-08,
      "logits/chosen": -2.7351415157318115,
      "logits/rejected": -2.679765224456787,
      "logps/chosen": -218.440673828125,
      "logps/rejected": -192.87567138671875,
      "loss": 0.6911,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.01345922239124775,
      "rewards/margins": 0.004528115503489971,
      "rewards/rejected": 0.008931105956435204,
      "step": 3780
    },
    {
      "epoch": 0.66,
      "learning_rate": 8.16624685138539e-08,
      "logits/chosen": -2.7771928310394287,
      "logits/rejected": -2.687178134918213,
      "logps/chosen": -257.64398193359375,
      "logps/rejected": -194.52593994140625,
      "loss": 0.691,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.01612105406820774,
      "rewards/margins": 0.0037273832131177187,
      "rewards/rejected": 0.012393670156598091,
      "step": 3790
    },
    {
      "epoch": 0.67,
      "learning_rate": 8.161209068010075e-08,
      "logits/chosen": -2.742800712585449,
      "logits/rejected": -2.764816999435425,
      "logps/chosen": -207.69876098632812,
      "logps/rejected": -197.75302124023438,
      "loss": 0.6926,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.015343104489147663,
      "rewards/margins": 0.004005838185548782,
      "rewards/rejected": 0.011337265372276306,
      "step": 3800
    },
    {
      "epoch": 0.67,
      "learning_rate": 8.15617128463476e-08,
      "logits/chosen": -2.7895233631134033,
      "logits/rejected": -2.6712045669555664,
      "logps/chosen": -276.7768249511719,
      "logps/rejected": -248.6250457763672,
      "loss": 0.6912,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.02004379592835903,
      "rewards/margins": 0.006672368384897709,
      "rewards/rejected": 0.013371428474783897,
      "step": 3810
    },
    {
      "epoch": 0.67,
      "learning_rate": 8.151133501259445e-08,
      "logits/chosen": -2.7735464572906494,
      "logits/rejected": -2.731327533721924,
      "logps/chosen": -252.3322296142578,
      "logps/rejected": -193.44198608398438,
      "loss": 0.6907,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.01836908422410488,
      "rewards/margins": 0.005864441394805908,
      "rewards/rejected": 0.012504642829298973,
      "step": 3820
    },
    {
      "epoch": 0.67,
      "learning_rate": 8.14609571788413e-08,
      "logits/chosen": -2.7429397106170654,
      "logits/rejected": -2.7197344303131104,
      "logps/chosen": -166.81492614746094,
      "logps/rejected": -153.70217895507812,
      "loss": 0.6909,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.00818006880581379,
      "rewards/margins": 0.001661596237681806,
      "rewards/rejected": 0.006518472917377949,
      "step": 3830
    },
    {
      "epoch": 0.67,
      "learning_rate": 8.141057934508817e-08,
      "logits/chosen": -2.663329839706421,
      "logits/rejected": -2.762928009033203,
      "logps/chosen": -230.36672973632812,
      "logps/rejected": -208.890380859375,
      "loss": 0.6908,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.019962593913078308,
      "rewards/margins": 0.005378905683755875,
      "rewards/rejected": 0.014583689160645008,
      "step": 3840
    },
    {
      "epoch": 0.67,
      "learning_rate": 8.136020151133502e-08,
      "logits/chosen": -2.7788891792297363,
      "logits/rejected": -2.7233879566192627,
      "logps/chosen": -222.71078491210938,
      "logps/rejected": -188.53250122070312,
      "loss": 0.6908,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.018942857161164284,
      "rewards/margins": 0.004116528667509556,
      "rewards/rejected": 0.014826327562332153,
      "step": 3850
    },
    {
      "epoch": 0.68,
      "learning_rate": 8.130982367758186e-08,
      "logits/chosen": -2.789405107498169,
      "logits/rejected": -2.8390228748321533,
      "logps/chosen": -235.29867553710938,
      "logps/rejected": -248.5530548095703,
      "loss": 0.6918,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.01704949513077736,
      "rewards/margins": -0.0013193708145990968,
      "rewards/rejected": 0.01836886629462242,
      "step": 3860
    },
    {
      "epoch": 0.68,
      "learning_rate": 8.125944584382871e-08,
      "logits/chosen": -2.72065806388855,
      "logits/rejected": -2.7855780124664307,
      "logps/chosen": -223.49081420898438,
      "logps/rejected": -193.26185607910156,
      "loss": 0.6912,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.013743147253990173,
      "rewards/margins": 0.0023424732498824596,
      "rewards/rejected": 0.011400674469769001,
      "step": 3870
    },
    {
      "epoch": 0.68,
      "learning_rate": 8.120906801007557e-08,
      "logits/chosen": -2.7632253170013428,
      "logits/rejected": -2.6923036575317383,
      "logps/chosen": -215.9404754638672,
      "logps/rejected": -166.86044311523438,
      "loss": 0.6916,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.017654119059443474,
      "rewards/margins": 0.005568289663642645,
      "rewards/rejected": 0.01208583079278469,
      "step": 3880
    },
    {
      "epoch": 0.68,
      "learning_rate": 8.115869017632242e-08,
      "logits/chosen": -2.7535698413848877,
      "logits/rejected": -2.740983724594116,
      "logps/chosen": -188.03590393066406,
      "logps/rejected": -160.87303161621094,
      "loss": 0.6924,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": 0.01075964979827404,
      "rewards/margins": 0.0006056831916794181,
      "rewards/rejected": 0.010153967887163162,
      "step": 3890
    },
    {
      "epoch": 0.68,
      "learning_rate": 8.110831234256926e-08,
      "logits/chosen": -2.7618038654327393,
      "logits/rejected": -2.7280309200286865,
      "logps/chosen": -224.43685913085938,
      "logps/rejected": -183.01840209960938,
      "loss": 0.6905,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.015426975674927235,
      "rewards/margins": 0.005461042746901512,
      "rewards/rejected": 0.009965931996703148,
      "step": 3900
    },
    {
      "epoch": 0.68,
      "learning_rate": 8.105793450881612e-08,
      "logits/chosen": -2.7761223316192627,
      "logits/rejected": -2.716975450515747,
      "logps/chosen": -215.36886596679688,
      "logps/rejected": -202.84988403320312,
      "loss": 0.691,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.016212482005357742,
      "rewards/margins": 0.0005874786293134093,
      "rewards/rejected": 0.0156250037252903,
      "step": 3910
    },
    {
      "epoch": 0.69,
      "learning_rate": 8.100755667506297e-08,
      "logits/chosen": -2.7833571434020996,
      "logits/rejected": -2.876706838607788,
      "logps/chosen": -259.8275451660156,
      "logps/rejected": -235.8184051513672,
      "loss": 0.6914,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.01803353801369667,
      "rewards/margins": -0.0025308975018560886,
      "rewards/rejected": 0.020564435049891472,
      "step": 3920
    },
    {
      "epoch": 0.69,
      "learning_rate": 8.095717884130982e-08,
      "logits/chosen": -2.835965156555176,
      "logits/rejected": -2.7779338359832764,
      "logps/chosen": -225.6724853515625,
      "logps/rejected": -188.37217712402344,
      "loss": 0.6919,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.01582942344248295,
      "rewards/margins": 0.0034069358371198177,
      "rewards/rejected": 0.012422487139701843,
      "step": 3930
    },
    {
      "epoch": 0.69,
      "learning_rate": 8.090680100755666e-08,
      "logits/chosen": -2.8340446949005127,
      "logits/rejected": -2.8134090900421143,
      "logps/chosen": -212.24960327148438,
      "logps/rejected": -182.80239868164062,
      "loss": 0.6913,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.015874182805418968,
      "rewards/margins": 0.004976223688572645,
      "rewards/rejected": 0.01089795958250761,
      "step": 3940
    },
    {
      "epoch": 0.69,
      "learning_rate": 8.085642317380352e-08,
      "logits/chosen": -2.7107186317443848,
      "logits/rejected": -2.705901622772217,
      "logps/chosen": -207.7930450439453,
      "logps/rejected": -200.37973022460938,
      "loss": 0.691,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.01649567112326622,
      "rewards/margins": 0.0022905156947672367,
      "rewards/rejected": 0.014205153100192547,
      "step": 3950
    },
    {
      "epoch": 0.69,
      "learning_rate": 8.080604534005038e-08,
      "logits/chosen": -2.7393903732299805,
      "logits/rejected": -2.8167338371276855,
      "logps/chosen": -239.1031951904297,
      "logps/rejected": -190.6559295654297,
      "loss": 0.6915,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.020263103768229485,
      "rewards/margins": 0.009479226544499397,
      "rewards/rejected": 0.010783876292407513,
      "step": 3960
    },
    {
      "epoch": 0.7,
      "learning_rate": 8.075566750629722e-08,
      "logits/chosen": -2.751710891723633,
      "logits/rejected": -2.804835796356201,
      "logps/chosen": -216.37417602539062,
      "logps/rejected": -194.0995330810547,
      "loss": 0.6913,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.01658977009356022,
      "rewards/margins": 0.00448748329654336,
      "rewards/rejected": 0.012102288194000721,
      "step": 3970
    },
    {
      "epoch": 0.7,
      "learning_rate": 8.070528967254408e-08,
      "logits/chosen": -2.7608554363250732,
      "logits/rejected": -2.8055953979492188,
      "logps/chosen": -191.1044158935547,
      "logps/rejected": -156.22952270507812,
      "loss": 0.6924,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.01224413700401783,
      "rewards/margins": 0.0022321634460240602,
      "rewards/rejected": 0.010011972859501839,
      "step": 3980
    },
    {
      "epoch": 0.7,
      "learning_rate": 8.065491183879093e-08,
      "logits/chosen": -2.8037281036376953,
      "logits/rejected": -2.816038131713867,
      "logps/chosen": -234.3374481201172,
      "logps/rejected": -193.92330932617188,
      "loss": 0.6919,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.01975713111460209,
      "rewards/margins": 0.006761783268302679,
      "rewards/rejected": 0.012995347380638123,
      "step": 3990
    },
    {
      "epoch": 0.7,
      "learning_rate": 8.060453400503778e-08,
      "logits/chosen": -2.775963068008423,
      "logits/rejected": -2.75541615486145,
      "logps/chosen": -242.49606323242188,
      "logps/rejected": -206.7489471435547,
      "loss": 0.6918,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": 0.014242658391594887,
      "rewards/margins": -0.001507654320448637,
      "rewards/rejected": 0.01575031317770481,
      "step": 4000
    },
    {
      "epoch": 0.7,
      "learning_rate": 8.055415617128462e-08,
      "logits/chosen": -2.755798816680908,
      "logits/rejected": -2.765866756439209,
      "logps/chosen": -191.8464813232422,
      "logps/rejected": -161.7691650390625,
      "loss": 0.692,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.01425481028854847,
      "rewards/margins": 0.0025988915003836155,
      "rewards/rejected": 0.011655919253826141,
      "step": 4010
    },
    {
      "epoch": 0.7,
      "learning_rate": 8.050377833753148e-08,
      "logits/chosen": -2.7568089962005615,
      "logits/rejected": -2.723388671875,
      "logps/chosen": -220.6444549560547,
      "logps/rejected": -215.7344207763672,
      "loss": 0.6921,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.017022155225276947,
      "rewards/margins": 0.004023154266178608,
      "rewards/rejected": 0.012999000959098339,
      "step": 4020
    },
    {
      "epoch": 0.71,
      "learning_rate": 8.045340050377833e-08,
      "logits/chosen": -2.7698051929473877,
      "logits/rejected": -2.76641583442688,
      "logps/chosen": -201.69302368164062,
      "logps/rejected": -167.20213317871094,
      "loss": 0.6917,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.01841864362359047,
      "rewards/margins": 0.004708954133093357,
      "rewards/rejected": 0.013709688559174538,
      "step": 4030
    },
    {
      "epoch": 0.71,
      "learning_rate": 8.040302267002519e-08,
      "logits/chosen": -2.764843225479126,
      "logits/rejected": -2.7504663467407227,
      "logps/chosen": -206.3531036376953,
      "logps/rejected": -201.3180694580078,
      "loss": 0.691,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.014857729896903038,
      "rewards/margins": 0.000497685105074197,
      "rewards/rejected": 0.014360045082867146,
      "step": 4040
    },
    {
      "epoch": 0.71,
      "learning_rate": 8.035264483627203e-08,
      "logits/chosen": -2.724370241165161,
      "logits/rejected": -2.7362284660339355,
      "logps/chosen": -238.0730743408203,
      "logps/rejected": -218.9322052001953,
      "loss": 0.6915,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.02089729532599449,
      "rewards/margins": 0.005764030385762453,
      "rewards/rejected": 0.0151332663372159,
      "step": 4050
    },
    {
      "epoch": 0.71,
      "learning_rate": 8.030226700251888e-08,
      "logits/chosen": -2.831298351287842,
      "logits/rejected": -2.840911865234375,
      "logps/chosen": -247.79464721679688,
      "logps/rejected": -244.6702117919922,
      "loss": 0.6903,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.016922039911150932,
      "rewards/margins": 0.001367938588373363,
      "rewards/rejected": 0.015554102137684822,
      "step": 4060
    },
    {
      "epoch": 0.71,
      "learning_rate": 8.025188916876575e-08,
      "logits/chosen": -2.758810043334961,
      "logits/rejected": -2.7609002590179443,
      "logps/chosen": -188.1548614501953,
      "logps/rejected": -150.46853637695312,
      "loss": 0.6911,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.015438064932823181,
      "rewards/margins": 0.00756922597065568,
      "rewards/rejected": 0.007868838496506214,
      "step": 4070
    },
    {
      "epoch": 0.71,
      "learning_rate": 8.02015113350126e-08,
      "logits/chosen": -2.7641913890838623,
      "logits/rejected": -2.794325113296509,
      "logps/chosen": -276.16259765625,
      "logps/rejected": -221.947509765625,
      "loss": 0.6908,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.01936369575560093,
      "rewards/margins": 0.004809045232832432,
      "rewards/rejected": 0.014554649591445923,
      "step": 4080
    },
    {
      "epoch": 0.72,
      "learning_rate": 8.015113350125944e-08,
      "logits/chosen": -2.7735092639923096,
      "logits/rejected": -2.7139530181884766,
      "logps/chosen": -217.0701904296875,
      "logps/rejected": -204.7127685546875,
      "loss": 0.6905,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.01837734319269657,
      "rewards/margins": 0.006126923952251673,
      "rewards/rejected": 0.012250418774783611,
      "step": 4090
    },
    {
      "epoch": 0.72,
      "learning_rate": 8.01007556675063e-08,
      "logits/chosen": -2.746734380722046,
      "logits/rejected": -2.779106616973877,
      "logps/chosen": -223.3635711669922,
      "logps/rejected": -210.4722137451172,
      "loss": 0.6915,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.022333238273859024,
      "rewards/margins": 0.005000376142561436,
      "rewards/rejected": 0.017332863062620163,
      "step": 4100
    },
    {
      "epoch": 0.72,
      "learning_rate": 8.005037783375315e-08,
      "logits/chosen": -2.761756420135498,
      "logits/rejected": -2.8026602268218994,
      "logps/chosen": -221.0587158203125,
      "logps/rejected": -202.43582153320312,
      "loss": 0.6917,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.016418827697634697,
      "rewards/margins": 0.0006193307344801724,
      "rewards/rejected": 0.015799496322870255,
      "step": 4110
    },
    {
      "epoch": 0.72,
      "learning_rate": 8e-08,
      "logits/chosen": -2.7710421085357666,
      "logits/rejected": -2.744896411895752,
      "logps/chosen": -226.13107299804688,
      "logps/rejected": -206.2400360107422,
      "loss": 0.6908,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.018721196800470352,
      "rewards/margins": 0.0020058993250131607,
      "rewards/rejected": 0.01671529933810234,
      "step": 4120
    },
    {
      "epoch": 0.72,
      "learning_rate": 7.994962216624684e-08,
      "logits/chosen": -2.695394992828369,
      "logits/rejected": -2.672240734100342,
      "logps/chosen": -301.8807067871094,
      "logps/rejected": -241.90768432617188,
      "loss": 0.6907,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.02527024783194065,
      "rewards/margins": 0.009704559110105038,
      "rewards/rejected": 0.01556568592786789,
      "step": 4130
    },
    {
      "epoch": 0.73,
      "learning_rate": 7.98992443324937e-08,
      "logits/chosen": -2.7234575748443604,
      "logits/rejected": -2.7253074645996094,
      "logps/chosen": -242.5963897705078,
      "logps/rejected": -214.54873657226562,
      "loss": 0.6907,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.019268224015831947,
      "rewards/margins": 0.0059552742168307304,
      "rewards/rejected": 0.013312948867678642,
      "step": 4140
    },
    {
      "epoch": 0.73,
      "learning_rate": 7.984886649874055e-08,
      "logits/chosen": -2.747512102127075,
      "logits/rejected": -2.750851631164551,
      "logps/chosen": -221.0081787109375,
      "logps/rejected": -218.8545684814453,
      "loss": 0.6919,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.0189188402146101,
      "rewards/margins": 0.0030791780445724726,
      "rewards/rejected": 0.01583966240286827,
      "step": 4150
    },
    {
      "epoch": 0.73,
      "learning_rate": 7.97984886649874e-08,
      "logits/chosen": -2.7558045387268066,
      "logits/rejected": -2.7282321453094482,
      "logps/chosen": -260.65093994140625,
      "logps/rejected": -205.7612762451172,
      "loss": 0.6902,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.01898707076907158,
      "rewards/margins": 0.004444524645805359,
      "rewards/rejected": 0.01454254798591137,
      "step": 4160
    },
    {
      "epoch": 0.73,
      "learning_rate": 7.974811083123424e-08,
      "logits/chosen": -2.740727663040161,
      "logits/rejected": -2.738755702972412,
      "logps/chosen": -255.1199951171875,
      "logps/rejected": -223.3599395751953,
      "loss": 0.6917,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.019392380490899086,
      "rewards/margins": 0.0023465738631784916,
      "rewards/rejected": 0.01704580709338188,
      "step": 4170
    },
    {
      "epoch": 0.73,
      "learning_rate": 7.969773299748111e-08,
      "logits/chosen": -2.7931790351867676,
      "logits/rejected": -2.751844644546509,
      "logps/chosen": -221.10107421875,
      "logps/rejected": -173.9921875,
      "loss": 0.6911,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.02068236656486988,
      "rewards/margins": 0.00878697820007801,
      "rewards/rejected": 0.011895387433469296,
      "step": 4180
    },
    {
      "epoch": 0.73,
      "learning_rate": 7.964735516372796e-08,
      "logits/chosen": -2.736713409423828,
      "logits/rejected": -2.7641730308532715,
      "logps/chosen": -243.8798370361328,
      "logps/rejected": -209.95767211914062,
      "loss": 0.6912,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.022618575021624565,
      "rewards/margins": 0.006286067422479391,
      "rewards/rejected": 0.016332510858774185,
      "step": 4190
    },
    {
      "epoch": 0.74,
      "learning_rate": 7.959697732997482e-08,
      "logits/chosen": -2.768643856048584,
      "logits/rejected": -2.768256902694702,
      "logps/chosen": -213.690673828125,
      "logps/rejected": -223.6483917236328,
      "loss": 0.6914,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.01731937564909458,
      "rewards/margins": 0.0022180068772286177,
      "rewards/rejected": 0.015101367607712746,
      "step": 4200
    },
    {
      "epoch": 0.74,
      "learning_rate": 7.954659949622166e-08,
      "logits/chosen": -2.7336461544036865,
      "logits/rejected": -2.7381973266601562,
      "logps/chosen": -266.1100769042969,
      "logps/rejected": -220.54769897460938,
      "loss": 0.6909,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.018683522939682007,
      "rewards/margins": 0.004539954476058483,
      "rewards/rejected": 0.014143568463623524,
      "step": 4210
    },
    {
      "epoch": 0.74,
      "learning_rate": 7.949622166246851e-08,
      "logits/chosen": -2.7625882625579834,
      "logits/rejected": -2.7351088523864746,
      "logps/chosen": -222.360107421875,
      "logps/rejected": -187.1444549560547,
      "loss": 0.6911,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": 0.015518203377723694,
      "rewards/margins": -0.0003775197546929121,
      "rewards/rejected": 0.015895724296569824,
      "step": 4220
    },
    {
      "epoch": 0.74,
      "learning_rate": 7.944584382871537e-08,
      "logits/chosen": -2.8258113861083984,
      "logits/rejected": -2.7792856693267822,
      "logps/chosen": -265.05535888671875,
      "logps/rejected": -232.38101196289062,
      "loss": 0.6922,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.021588006988167763,
      "rewards/margins": 0.001263659680262208,
      "rewards/rejected": 0.02032434567809105,
      "step": 4230
    },
    {
      "epoch": 0.74,
      "learning_rate": 7.939546599496222e-08,
      "logits/chosen": -2.855884075164795,
      "logits/rejected": -2.7981677055358887,
      "logps/chosen": -241.37246704101562,
      "logps/rejected": -197.73507690429688,
      "loss": 0.6913,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": 0.017666930332779884,
      "rewards/margins": -0.0018121827160939574,
      "rewards/rejected": 0.019479114562273026,
      "step": 4240
    },
    {
      "epoch": 0.74,
      "learning_rate": 7.934508816120906e-08,
      "logits/chosen": -2.769641160964966,
      "logits/rejected": -2.7314181327819824,
      "logps/chosen": -241.15615844726562,
      "logps/rejected": -221.33547973632812,
      "loss": 0.6911,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.023081984370946884,
      "rewards/margins": 0.005661494564265013,
      "rewards/rejected": 0.017420491203665733,
      "step": 4250
    },
    {
      "epoch": 0.75,
      "learning_rate": 7.929471032745591e-08,
      "logits/chosen": -2.743436813354492,
      "logits/rejected": -2.6626510620117188,
      "logps/chosen": -228.18014526367188,
      "logps/rejected": -220.50277709960938,
      "loss": 0.6914,
      "rewards/accuracies": 0.3499999940395355,
      "rewards/chosen": 0.01799473538994789,
      "rewards/margins": -0.0007408105302602053,
      "rewards/rejected": 0.018735546618700027,
      "step": 4260
    },
    {
      "epoch": 0.75,
      "learning_rate": 7.924433249370277e-08,
      "logits/chosen": -2.8057007789611816,
      "logits/rejected": -2.7800495624542236,
      "logps/chosen": -263.08514404296875,
      "logps/rejected": -211.67666625976562,
      "loss": 0.6907,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.02124876156449318,
      "rewards/margins": 0.009110869839787483,
      "rewards/rejected": 0.012137891724705696,
      "step": 4270
    },
    {
      "epoch": 0.75,
      "learning_rate": 7.919395465994961e-08,
      "logits/chosen": -2.7258174419403076,
      "logits/rejected": -2.7756786346435547,
      "logps/chosen": -193.611083984375,
      "logps/rejected": -179.30149841308594,
      "loss": 0.6917,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.016186311841011047,
      "rewards/margins": 0.0020093251951038837,
      "rewards/rejected": 0.0141769889742136,
      "step": 4280
    },
    {
      "epoch": 0.75,
      "learning_rate": 7.914357682619647e-08,
      "logits/chosen": -2.7517471313476562,
      "logits/rejected": -2.734189987182617,
      "logps/chosen": -230.6570281982422,
      "logps/rejected": -202.70420837402344,
      "loss": 0.6913,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.02114647440612316,
      "rewards/margins": 0.0036278795450925827,
      "rewards/rejected": 0.01751859486103058,
      "step": 4290
    },
    {
      "epoch": 0.75,
      "learning_rate": 7.909319899244333e-08,
      "logits/chosen": -2.730635166168213,
      "logits/rejected": -2.7385268211364746,
      "logps/chosen": -198.06155395507812,
      "logps/rejected": -176.04498291015625,
      "loss": 0.6911,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.016498813405632973,
      "rewards/margins": 0.001804836094379425,
      "rewards/rejected": 0.014693977311253548,
      "step": 4300
    },
    {
      "epoch": 0.75,
      "learning_rate": 7.904282115869018e-08,
      "logits/chosen": -2.7733383178710938,
      "logits/rejected": -2.792484998703003,
      "logps/chosen": -191.0068817138672,
      "logps/rejected": -161.52700805664062,
      "loss": 0.6915,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.014926761388778687,
      "rewards/margins": 0.004622498992830515,
      "rewards/rejected": 0.010304262861609459,
      "step": 4310
    },
    {
      "epoch": 0.76,
      "learning_rate": 7.899244332493702e-08,
      "logits/chosen": -2.768362522125244,
      "logits/rejected": -2.8112378120422363,
      "logps/chosen": -214.1084747314453,
      "logps/rejected": -185.85166931152344,
      "loss": 0.6909,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.018819646909832954,
      "rewards/margins": 0.007635760121047497,
      "rewards/rejected": 0.011183887720108032,
      "step": 4320
    },
    {
      "epoch": 0.76,
      "learning_rate": 7.894206549118388e-08,
      "logits/chosen": -2.7518534660339355,
      "logits/rejected": -2.765934467315674,
      "logps/chosen": -199.8416748046875,
      "logps/rejected": -181.0721893310547,
      "loss": 0.6917,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.018102090805768967,
      "rewards/margins": -0.00039173493860289454,
      "rewards/rejected": 0.01849382556974888,
      "step": 4330
    },
    {
      "epoch": 0.76,
      "learning_rate": 7.889168765743073e-08,
      "logits/chosen": -2.7839748859405518,
      "logits/rejected": -2.7550039291381836,
      "logps/chosen": -263.43011474609375,
      "logps/rejected": -206.5287322998047,
      "loss": 0.6909,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": 0.02562366984784603,
      "rewards/margins": 0.008644169196486473,
      "rewards/rejected": 0.01697949692606926,
      "step": 4340
    },
    {
      "epoch": 0.76,
      "learning_rate": 7.884130982367758e-08,
      "logits/chosen": -2.734191417694092,
      "logits/rejected": -2.6990275382995605,
      "logps/chosen": -223.6094970703125,
      "logps/rejected": -209.64791870117188,
      "loss": 0.6911,
      "rewards/accuracies": 0.42500001192092896,
      "rewards/chosen": 0.01814252696931362,
      "rewards/margins": 0.0009969560196623206,
      "rewards/rejected": 0.017145570367574692,
      "step": 4350
    },
    {
      "epoch": 0.76,
      "learning_rate": 7.879093198992442e-08,
      "logits/chosen": -2.7549777030944824,
      "logits/rejected": -2.7530622482299805,
      "logps/chosen": -267.322998046875,
      "logps/rejected": -225.5402069091797,
      "loss": 0.6912,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.018788613379001617,
      "rewards/margins": -0.0013716205721721053,
      "rewards/rejected": 0.020160233601927757,
      "step": 4360
    },
    {
      "epoch": 0.77,
      "learning_rate": 7.874055415617128e-08,
      "logits/chosen": -2.8316569328308105,
      "logits/rejected": -2.7772135734558105,
      "logps/chosen": -238.33383178710938,
      "logps/rejected": -207.7853240966797,
      "loss": 0.6916,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.021232271566987038,
      "rewards/margins": 0.0057861944660544395,
      "rewards/rejected": 0.015446076169610023,
      "step": 4370
    },
    {
      "epoch": 0.77,
      "learning_rate": 7.869017632241813e-08,
      "logits/chosen": -2.7817704677581787,
      "logits/rejected": -2.724588632583618,
      "logps/chosen": -211.82650756835938,
      "logps/rejected": -178.63743591308594,
      "loss": 0.6916,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.01910659857094288,
      "rewards/margins": 0.004909339360892773,
      "rewards/rejected": 0.014197258278727531,
      "step": 4380
    },
    {
      "epoch": 0.77,
      "learning_rate": 7.863979848866498e-08,
      "logits/chosen": -2.7254724502563477,
      "logits/rejected": -2.802567958831787,
      "logps/chosen": -213.8620147705078,
      "logps/rejected": -179.79685974121094,
      "loss": 0.6911,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.02377455309033394,
      "rewards/margins": 0.004999110009521246,
      "rewards/rejected": 0.018775442615151405,
      "step": 4390
    },
    {
      "epoch": 0.77,
      "learning_rate": 7.858942065491184e-08,
      "logits/chosen": -2.7992103099823,
      "logits/rejected": -2.742579698562622,
      "logps/chosen": -259.1708984375,
      "logps/rejected": -212.4176483154297,
      "loss": 0.6905,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.025623898953199387,
      "rewards/margins": 0.008028807118535042,
      "rewards/rejected": 0.017595095559954643,
      "step": 4400
    },
    {
      "epoch": 0.77,
      "learning_rate": 7.853904282115869e-08,
      "logits/chosen": -2.7983293533325195,
      "logits/rejected": -2.760340929031372,
      "logps/chosen": -238.68203735351562,
      "logps/rejected": -205.7268524169922,
      "loss": 0.6889,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.022161977365612984,
      "rewards/margins": 0.008466407656669617,
      "rewards/rejected": 0.013695569708943367,
      "step": 4410
    },
    {
      "epoch": 0.77,
      "learning_rate": 7.848866498740554e-08,
      "logits/chosen": -2.7707414627075195,
      "logits/rejected": -2.666015148162842,
      "logps/chosen": -267.7262268066406,
      "logps/rejected": -229.92568969726562,
      "loss": 0.6911,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.02798035740852356,
      "rewards/margins": 0.0134460823610425,
      "rewards/rejected": 0.01453427691012621,
      "step": 4420
    },
    {
      "epoch": 0.78,
      "learning_rate": 7.84382871536524e-08,
      "logits/chosen": -2.7728307247161865,
      "logits/rejected": -2.748939275741577,
      "logps/chosen": -241.35791015625,
      "logps/rejected": -208.1006622314453,
      "loss": 0.6919,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.02361791953444481,
      "rewards/margins": 0.006282551679760218,
      "rewards/rejected": 0.01733536645770073,
      "step": 4430
    },
    {
      "epoch": 0.78,
      "learning_rate": 7.838790931989924e-08,
      "logits/chosen": -2.751283645629883,
      "logits/rejected": -2.7884583473205566,
      "logps/chosen": -230.86865234375,
      "logps/rejected": -205.05746459960938,
      "loss": 0.6905,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.01816331408917904,
      "rewards/margins": 0.0020460118539631367,
      "rewards/rejected": 0.01611730083823204,
      "step": 4440
    },
    {
      "epoch": 0.78,
      "learning_rate": 7.833753148614609e-08,
      "logits/chosen": -2.7546162605285645,
      "logits/rejected": -2.7503271102905273,
      "logps/chosen": -237.634765625,
      "logps/rejected": -209.13125610351562,
      "loss": 0.6905,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.021774889901280403,
      "rewards/margins": 0.007742063608020544,
      "rewards/rejected": 0.014032823964953423,
      "step": 4450
    },
    {
      "epoch": 0.78,
      "learning_rate": 7.828715365239295e-08,
      "logits/chosen": -2.7621121406555176,
      "logits/rejected": -2.7742960453033447,
      "logps/chosen": -217.29983520507812,
      "logps/rejected": -208.7975616455078,
      "loss": 0.6914,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.023997284471988678,
      "rewards/margins": 0.005684687290340662,
      "rewards/rejected": 0.018312597647309303,
      "step": 4460
    },
    {
      "epoch": 0.78,
      "learning_rate": 7.82367758186398e-08,
      "logits/chosen": -2.711585521697998,
      "logits/rejected": -2.713052988052368,
      "logps/chosen": -249.82479858398438,
      "logps/rejected": -200.6315155029297,
      "loss": 0.692,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.019304895773530006,
      "rewards/margins": 0.0010892207501456141,
      "rewards/rejected": 0.01821567490696907,
      "step": 4470
    },
    {
      "epoch": 0.78,
      "learning_rate": 7.818639798488664e-08,
      "logits/chosen": -2.705024242401123,
      "logits/rejected": -2.74556565284729,
      "logps/chosen": -194.25546264648438,
      "logps/rejected": -171.3458251953125,
      "loss": 0.6905,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.011694484390318394,
      "rewards/margins": 0.0016950942808762193,
      "rewards/rejected": 0.009999390691518784,
      "step": 4480
    },
    {
      "epoch": 0.79,
      "learning_rate": 7.81360201511335e-08,
      "logits/chosen": -2.7292304039001465,
      "logits/rejected": -2.7129428386688232,
      "logps/chosen": -209.78079223632812,
      "logps/rejected": -241.6830291748047,
      "loss": 0.6911,
      "rewards/accuracies": 0.42500001192092896,
      "rewards/chosen": 0.012067977339029312,
      "rewards/margins": -0.0037381560541689396,
      "rewards/rejected": 0.015806132927536964,
      "step": 4490
    },
    {
      "epoch": 0.79,
      "learning_rate": 7.808564231738035e-08,
      "logits/chosen": -2.8351051807403564,
      "logits/rejected": -2.7264862060546875,
      "logps/chosen": -220.93252563476562,
      "logps/rejected": -205.50259399414062,
      "loss": 0.6917,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.02219536155462265,
      "rewards/margins": 0.00513820443302393,
      "rewards/rejected": 0.017057154327630997,
      "step": 4500
    },
    {
      "epoch": 0.79,
      "learning_rate": 7.80352644836272e-08,
      "logits/chosen": -2.785386323928833,
      "logits/rejected": -2.7174456119537354,
      "logps/chosen": -234.05874633789062,
      "logps/rejected": -205.1448974609375,
      "loss": 0.6909,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.018111255019903183,
      "rewards/margins": 0.004342755768448114,
      "rewards/rejected": 0.013768496923148632,
      "step": 4510
    },
    {
      "epoch": 0.79,
      "learning_rate": 7.798488664987405e-08,
      "logits/chosen": -2.7610111236572266,
      "logits/rejected": -2.6893057823181152,
      "logps/chosen": -215.45077514648438,
      "logps/rejected": -212.09848022460938,
      "loss": 0.6914,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": 0.01872086152434349,
      "rewards/margins": -0.00038822871283628047,
      "rewards/rejected": 0.01910909079015255,
      "step": 4520
    },
    {
      "epoch": 0.79,
      "learning_rate": 7.793450881612091e-08,
      "logits/chosen": -2.790309429168701,
      "logits/rejected": -2.8239731788635254,
      "logps/chosen": -251.9709014892578,
      "logps/rejected": -204.56088256835938,
      "loss": 0.6902,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.02696056105196476,
      "rewards/margins": 0.0066688829101622105,
      "rewards/rejected": 0.020291678607463837,
      "step": 4530
    },
    {
      "epoch": 0.8,
      "learning_rate": 7.788413098236776e-08,
      "logits/chosen": -2.8395566940307617,
      "logits/rejected": -2.781035900115967,
      "logps/chosen": -239.2814483642578,
      "logps/rejected": -189.88824462890625,
      "loss": 0.6907,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.022276435047388077,
      "rewards/margins": 0.00612932164222002,
      "rewards/rejected": 0.01614711619913578,
      "step": 4540
    },
    {
      "epoch": 0.8,
      "learning_rate": 7.783375314861462e-08,
      "logits/chosen": -2.777453899383545,
      "logits/rejected": -2.7720651626586914,
      "logps/chosen": -277.6221008300781,
      "logps/rejected": -232.5399932861328,
      "loss": 0.6914,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.022800957784056664,
      "rewards/margins": 0.006417891476303339,
      "rewards/rejected": 0.016383066773414612,
      "step": 4550
    },
    {
      "epoch": 0.8,
      "learning_rate": 7.778337531486146e-08,
      "logits/chosen": -2.8126351833343506,
      "logits/rejected": -2.748579263687134,
      "logps/chosen": -235.12881469726562,
      "logps/rejected": -250.25527954101562,
      "loss": 0.6913,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.024302994832396507,
      "rewards/margins": 0.005716138519346714,
      "rewards/rejected": 0.01858685538172722,
      "step": 4560
    },
    {
      "epoch": 0.8,
      "learning_rate": 7.773299748110831e-08,
      "logits/chosen": -2.7417664527893066,
      "logits/rejected": -2.736534833908081,
      "logps/chosen": -251.11880493164062,
      "logps/rejected": -205.9317169189453,
      "loss": 0.6902,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.021253764629364014,
      "rewards/margins": 0.006818639580160379,
      "rewards/rejected": 0.014435125514864922,
      "step": 4570
    },
    {
      "epoch": 0.8,
      "learning_rate": 7.768261964735516e-08,
      "logits/chosen": -2.879304885864258,
      "logits/rejected": -2.8242783546447754,
      "logps/chosen": -260.59454345703125,
      "logps/rejected": -209.79367065429688,
      "loss": 0.6904,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.0261318888515234,
      "rewards/margins": 0.007015122566372156,
      "rewards/rejected": 0.01911676675081253,
      "step": 4580
    },
    {
      "epoch": 0.8,
      "learning_rate": 7.7632241813602e-08,
      "logits/chosen": -2.7443363666534424,
      "logits/rejected": -2.663283348083496,
      "logps/chosen": -227.0984649658203,
      "logps/rejected": -207.4261016845703,
      "loss": 0.691,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.020075734704732895,
      "rewards/margins": 0.0018470725044608116,
      "rewards/rejected": 0.01822866126894951,
      "step": 4590
    },
    {
      "epoch": 0.81,
      "learning_rate": 7.758186397984886e-08,
      "logits/chosen": -2.765794277191162,
      "logits/rejected": -2.7267675399780273,
      "logps/chosen": -227.9505615234375,
      "logps/rejected": -175.19935607910156,
      "loss": 0.6886,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.01690318062901497,
      "rewards/margins": 0.005889912601560354,
      "rewards/rejected": 0.011013267561793327,
      "step": 4600
    },
    {
      "epoch": 0.81,
      "learning_rate": 7.753148614609571e-08,
      "logits/chosen": -2.7460758686065674,
      "logits/rejected": -2.7646918296813965,
      "logps/chosen": -255.68576049804688,
      "logps/rejected": -223.80899047851562,
      "loss": 0.6902,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.025373512879014015,
      "rewards/margins": 0.004764680750668049,
      "rewards/rejected": 0.02060883305966854,
      "step": 4610
    },
    {
      "epoch": 0.81,
      "learning_rate": 7.748110831234256e-08,
      "logits/chosen": -2.7983150482177734,
      "logits/rejected": -2.815904140472412,
      "logps/chosen": -246.80892944335938,
      "logps/rejected": -207.28793334960938,
      "loss": 0.6924,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.02356107532978058,
      "rewards/margins": 0.0033489782363176346,
      "rewards/rejected": 0.020212098956108093,
      "step": 4620
    },
    {
      "epoch": 0.81,
      "learning_rate": 7.743073047858942e-08,
      "logits/chosen": -2.726724147796631,
      "logits/rejected": -2.7251830101013184,
      "logps/chosen": -203.96339416503906,
      "logps/rejected": -179.08773803710938,
      "loss": 0.6912,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.018438972532749176,
      "rewards/margins": 0.0032338383607566357,
      "rewards/rejected": 0.015205135568976402,
      "step": 4630
    },
    {
      "epoch": 0.81,
      "learning_rate": 7.738035264483627e-08,
      "logits/chosen": -2.799243688583374,
      "logits/rejected": -2.757625102996826,
      "logps/chosen": -231.72024536132812,
      "logps/rejected": -194.36715698242188,
      "loss": 0.6908,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.02035854198038578,
      "rewards/margins": 0.003642552299425006,
      "rewards/rejected": 0.016715990379452705,
      "step": 4640
    },
    {
      "epoch": 0.81,
      "learning_rate": 7.732997481108313e-08,
      "logits/chosen": -2.8015732765197754,
      "logits/rejected": -2.7627758979797363,
      "logps/chosen": -225.84866333007812,
      "logps/rejected": -202.64268493652344,
      "loss": 0.6912,
      "rewards/accuracies": 0.42500001192092896,
      "rewards/chosen": 0.017703380435705185,
      "rewards/margins": 0.0011888284934684634,
      "rewards/rejected": 0.016514552757143974,
      "step": 4650
    },
    {
      "epoch": 0.82,
      "learning_rate": 7.727959697732998e-08,
      "logits/chosen": -2.7188124656677246,
      "logits/rejected": -2.758882522583008,
      "logps/chosen": -228.21188354492188,
      "logps/rejected": -214.9092254638672,
      "loss": 0.6903,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.024845756590366364,
      "rewards/margins": 0.006363884545862675,
      "rewards/rejected": 0.018481872975826263,
      "step": 4660
    },
    {
      "epoch": 0.82,
      "learning_rate": 7.722921914357682e-08,
      "logits/chosen": -2.81644344329834,
      "logits/rejected": -2.7707905769348145,
      "logps/chosen": -243.9435272216797,
      "logps/rejected": -188.6251983642578,
      "loss": 0.69,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.026427235454320908,
      "rewards/margins": 0.011382916942238808,
      "rewards/rejected": 0.015044321306049824,
      "step": 4670
    },
    {
      "epoch": 0.82,
      "learning_rate": 7.717884130982367e-08,
      "logits/chosen": -2.7194998264312744,
      "logits/rejected": -2.8078126907348633,
      "logps/chosen": -220.48046875,
      "logps/rejected": -195.44012451171875,
      "loss": 0.6904,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.019193807616829872,
      "rewards/margins": 0.003940016962587833,
      "rewards/rejected": 0.015253791585564613,
      "step": 4680
    },
    {
      "epoch": 0.82,
      "learning_rate": 7.712846347607053e-08,
      "logits/chosen": -2.745544910430908,
      "logits/rejected": -2.796077013015747,
      "logps/chosen": -242.34103393554688,
      "logps/rejected": -186.40908813476562,
      "loss": 0.6915,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.02268916554749012,
      "rewards/margins": 0.006848675664514303,
      "rewards/rejected": 0.01584048941731453,
      "step": 4690
    },
    {
      "epoch": 0.82,
      "learning_rate": 7.707808564231738e-08,
      "logits/chosen": -2.7313311100006104,
      "logits/rejected": -2.718033790588379,
      "logps/chosen": -180.11282348632812,
      "logps/rejected": -193.9093780517578,
      "loss": 0.691,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.013201892375946045,
      "rewards/margins": 0.001982487738132477,
      "rewards/rejected": 0.011219402775168419,
      "step": 4700
    },
    {
      "epoch": 0.83,
      "learning_rate": 7.702770780856422e-08,
      "logits/chosen": -2.7362091541290283,
      "logits/rejected": -2.6564507484436035,
      "logps/chosen": -245.69277954101562,
      "logps/rejected": -222.1023406982422,
      "loss": 0.6913,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.024562204256653786,
      "rewards/margins": 0.0058278171345591545,
      "rewards/rejected": 0.018734386190772057,
      "step": 4710
    },
    {
      "epoch": 0.83,
      "learning_rate": 7.697732997481107e-08,
      "logits/chosen": -2.7647807598114014,
      "logits/rejected": -2.748202323913574,
      "logps/chosen": -207.46826171875,
      "logps/rejected": -204.72366333007812,
      "loss": 0.6903,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.017461542040109634,
      "rewards/margins": -1.7283111446886323e-05,
      "rewards/rejected": 0.01747882552444935,
      "step": 4720
    },
    {
      "epoch": 0.83,
      "learning_rate": 7.692695214105793e-08,
      "logits/chosen": -2.8048958778381348,
      "logits/rejected": -2.830275058746338,
      "logps/chosen": -256.74993896484375,
      "logps/rejected": -186.250732421875,
      "loss": 0.691,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.02503347396850586,
      "rewards/margins": 0.007637057453393936,
      "rewards/rejected": 0.017396416515111923,
      "step": 4730
    },
    {
      "epoch": 0.83,
      "learning_rate": 7.687657430730478e-08,
      "logits/chosen": -2.7937734127044678,
      "logits/rejected": -2.771660089492798,
      "logps/chosen": -243.90585327148438,
      "logps/rejected": -215.63577270507812,
      "loss": 0.6915,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": 0.021137814968824387,
      "rewards/margins": -0.0011193671962246299,
      "rewards/rejected": 0.022257182747125626,
      "step": 4740
    },
    {
      "epoch": 0.83,
      "learning_rate": 7.682619647355164e-08,
      "logits/chosen": -2.7488274574279785,
      "logits/rejected": -2.817153215408325,
      "logps/chosen": -236.9940643310547,
      "logps/rejected": -228.24642944335938,
      "loss": 0.6908,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.024236269295215607,
      "rewards/margins": 0.002860927488654852,
      "rewards/rejected": 0.021375341340899467,
      "step": 4750
    },
    {
      "epoch": 0.83,
      "learning_rate": 7.677581863979849e-08,
      "logits/chosen": -2.7421677112579346,
      "logits/rejected": -2.767341375350952,
      "logps/chosen": -226.1661834716797,
      "logps/rejected": -179.6350860595703,
      "loss": 0.6901,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.027445446699857712,
      "rewards/margins": 0.007057595998048782,
      "rewards/rejected": 0.02038785070180893,
      "step": 4760
    },
    {
      "epoch": 0.84,
      "learning_rate": 7.672544080604534e-08,
      "logits/chosen": -2.817746877670288,
      "logits/rejected": -2.760152578353882,
      "logps/chosen": -229.91854858398438,
      "logps/rejected": -194.99644470214844,
      "loss": 0.6915,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.02216922864317894,
      "rewards/margins": 0.008342455141246319,
      "rewards/rejected": 0.013826772570610046,
      "step": 4770
    },
    {
      "epoch": 0.84,
      "learning_rate": 7.66750629722922e-08,
      "logits/chosen": -2.8346447944641113,
      "logits/rejected": -2.751246690750122,
      "logps/chosen": -250.40609741210938,
      "logps/rejected": -210.5948944091797,
      "loss": 0.6904,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.024279357865452766,
      "rewards/margins": 0.005712731741368771,
      "rewards/rejected": 0.01856662891805172,
      "step": 4780
    },
    {
      "epoch": 0.84,
      "learning_rate": 7.662468513853904e-08,
      "logits/chosen": -2.7878410816192627,
      "logits/rejected": -2.7772955894470215,
      "logps/chosen": -237.049560546875,
      "logps/rejected": -193.94973754882812,
      "loss": 0.6915,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.021299934014678,
      "rewards/margins": 0.004049413371831179,
      "rewards/rejected": 0.01725051924586296,
      "step": 4790
    },
    {
      "epoch": 0.84,
      "learning_rate": 7.657430730478589e-08,
      "logits/chosen": -2.7216315269470215,
      "logits/rejected": -2.725965976715088,
      "logps/chosen": -260.41021728515625,
      "logps/rejected": -253.3103790283203,
      "loss": 0.692,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": 0.025030773133039474,
      "rewards/margins": 0.001364406431093812,
      "rewards/rejected": 0.023666368797421455,
      "step": 4800
    },
    {
      "epoch": 0.84,
      "learning_rate": 7.652392947103274e-08,
      "logits/chosen": -2.7526895999908447,
      "logits/rejected": -2.783440589904785,
      "logps/chosen": -233.69204711914062,
      "logps/rejected": -257.5528564453125,
      "loss": 0.6915,
      "rewards/accuracies": 0.42500001192092896,
      "rewards/chosen": 0.0172027088701725,
      "rewards/margins": -0.000673537200782448,
      "rewards/rejected": 0.017876241356134415,
      "step": 4810
    },
    {
      "epoch": 0.84,
      "learning_rate": 7.64735516372796e-08,
      "logits/chosen": -2.697016477584839,
      "logits/rejected": -2.7442197799682617,
      "logps/chosen": -241.70361328125,
      "logps/rejected": -219.9405059814453,
      "loss": 0.6904,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.028333717957139015,
      "rewards/margins": 0.009502708911895752,
      "rewards/rejected": 0.018831010907888412,
      "step": 4820
    },
    {
      "epoch": 0.85,
      "learning_rate": 7.642317380352644e-08,
      "logits/chosen": -2.792147397994995,
      "logits/rejected": -2.752933979034424,
      "logps/chosen": -263.72259521484375,
      "logps/rejected": -222.4014434814453,
      "loss": 0.6912,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": 0.023669295012950897,
      "rewards/margins": 0.0015563804190605879,
      "rewards/rejected": 0.02211291715502739,
      "step": 4830
    },
    {
      "epoch": 0.85,
      "learning_rate": 7.637279596977329e-08,
      "logits/chosen": -2.744185447692871,
      "logits/rejected": -2.718427896499634,
      "logps/chosen": -266.70318603515625,
      "logps/rejected": -220.90786743164062,
      "loss": 0.6904,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.02698015235364437,
      "rewards/margins": 0.007942819967865944,
      "rewards/rejected": 0.019037332385778427,
      "step": 4840
    },
    {
      "epoch": 0.85,
      "learning_rate": 7.632241813602014e-08,
      "logits/chosen": -2.786680221557617,
      "logits/rejected": -2.7786288261413574,
      "logps/chosen": -223.2958221435547,
      "logps/rejected": -175.2308807373047,
      "loss": 0.6914,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.019782308489084244,
      "rewards/margins": 0.007503434084355831,
      "rewards/rejected": 0.012278875336050987,
      "step": 4850
    },
    {
      "epoch": 0.85,
      "learning_rate": 7.6272040302267e-08,
      "logits/chosen": -2.7439236640930176,
      "logits/rejected": -2.754936695098877,
      "logps/chosen": -196.223876953125,
      "logps/rejected": -198.14244079589844,
      "loss": 0.6913,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.021982144564390182,
      "rewards/margins": 0.003115237457677722,
      "rewards/rejected": 0.018866905942559242,
      "step": 4860
    },
    {
      "epoch": 0.85,
      "learning_rate": 7.622166246851385e-08,
      "logits/chosen": -2.718632221221924,
      "logits/rejected": -2.7445740699768066,
      "logps/chosen": -230.5083465576172,
      "logps/rejected": -186.44937133789062,
      "loss": 0.6923,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": 0.026937445625662804,
      "rewards/margins": 0.0090105552226305,
      "rewards/rejected": 0.017926890403032303,
      "step": 4870
    },
    {
      "epoch": 0.85,
      "learning_rate": 7.61712846347607e-08,
      "logits/chosen": -2.8068249225616455,
      "logits/rejected": -2.7692713737487793,
      "logps/chosen": -246.0875701904297,
      "logps/rejected": -199.01675415039062,
      "loss": 0.6914,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.02345597743988037,
      "rewards/margins": 0.007212613709270954,
      "rewards/rejected": 0.01624336466193199,
      "step": 4880
    },
    {
      "epoch": 0.86,
      "learning_rate": 7.612090680100756e-08,
      "logits/chosen": -2.745042324066162,
      "logits/rejected": -2.815016746520996,
      "logps/chosen": -257.94610595703125,
      "logps/rejected": -189.48870849609375,
      "loss": 0.6908,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.023482834920287132,
      "rewards/margins": 0.006505007389932871,
      "rewards/rejected": 0.01697782799601555,
      "step": 4890
    },
    {
      "epoch": 0.86,
      "learning_rate": 7.60705289672544e-08,
      "logits/chosen": -2.661994695663452,
      "logits/rejected": -2.670673131942749,
      "logps/chosen": -239.2667236328125,
      "logps/rejected": -215.118408203125,
      "loss": 0.6902,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.029827941209077835,
      "rewards/margins": 0.0076766968704760075,
      "rewards/rejected": 0.022151242941617966,
      "step": 4900
    },
    {
      "epoch": 0.86,
      "learning_rate": 7.602015113350125e-08,
      "logits/chosen": -2.7965893745422363,
      "logits/rejected": -2.8132386207580566,
      "logps/chosen": -246.4945831298828,
      "logps/rejected": -211.777587890625,
      "loss": 0.6912,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.0261533260345459,
      "rewards/margins": 0.006919356994330883,
      "rewards/rejected": 0.01923396997153759,
      "step": 4910
    },
    {
      "epoch": 0.86,
      "learning_rate": 7.596977329974811e-08,
      "logits/chosen": -2.790916919708252,
      "logits/rejected": -2.753495693206787,
      "logps/chosen": -231.7954864501953,
      "logps/rejected": -226.02127075195312,
      "loss": 0.6912,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.021478954702615738,
      "rewards/margins": 0.001239580800756812,
      "rewards/rejected": 0.020239371806383133,
      "step": 4920
    },
    {
      "epoch": 0.86,
      "learning_rate": 7.591939546599496e-08,
      "logits/chosen": -2.7043099403381348,
      "logits/rejected": -2.789663314819336,
      "logps/chosen": -261.9654541015625,
      "logps/rejected": -234.9620361328125,
      "loss": 0.6906,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.03034358285367489,
      "rewards/margins": 0.005865463055670261,
      "rewards/rejected": 0.024478118866682053,
      "step": 4930
    },
    {
      "epoch": 0.87,
      "learning_rate": 7.58690176322418e-08,
      "logits/chosen": -2.813978672027588,
      "logits/rejected": -2.751081943511963,
      "logps/chosen": -250.8798370361328,
      "logps/rejected": -239.74002075195312,
      "loss": 0.6918,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.028360417112708092,
      "rewards/margins": 0.006048205774277449,
      "rewards/rejected": 0.022312210872769356,
      "step": 4940
    },
    {
      "epoch": 0.87,
      "learning_rate": 7.581863979848865e-08,
      "logits/chosen": -2.7854201793670654,
      "logits/rejected": -2.7969956398010254,
      "logps/chosen": -218.26541137695312,
      "logps/rejected": -190.56585693359375,
      "loss": 0.6916,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.02540905773639679,
      "rewards/margins": 0.004594909492880106,
      "rewards/rejected": 0.02081414684653282,
      "step": 4950
    },
    {
      "epoch": 0.87,
      "learning_rate": 7.576826196473551e-08,
      "logits/chosen": -2.758981943130493,
      "logits/rejected": -2.732278347015381,
      "logps/chosen": -238.2212371826172,
      "logps/rejected": -241.3662872314453,
      "loss": 0.6913,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.024932190775871277,
      "rewards/margins": 0.002725116442888975,
      "rewards/rejected": 0.022207075729966164,
      "step": 4960
    },
    {
      "epoch": 0.87,
      "learning_rate": 7.571788413098236e-08,
      "logits/chosen": -2.749824285507202,
      "logits/rejected": -2.7466375827789307,
      "logps/chosen": -263.97076416015625,
      "logps/rejected": -213.78076171875,
      "loss": 0.6899,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": 0.03015116974711418,
      "rewards/margins": 0.01208040677011013,
      "rewards/rejected": 0.0180707648396492,
      "step": 4970
    },
    {
      "epoch": 0.87,
      "learning_rate": 7.566750629722922e-08,
      "logits/chosen": -2.728415012359619,
      "logits/rejected": -2.7121329307556152,
      "logps/chosen": -240.95773315429688,
      "logps/rejected": -253.390380859375,
      "loss": 0.6916,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.02616221271455288,
      "rewards/margins": 0.0012474370887503028,
      "rewards/rejected": 0.024914775043725967,
      "step": 4980
    },
    {
      "epoch": 0.87,
      "learning_rate": 7.561712846347607e-08,
      "logits/chosen": -2.7221524715423584,
      "logits/rejected": -2.7137563228607178,
      "logps/chosen": -244.83627319335938,
      "logps/rejected": -194.1187744140625,
      "loss": 0.6905,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.023836804553866386,
      "rewards/margins": 0.005772692617028952,
      "rewards/rejected": 0.018064109608530998,
      "step": 4990
    },
    {
      "epoch": 0.88,
      "learning_rate": 7.556675062972292e-08,
      "logits/chosen": -2.7389016151428223,
      "logits/rejected": -2.7844491004943848,
      "logps/chosen": -258.43865966796875,
      "logps/rejected": -235.6800079345703,
      "loss": 0.6902,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.024131130427122116,
      "rewards/margins": 0.007521049585193396,
      "rewards/rejected": 0.016610082238912582,
      "step": 5000
    },
    {
      "epoch": 0.88,
      "learning_rate": 7.551637279596978e-08,
      "logits/chosen": -2.7731165885925293,
      "logits/rejected": -2.6811578273773193,
      "logps/chosen": -205.5344696044922,
      "logps/rejected": -183.38037109375,
      "loss": 0.6902,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.016569796949625015,
      "rewards/margins": -0.002280243206769228,
      "rewards/rejected": 0.018850039690732956,
      "step": 5010
    },
    {
      "epoch": 0.88,
      "learning_rate": 7.546599496221662e-08,
      "logits/chosen": -2.7271320819854736,
      "logits/rejected": -2.809903144836426,
      "logps/chosen": -206.69082641601562,
      "logps/rejected": -222.5525360107422,
      "loss": 0.6902,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.017895519733428955,
      "rewards/margins": 0.00183036585804075,
      "rewards/rejected": 0.01606515422463417,
      "step": 5020
    },
    {
      "epoch": 0.88,
      "learning_rate": 7.541561712846347e-08,
      "logits/chosen": -2.7593743801116943,
      "logits/rejected": -2.7369823455810547,
      "logps/chosen": -215.10879516601562,
      "logps/rejected": -209.2699737548828,
      "loss": 0.6911,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": 0.023434340953826904,
      "rewards/margins": 0.00012367055751383305,
      "rewards/rejected": 0.023310670629143715,
      "step": 5030
    },
    {
      "epoch": 0.88,
      "learning_rate": 7.536523929471032e-08,
      "logits/chosen": -2.748511791229248,
      "logits/rejected": -2.74576997756958,
      "logps/chosen": -233.9241180419922,
      "logps/rejected": -241.55917358398438,
      "loss": 0.6903,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": 0.02254708856344223,
      "rewards/margins": 0.002134478185325861,
      "rewards/rejected": 0.020412612706422806,
      "step": 5040
    },
    {
      "epoch": 0.88,
      "learning_rate": 7.531486146095718e-08,
      "logits/chosen": -2.7883503437042236,
      "logits/rejected": -2.765450954437256,
      "logps/chosen": -194.64598083496094,
      "logps/rejected": -229.24850463867188,
      "loss": 0.6905,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.024526655673980713,
      "rewards/margins": -0.0027814474888145924,
      "rewards/rejected": 0.027308102697134018,
      "step": 5050
    },
    {
      "epoch": 0.89,
      "learning_rate": 7.526448362720402e-08,
      "logits/chosen": -2.714806318283081,
      "logits/rejected": -2.6766140460968018,
      "logps/chosen": -248.44253540039062,
      "logps/rejected": -229.79556274414062,
      "loss": 0.6907,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.027913689613342285,
      "rewards/margins": 0.005528334993869066,
      "rewards/rejected": 0.022385356947779655,
      "step": 5060
    },
    {
      "epoch": 0.89,
      "learning_rate": 7.521410579345087e-08,
      "logits/chosen": -2.7708027362823486,
      "logits/rejected": -2.7898004055023193,
      "logps/chosen": -224.08193969726562,
      "logps/rejected": -220.9014129638672,
      "loss": 0.6907,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.027087047696113586,
      "rewards/margins": 0.004851869307458401,
      "rewards/rejected": 0.02223517931997776,
      "step": 5070
    },
    {
      "epoch": 0.89,
      "learning_rate": 7.516372795969773e-08,
      "logits/chosen": -2.7174248695373535,
      "logits/rejected": -2.741032123565674,
      "logps/chosen": -198.52511596679688,
      "logps/rejected": -177.54678344726562,
      "loss": 0.6893,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.020215127617120743,
      "rewards/margins": 0.0038847383111715317,
      "rewards/rejected": 0.016330387443304062,
      "step": 5080
    },
    {
      "epoch": 0.89,
      "learning_rate": 7.511335012594458e-08,
      "logits/chosen": -2.718716859817505,
      "logits/rejected": -2.748931407928467,
      "logps/chosen": -251.72378540039062,
      "logps/rejected": -226.3111114501953,
      "loss": 0.6916,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.02789355255663395,
      "rewards/margins": 0.0028011654503643513,
      "rewards/rejected": 0.025092383846640587,
      "step": 5090
    },
    {
      "epoch": 0.89,
      "learning_rate": 7.506297229219143e-08,
      "logits/chosen": -2.720767021179199,
      "logits/rejected": -2.744206666946411,
      "logps/chosen": -257.54852294921875,
      "logps/rejected": -162.34461975097656,
      "loss": 0.6903,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.029418539255857468,
      "rewards/margins": 0.013503951020538807,
      "rewards/rejected": 0.015914589166641235,
      "step": 5100
    },
    {
      "epoch": 0.9,
      "learning_rate": 7.501259445843829e-08,
      "logits/chosen": -2.801480293273926,
      "logits/rejected": -2.8004708290100098,
      "logps/chosen": -250.4967803955078,
      "logps/rejected": -197.5179901123047,
      "loss": 0.6895,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.024577541276812553,
      "rewards/margins": 0.010709354653954506,
      "rewards/rejected": 0.013868187554180622,
      "step": 5110
    },
    {
      "epoch": 0.9,
      "learning_rate": 7.496221662468514e-08,
      "logits/chosen": -2.754500389099121,
      "logits/rejected": -2.7321605682373047,
      "logps/chosen": -211.9841766357422,
      "logps/rejected": -153.45176696777344,
      "loss": 0.6904,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": 0.02272544428706169,
      "rewards/margins": 0.012496823444962502,
      "rewards/rejected": 0.01022862084209919,
      "step": 5120
    },
    {
      "epoch": 0.9,
      "learning_rate": 7.4911838790932e-08,
      "logits/chosen": -2.765127658843994,
      "logits/rejected": -2.803529739379883,
      "logps/chosen": -265.0037841796875,
      "logps/rejected": -221.8296661376953,
      "loss": 0.69,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.03228873759508133,
      "rewards/margins": 0.007724687457084656,
      "rewards/rejected": 0.024564048275351524,
      "step": 5130
    },
    {
      "epoch": 0.9,
      "learning_rate": 7.486146095717883e-08,
      "logits/chosen": -2.739295482635498,
      "logits/rejected": -2.737955093383789,
      "logps/chosen": -210.24307250976562,
      "logps/rejected": -197.39566040039062,
      "loss": 0.691,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.01834609918296337,
      "rewards/margins": 0.0016159085789695382,
      "rewards/rejected": 0.016730191186070442,
      "step": 5140
    },
    {
      "epoch": 0.9,
      "learning_rate": 7.481108312342569e-08,
      "logits/chosen": -2.8013620376586914,
      "logits/rejected": -2.766737461090088,
      "logps/chosen": -246.08480834960938,
      "logps/rejected": -205.478759765625,
      "loss": 0.6912,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.024060197174549103,
      "rewards/margins": 0.003503562416881323,
      "rewards/rejected": 0.020556632429361343,
      "step": 5150
    },
    {
      "epoch": 0.9,
      "learning_rate": 7.476070528967254e-08,
      "logits/chosen": -2.8327503204345703,
      "logits/rejected": -2.825082302093506,
      "logps/chosen": -218.51394653320312,
      "logps/rejected": -197.2113800048828,
      "loss": 0.6913,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.025230538100004196,
      "rewards/margins": 0.002071975963190198,
      "rewards/rejected": 0.023158561438322067,
      "step": 5160
    },
    {
      "epoch": 0.91,
      "learning_rate": 7.47103274559194e-08,
      "logits/chosen": -2.721312999725342,
      "logits/rejected": -2.7535860538482666,
      "logps/chosen": -210.86325073242188,
      "logps/rejected": -202.97125244140625,
      "loss": 0.6902,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.01850944198668003,
      "rewards/margins": 0.002257524523884058,
      "rewards/rejected": 0.01625191606581211,
      "step": 5170
    },
    {
      "epoch": 0.91,
      "learning_rate": 7.465994962216624e-08,
      "logits/chosen": -2.7364120483398438,
      "logits/rejected": -2.808478355407715,
      "logps/chosen": -236.8877716064453,
      "logps/rejected": -233.90103149414062,
      "loss": 0.6913,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.02594706416130066,
      "rewards/margins": 0.0006133163697086275,
      "rewards/rejected": 0.025333750993013382,
      "step": 5180
    },
    {
      "epoch": 0.91,
      "learning_rate": 7.460957178841309e-08,
      "logits/chosen": -2.789334535598755,
      "logits/rejected": -2.7878098487854004,
      "logps/chosen": -246.96987915039062,
      "logps/rejected": -218.8030548095703,
      "loss": 0.6911,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.02684968151152134,
      "rewards/margins": 0.00774233415722847,
      "rewards/rejected": 0.019107351079583168,
      "step": 5190
    },
    {
      "epoch": 0.91,
      "learning_rate": 7.455919395465994e-08,
      "logits/chosen": -2.6209568977355957,
      "logits/rejected": -2.766512870788574,
      "logps/chosen": -260.9529113769531,
      "logps/rejected": -207.5177459716797,
      "loss": 0.6897,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.02858014777302742,
      "rewards/margins": 0.006817141082137823,
      "rewards/rejected": 0.02176300808787346,
      "step": 5200
    },
    {
      "epoch": 0.91,
      "learning_rate": 7.45088161209068e-08,
      "logits/chosen": -2.7715020179748535,
      "logits/rejected": -2.7795920372009277,
      "logps/chosen": -250.96273803710938,
      "logps/rejected": -235.0276336669922,
      "loss": 0.6909,
      "rewards/accuracies": 0.42500001192092896,
      "rewards/chosen": 0.027890939265489578,
      "rewards/margins": -0.0006950365495868027,
      "rewards/rejected": 0.028585975989699364,
      "step": 5210
    },
    {
      "epoch": 0.91,
      "learning_rate": 7.445843828715365e-08,
      "logits/chosen": -2.709684133529663,
      "logits/rejected": -2.7793078422546387,
      "logps/chosen": -239.56918334960938,
      "logps/rejected": -206.3387908935547,
      "loss": 0.6904,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.027700409293174744,
      "rewards/margins": 0.007622907869517803,
      "rewards/rejected": 0.020077500492334366,
      "step": 5220
    },
    {
      "epoch": 0.92,
      "learning_rate": 7.44080604534005e-08,
      "logits/chosen": -2.7601256370544434,
      "logits/rejected": -2.716702461242676,
      "logps/chosen": -233.1493377685547,
      "logps/rejected": -227.4046173095703,
      "loss": 0.6898,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.028554772958159447,
      "rewards/margins": 0.009560917504131794,
      "rewards/rejected": 0.018993858247995377,
      "step": 5230
    },
    {
      "epoch": 0.92,
      "learning_rate": 7.435768261964736e-08,
      "logits/chosen": -2.7953238487243652,
      "logits/rejected": -2.7698287963867188,
      "logps/chosen": -253.69473266601562,
      "logps/rejected": -218.02468872070312,
      "loss": 0.6901,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.03720221668481827,
      "rewards/margins": 0.011492163874208927,
      "rewards/rejected": 0.025710051879286766,
      "step": 5240
    },
    {
      "epoch": 0.92,
      "learning_rate": 7.43073047858942e-08,
      "logits/chosen": -2.7610411643981934,
      "logits/rejected": -2.7470200061798096,
      "logps/chosen": -214.7230987548828,
      "logps/rejected": -183.76800537109375,
      "loss": 0.6901,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.02231210097670555,
      "rewards/margins": 0.00498523935675621,
      "rewards/rejected": 0.01732686348259449,
      "step": 5250
    },
    {
      "epoch": 0.92,
      "learning_rate": 7.425692695214105e-08,
      "logits/chosen": -2.7620432376861572,
      "logits/rejected": -2.724316358566284,
      "logps/chosen": -206.0223388671875,
      "logps/rejected": -204.09695434570312,
      "loss": 0.6909,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.021319400519132614,
      "rewards/margins": 0.007719547487795353,
      "rewards/rejected": 0.01359985489398241,
      "step": 5260
    },
    {
      "epoch": 0.92,
      "learning_rate": 7.42065491183879e-08,
      "logits/chosen": -2.7803006172180176,
      "logits/rejected": -2.7820000648498535,
      "logps/chosen": -175.95228576660156,
      "logps/rejected": -147.31631469726562,
      "loss": 0.6906,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.01864704303443432,
      "rewards/margins": 0.005782184191048145,
      "rewards/rejected": 0.012864859774708748,
      "step": 5270
    },
    {
      "epoch": 0.92,
      "learning_rate": 7.415617128463476e-08,
      "logits/chosen": -2.7566115856170654,
      "logits/rejected": -2.7493796348571777,
      "logps/chosen": -277.02899169921875,
      "logps/rejected": -236.46224975585938,
      "loss": 0.6905,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 0.037890736013650894,
      "rewards/margins": 0.00899316743016243,
      "rewards/rejected": 0.028897572308778763,
      "step": 5280
    },
    {
      "epoch": 0.93,
      "learning_rate": 7.41057934508816e-08,
      "logits/chosen": -2.733806610107422,
      "logits/rejected": -2.809513807296753,
      "logps/chosen": -249.7975311279297,
      "logps/rejected": -216.98861694335938,
      "loss": 0.6891,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.02854977548122406,
      "rewards/margins": 0.007073344197124243,
      "rewards/rejected": 0.021476436406373978,
      "step": 5290
    },
    {
      "epoch": 0.93,
      "learning_rate": 7.405541561712845e-08,
      "logits/chosen": -2.793487071990967,
      "logits/rejected": -2.745420217514038,
      "logps/chosen": -219.4196319580078,
      "logps/rejected": -186.6765594482422,
      "loss": 0.6899,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.023922424763441086,
      "rewards/margins": 0.007303243037313223,
      "rewards/rejected": 0.016619181260466576,
      "step": 5300
    },
    {
      "epoch": 0.93,
      "learning_rate": 7.40050377833753e-08,
      "logits/chosen": -2.798801898956299,
      "logits/rejected": -2.8438451290130615,
      "logps/chosen": -214.73941040039062,
      "logps/rejected": -186.37538146972656,
      "loss": 0.6908,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.02055622637271881,
      "rewards/margins": 0.003379581030458212,
      "rewards/rejected": 0.017176643013954163,
      "step": 5310
    },
    {
      "epoch": 0.93,
      "learning_rate": 7.395465994962217e-08,
      "logits/chosen": -2.7654082775115967,
      "logits/rejected": -2.7806427478790283,
      "logps/chosen": -219.7326202392578,
      "logps/rejected": -204.74957275390625,
      "loss": 0.6899,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.02500101923942566,
      "rewards/margins": 0.0017423263052478433,
      "rewards/rejected": 0.023258691653609276,
      "step": 5320
    },
    {
      "epoch": 0.93,
      "learning_rate": 7.390428211586901e-08,
      "logits/chosen": -2.636276960372925,
      "logits/rejected": -2.816444158554077,
      "logps/chosen": -206.4623565673828,
      "logps/rejected": -154.90005493164062,
      "loss": 0.6904,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.02116440422832966,
      "rewards/margins": 0.004634098149836063,
      "rewards/rejected": 0.01653030514717102,
      "step": 5330
    },
    {
      "epoch": 0.94,
      "learning_rate": 7.385390428211587e-08,
      "logits/chosen": -2.71712589263916,
      "logits/rejected": -2.712641477584839,
      "logps/chosen": -234.46609497070312,
      "logps/rejected": -188.48818969726562,
      "loss": 0.6926,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.020298270508646965,
      "rewards/margins": 0.0009019045392051339,
      "rewards/rejected": 0.01939636841416359,
      "step": 5340
    },
    {
      "epoch": 0.94,
      "learning_rate": 7.380352644836272e-08,
      "logits/chosen": -2.813690662384033,
      "logits/rejected": -2.7450575828552246,
      "logps/chosen": -229.24288940429688,
      "logps/rejected": -251.2069549560547,
      "loss": 0.6904,
      "rewards/accuracies": 0.42500001192092896,
      "rewards/chosen": 0.02722129225730896,
      "rewards/margins": -0.0024374243803322315,
      "rewards/rejected": 0.029658714309334755,
      "step": 5350
    },
    {
      "epoch": 0.94,
      "learning_rate": 7.375314861460957e-08,
      "logits/chosen": -2.7050914764404297,
      "logits/rejected": -2.6738526821136475,
      "logps/chosen": -209.76651000976562,
      "logps/rejected": -210.8341064453125,
      "loss": 0.6913,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": 0.018942106515169144,
      "rewards/margins": -0.0005925153382122517,
      "rewards/rejected": 0.019534621387720108,
      "step": 5360
    },
    {
      "epoch": 0.94,
      "learning_rate": 7.370277078085641e-08,
      "logits/chosen": -2.723813533782959,
      "logits/rejected": -2.7255825996398926,
      "logps/chosen": -245.219482421875,
      "logps/rejected": -260.9398498535156,
      "loss": 0.6912,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.02663588896393776,
      "rewards/margins": -0.0023054510820657015,
      "rewards/rejected": 0.02894134260714054,
      "step": 5370
    },
    {
      "epoch": 0.94,
      "learning_rate": 7.365239294710327e-08,
      "logits/chosen": -2.768441677093506,
      "logits/rejected": -2.7272372245788574,
      "logps/chosen": -226.75308227539062,
      "logps/rejected": -202.91358947753906,
      "loss": 0.6912,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.026924926787614822,
      "rewards/margins": 0.00796701293438673,
      "rewards/rejected": 0.018957912921905518,
      "step": 5380
    },
    {
      "epoch": 0.94,
      "learning_rate": 7.360201511335012e-08,
      "logits/chosen": -2.7350900173187256,
      "logits/rejected": -2.7533819675445557,
      "logps/chosen": -191.90249633789062,
      "logps/rejected": -169.1590118408203,
      "loss": 0.6904,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.018432730808854103,
      "rewards/margins": 0.006261094473302364,
      "rewards/rejected": 0.012171637266874313,
      "step": 5390
    },
    {
      "epoch": 0.95,
      "learning_rate": 7.355163727959698e-08,
      "logits/chosen": -2.700066089630127,
      "logits/rejected": -2.7570040225982666,
      "logps/chosen": -263.6595153808594,
      "logps/rejected": -196.6237335205078,
      "loss": 0.6895,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.030814606696367264,
      "rewards/margins": 0.012204675003886223,
      "rewards/rejected": 0.01860993169248104,
      "step": 5400
    },
    {
      "epoch": 0.95,
      "learning_rate": 7.350125944584382e-08,
      "logits/chosen": -2.8282995223999023,
      "logits/rejected": -2.7629384994506836,
      "logps/chosen": -218.19076538085938,
      "logps/rejected": -225.69961547851562,
      "loss": 0.6907,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.025148767977952957,
      "rewards/margins": 0.0035461492370814085,
      "rewards/rejected": 0.02160261943936348,
      "step": 5410
    },
    {
      "epoch": 0.95,
      "learning_rate": 7.345088161209067e-08,
      "logits/chosen": -2.7315096855163574,
      "logits/rejected": -2.767759323120117,
      "logps/chosen": -221.05850219726562,
      "logps/rejected": -205.8386688232422,
      "loss": 0.6912,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.025065699592232704,
      "rewards/margins": 0.004063331056386232,
      "rewards/rejected": 0.02100237086415291,
      "step": 5420
    },
    {
      "epoch": 0.95,
      "learning_rate": 7.340050377833754e-08,
      "logits/chosen": -2.754154682159424,
      "logits/rejected": -2.780801296234131,
      "logps/chosen": -197.50173950195312,
      "logps/rejected": -179.8265380859375,
      "loss": 0.6907,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.02420850098133087,
      "rewards/margins": 0.010105123743414879,
      "rewards/rejected": 0.014103377237915993,
      "step": 5430
    },
    {
      "epoch": 0.95,
      "learning_rate": 7.335012594458439e-08,
      "logits/chosen": -2.699470043182373,
      "logits/rejected": -2.740807294845581,
      "logps/chosen": -234.15097045898438,
      "logps/rejected": -194.5983428955078,
      "loss": 0.6902,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.02444395422935486,
      "rewards/margins": 0.007107601501047611,
      "rewards/rejected": 0.017336349934339523,
      "step": 5440
    },
    {
      "epoch": 0.95,
      "learning_rate": 7.329974811083123e-08,
      "logits/chosen": -2.7896604537963867,
      "logits/rejected": -2.846015214920044,
      "logps/chosen": -242.762939453125,
      "logps/rejected": -184.61318969726562,
      "loss": 0.6902,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.02861708402633667,
      "rewards/margins": 0.012237682938575745,
      "rewards/rejected": 0.016379402950406075,
      "step": 5450
    },
    {
      "epoch": 0.96,
      "learning_rate": 7.324937027707808e-08,
      "logits/chosen": -2.7799696922302246,
      "logits/rejected": -2.8197786808013916,
      "logps/chosen": -238.2605743408203,
      "logps/rejected": -217.13143920898438,
      "loss": 0.6904,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.02993357554078102,
      "rewards/margins": 0.008179426193237305,
      "rewards/rejected": 0.021754145622253418,
      "step": 5460
    },
    {
      "epoch": 0.96,
      "learning_rate": 7.319899244332494e-08,
      "logits/chosen": -2.792241096496582,
      "logits/rejected": -2.817352294921875,
      "logps/chosen": -229.5667266845703,
      "logps/rejected": -233.867431640625,
      "loss": 0.6899,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.028801094740629196,
      "rewards/margins": 0.0033335580956190825,
      "rewards/rejected": 0.025467539206147194,
      "step": 5470
    },
    {
      "epoch": 0.96,
      "learning_rate": 7.314861460957179e-08,
      "logits/chosen": -2.7579219341278076,
      "logits/rejected": -2.7147867679595947,
      "logps/chosen": -258.652587890625,
      "logps/rejected": -257.3244934082031,
      "loss": 0.6907,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": 0.02721872553229332,
      "rewards/margins": 0.00041804014472290874,
      "rewards/rejected": 0.02680068649351597,
      "step": 5480
    },
    {
      "epoch": 0.96,
      "learning_rate": 7.309823677581863e-08,
      "logits/chosen": -2.8154964447021484,
      "logits/rejected": -2.8179690837860107,
      "logps/chosen": -223.1553192138672,
      "logps/rejected": -172.56883239746094,
      "loss": 0.6907,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.025681037455797195,
      "rewards/margins": 0.011194641701877117,
      "rewards/rejected": 0.014486396685242653,
      "step": 5490
    },
    {
      "epoch": 0.96,
      "learning_rate": 7.304785894206549e-08,
      "logits/chosen": -2.7281696796417236,
      "logits/rejected": -2.7027804851531982,
      "logps/chosen": -190.97433471679688,
      "logps/rejected": -182.0121307373047,
      "loss": 0.6895,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.023420389741659164,
      "rewards/margins": 0.006670433096587658,
      "rewards/rejected": 0.01674995757639408,
      "step": 5500
    },
    {
      "epoch": 0.97,
      "learning_rate": 7.299748110831234e-08,
      "logits/chosen": -2.6794257164001465,
      "logits/rejected": -2.696403980255127,
      "logps/chosen": -214.8280029296875,
      "logps/rejected": -171.09063720703125,
      "loss": 0.6898,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.026057744398713112,
      "rewards/margins": 0.010476592928171158,
      "rewards/rejected": 0.015581147745251656,
      "step": 5510
    },
    {
      "epoch": 0.97,
      "learning_rate": 7.294710327455918e-08,
      "logits/chosen": -2.7246193885803223,
      "logits/rejected": -2.7311501502990723,
      "logps/chosen": -266.88446044921875,
      "logps/rejected": -221.1366424560547,
      "loss": 0.6903,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.028967270627617836,
      "rewards/margins": 0.006662419531494379,
      "rewards/rejected": 0.022304847836494446,
      "step": 5520
    },
    {
      "epoch": 0.97,
      "learning_rate": 7.289672544080603e-08,
      "logits/chosen": -2.7876334190368652,
      "logits/rejected": -2.8115463256835938,
      "logps/chosen": -240.1504669189453,
      "logps/rejected": -210.5426788330078,
      "loss": 0.69,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.027545759454369545,
      "rewards/margins": 0.009936556220054626,
      "rewards/rejected": 0.01760920323431492,
      "step": 5530
    },
    {
      "epoch": 0.97,
      "learning_rate": 7.28463476070529e-08,
      "logits/chosen": -2.799808979034424,
      "logits/rejected": -2.8136532306671143,
      "logps/chosen": -222.64321899414062,
      "logps/rejected": -197.42601013183594,
      "loss": 0.6918,
      "rewards/accuracies": 0.42500001192092896,
      "rewards/chosen": 0.021958649158477783,
      "rewards/margins": -0.0008252925472334027,
      "rewards/rejected": 0.02278394252061844,
      "step": 5540
    },
    {
      "epoch": 0.97,
      "learning_rate": 7.279596977329975e-08,
      "logits/chosen": -2.6864452362060547,
      "logits/rejected": -2.707339286804199,
      "logps/chosen": -233.6274871826172,
      "logps/rejected": -208.48046875,
      "loss": 0.6901,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.03295878320932388,
      "rewards/margins": 0.011183591559529305,
      "rewards/rejected": 0.021775193512439728,
      "step": 5550
    },
    {
      "epoch": 0.97,
      "learning_rate": 7.27455919395466e-08,
      "logits/chosen": -2.8088607788085938,
      "logits/rejected": -2.825002431869507,
      "logps/chosen": -218.2990264892578,
      "logps/rejected": -191.56271362304688,
      "loss": 0.6911,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.02727615274488926,
      "rewards/margins": 0.0035555907525122166,
      "rewards/rejected": 0.02372056432068348,
      "step": 5560
    },
    {
      "epoch": 0.98,
      "learning_rate": 7.269521410579345e-08,
      "logits/chosen": -2.7786643505096436,
      "logits/rejected": -2.771984100341797,
      "logps/chosen": -216.9097442626953,
      "logps/rejected": -186.2939910888672,
      "loss": 0.6897,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.023232027888298035,
      "rewards/margins": 0.009016161784529686,
      "rewards/rejected": 0.014215867035090923,
      "step": 5570
    },
    {
      "epoch": 0.98,
      "learning_rate": 7.26448362720403e-08,
      "logits/chosen": -2.739405393600464,
      "logits/rejected": -2.7287230491638184,
      "logps/chosen": -203.91127014160156,
      "logps/rejected": -170.11453247070312,
      "loss": 0.6909,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.020928533747792244,
      "rewards/margins": 0.00953659974038601,
      "rewards/rejected": 0.011391934007406235,
      "step": 5580
    },
    {
      "epoch": 0.98,
      "learning_rate": 7.259445843828716e-08,
      "logits/chosen": -2.740872383117676,
      "logits/rejected": -2.7223007678985596,
      "logps/chosen": -233.0533905029297,
      "logps/rejected": -194.1956024169922,
      "loss": 0.6897,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.030118608847260475,
      "rewards/margins": 0.010699293576180935,
      "rewards/rejected": 0.019419312477111816,
      "step": 5590
    },
    {
      "epoch": 0.98,
      "learning_rate": 7.2544080604534e-08,
      "logits/chosen": -2.7146565914154053,
      "logits/rejected": -2.750152349472046,
      "logps/chosen": -256.93853759765625,
      "logps/rejected": -221.79159545898438,
      "loss": 0.6904,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": 0.02939864620566368,
      "rewards/margins": 0.0112245362251997,
      "rewards/rejected": 0.01817411556839943,
      "step": 5600
    },
    {
      "epoch": 0.98,
      "learning_rate": 7.249370277078085e-08,
      "logits/chosen": -2.8190269470214844,
      "logits/rejected": -2.767198085784912,
      "logps/chosen": -229.6221923828125,
      "logps/rejected": -172.2571258544922,
      "loss": 0.6898,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.028731968253850937,
      "rewards/margins": 0.008819608017802238,
      "rewards/rejected": 0.019912362098693848,
      "step": 5610
    },
    {
      "epoch": 0.98,
      "learning_rate": 7.24433249370277e-08,
      "logits/chosen": -2.7028396129608154,
      "logits/rejected": -2.7214255332946777,
      "logps/chosen": -250.2680206298828,
      "logps/rejected": -220.27035522460938,
      "loss": 0.6902,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.025800978764891624,
      "rewards/margins": 0.005141588859260082,
      "rewards/rejected": 0.02065938711166382,
      "step": 5620
    },
    {
      "epoch": 0.99,
      "learning_rate": 7.239294710327456e-08,
      "logits/chosen": -2.701507806777954,
      "logits/rejected": -2.7152953147888184,
      "logps/chosen": -232.4032745361328,
      "logps/rejected": -196.23092651367188,
      "loss": 0.6896,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.02751338854432106,
      "rewards/margins": 0.009677217341959476,
      "rewards/rejected": 0.01783617213368416,
      "step": 5630
    },
    {
      "epoch": 0.99,
      "learning_rate": 7.23425692695214e-08,
      "logits/chosen": -2.7169084548950195,
      "logits/rejected": -2.7731575965881348,
      "logps/chosen": -185.28652954101562,
      "logps/rejected": -152.3063507080078,
      "loss": 0.6894,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.019060537219047546,
      "rewards/margins": 0.005134892649948597,
      "rewards/rejected": 0.01392564456909895,
      "step": 5640
    },
    {
      "epoch": 0.99,
      "learning_rate": 7.229219143576826e-08,
      "logits/chosen": -2.6719276905059814,
      "logits/rejected": -2.738537549972534,
      "logps/chosen": -235.8744354248047,
      "logps/rejected": -199.79006958007812,
      "loss": 0.6906,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.020635856315493584,
      "rewards/margins": 0.0033695294987410307,
      "rewards/rejected": 0.017266323789954185,
      "step": 5650
    },
    {
      "epoch": 0.99,
      "learning_rate": 7.224181360201512e-08,
      "logits/chosen": -2.7093288898468018,
      "logits/rejected": -2.734323740005493,
      "logps/chosen": -250.06448364257812,
      "logps/rejected": -218.0438995361328,
      "loss": 0.6891,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.03046129085123539,
      "rewards/margins": 0.009602924808859825,
      "rewards/rejected": 0.020858367905020714,
      "step": 5660
    },
    {
      "epoch": 0.99,
      "learning_rate": 7.219143576826197e-08,
      "logits/chosen": -2.734480381011963,
      "logits/rejected": -2.7367444038391113,
      "logps/chosen": -229.1237030029297,
      "logps/rejected": -193.18789672851562,
      "loss": 0.6905,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.030041057616472244,
      "rewards/margins": 0.01030112337321043,
      "rewards/rejected": 0.01973992958664894,
      "step": 5670
    },
    {
      "epoch": 0.99,
      "learning_rate": 7.214105793450881e-08,
      "logits/chosen": -2.8194003105163574,
      "logits/rejected": -2.806227922439575,
      "logps/chosen": -236.7169647216797,
      "logps/rejected": -207.375244140625,
      "loss": 0.6903,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.03411722555756569,
      "rewards/margins": 0.006585100200027227,
      "rewards/rejected": 0.027532124891877174,
      "step": 5680
    },
    {
      "epoch": 1.0,
      "learning_rate": 7.209068010075566e-08,
      "logits/chosen": -2.7609643936157227,
      "logits/rejected": -2.717151165008545,
      "logps/chosen": -202.71209716796875,
      "logps/rejected": -206.85635375976562,
      "loss": 0.6906,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.021368782967329025,
      "rewards/margins": 0.0015123977791517973,
      "rewards/rejected": 0.01985638588666916,
      "step": 5690
    },
    {
      "epoch": 1.0,
      "learning_rate": 7.204030226700252e-08,
      "logits/chosen": -2.7241241931915283,
      "logits/rejected": -2.7976861000061035,
      "logps/chosen": -214.01611328125,
      "logps/rejected": -226.95248413085938,
      "loss": 0.6895,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.02871261164546013,
      "rewards/margins": 0.0027239874470978975,
      "rewards/rejected": 0.025988629087805748,
      "step": 5700
    },
    {
      "epoch": 1.0,
      "learning_rate": 7.198992443324937e-08,
      "logits/chosen": -2.7344629764556885,
      "logits/rejected": -2.7737178802490234,
      "logps/chosen": -182.60733032226562,
      "logps/rejected": -152.49124145507812,
      "loss": 0.6898,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.017043083906173706,
      "rewards/margins": 0.007706266827881336,
      "rewards/rejected": 0.00933681707829237,
      "step": 5710
    },
    {
      "epoch": 1.0,
      "learning_rate": 7.193954659949621e-08,
      "logits/chosen": -2.763538360595703,
      "logits/rejected": -2.748915195465088,
      "logps/chosen": -231.2855224609375,
      "logps/rejected": -191.68783569335938,
      "loss": 0.6902,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.025866854935884476,
      "rewards/margins": 0.006101249251514673,
      "rewards/rejected": 0.01976560428738594,
      "step": 5720
    },
    {
      "epoch": 1.0,
      "learning_rate": 7.188916876574307e-08,
      "logits/chosen": -2.7810988426208496,
      "logits/rejected": -2.7703845500946045,
      "logps/chosen": -244.68374633789062,
      "logps/rejected": -209.5547637939453,
      "loss": 0.6905,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.02857055887579918,
      "rewards/margins": 0.007741375360637903,
      "rewards/rejected": 0.020829183980822563,
      "step": 5730
    },
    {
      "epoch": 1.01,
      "learning_rate": 7.183879093198992e-08,
      "logits/chosen": -2.7122907638549805,
      "logits/rejected": -2.73596453666687,
      "logps/chosen": -255.967041015625,
      "logps/rejected": -266.4009094238281,
      "loss": 0.6912,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.028105050325393677,
      "rewards/margins": -0.008025139570236206,
      "rewards/rejected": 0.03613018989562988,
      "step": 5740
    },
    {
      "epoch": 1.01,
      "learning_rate": 7.178841309823677e-08,
      "logits/chosen": -2.866520404815674,
      "logits/rejected": -2.8223705291748047,
      "logps/chosen": -247.6078643798828,
      "logps/rejected": -226.6530303955078,
      "loss": 0.6899,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.03066597506403923,
      "rewards/margins": 0.0038692415691912174,
      "rewards/rejected": 0.0267967339605093,
      "step": 5750
    },
    {
      "epoch": 1.01,
      "learning_rate": 7.173803526448363e-08,
      "logits/chosen": -2.6920933723449707,
      "logits/rejected": -2.7121245861053467,
      "logps/chosen": -215.04800415039062,
      "logps/rejected": -174.98876953125,
      "loss": 0.6894,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.02779332920908928,
      "rewards/margins": 0.01028980128467083,
      "rewards/rejected": 0.0175035297870636,
      "step": 5760
    },
    {
      "epoch": 1.01,
      "learning_rate": 7.168765743073048e-08,
      "logits/chosen": -2.8041000366210938,
      "logits/rejected": -2.824524164199829,
      "logps/chosen": -218.9689483642578,
      "logps/rejected": -179.63414001464844,
      "loss": 0.6899,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.03265921026468277,
      "rewards/margins": 0.011257478967308998,
      "rewards/rejected": 0.02140173129737377,
      "step": 5770
    },
    {
      "epoch": 1.01,
      "learning_rate": 7.163727959697733e-08,
      "logits/chosen": -2.758688449859619,
      "logits/rejected": -2.7820353507995605,
      "logps/chosen": -208.04373168945312,
      "logps/rejected": -205.10140991210938,
      "loss": 0.6901,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": 0.02206188254058361,
      "rewards/margins": -0.00437159975990653,
      "rewards/rejected": 0.026433482766151428,
      "step": 5780
    },
    {
      "epoch": 1.01,
      "learning_rate": 7.158690176322419e-08,
      "logits/chosen": -2.7041327953338623,
      "logits/rejected": -2.728461503982544,
      "logps/chosen": -280.71221923828125,
      "logps/rejected": -219.6815643310547,
      "loss": 0.689,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": 0.03200947865843773,
      "rewards/margins": 0.012490824796259403,
      "rewards/rejected": 0.019518649205565453,
      "step": 5790
    },
    {
      "epoch": 1.02,
      "learning_rate": 7.153652392947103e-08,
      "logits/chosen": -2.7607803344726562,
      "logits/rejected": -2.7257144451141357,
      "logps/chosen": -190.4318084716797,
      "logps/rejected": -195.79026794433594,
      "loss": 0.6905,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.02486034855246544,
      "rewards/margins": 0.004990352317690849,
      "rewards/rejected": 0.01986999623477459,
      "step": 5800
    },
    {
      "epoch": 1.02,
      "learning_rate": 7.148614609571788e-08,
      "logits/chosen": -2.792731761932373,
      "logits/rejected": -2.72947096824646,
      "logps/chosen": -285.7711486816406,
      "logps/rejected": -224.35043334960938,
      "loss": 0.6903,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": 0.038356930017471313,
      "rewards/margins": 0.013529865071177483,
      "rewards/rejected": 0.02482706308364868,
      "step": 5810
    },
    {
      "epoch": 1.02,
      "learning_rate": 7.143576826196474e-08,
      "logits/chosen": -2.727290630340576,
      "logits/rejected": -2.7715020179748535,
      "logps/chosen": -213.47970581054688,
      "logps/rejected": -171.37945556640625,
      "loss": 0.6895,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.029690707102417946,
      "rewards/margins": 0.009509574621915817,
      "rewards/rejected": 0.02018113061785698,
      "step": 5820
    },
    {
      "epoch": 1.02,
      "learning_rate": 7.138539042821158e-08,
      "logits/chosen": -2.8073341846466064,
      "logits/rejected": -2.7381162643432617,
      "logps/chosen": -247.31967163085938,
      "logps/rejected": -208.36679077148438,
      "loss": 0.6895,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.026355374604463577,
      "rewards/margins": 0.008763824589550495,
      "rewards/rejected": 0.017591550946235657,
      "step": 5830
    },
    {
      "epoch": 1.02,
      "learning_rate": 7.133501259445843e-08,
      "logits/chosen": -2.730480670928955,
      "logits/rejected": -2.8047595024108887,
      "logps/chosen": -233.4124298095703,
      "logps/rejected": -195.85726928710938,
      "loss": 0.6887,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": 0.03406389430165291,
      "rewards/margins": 0.012370677664875984,
      "rewards/rejected": 0.021693218499422073,
      "step": 5840
    },
    {
      "epoch": 1.02,
      "learning_rate": 7.128463476070528e-08,
      "logits/chosen": -2.808271884918213,
      "logits/rejected": -2.814162015914917,
      "logps/chosen": -236.5506134033203,
      "logps/rejected": -254.416748046875,
      "loss": 0.6909,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.034293659031391144,
      "rewards/margins": -0.00255168997682631,
      "rewards/rejected": 0.03684535250067711,
      "step": 5850
    },
    {
      "epoch": 1.03,
      "learning_rate": 7.123425692695214e-08,
      "logits/chosen": -2.7728164196014404,
      "logits/rejected": -2.8868014812469482,
      "logps/chosen": -273.46490478515625,
      "logps/rejected": -276.16217041015625,
      "loss": 0.6904,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.032950371503829956,
      "rewards/margins": -0.0058128563687205315,
      "rewards/rejected": 0.038763221353292465,
      "step": 5860
    },
    {
      "epoch": 1.03,
      "learning_rate": 7.118387909319899e-08,
      "logits/chosen": -2.7602274417877197,
      "logits/rejected": -2.7588744163513184,
      "logps/chosen": -227.1925048828125,
      "logps/rejected": -201.4271240234375,
      "loss": 0.69,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.027265915647149086,
      "rewards/margins": 0.004671896807849407,
      "rewards/rejected": 0.022594016045331955,
      "step": 5870
    },
    {
      "epoch": 1.03,
      "learning_rate": 7.113350125944584e-08,
      "logits/chosen": -2.844517230987549,
      "logits/rejected": -2.8165762424468994,
      "logps/chosen": -264.95025634765625,
      "logps/rejected": -215.6854705810547,
      "loss": 0.6902,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.03531178832054138,
      "rewards/margins": 0.013238337822258472,
      "rewards/rejected": 0.022073451429605484,
      "step": 5880
    },
    {
      "epoch": 1.03,
      "learning_rate": 7.10831234256927e-08,
      "logits/chosen": -2.6709351539611816,
      "logits/rejected": -2.7894115447998047,
      "logps/chosen": -224.2041473388672,
      "logps/rejected": -195.13259887695312,
      "loss": 0.6913,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.030605291947722435,
      "rewards/margins": 0.006223782896995544,
      "rewards/rejected": 0.02438150718808174,
      "step": 5890
    },
    {
      "epoch": 1.03,
      "learning_rate": 7.103274559193955e-08,
      "logits/chosen": -2.7667765617370605,
      "logits/rejected": -2.8099000453948975,
      "logps/chosen": -275.52734375,
      "logps/rejected": -254.30813598632812,
      "loss": 0.6897,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.03211339935660362,
      "rewards/margins": 0.007406218443065882,
      "rewards/rejected": 0.02470717951655388,
      "step": 5900
    },
    {
      "epoch": 1.04,
      "learning_rate": 7.098236775818639e-08,
      "logits/chosen": -2.7795968055725098,
      "logits/rejected": -2.7805514335632324,
      "logps/chosen": -233.39816284179688,
      "logps/rejected": -226.5900115966797,
      "loss": 0.6904,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.032585471868515015,
      "rewards/margins": 0.0037678040098398924,
      "rewards/rejected": 0.02881767228245735,
      "step": 5910
    },
    {
      "epoch": 1.04,
      "learning_rate": 7.093198992443325e-08,
      "logits/chosen": -2.794847249984741,
      "logits/rejected": -2.7636165618896484,
      "logps/chosen": -205.1078338623047,
      "logps/rejected": -182.61715698242188,
      "loss": 0.6898,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.030280139297246933,
      "rewards/margins": 0.009688856080174446,
      "rewards/rejected": 0.020591281354427338,
      "step": 5920
    },
    {
      "epoch": 1.04,
      "learning_rate": 7.08816120906801e-08,
      "logits/chosen": -2.6991028785705566,
      "logits/rejected": -2.7790234088897705,
      "logps/chosen": -191.94183349609375,
      "logps/rejected": -199.76925659179688,
      "loss": 0.691,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.023268550634384155,
      "rewards/margins": -0.0027100746519863605,
      "rewards/rejected": 0.02597862482070923,
      "step": 5930
    },
    {
      "epoch": 1.04,
      "learning_rate": 7.083123425692695e-08,
      "logits/chosen": -2.7407565116882324,
      "logits/rejected": -2.7216477394104004,
      "logps/chosen": -215.4610595703125,
      "logps/rejected": -210.43701171875,
      "loss": 0.691,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.027795767411589622,
      "rewards/margins": 0.008198624476790428,
      "rewards/rejected": 0.019597141072154045,
      "step": 5940
    },
    {
      "epoch": 1.04,
      "learning_rate": 7.078085642317379e-08,
      "logits/chosen": -2.7855286598205566,
      "logits/rejected": -2.8717663288116455,
      "logps/chosen": -264.8656005859375,
      "logps/rejected": -259.3266906738281,
      "loss": 0.6915,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.03862554579973221,
      "rewards/margins": 0.004008482210338116,
      "rewards/rejected": 0.03461706265807152,
      "step": 5950
    },
    {
      "epoch": 1.04,
      "learning_rate": 7.073047858942065e-08,
      "logits/chosen": -2.6747562885284424,
      "logits/rejected": -2.7460849285125732,
      "logps/chosen": -285.8426513671875,
      "logps/rejected": -243.5648193359375,
      "loss": 0.6901,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.04209999367594719,
      "rewards/margins": 0.006486804690212011,
      "rewards/rejected": 0.03561318665742874,
      "step": 5960
    },
    {
      "epoch": 1.05,
      "learning_rate": 7.06801007556675e-08,
      "logits/chosen": -2.763223171234131,
      "logits/rejected": -2.7372517585754395,
      "logps/chosen": -245.48654174804688,
      "logps/rejected": -239.01199340820312,
      "loss": 0.6895,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.03261946514248848,
      "rewards/margins": 0.009107634425163269,
      "rewards/rejected": 0.02351183071732521,
      "step": 5970
    },
    {
      "epoch": 1.05,
      "learning_rate": 7.062972292191435e-08,
      "logits/chosen": -2.7299296855926514,
      "logits/rejected": -2.765787363052368,
      "logps/chosen": -198.8389129638672,
      "logps/rejected": -203.43792724609375,
      "loss": 0.6911,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.02900700271129608,
      "rewards/margins": 0.0003943299816455692,
      "rewards/rejected": 0.028612669557332993,
      "step": 5980
    },
    {
      "epoch": 1.05,
      "learning_rate": 7.057934508816121e-08,
      "logits/chosen": -2.7050979137420654,
      "logits/rejected": -2.739762544631958,
      "logps/chosen": -308.5540466308594,
      "logps/rejected": -277.5274658203125,
      "loss": 0.6896,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.03356204554438591,
      "rewards/margins": 0.0038282524328678846,
      "rewards/rejected": 0.029733791947364807,
      "step": 5990
    },
    {
      "epoch": 1.05,
      "learning_rate": 7.052896725440806e-08,
      "logits/chosen": -2.6613450050354004,
      "logits/rejected": -2.6997017860412598,
      "logps/chosen": -219.81979370117188,
      "logps/rejected": -204.18075561523438,
      "loss": 0.6898,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.029529878869652748,
      "rewards/margins": 0.0035093098413199186,
      "rewards/rejected": 0.02602056786417961,
      "step": 6000
    },
    {
      "epoch": 1.05,
      "learning_rate": 7.047858942065492e-08,
      "logits/chosen": -2.6856908798217773,
      "logits/rejected": -2.740572452545166,
      "logps/chosen": -281.0226135253906,
      "logps/rejected": -247.1945037841797,
      "loss": 0.6903,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.03560008853673935,
      "rewards/margins": 0.007806276436895132,
      "rewards/rejected": 0.02779381349682808,
      "step": 6010
    },
    {
      "epoch": 1.05,
      "learning_rate": 7.042821158690177e-08,
      "logits/chosen": -2.833984851837158,
      "logits/rejected": -2.7636830806732178,
      "logps/chosen": -253.18115234375,
      "logps/rejected": -197.1452178955078,
      "loss": 0.6892,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.03183367848396301,
      "rewards/margins": 0.0074056894518435,
      "rewards/rejected": 0.0244279894977808,
      "step": 6020
    },
    {
      "epoch": 1.06,
      "learning_rate": 7.037783375314861e-08,
      "logits/chosen": -2.7777915000915527,
      "logits/rejected": -2.8554296493530273,
      "logps/chosen": -242.44552612304688,
      "logps/rejected": -209.59677124023438,
      "loss": 0.6903,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.03452635556459427,
      "rewards/margins": 0.002237586071714759,
      "rewards/rejected": 0.03228876739740372,
      "step": 6030
    },
    {
      "epoch": 1.06,
      "learning_rate": 7.032745591939546e-08,
      "logits/chosen": -2.7216124534606934,
      "logits/rejected": -2.7077813148498535,
      "logps/chosen": -227.49072265625,
      "logps/rejected": -183.86502075195312,
      "loss": 0.6903,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": 0.0319247804582119,
      "rewards/margins": 0.00687766820192337,
      "rewards/rejected": 0.025047114118933678,
      "step": 6040
    },
    {
      "epoch": 1.06,
      "learning_rate": 7.027707808564232e-08,
      "logits/chosen": -2.730086088180542,
      "logits/rejected": -2.7649378776550293,
      "logps/chosen": -238.0532989501953,
      "logps/rejected": -194.22817993164062,
      "loss": 0.6897,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.029018903151154518,
      "rewards/margins": 0.002032119780778885,
      "rewards/rejected": 0.026986781507730484,
      "step": 6050
    },
    {
      "epoch": 1.06,
      "learning_rate": 7.022670025188917e-08,
      "logits/chosen": -2.7800655364990234,
      "logits/rejected": -2.7575435638427734,
      "logps/chosen": -224.218017578125,
      "logps/rejected": -186.80465698242188,
      "loss": 0.69,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.029716094955801964,
      "rewards/margins": 0.0046003833413124084,
      "rewards/rejected": 0.025115709751844406,
      "step": 6060
    },
    {
      "epoch": 1.06,
      "learning_rate": 7.017632241813601e-08,
      "logits/chosen": -2.7532472610473633,
      "logits/rejected": -2.775247573852539,
      "logps/chosen": -232.7293701171875,
      "logps/rejected": -259.34393310546875,
      "loss": 0.69,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.033128730952739716,
      "rewards/margins": -0.0032485132105648518,
      "rewards/rejected": 0.03637724369764328,
      "step": 6070
    },
    {
      "epoch": 1.06,
      "learning_rate": 7.012594458438286e-08,
      "logits/chosen": -2.722989559173584,
      "logits/rejected": -2.778897285461426,
      "logps/chosen": -227.8129119873047,
      "logps/rejected": -208.2938232421875,
      "loss": 0.6896,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": 0.028836023062467575,
      "rewards/margins": 0.0021686807740479708,
      "rewards/rejected": 0.026667345315217972,
      "step": 6080
    },
    {
      "epoch": 1.07,
      "learning_rate": 7.007556675062972e-08,
      "logits/chosen": -2.8650097846984863,
      "logits/rejected": -2.8455333709716797,
      "logps/chosen": -229.21517944335938,
      "logps/rejected": -196.00250244140625,
      "loss": 0.6896,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.032781932502985,
      "rewards/margins": 0.013185905292630196,
      "rewards/rejected": 0.019596025347709656,
      "step": 6090
    },
    {
      "epoch": 1.07,
      "learning_rate": 7.002518891687657e-08,
      "logits/chosen": -2.674905300140381,
      "logits/rejected": -2.693514585494995,
      "logps/chosen": -216.5654754638672,
      "logps/rejected": -172.6519012451172,
      "loss": 0.6906,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.02908332273364067,
      "rewards/margins": 0.004312724806368351,
      "rewards/rejected": 0.024770598858594894,
      "step": 6100
    },
    {
      "epoch": 1.07,
      "learning_rate": 6.997481108312342e-08,
      "logits/chosen": -2.792025566101074,
      "logits/rejected": -2.8154520988464355,
      "logps/chosen": -285.5600280761719,
      "logps/rejected": -229.28829956054688,
      "loss": 0.6898,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.034517981112003326,
      "rewards/margins": 0.0033892481587827206,
      "rewards/rejected": 0.03112873062491417,
      "step": 6110
    },
    {
      "epoch": 1.07,
      "learning_rate": 6.992443324937028e-08,
      "logits/chosen": -2.7616074085235596,
      "logits/rejected": -2.837329864501953,
      "logps/chosen": -221.0269317626953,
      "logps/rejected": -172.927978515625,
      "loss": 0.6897,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.028237927705049515,
      "rewards/margins": 0.00989912636578083,
      "rewards/rejected": 0.018338803201913834,
      "step": 6120
    },
    {
      "epoch": 1.07,
      "learning_rate": 6.987405541561713e-08,
      "logits/chosen": -2.712308168411255,
      "logits/rejected": -2.7479236125946045,
      "logps/chosen": -242.038818359375,
      "logps/rejected": -203.10403442382812,
      "loss": 0.6898,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.032026901841163635,
      "rewards/margins": 0.007596462965011597,
      "rewards/rejected": 0.02443043515086174,
      "step": 6130
    },
    {
      "epoch": 1.08,
      "learning_rate": 6.982367758186397e-08,
      "logits/chosen": -2.7296433448791504,
      "logits/rejected": -2.7758820056915283,
      "logps/chosen": -241.84091186523438,
      "logps/rejected": -200.144287109375,
      "loss": 0.6892,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.03152359649538994,
      "rewards/margins": 0.009202331304550171,
      "rewards/rejected": 0.022321265190839767,
      "step": 6140
    },
    {
      "epoch": 1.08,
      "learning_rate": 6.977329974811083e-08,
      "logits/chosen": -2.761535406112671,
      "logits/rejected": -2.720188856124878,
      "logps/chosen": -230.50210571289062,
      "logps/rejected": -198.60598754882812,
      "loss": 0.6898,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.028255242854356766,
      "rewards/margins": 0.002794310450553894,
      "rewards/rejected": 0.025460928678512573,
      "step": 6150
    },
    {
      "epoch": 1.08,
      "learning_rate": 6.972292191435768e-08,
      "logits/chosen": -2.8285574913024902,
      "logits/rejected": -2.7832274436950684,
      "logps/chosen": -254.6967315673828,
      "logps/rejected": -234.7633056640625,
      "loss": 0.6903,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.03328879922628403,
      "rewards/margins": 0.005169660784304142,
      "rewards/rejected": 0.028119131922721863,
      "step": 6160
    },
    {
      "epoch": 1.08,
      "learning_rate": 6.967254408060453e-08,
      "logits/chosen": -2.7056522369384766,
      "logits/rejected": -2.7208755016326904,
      "logps/chosen": -209.24563598632812,
      "logps/rejected": -197.8179473876953,
      "loss": 0.6898,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.02616763673722744,
      "rewards/margins": 0.004211708437651396,
      "rewards/rejected": 0.021955927833914757,
      "step": 6170
    },
    {
      "epoch": 1.08,
      "learning_rate": 6.962216624685137e-08,
      "logits/chosen": -2.662492275238037,
      "logits/rejected": -2.671931028366089,
      "logps/chosen": -261.2315979003906,
      "logps/rejected": -233.17416381835938,
      "loss": 0.6897,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.03277582675218582,
      "rewards/margins": 0.004679680801928043,
      "rewards/rejected": 0.02809615060687065,
      "step": 6180
    },
    {
      "epoch": 1.08,
      "learning_rate": 6.957178841309823e-08,
      "logits/chosen": -2.707197427749634,
      "logits/rejected": -2.730229616165161,
      "logps/chosen": -252.9608154296875,
      "logps/rejected": -213.77774047851562,
      "loss": 0.6906,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.036902423948049545,
      "rewards/margins": 0.009274457581341267,
      "rewards/rejected": 0.027627963572740555,
      "step": 6190
    },
    {
      "epoch": 1.09,
      "learning_rate": 6.952141057934508e-08,
      "logits/chosen": -2.769176959991455,
      "logits/rejected": -2.7323288917541504,
      "logps/chosen": -218.0635986328125,
      "logps/rejected": -207.66445922851562,
      "loss": 0.6892,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.026653211563825607,
      "rewards/margins": 0.00453142449259758,
      "rewards/rejected": 0.022121790796518326,
      "step": 6200
    },
    {
      "epoch": 1.09,
      "learning_rate": 6.947103274559193e-08,
      "logits/chosen": -2.806384325027466,
      "logits/rejected": -2.756983518600464,
      "logps/chosen": -237.72232055664062,
      "logps/rejected": -252.24459838867188,
      "loss": 0.6915,
      "rewards/accuracies": 0.42500001192092896,
      "rewards/chosen": 0.03373720124363899,
      "rewards/margins": -0.0009608466061763465,
      "rewards/rejected": 0.03469805419445038,
      "step": 6210
    },
    {
      "epoch": 1.09,
      "learning_rate": 6.942065491183879e-08,
      "logits/chosen": -2.7189059257507324,
      "logits/rejected": -2.69968843460083,
      "logps/chosen": -186.25367736816406,
      "logps/rejected": -182.58639526367188,
      "loss": 0.6907,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.02435891702771187,
      "rewards/margins": 0.0031697209924459457,
      "rewards/rejected": 0.021189197897911072,
      "step": 6220
    },
    {
      "epoch": 1.09,
      "learning_rate": 6.937027707808564e-08,
      "logits/chosen": -2.82783579826355,
      "logits/rejected": -2.7836272716522217,
      "logps/chosen": -247.9126739501953,
      "logps/rejected": -207.5823974609375,
      "loss": 0.6892,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.03497808799147606,
      "rewards/margins": 0.012263385578989983,
      "rewards/rejected": 0.022714700549840927,
      "step": 6230
    },
    {
      "epoch": 1.09,
      "learning_rate": 6.93198992443325e-08,
      "logits/chosen": -2.6979732513427734,
      "logits/rejected": -2.7468159198760986,
      "logps/chosen": -195.87741088867188,
      "logps/rejected": -199.96136474609375,
      "loss": 0.6895,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": 0.027930688112974167,
      "rewards/margins": -0.0017716297879815102,
      "rewards/rejected": 0.0297023206949234,
      "step": 6240
    },
    {
      "epoch": 1.09,
      "learning_rate": 6.926952141057935e-08,
      "logits/chosen": -2.757378101348877,
      "logits/rejected": -2.8484787940979004,
      "logps/chosen": -252.18197631835938,
      "logps/rejected": -213.80307006835938,
      "loss": 0.6905,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.032926805317401886,
      "rewards/margins": 0.007040529511868954,
      "rewards/rejected": 0.025886276736855507,
      "step": 6250
    },
    {
      "epoch": 1.1,
      "learning_rate": 6.921914357682619e-08,
      "logits/chosen": -2.7444984912872314,
      "logits/rejected": -2.7312824726104736,
      "logps/chosen": -244.59176635742188,
      "logps/rejected": -209.32861328125,
      "loss": 0.6902,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.03717253729701042,
      "rewards/margins": 0.011591591872274876,
      "rewards/rejected": 0.02558094635605812,
      "step": 6260
    },
    {
      "epoch": 1.1,
      "learning_rate": 6.916876574307304e-08,
      "logits/chosen": -2.8128814697265625,
      "logits/rejected": -2.7963852882385254,
      "logps/chosen": -273.450927734375,
      "logps/rejected": -212.9509735107422,
      "loss": 0.69,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": 0.037680648267269135,
      "rewards/margins": 0.013447249308228493,
      "rewards/rejected": 0.024233397096395493,
      "step": 6270
    },
    {
      "epoch": 1.1,
      "learning_rate": 6.91183879093199e-08,
      "logits/chosen": -2.8444695472717285,
      "logits/rejected": -2.833711862564087,
      "logps/chosen": -237.81787109375,
      "logps/rejected": -202.39505004882812,
      "loss": 0.6896,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.03637075051665306,
      "rewards/margins": 0.012740323320031166,
      "rewards/rejected": 0.023630429059267044,
      "step": 6280
    },
    {
      "epoch": 1.1,
      "learning_rate": 6.906801007556675e-08,
      "logits/chosen": -2.8796963691711426,
      "logits/rejected": -2.8374648094177246,
      "logps/chosen": -218.6466064453125,
      "logps/rejected": -233.56826782226562,
      "loss": 0.6893,
      "rewards/accuracies": 0.42500001192092896,
      "rewards/chosen": 0.027191419154405594,
      "rewards/margins": -0.00017332080460619181,
      "rewards/rejected": 0.027364740148186684,
      "step": 6290
    },
    {
      "epoch": 1.1,
      "learning_rate": 6.901763224181359e-08,
      "logits/chosen": -2.7870595455169678,
      "logits/rejected": -2.726167678833008,
      "logps/chosen": -234.41732788085938,
      "logps/rejected": -195.29246520996094,
      "loss": 0.6901,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.032856278121471405,
      "rewards/margins": 0.009746129624545574,
      "rewards/rejected": 0.023110145702958107,
      "step": 6300
    },
    {
      "epoch": 1.11,
      "learning_rate": 6.896725440806044e-08,
      "logits/chosen": -2.7080512046813965,
      "logits/rejected": -2.6986517906188965,
      "logps/chosen": -214.7272491455078,
      "logps/rejected": -181.70693969726562,
      "loss": 0.6897,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.028171220794320107,
      "rewards/margins": 0.008141187019646168,
      "rewards/rejected": 0.020030032843351364,
      "step": 6310
    },
    {
      "epoch": 1.11,
      "learning_rate": 6.89168765743073e-08,
      "logits/chosen": -2.8024163246154785,
      "logits/rejected": -2.761427164077759,
      "logps/chosen": -221.8937530517578,
      "logps/rejected": -207.2194366455078,
      "loss": 0.6912,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.02579706907272339,
      "rewards/margins": 0.005380997434258461,
      "rewards/rejected": 0.02041606977581978,
      "step": 6320
    },
    {
      "epoch": 1.11,
      "learning_rate": 6.886649874055415e-08,
      "logits/chosen": -2.7723660469055176,
      "logits/rejected": -2.7853739261627197,
      "logps/chosen": -264.2718811035156,
      "logps/rejected": -234.5113067626953,
      "loss": 0.6895,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.03535057231783867,
      "rewards/margins": 0.0035186149179935455,
      "rewards/rejected": 0.03183195739984512,
      "step": 6330
    },
    {
      "epoch": 1.11,
      "learning_rate": 6.8816120906801e-08,
      "logits/chosen": -2.703395366668701,
      "logits/rejected": -2.809828519821167,
      "logps/chosen": -301.8238525390625,
      "logps/rejected": -219.3530731201172,
      "loss": 0.6898,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.0384126715362072,
      "rewards/margins": 0.007866952568292618,
      "rewards/rejected": 0.03054572269320488,
      "step": 6340
    },
    {
      "epoch": 1.11,
      "learning_rate": 6.876574307304786e-08,
      "logits/chosen": -2.8216795921325684,
      "logits/rejected": -2.800515651702881,
      "logps/chosen": -247.65792846679688,
      "logps/rejected": -247.12326049804688,
      "loss": 0.6906,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.03470718860626221,
      "rewards/margins": 0.0031642660032957792,
      "rewards/rejected": 0.03154291957616806,
      "step": 6350
    },
    {
      "epoch": 1.11,
      "learning_rate": 6.871536523929471e-08,
      "logits/chosen": -2.7862982749938965,
      "logits/rejected": -2.735156297683716,
      "logps/chosen": -187.41934204101562,
      "logps/rejected": -175.93814086914062,
      "loss": 0.6894,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.02837715670466423,
      "rewards/margins": 0.007649431936442852,
      "rewards/rejected": 0.020727727562189102,
      "step": 6360
    },
    {
      "epoch": 1.12,
      "learning_rate": 6.866498740554157e-08,
      "logits/chosen": -2.718940258026123,
      "logits/rejected": -2.647468090057373,
      "logps/chosen": -211.13345336914062,
      "logps/rejected": -231.8296356201172,
      "loss": 0.6903,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.027587344869971275,
      "rewards/margins": 0.00827721692621708,
      "rewards/rejected": 0.019310127943754196,
      "step": 6370
    },
    {
      "epoch": 1.12,
      "learning_rate": 6.861460957178841e-08,
      "logits/chosen": -2.73569917678833,
      "logits/rejected": -2.6965410709381104,
      "logps/chosen": -213.14785766601562,
      "logps/rejected": -239.1310577392578,
      "loss": 0.6904,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.033782411366701126,
      "rewards/margins": 0.004613696597516537,
      "rewards/rejected": 0.029168713837862015,
      "step": 6380
    },
    {
      "epoch": 1.12,
      "learning_rate": 6.856423173803526e-08,
      "logits/chosen": -2.781935691833496,
      "logits/rejected": -2.749058246612549,
      "logps/chosen": -212.47116088867188,
      "logps/rejected": -164.14442443847656,
      "loss": 0.6896,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.02552088536322117,
      "rewards/margins": 0.009579035453498363,
      "rewards/rejected": 0.01594184711575508,
      "step": 6390
    },
    {
      "epoch": 1.12,
      "learning_rate": 6.851385390428211e-08,
      "logits/chosen": -2.7572851181030273,
      "logits/rejected": -2.7509846687316895,
      "logps/chosen": -216.94027709960938,
      "logps/rejected": -181.42041015625,
      "loss": 0.688,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.02696484699845314,
      "rewards/margins": 0.009268132969737053,
      "rewards/rejected": 0.017696712166070938,
      "step": 6400
    },
    {
      "epoch": 1.12,
      "learning_rate": 6.846347607052897e-08,
      "logits/chosen": -2.7396633625030518,
      "logits/rejected": -2.7270400524139404,
      "logps/chosen": -205.31503295898438,
      "logps/rejected": -208.9713134765625,
      "loss": 0.6897,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.027754198759794235,
      "rewards/margins": 0.00899929367005825,
      "rewards/rejected": 0.018754903227090836,
      "step": 6410
    },
    {
      "epoch": 1.12,
      "learning_rate": 6.841309823677581e-08,
      "logits/chosen": -2.7138712406158447,
      "logits/rejected": -2.6797842979431152,
      "logps/chosen": -195.82118225097656,
      "logps/rejected": -187.37338256835938,
      "loss": 0.6902,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.024918612092733383,
      "rewards/margins": 0.0022410769015550613,
      "rewards/rejected": 0.022677535191178322,
      "step": 6420
    },
    {
      "epoch": 1.13,
      "learning_rate": 6.836272040302266e-08,
      "logits/chosen": -2.751479387283325,
      "logits/rejected": -2.7326974868774414,
      "logps/chosen": -228.85043334960938,
      "logps/rejected": -207.35287475585938,
      "loss": 0.6904,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.033883582800626755,
      "rewards/margins": 0.0020071747712790966,
      "rewards/rejected": 0.03187640756368637,
      "step": 6430
    },
    {
      "epoch": 1.13,
      "learning_rate": 6.831234256926952e-08,
      "logits/chosen": -2.7368900775909424,
      "logits/rejected": -2.7022595405578613,
      "logps/chosen": -234.35888671875,
      "logps/rejected": -216.0355682373047,
      "loss": 0.6898,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.03236772492527962,
      "rewards/margins": 0.006880121771246195,
      "rewards/rejected": 0.02548760175704956,
      "step": 6440
    },
    {
      "epoch": 1.13,
      "learning_rate": 6.826196473551637e-08,
      "logits/chosen": -2.8036484718322754,
      "logits/rejected": -2.8090319633483887,
      "logps/chosen": -221.8217010498047,
      "logps/rejected": -197.96914672851562,
      "loss": 0.6894,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.03676612302660942,
      "rewards/margins": 0.00784066691994667,
      "rewards/rejected": 0.0289254579693079,
      "step": 6450
    },
    {
      "epoch": 1.13,
      "learning_rate": 6.821158690176322e-08,
      "logits/chosen": -2.7491366863250732,
      "logits/rejected": -2.7955727577209473,
      "logps/chosen": -246.05380249023438,
      "logps/rejected": -199.51351928710938,
      "loss": 0.6887,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.03833184391260147,
      "rewards/margins": 0.010535829700529575,
      "rewards/rejected": 0.02779601886868477,
      "step": 6460
    },
    {
      "epoch": 1.13,
      "learning_rate": 6.816120906801008e-08,
      "logits/chosen": -2.7315986156463623,
      "logits/rejected": -2.788578987121582,
      "logps/chosen": -293.5569763183594,
      "logps/rejected": -195.04043579101562,
      "loss": 0.689,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.041501227766275406,
      "rewards/margins": 0.01331830583512783,
      "rewards/rejected": 0.028182918205857277,
      "step": 6470
    },
    {
      "epoch": 1.14,
      "learning_rate": 6.811083123425693e-08,
      "logits/chosen": -2.735344409942627,
      "logits/rejected": -2.812255859375,
      "logps/chosen": -218.69607543945312,
      "logps/rejected": -176.1515350341797,
      "loss": 0.6888,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.029728304594755173,
      "rewards/margins": 0.007465772330760956,
      "rewards/rejected": 0.022262532263994217,
      "step": 6480
    },
    {
      "epoch": 1.14,
      "learning_rate": 6.806045340050377e-08,
      "logits/chosen": -2.7940878868103027,
      "logits/rejected": -2.7996413707733154,
      "logps/chosen": -222.28524780273438,
      "logps/rejected": -160.18359375,
      "loss": 0.6886,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.03442731127142906,
      "rewards/margins": 0.01763591542840004,
      "rewards/rejected": 0.01679139956831932,
      "step": 6490
    },
    {
      "epoch": 1.14,
      "learning_rate": 6.801007556675062e-08,
      "logits/chosen": -2.800783157348633,
      "logits/rejected": -2.8015666007995605,
      "logps/chosen": -258.265625,
      "logps/rejected": -225.9890594482422,
      "loss": 0.6912,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.027882227674126625,
      "rewards/margins": -0.0033497277181595564,
      "rewards/rejected": 0.03123195841908455,
      "step": 6500
    },
    {
      "epoch": 1.14,
      "learning_rate": 6.795969773299748e-08,
      "logits/chosen": -2.7174441814422607,
      "logits/rejected": -2.72218656539917,
      "logps/chosen": -224.7734832763672,
      "logps/rejected": -211.9218292236328,
      "loss": 0.6896,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.030056113377213478,
      "rewards/margins": 0.001766999950632453,
      "rewards/rejected": 0.02828911505639553,
      "step": 6510
    },
    {
      "epoch": 1.14,
      "learning_rate": 6.790931989924433e-08,
      "logits/chosen": -2.7325847148895264,
      "logits/rejected": -2.677220582962036,
      "logps/chosen": -209.3241729736328,
      "logps/rejected": -201.1617889404297,
      "loss": 0.6901,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": 0.03640920668840408,
      "rewards/margins": 0.004497000481933355,
      "rewards/rejected": 0.03191220760345459,
      "step": 6520
    },
    {
      "epoch": 1.14,
      "learning_rate": 6.785894206549117e-08,
      "logits/chosen": -2.7233242988586426,
      "logits/rejected": -2.754073143005371,
      "logps/chosen": -261.9200439453125,
      "logps/rejected": -231.17337036132812,
      "loss": 0.6898,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.033219657838344574,
      "rewards/margins": 0.005315340589731932,
      "rewards/rejected": 0.027904313057661057,
      "step": 6530
    },
    {
      "epoch": 1.15,
      "learning_rate": 6.780856423173803e-08,
      "logits/chosen": -2.76570463180542,
      "logits/rejected": -2.751070022583008,
      "logps/chosen": -222.1561279296875,
      "logps/rejected": -208.1566162109375,
      "loss": 0.6888,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": 0.03213818743824959,
      "rewards/margins": 0.011885440908372402,
      "rewards/rejected": 0.02025274559855461,
      "step": 6540
    },
    {
      "epoch": 1.15,
      "learning_rate": 6.775818639798488e-08,
      "logits/chosen": -2.7724246978759766,
      "logits/rejected": -2.75826358795166,
      "logps/chosen": -221.2061767578125,
      "logps/rejected": -196.8527069091797,
      "loss": 0.6906,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.03563159704208374,
      "rewards/margins": 0.00219206721521914,
      "rewards/rejected": 0.03343953564763069,
      "step": 6550
    },
    {
      "epoch": 1.15,
      "learning_rate": 6.770780856423173e-08,
      "logits/chosen": -2.753361225128174,
      "logits/rejected": -2.754962921142578,
      "logps/chosen": -238.34725952148438,
      "logps/rejected": -216.7457733154297,
      "loss": 0.6886,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.036745570600032806,
      "rewards/margins": 0.006122713442891836,
      "rewards/rejected": 0.030622858554124832,
      "step": 6560
    },
    {
      "epoch": 1.15,
      "learning_rate": 6.765743073047859e-08,
      "logits/chosen": -2.726283550262451,
      "logits/rejected": -2.680886745452881,
      "logps/chosen": -244.8686065673828,
      "logps/rejected": -183.89035034179688,
      "loss": 0.6879,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": 0.03608755022287369,
      "rewards/margins": 0.011775456368923187,
      "rewards/rejected": 0.0243120975792408,
      "step": 6570
    },
    {
      "epoch": 1.15,
      "learning_rate": 6.760705289672544e-08,
      "logits/chosen": -2.6707301139831543,
      "logits/rejected": -2.8088834285736084,
      "logps/chosen": -267.09796142578125,
      "logps/rejected": -217.032958984375,
      "loss": 0.6887,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.03969926759600639,
      "rewards/margins": 0.006029042415320873,
      "rewards/rejected": 0.033670224249362946,
      "step": 6580
    },
    {
      "epoch": 1.15,
      "learning_rate": 6.75566750629723e-08,
      "logits/chosen": -2.808931827545166,
      "logits/rejected": -2.7902331352233887,
      "logps/chosen": -247.80078125,
      "logps/rejected": -202.20140075683594,
      "loss": 0.6901,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.03151538223028183,
      "rewards/margins": 0.003934096544981003,
      "rewards/rejected": 0.027581283822655678,
      "step": 6590
    },
    {
      "epoch": 1.16,
      "learning_rate": 6.750629722921915e-08,
      "logits/chosen": -2.787019968032837,
      "logits/rejected": -2.751845359802246,
      "logps/chosen": -209.40011596679688,
      "logps/rejected": -177.8138885498047,
      "loss": 0.6893,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.025737786665558815,
      "rewards/margins": 0.008292414247989655,
      "rewards/rejected": 0.01744537055492401,
      "step": 6600
    },
    {
      "epoch": 1.16,
      "learning_rate": 6.745591939546599e-08,
      "logits/chosen": -2.734375476837158,
      "logits/rejected": -2.83819842338562,
      "logps/chosen": -279.52569580078125,
      "logps/rejected": -200.38143920898438,
      "loss": 0.689,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.045416612178087234,
      "rewards/margins": 0.015584426000714302,
      "rewards/rejected": 0.02983218990266323,
      "step": 6610
    },
    {
      "epoch": 1.16,
      "learning_rate": 6.740554156171284e-08,
      "logits/chosen": -2.7101967334747314,
      "logits/rejected": -2.7053380012512207,
      "logps/chosen": -208.09811401367188,
      "logps/rejected": -186.99386596679688,
      "loss": 0.6893,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.03134473040699959,
      "rewards/margins": 0.013868039473891258,
      "rewards/rejected": 0.01747668907046318,
      "step": 6620
    },
    {
      "epoch": 1.16,
      "learning_rate": 6.73551637279597e-08,
      "logits/chosen": -2.791965961456299,
      "logits/rejected": -2.7837865352630615,
      "logps/chosen": -208.32333374023438,
      "logps/rejected": -211.68014526367188,
      "loss": 0.6906,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.02462940290570259,
      "rewards/margins": 0.0018524207407608628,
      "rewards/rejected": 0.022776981815695763,
      "step": 6630
    },
    {
      "epoch": 1.16,
      "learning_rate": 6.730478589420655e-08,
      "logits/chosen": -2.769686222076416,
      "logits/rejected": -2.740999937057495,
      "logps/chosen": -241.9813690185547,
      "logps/rejected": -229.87850952148438,
      "loss": 0.6885,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.03970779478549957,
      "rewards/margins": 0.005852693226188421,
      "rewards/rejected": 0.03385510295629501,
      "step": 6640
    },
    {
      "epoch": 1.16,
      "learning_rate": 6.725440806045339e-08,
      "logits/chosen": -2.763948440551758,
      "logits/rejected": -2.793121814727783,
      "logps/chosen": -297.51629638671875,
      "logps/rejected": -206.64999389648438,
      "loss": 0.6892,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": 0.04574600234627724,
      "rewards/margins": 0.01908976212143898,
      "rewards/rejected": 0.02665623463690281,
      "step": 6650
    },
    {
      "epoch": 1.17,
      "learning_rate": 6.720403022670024e-08,
      "logits/chosen": -2.7918996810913086,
      "logits/rejected": -2.747771978378296,
      "logps/chosen": -217.94424438476562,
      "logps/rejected": -168.6347198486328,
      "loss": 0.6898,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.02763747237622738,
      "rewards/margins": 0.00968779344111681,
      "rewards/rejected": 0.017949678003787994,
      "step": 6660
    },
    {
      "epoch": 1.17,
      "learning_rate": 6.71536523929471e-08,
      "logits/chosen": -2.796182155609131,
      "logits/rejected": -2.7847232818603516,
      "logps/chosen": -263.08062744140625,
      "logps/rejected": -272.1901550292969,
      "loss": 0.6905,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.036897528916597366,
      "rewards/margins": 0.00214485265314579,
      "rewards/rejected": 0.034752678126096725,
      "step": 6670
    },
    {
      "epoch": 1.17,
      "learning_rate": 6.710327455919395e-08,
      "logits/chosen": -2.7384700775146484,
      "logits/rejected": -2.7586898803710938,
      "logps/chosen": -263.4530029296875,
      "logps/rejected": -232.83840942382812,
      "loss": 0.6897,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.035116128623485565,
      "rewards/margins": 0.007391026709228754,
      "rewards/rejected": 0.027725106105208397,
      "step": 6680
    },
    {
      "epoch": 1.17,
      "learning_rate": 6.70528967254408e-08,
      "logits/chosen": -2.75610089302063,
      "logits/rejected": -2.781942367553711,
      "logps/chosen": -215.61819458007812,
      "logps/rejected": -199.6807098388672,
      "loss": 0.6886,
      "rewards/accuracies": 0.42500001192092896,
      "rewards/chosen": 0.028895875439047813,
      "rewards/margins": -0.0010175227653235197,
      "rewards/rejected": 0.02991339936852455,
      "step": 6690
    },
    {
      "epoch": 1.17,
      "learning_rate": 6.700251889168766e-08,
      "logits/chosen": -2.722428798675537,
      "logits/rejected": -2.7623000144958496,
      "logps/chosen": -260.5478210449219,
      "logps/rejected": -204.60206604003906,
      "loss": 0.6896,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.04780068248510361,
      "rewards/margins": 0.013338456861674786,
      "rewards/rejected": 0.03446222469210625,
      "step": 6700
    },
    {
      "epoch": 1.18,
      "learning_rate": 6.695214105793451e-08,
      "logits/chosen": -2.8044791221618652,
      "logits/rejected": -2.7793936729431152,
      "logps/chosen": -286.1170959472656,
      "logps/rejected": -230.1556396484375,
      "loss": 0.6905,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.043489281088113785,
      "rewards/margins": 0.013985136523842812,
      "rewards/rejected": 0.029504140838980675,
      "step": 6710
    },
    {
      "epoch": 1.18,
      "learning_rate": 6.690176322418136e-08,
      "logits/chosen": -2.701599597930908,
      "logits/rejected": -2.7080159187316895,
      "logps/chosen": -230.49649047851562,
      "logps/rejected": -155.5298309326172,
      "loss": 0.6887,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": 0.034238483756780624,
      "rewards/margins": 0.015169240534305573,
      "rewards/rejected": 0.0190692450851202,
      "step": 6720
    },
    {
      "epoch": 1.18,
      "learning_rate": 6.68513853904282e-08,
      "logits/chosen": -2.689814329147339,
      "logits/rejected": -2.642368793487549,
      "logps/chosen": -195.3006134033203,
      "logps/rejected": -173.03106689453125,
      "loss": 0.6876,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.029249629005789757,
      "rewards/margins": 0.012950867414474487,
      "rewards/rejected": 0.01629875972867012,
      "step": 6730
    },
    {
      "epoch": 1.18,
      "learning_rate": 6.680100755667506e-08,
      "logits/chosen": -2.7278635501861572,
      "logits/rejected": -2.732556104660034,
      "logps/chosen": -243.13375854492188,
      "logps/rejected": -198.23207092285156,
      "loss": 0.6898,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.03910071402788162,
      "rewards/margins": 0.012855380773544312,
      "rewards/rejected": 0.02624533139169216,
      "step": 6740
    },
    {
      "epoch": 1.18,
      "learning_rate": 6.675062972292191e-08,
      "logits/chosen": -2.7523670196533203,
      "logits/rejected": -2.750101089477539,
      "logps/chosen": -240.2274627685547,
      "logps/rejected": -221.08053588867188,
      "loss": 0.6893,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.040501974523067474,
      "rewards/margins": 0.01037162821739912,
      "rewards/rejected": 0.030130350962281227,
      "step": 6750
    },
    {
      "epoch": 1.18,
      "learning_rate": 6.670025188916877e-08,
      "logits/chosen": -2.802891254425049,
      "logits/rejected": -2.7591326236724854,
      "logps/chosen": -266.6939697265625,
      "logps/rejected": -221.074462890625,
      "loss": 0.6898,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.037126895040273666,
      "rewards/margins": 0.00036938933772034943,
      "rewards/rejected": 0.03675750270485878,
      "step": 6760
    },
    {
      "epoch": 1.19,
      "learning_rate": 6.66498740554156e-08,
      "logits/chosen": -2.8116838932037354,
      "logits/rejected": -2.735592842102051,
      "logps/chosen": -245.67007446289062,
      "logps/rejected": -214.1964569091797,
      "loss": 0.6882,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.0364735908806324,
      "rewards/margins": 0.009604789316654205,
      "rewards/rejected": 0.026868799701333046,
      "step": 6770
    },
    {
      "epoch": 1.19,
      "learning_rate": 6.659949622166246e-08,
      "logits/chosen": -2.8009650707244873,
      "logits/rejected": -2.7831168174743652,
      "logps/chosen": -213.4352569580078,
      "logps/rejected": -164.82901000976562,
      "loss": 0.6883,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.02785235084593296,
      "rewards/margins": 0.01253105141222477,
      "rewards/rejected": 0.01532130129635334,
      "step": 6780
    },
    {
      "epoch": 1.19,
      "learning_rate": 6.654911838790931e-08,
      "logits/chosen": -2.6899755001068115,
      "logits/rejected": -2.7289154529571533,
      "logps/chosen": -199.33334350585938,
      "logps/rejected": -205.575439453125,
      "loss": 0.6909,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": 0.03206375241279602,
      "rewards/margins": 0.001432869117707014,
      "rewards/rejected": 0.03063088282942772,
      "step": 6790
    },
    {
      "epoch": 1.19,
      "learning_rate": 6.649874055415617e-08,
      "logits/chosen": -2.6635406017303467,
      "logits/rejected": -2.7157580852508545,
      "logps/chosen": -196.7146453857422,
      "logps/rejected": -176.63833618164062,
      "loss": 0.6903,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.031799446791410446,
      "rewards/margins": 0.008729107677936554,
      "rewards/rejected": 0.02307034283876419,
      "step": 6800
    },
    {
      "epoch": 1.19,
      "learning_rate": 6.644836272040302e-08,
      "logits/chosen": -2.8116531372070312,
      "logits/rejected": -2.779773235321045,
      "logps/chosen": -196.38583374023438,
      "logps/rejected": -194.5372314453125,
      "loss": 0.6897,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.03152305632829666,
      "rewards/margins": 0.006186535116285086,
      "rewards/rejected": 0.02533651888370514,
      "step": 6810
    },
    {
      "epoch": 1.19,
      "learning_rate": 6.639798488664987e-08,
      "logits/chosen": -2.771275043487549,
      "logits/rejected": -2.77402925491333,
      "logps/chosen": -235.0574493408203,
      "logps/rejected": -201.66380310058594,
      "loss": 0.6903,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.040426142513751984,
      "rewards/margins": 0.007161187473684549,
      "rewards/rejected": 0.033264946192502975,
      "step": 6820
    },
    {
      "epoch": 1.2,
      "learning_rate": 6.634760705289673e-08,
      "logits/chosen": -2.8188562393188477,
      "logits/rejected": -2.7583014965057373,
      "logps/chosen": -241.53237915039062,
      "logps/rejected": -216.1865997314453,
      "loss": 0.6895,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": 0.037272509187459946,
      "rewards/margins": 0.004425130784511566,
      "rewards/rejected": 0.03284738212823868,
      "step": 6830
    },
    {
      "epoch": 1.2,
      "learning_rate": 6.629722921914357e-08,
      "logits/chosen": -2.8168251514434814,
      "logits/rejected": -2.748887062072754,
      "logps/chosen": -261.4979248046875,
      "logps/rejected": -232.7153778076172,
      "loss": 0.6889,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.041307780891656876,
      "rewards/margins": 0.01554332859814167,
      "rewards/rejected": 0.025764450430870056,
      "step": 6840
    },
    {
      "epoch": 1.2,
      "learning_rate": 6.624685138539042e-08,
      "logits/chosen": -2.751751661300659,
      "logits/rejected": -2.7708382606506348,
      "logps/chosen": -273.7115783691406,
      "logps/rejected": -220.8414306640625,
      "loss": 0.6894,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.037936046719551086,
      "rewards/margins": 0.006723390426486731,
      "rewards/rejected": 0.03121265396475792,
      "step": 6850
    },
    {
      "epoch": 1.2,
      "learning_rate": 6.619647355163728e-08,
      "logits/chosen": -2.800870180130005,
      "logits/rejected": -2.7695047855377197,
      "logps/chosen": -238.0107421875,
      "logps/rejected": -195.6055450439453,
      "loss": 0.6899,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.03982401639223099,
      "rewards/margins": 0.009933261200785637,
      "rewards/rejected": 0.0298907570540905,
      "step": 6860
    },
    {
      "epoch": 1.2,
      "learning_rate": 6.614609571788413e-08,
      "logits/chosen": -2.7755208015441895,
      "logits/rejected": -2.731126070022583,
      "logps/chosen": -205.5043487548828,
      "logps/rejected": -194.66209411621094,
      "loss": 0.6888,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.03579959273338318,
      "rewards/margins": 0.009841179475188255,
      "rewards/rejected": 0.025958415120840073,
      "step": 6870
    },
    {
      "epoch": 1.21,
      "learning_rate": 6.609571788413097e-08,
      "logits/chosen": -2.7811360359191895,
      "logits/rejected": -2.747478485107422,
      "logps/chosen": -218.49624633789062,
      "logps/rejected": -224.67333984375,
      "loss": 0.6916,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": 0.03223881870508194,
      "rewards/margins": -0.00041040778160095215,
      "rewards/rejected": 0.03264923021197319,
      "step": 6880
    },
    {
      "epoch": 1.21,
      "learning_rate": 6.604534005037782e-08,
      "logits/chosen": -2.829068660736084,
      "logits/rejected": -2.851719617843628,
      "logps/chosen": -255.1793670654297,
      "logps/rejected": -238.48300170898438,
      "loss": 0.6894,
      "rewards/accuracies": 0.42500001192092896,
      "rewards/chosen": 0.031496562063694,
      "rewards/margins": 0.0001958387001650408,
      "rewards/rejected": 0.03130072355270386,
      "step": 6890
    },
    {
      "epoch": 1.21,
      "learning_rate": 6.599496221662468e-08,
      "logits/chosen": -2.751180410385132,
      "logits/rejected": -2.813422679901123,
      "logps/chosen": -252.8855438232422,
      "logps/rejected": -217.68637084960938,
      "loss": 0.6894,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": 0.03818538039922714,
      "rewards/margins": 0.0018034654203802347,
      "rewards/rejected": 0.03638191893696785,
      "step": 6900
    },
    {
      "epoch": 1.21,
      "learning_rate": 6.594458438287154e-08,
      "logits/chosen": -2.712439775466919,
      "logits/rejected": -2.7637240886688232,
      "logps/chosen": -294.4551086425781,
      "logps/rejected": -206.96664428710938,
      "loss": 0.69,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.03531162813305855,
      "rewards/margins": 0.006781014613807201,
      "rewards/rejected": 0.028530608862638474,
      "step": 6910
    },
    {
      "epoch": 1.21,
      "learning_rate": 6.589420654911838e-08,
      "logits/chosen": -2.796703577041626,
      "logits/rejected": -2.817638635635376,
      "logps/chosen": -186.1031951904297,
      "logps/rejected": -150.4288787841797,
      "loss": 0.688,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.027689915150403976,
      "rewards/margins": 0.010430497117340565,
      "rewards/rejected": 0.017259418964385986,
      "step": 6920
    },
    {
      "epoch": 1.21,
      "learning_rate": 6.584382871536524e-08,
      "logits/chosen": -2.7453088760375977,
      "logits/rejected": -2.776571750640869,
      "logps/chosen": -225.50698852539062,
      "logps/rejected": -196.79696655273438,
      "loss": 0.6888,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.03130626305937767,
      "rewards/margins": 0.006553110666573048,
      "rewards/rejected": 0.0247531495988369,
      "step": 6930
    },
    {
      "epoch": 1.22,
      "learning_rate": 6.579345088161209e-08,
      "logits/chosen": -2.7517857551574707,
      "logits/rejected": -2.759016513824463,
      "logps/chosen": -189.88516235351562,
      "logps/rejected": -184.5245819091797,
      "loss": 0.6896,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.02992168627679348,
      "rewards/margins": 0.001992257311940193,
      "rewards/rejected": 0.027929430827498436,
      "step": 6940
    },
    {
      "epoch": 1.22,
      "learning_rate": 6.574307304785894e-08,
      "logits/chosen": -2.815490245819092,
      "logits/rejected": -2.803900718688965,
      "logps/chosen": -210.9685821533203,
      "logps/rejected": -191.39927673339844,
      "loss": 0.6907,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": 0.034951768815517426,
      "rewards/margins": 0.0016877256566658616,
      "rewards/rejected": 0.03326404467225075,
      "step": 6950
    },
    {
      "epoch": 1.22,
      "learning_rate": 6.569269521410579e-08,
      "logits/chosen": -2.803596019744873,
      "logits/rejected": -2.786287546157837,
      "logps/chosen": -248.3104248046875,
      "logps/rejected": -213.3457489013672,
      "loss": 0.689,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.04742354154586792,
      "rewards/margins": 0.01468456070870161,
      "rewards/rejected": 0.032738976180553436,
      "step": 6960
    },
    {
      "epoch": 1.22,
      "learning_rate": 6.564231738035264e-08,
      "logits/chosen": -2.7051408290863037,
      "logits/rejected": -2.6857523918151855,
      "logps/chosen": -226.7938232421875,
      "logps/rejected": -225.24612426757812,
      "loss": 0.6894,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.03250659257173538,
      "rewards/margins": -0.001109436503611505,
      "rewards/rejected": 0.03361602872610092,
      "step": 6970
    },
    {
      "epoch": 1.22,
      "learning_rate": 6.559193954659949e-08,
      "logits/chosen": -2.7691562175750732,
      "logits/rejected": -2.7059037685394287,
      "logps/chosen": -235.966552734375,
      "logps/rejected": -244.332275390625,
      "loss": 0.691,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.03826393932104111,
      "rewards/margins": 0.002380301710218191,
      "rewards/rejected": 0.03588363528251648,
      "step": 6980
    },
    {
      "epoch": 1.22,
      "learning_rate": 6.554156171284635e-08,
      "logits/chosen": -2.773897647857666,
      "logits/rejected": -2.8118696212768555,
      "logps/chosen": -279.00213623046875,
      "logps/rejected": -199.22181701660156,
      "loss": 0.6897,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.03939899057149887,
      "rewards/margins": 0.010439836420118809,
      "rewards/rejected": 0.028959155082702637,
      "step": 6990
    },
    {
      "epoch": 1.23,
      "learning_rate": 6.549118387909319e-08,
      "logits/chosen": -2.78377366065979,
      "logits/rejected": -2.8232734203338623,
      "logps/chosen": -222.0625,
      "logps/rejected": -167.61061096191406,
      "loss": 0.6889,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.03677304834127426,
      "rewards/margins": 0.011829810217022896,
      "rewards/rejected": 0.024943236261606216,
      "step": 7000
    },
    {
      "epoch": 1.23,
      "learning_rate": 6.544080604534004e-08,
      "logits/chosen": -2.801302671432495,
      "logits/rejected": -2.7772815227508545,
      "logps/chosen": -223.4099884033203,
      "logps/rejected": -201.2372283935547,
      "loss": 0.6891,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.034436147660017014,
      "rewards/margins": 0.012764090672135353,
      "rewards/rejected": 0.02167206071317196,
      "step": 7010
    },
    {
      "epoch": 1.23,
      "learning_rate": 6.539042821158691e-08,
      "logits/chosen": -2.726637601852417,
      "logits/rejected": -2.7596089839935303,
      "logps/chosen": -244.93588256835938,
      "logps/rejected": -204.8834991455078,
      "loss": 0.6892,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.03578575700521469,
      "rewards/margins": 0.005086420103907585,
      "rewards/rejected": 0.030699338763952255,
      "step": 7020
    },
    {
      "epoch": 1.23,
      "learning_rate": 6.534005037783376e-08,
      "logits/chosen": -2.778423547744751,
      "logits/rejected": -2.8102736473083496,
      "logps/chosen": -235.8497772216797,
      "logps/rejected": -181.15029907226562,
      "loss": 0.6885,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": 0.0325014665722847,
      "rewards/margins": 0.010433828458189964,
      "rewards/rejected": 0.022067639976739883,
      "step": 7030
    },
    {
      "epoch": 1.23,
      "learning_rate": 6.52896725440806e-08,
      "logits/chosen": -2.790149211883545,
      "logits/rejected": -2.782501459121704,
      "logps/chosen": -244.16952514648438,
      "logps/rejected": -217.7505340576172,
      "loss": 0.6898,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.032779667526483536,
      "rewards/margins": 0.004313626326620579,
      "rewards/rejected": 0.028466040268540382,
      "step": 7040
    },
    {
      "epoch": 1.23,
      "learning_rate": 6.523929471032745e-08,
      "logits/chosen": -2.685401201248169,
      "logits/rejected": -2.7205746173858643,
      "logps/chosen": -246.46542358398438,
      "logps/rejected": -218.16384887695312,
      "loss": 0.6883,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.03928607702255249,
      "rewards/margins": 0.012703100219368935,
      "rewards/rejected": 0.026582980528473854,
      "step": 7050
    },
    {
      "epoch": 1.24,
      "learning_rate": 6.518891687657431e-08,
      "logits/chosen": -2.763960599899292,
      "logits/rejected": -2.6985137462615967,
      "logps/chosen": -219.4324188232422,
      "logps/rejected": -199.53598022460938,
      "loss": 0.6894,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.03679573908448219,
      "rewards/margins": 0.003578348783776164,
      "rewards/rejected": 0.03321739658713341,
      "step": 7060
    },
    {
      "epoch": 1.24,
      "learning_rate": 6.513853904282116e-08,
      "logits/chosen": -2.739204168319702,
      "logits/rejected": -2.721792221069336,
      "logps/chosen": -221.15011596679688,
      "logps/rejected": -224.3832550048828,
      "loss": 0.6879,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.03717418760061264,
      "rewards/margins": 0.006488792598247528,
      "rewards/rejected": 0.030685395002365112,
      "step": 7070
    },
    {
      "epoch": 1.24,
      "learning_rate": 6.5088161209068e-08,
      "logits/chosen": -2.763201951980591,
      "logits/rejected": -2.816208839416504,
      "logps/chosen": -219.6378631591797,
      "logps/rejected": -171.7489471435547,
      "loss": 0.6893,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": 0.03814969211816788,
      "rewards/margins": 0.015910113230347633,
      "rewards/rejected": 0.02223958447575569,
      "step": 7080
    },
    {
      "epoch": 1.24,
      "learning_rate": 6.503778337531486e-08,
      "logits/chosen": -2.749999523162842,
      "logits/rejected": -2.7644400596618652,
      "logps/chosen": -256.59527587890625,
      "logps/rejected": -254.48568725585938,
      "loss": 0.6909,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": 0.029433051124215126,
      "rewards/margins": -0.0038326564244925976,
      "rewards/rejected": 0.03326570615172386,
      "step": 7090
    },
    {
      "epoch": 1.24,
      "learning_rate": 6.498740554156171e-08,
      "logits/chosen": -2.7424628734588623,
      "logits/rejected": -2.669203281402588,
      "logps/chosen": -217.8045654296875,
      "logps/rejected": -284.6188049316406,
      "loss": 0.6892,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.03091282583773136,
      "rewards/margins": -0.0043990714475512505,
      "rewards/rejected": 0.03531189635396004,
      "step": 7100
    },
    {
      "epoch": 1.25,
      "learning_rate": 6.493702770780855e-08,
      "logits/chosen": -2.7484242916107178,
      "logits/rejected": -2.7699131965637207,
      "logps/chosen": -222.022705078125,
      "logps/rejected": -180.71083068847656,
      "loss": 0.6901,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.03578049689531326,
      "rewards/margins": 0.007164050824940205,
      "rewards/rejected": 0.028616448864340782,
      "step": 7110
    },
    {
      "epoch": 1.25,
      "learning_rate": 6.48866498740554e-08,
      "logits/chosen": -2.7241392135620117,
      "logits/rejected": -2.7856979370117188,
      "logps/chosen": -220.6631317138672,
      "logps/rejected": -172.29763793945312,
      "loss": 0.6881,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.03620423376560211,
      "rewards/margins": 0.01442769169807434,
      "rewards/rejected": 0.02177654579281807,
      "step": 7120
    },
    {
      "epoch": 1.25,
      "learning_rate": 6.483627204030227e-08,
      "logits/chosen": -2.6797337532043457,
      "logits/rejected": -2.677027702331543,
      "logps/chosen": -216.7772979736328,
      "logps/rejected": -199.2020721435547,
      "loss": 0.6881,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.03498489409685135,
      "rewards/margins": 0.0047217002138495445,
      "rewards/rejected": 0.03026319481432438,
      "step": 7130
    },
    {
      "epoch": 1.25,
      "learning_rate": 6.478589420654912e-08,
      "logits/chosen": -2.7731869220733643,
      "logits/rejected": -2.758033275604248,
      "logps/chosen": -210.3363037109375,
      "logps/rejected": -220.9199676513672,
      "loss": 0.688,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.03830721229314804,
      "rewards/margins": 0.009248893707990646,
      "rewards/rejected": 0.029058322310447693,
      "step": 7140
    },
    {
      "epoch": 1.25,
      "learning_rate": 6.473551637279596e-08,
      "logits/chosen": -2.840115547180176,
      "logits/rejected": -2.8175790309906006,
      "logps/chosen": -228.0071258544922,
      "logps/rejected": -200.12808227539062,
      "loss": 0.6891,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.03204935044050217,
      "rewards/margins": 0.0035434868186712265,
      "rewards/rejected": 0.02850586175918579,
      "step": 7150
    },
    {
      "epoch": 1.25,
      "learning_rate": 6.468513853904282e-08,
      "logits/chosen": -2.7763142585754395,
      "logits/rejected": -2.729067802429199,
      "logps/chosen": -198.84445190429688,
      "logps/rejected": -188.24356079101562,
      "loss": 0.6889,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.02600068226456642,
      "rewards/margins": 0.002334971446543932,
      "rewards/rejected": 0.023665711283683777,
      "step": 7160
    },
    {
      "epoch": 1.26,
      "learning_rate": 6.463476070528967e-08,
      "logits/chosen": -2.790109395980835,
      "logits/rejected": -2.776125431060791,
      "logps/chosen": -250.983642578125,
      "logps/rejected": -225.20327758789062,
      "loss": 0.6897,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.05125053972005844,
      "rewards/margins": 0.00900472141802311,
      "rewards/rejected": 0.042245812714099884,
      "step": 7170
    },
    {
      "epoch": 1.26,
      "learning_rate": 6.458438287153653e-08,
      "logits/chosen": -2.85467529296875,
      "logits/rejected": -2.8630192279815674,
      "logps/chosen": -248.15420532226562,
      "logps/rejected": -203.410400390625,
      "loss": 0.6895,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.0435900017619133,
      "rewards/margins": 0.013718393631279469,
      "rewards/rejected": 0.02987160161137581,
      "step": 7180
    },
    {
      "epoch": 1.26,
      "learning_rate": 6.453400503778337e-08,
      "logits/chosen": -2.787590503692627,
      "logits/rejected": -2.786054849624634,
      "logps/chosen": -215.96484375,
      "logps/rejected": -205.36135864257812,
      "loss": 0.6911,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.030155668035149574,
      "rewards/margins": 0.001419505337253213,
      "rewards/rejected": 0.028736162930727005,
      "step": 7190
    },
    {
      "epoch": 1.26,
      "learning_rate": 6.448362720403022e-08,
      "logits/chosen": -2.7419791221618652,
      "logits/rejected": -2.7809367179870605,
      "logps/chosen": -231.2300262451172,
      "logps/rejected": -209.7133026123047,
      "loss": 0.689,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.043003469705581665,
      "rewards/margins": 0.012471351772546768,
      "rewards/rejected": 0.030532116070389748,
      "step": 7200
    },
    {
      "epoch": 1.26,
      "learning_rate": 6.443324937027707e-08,
      "logits/chosen": -2.771419048309326,
      "logits/rejected": -2.7687599658966064,
      "logps/chosen": -266.37310791015625,
      "logps/rejected": -210.47946166992188,
      "loss": 0.6914,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.035980530083179474,
      "rewards/margins": 0.006279982626438141,
      "rewards/rejected": 0.02970055118203163,
      "step": 7210
    },
    {
      "epoch": 1.26,
      "learning_rate": 6.438287153652393e-08,
      "logits/chosen": -2.7209925651550293,
      "logits/rejected": -2.753868579864502,
      "logps/chosen": -244.1683349609375,
      "logps/rejected": -209.34567260742188,
      "loss": 0.6902,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.04534623771905899,
      "rewards/margins": 0.00936298631131649,
      "rewards/rejected": 0.03598325327038765,
      "step": 7220
    },
    {
      "epoch": 1.27,
      "learning_rate": 6.433249370277077e-08,
      "logits/chosen": -2.7852845191955566,
      "logits/rejected": -2.7609314918518066,
      "logps/chosen": -219.0021209716797,
      "logps/rejected": -212.36203002929688,
      "loss": 0.6908,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.034276049584150314,
      "rewards/margins": 0.008765382692217827,
      "rewards/rejected": 0.025510672479867935,
      "step": 7230
    },
    {
      "epoch": 1.27,
      "learning_rate": 6.428211586901763e-08,
      "logits/chosen": -2.7715468406677246,
      "logits/rejected": -2.7390708923339844,
      "logps/chosen": -244.0713348388672,
      "logps/rejected": -203.73193359375,
      "loss": 0.6893,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.04323107749223709,
      "rewards/margins": 0.014766479842364788,
      "rewards/rejected": 0.02846459485590458,
      "step": 7240
    },
    {
      "epoch": 1.27,
      "learning_rate": 6.423173803526449e-08,
      "logits/chosen": -2.7359414100646973,
      "logits/rejected": -2.7200896739959717,
      "logps/chosen": -236.83010864257812,
      "logps/rejected": -222.8236083984375,
      "loss": 0.6886,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.03487788513302803,
      "rewards/margins": 0.003767541376873851,
      "rewards/rejected": 0.03111034631729126,
      "step": 7250
    },
    {
      "epoch": 1.27,
      "learning_rate": 6.418136020151134e-08,
      "logits/chosen": -2.7496371269226074,
      "logits/rejected": -2.7715988159179688,
      "logps/chosen": -237.1099395751953,
      "logps/rejected": -190.89756774902344,
      "loss": 0.6883,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": 0.04079810902476311,
      "rewards/margins": 0.012925440445542336,
      "rewards/rejected": 0.027872666716575623,
      "step": 7260
    },
    {
      "epoch": 1.27,
      "learning_rate": 6.413098236775818e-08,
      "logits/chosen": -2.79183292388916,
      "logits/rejected": -2.8067638874053955,
      "logps/chosen": -239.8271484375,
      "logps/rejected": -196.8553924560547,
      "loss": 0.6886,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.04036902263760567,
      "rewards/margins": 0.008599973283708096,
      "rewards/rejected": 0.031769048422575,
      "step": 7270
    },
    {
      "epoch": 1.28,
      "learning_rate": 6.408060453400504e-08,
      "logits/chosen": -2.7779862880706787,
      "logits/rejected": -2.7364373207092285,
      "logps/chosen": -215.1895294189453,
      "logps/rejected": -221.6739501953125,
      "loss": 0.6881,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.03108001872897148,
      "rewards/margins": 0.007479669060558081,
      "rewards/rejected": 0.023600349202752113,
      "step": 7280
    },
    {
      "epoch": 1.28,
      "learning_rate": 6.403022670025189e-08,
      "logits/chosen": -2.7336158752441406,
      "logits/rejected": -2.7742183208465576,
      "logps/chosen": -224.93276977539062,
      "logps/rejected": -220.05130004882812,
      "loss": 0.6887,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.032671328634023666,
      "rewards/margins": 0.004601246677339077,
      "rewards/rejected": 0.028070081025362015,
      "step": 7290
    },
    {
      "epoch": 1.28,
      "learning_rate": 6.397984886649874e-08,
      "logits/chosen": -2.704530715942383,
      "logits/rejected": -2.6919538974761963,
      "logps/chosen": -196.310546875,
      "logps/rejected": -192.64761352539062,
      "loss": 0.6893,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.034642450511455536,
      "rewards/margins": 0.008061992935836315,
      "rewards/rejected": 0.026580458506941795,
      "step": 7300
    },
    {
      "epoch": 1.28,
      "learning_rate": 6.392947103274558e-08,
      "logits/chosen": -2.7123913764953613,
      "logits/rejected": -2.692824602127075,
      "logps/chosen": -200.9581756591797,
      "logps/rejected": -160.29251098632812,
      "loss": 0.6897,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.030062520876526833,
      "rewards/margins": 0.012599813751876354,
      "rewards/rejected": 0.017462706193327904,
      "step": 7310
    },
    {
      "epoch": 1.28,
      "learning_rate": 6.387909319899244e-08,
      "logits/chosen": -2.7910399436950684,
      "logits/rejected": -2.8223717212677,
      "logps/chosen": -243.82861328125,
      "logps/rejected": -190.62478637695312,
      "loss": 0.6883,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": 0.05294889211654663,
      "rewards/margins": 0.02855229750275612,
      "rewards/rejected": 0.024396590888500214,
      "step": 7320
    },
    {
      "epoch": 1.28,
      "learning_rate": 6.382871536523929e-08,
      "logits/chosen": -2.7878623008728027,
      "logits/rejected": -2.789745330810547,
      "logps/chosen": -279.63775634765625,
      "logps/rejected": -233.55050659179688,
      "loss": 0.6891,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.04993782192468643,
      "rewards/margins": 0.010094448924064636,
      "rewards/rejected": 0.039843373000621796,
      "step": 7330
    },
    {
      "epoch": 1.29,
      "learning_rate": 6.377833753148614e-08,
      "logits/chosen": -2.8130502700805664,
      "logits/rejected": -2.828782796859741,
      "logps/chosen": -216.92153930664062,
      "logps/rejected": -179.07345581054688,
      "loss": 0.6902,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.037634026259183884,
      "rewards/margins": 0.011927349492907524,
      "rewards/rejected": 0.02570667862892151,
      "step": 7340
    },
    {
      "epoch": 1.29,
      "learning_rate": 6.3727959697733e-08,
      "logits/chosen": -2.717914342880249,
      "logits/rejected": -2.771929979324341,
      "logps/chosen": -272.27349853515625,
      "logps/rejected": -217.73965454101562,
      "loss": 0.6891,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.04304666072130203,
      "rewards/margins": 0.01667637750506401,
      "rewards/rejected": 0.026370281353592873,
      "step": 7350
    },
    {
      "epoch": 1.29,
      "learning_rate": 6.367758186397985e-08,
      "logits/chosen": -2.7765262126922607,
      "logits/rejected": -2.756239175796509,
      "logps/chosen": -245.5876922607422,
      "logps/rejected": -211.84426879882812,
      "loss": 0.6886,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.04092712327837944,
      "rewards/margins": 0.010894590988755226,
      "rewards/rejected": 0.030032526701688766,
      "step": 7360
    },
    {
      "epoch": 1.29,
      "learning_rate": 6.36272040302267e-08,
      "logits/chosen": -2.7805514335632324,
      "logits/rejected": -2.779824733734131,
      "logps/chosen": -232.1643829345703,
      "logps/rejected": -227.3907470703125,
      "loss": 0.6903,
      "rewards/accuracies": 0.42500001192092896,
      "rewards/chosen": 0.04033838212490082,
      "rewards/margins": -0.0015016479883342981,
      "rewards/rejected": 0.041840024292469025,
      "step": 7370
    },
    {
      "epoch": 1.29,
      "learning_rate": 6.357682619647356e-08,
      "logits/chosen": -2.715909004211426,
      "logits/rejected": -2.6816649436950684,
      "logps/chosen": -183.8916473388672,
      "logps/rejected": -177.9539794921875,
      "loss": 0.69,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.02817304991185665,
      "rewards/margins": 0.002602068241685629,
      "rewards/rejected": 0.025570983067154884,
      "step": 7380
    },
    {
      "epoch": 1.29,
      "learning_rate": 6.35264483627204e-08,
      "logits/chosen": -2.764040470123291,
      "logits/rejected": -2.7185869216918945,
      "logps/chosen": -279.495361328125,
      "logps/rejected": -254.57553100585938,
      "loss": 0.688,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.04607158154249191,
      "rewards/margins": 0.00835057720541954,
      "rewards/rejected": 0.037721000611782074,
      "step": 7390
    },
    {
      "epoch": 1.3,
      "learning_rate": 6.347607052896725e-08,
      "logits/chosen": -2.691213607788086,
      "logits/rejected": -2.580820083618164,
      "logps/chosen": -216.9579620361328,
      "logps/rejected": -203.49806213378906,
      "loss": 0.6893,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.03406810015439987,
      "rewards/margins": 0.004827213007956743,
      "rewards/rejected": 0.029240887612104416,
      "step": 7400
    },
    {
      "epoch": 1.3,
      "learning_rate": 6.34256926952141e-08,
      "logits/chosen": -2.789217710494995,
      "logits/rejected": -2.7983765602111816,
      "logps/chosen": -259.81915283203125,
      "logps/rejected": -181.50772094726562,
      "loss": 0.6889,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": 0.038249146193265915,
      "rewards/margins": 0.019794996827840805,
      "rewards/rejected": 0.01845415122807026,
      "step": 7410
    },
    {
      "epoch": 1.3,
      "learning_rate": 6.337531486146095e-08,
      "logits/chosen": -2.7692363262176514,
      "logits/rejected": -2.7888898849487305,
      "logps/chosen": -251.02481079101562,
      "logps/rejected": -207.87417602539062,
      "loss": 0.689,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.0431194081902504,
      "rewards/margins": 0.009726546704769135,
      "rewards/rejected": 0.03339286148548126,
      "step": 7420
    },
    {
      "epoch": 1.3,
      "learning_rate": 6.33249370277078e-08,
      "logits/chosen": -2.8098390102386475,
      "logits/rejected": -2.824765682220459,
      "logps/chosen": -288.68280029296875,
      "logps/rejected": -209.61767578125,
      "loss": 0.6864,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.048886384814977646,
      "rewards/margins": 0.018948834389448166,
      "rewards/rejected": 0.02993755042552948,
      "step": 7430
    },
    {
      "epoch": 1.3,
      "learning_rate": 6.327455919395465e-08,
      "logits/chosen": -2.7200918197631836,
      "logits/rejected": -2.791701316833496,
      "logps/chosen": -197.84213256835938,
      "logps/rejected": -163.2316436767578,
      "loss": 0.69,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.03175193443894386,
      "rewards/margins": 0.009572578594088554,
      "rewards/rejected": 0.022179359570145607,
      "step": 7440
    },
    {
      "epoch": 1.3,
      "learning_rate": 6.322418136020151e-08,
      "logits/chosen": -2.7373557090759277,
      "logits/rejected": -2.7854480743408203,
      "logps/chosen": -183.2852325439453,
      "logps/rejected": -177.6833038330078,
      "loss": 0.6893,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.03157498314976692,
      "rewards/margins": 0.005984650459140539,
      "rewards/rejected": 0.025590334087610245,
      "step": 7450
    },
    {
      "epoch": 1.31,
      "learning_rate": 6.317380352644836e-08,
      "logits/chosen": -2.798292875289917,
      "logits/rejected": -2.690913677215576,
      "logps/chosen": -223.384033203125,
      "logps/rejected": -191.24310302734375,
      "loss": 0.6894,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.03310397267341614,
      "rewards/margins": 0.01318474393337965,
      "rewards/rejected": 0.019919229671359062,
      "step": 7460
    },
    {
      "epoch": 1.31,
      "learning_rate": 6.312342569269521e-08,
      "logits/chosen": -2.8380043506622314,
      "logits/rejected": -2.7938342094421387,
      "logps/chosen": -252.09130859375,
      "logps/rejected": -234.51791381835938,
      "loss": 0.6881,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.04607817903161049,
      "rewards/margins": 0.009110006503760815,
      "rewards/rejected": 0.0369681641459465,
      "step": 7470
    },
    {
      "epoch": 1.31,
      "learning_rate": 6.307304785894207e-08,
      "logits/chosen": -2.7199976444244385,
      "logits/rejected": -2.7746763229370117,
      "logps/chosen": -224.08078002929688,
      "logps/rejected": -199.91787719726562,
      "loss": 0.6891,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.03400200232863426,
      "rewards/margins": 0.0022418289445340633,
      "rewards/rejected": 0.03176017105579376,
      "step": 7480
    },
    {
      "epoch": 1.31,
      "learning_rate": 6.302267002518892e-08,
      "logits/chosen": -2.731687068939209,
      "logits/rejected": -2.7257843017578125,
      "logps/chosen": -239.6183624267578,
      "logps/rejected": -202.96115112304688,
      "loss": 0.6883,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.03186950460076332,
      "rewards/margins": 0.008324464783072472,
      "rewards/rejected": 0.023545041680336,
      "step": 7490
    },
    {
      "epoch": 1.31,
      "learning_rate": 6.297229219143576e-08,
      "logits/chosen": -2.738438129425049,
      "logits/rejected": -2.806363821029663,
      "logps/chosen": -176.00735473632812,
      "logps/rejected": -169.10317993164062,
      "loss": 0.6886,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.033585112541913986,
      "rewards/margins": 0.007096168585121632,
      "rewards/rejected": 0.02648894488811493,
      "step": 7500
    },
    {
      "epoch": 1.32,
      "learning_rate": 6.292191435768262e-08,
      "logits/chosen": -2.7676918506622314,
      "logits/rejected": -2.723275899887085,
      "logps/chosen": -244.68380737304688,
      "logps/rejected": -214.3140869140625,
      "loss": 0.6881,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": 0.049413811415433884,
      "rewards/margins": 0.013775454834103584,
      "rewards/rejected": 0.03563835844397545,
      "step": 7510
    },
    {
      "epoch": 1.32,
      "learning_rate": 6.287153652392947e-08,
      "logits/chosen": -2.7488560676574707,
      "logits/rejected": -2.771021842956543,
      "logps/chosen": -209.1370849609375,
      "logps/rejected": -213.5133056640625,
      "loss": 0.6881,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.033164605498313904,
      "rewards/margins": 0.0037708343006670475,
      "rewards/rejected": 0.029393767938017845,
      "step": 7520
    },
    {
      "epoch": 1.32,
      "learning_rate": 6.282115869017632e-08,
      "logits/chosen": -2.7546322345733643,
      "logits/rejected": -2.8115854263305664,
      "logps/chosen": -228.2063751220703,
      "logps/rejected": -213.37265014648438,
      "loss": 0.6904,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.042360819876194,
      "rewards/margins": 0.0021514396648854017,
      "rewards/rejected": 0.04020937532186508,
      "step": 7530
    },
    {
      "epoch": 1.32,
      "learning_rate": 6.277078085642316e-08,
      "logits/chosen": -2.8055498600006104,
      "logits/rejected": -2.8536624908447266,
      "logps/chosen": -249.27444458007812,
      "logps/rejected": -214.2854766845703,
      "loss": 0.6912,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.03579330071806908,
      "rewards/margins": 0.0017019972437992692,
      "rewards/rejected": 0.03409130498766899,
      "step": 7540
    },
    {
      "epoch": 1.32,
      "learning_rate": 6.272040302267002e-08,
      "logits/chosen": -2.7170093059539795,
      "logits/rejected": -2.7954814434051514,
      "logps/chosen": -202.75704956054688,
      "logps/rejected": -189.86465454101562,
      "loss": 0.6892,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.03642931208014488,
      "rewards/margins": 0.004201197065412998,
      "rewards/rejected": 0.03222811967134476,
      "step": 7550
    },
    {
      "epoch": 1.32,
      "learning_rate": 6.267002518891687e-08,
      "logits/chosen": -2.6571552753448486,
      "logits/rejected": -2.666952133178711,
      "logps/chosen": -255.24887084960938,
      "logps/rejected": -238.71731567382812,
      "loss": 0.6889,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.03816863149404526,
      "rewards/margins": 0.0025174126494675875,
      "rewards/rejected": 0.03565122187137604,
      "step": 7560
    },
    {
      "epoch": 1.33,
      "learning_rate": 6.261964735516372e-08,
      "logits/chosen": -2.7783854007720947,
      "logits/rejected": -2.782365322113037,
      "logps/chosen": -278.375732421875,
      "logps/rejected": -238.59201049804688,
      "loss": 0.6902,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.05053055286407471,
      "rewards/margins": 0.008491361513733864,
      "rewards/rejected": 0.042039185762405396,
      "step": 7570
    },
    {
      "epoch": 1.33,
      "learning_rate": 6.256926952141058e-08,
      "logits/chosen": -2.7886173725128174,
      "logits/rejected": -2.8333420753479004,
      "logps/chosen": -265.0281677246094,
      "logps/rejected": -229.58602905273438,
      "loss": 0.6894,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.03801819309592247,
      "rewards/margins": 0.005499082151800394,
      "rewards/rejected": 0.032519109547138214,
      "step": 7580
    },
    {
      "epoch": 1.33,
      "learning_rate": 6.251889168765743e-08,
      "logits/chosen": -2.7609009742736816,
      "logits/rejected": -2.722909450531006,
      "logps/chosen": -224.86181640625,
      "logps/rejected": -174.26095581054688,
      "loss": 0.6867,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.04278299957513809,
      "rewards/margins": 0.019222671166062355,
      "rewards/rejected": 0.023560328409075737,
      "step": 7590
    },
    {
      "epoch": 1.33,
      "learning_rate": 6.246851385390429e-08,
      "logits/chosen": -2.7045106887817383,
      "logits/rejected": -2.762742519378662,
      "logps/chosen": -226.1327362060547,
      "logps/rejected": -192.94805908203125,
      "loss": 0.6891,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.03624294325709343,
      "rewards/margins": 0.008845340460538864,
      "rewards/rejected": 0.027397602796554565,
      "step": 7600
    },
    {
      "epoch": 1.33,
      "learning_rate": 6.241813602015114e-08,
      "logits/chosen": -2.7494304180145264,
      "logits/rejected": -2.7993853092193604,
      "logps/chosen": -240.58883666992188,
      "logps/rejected": -228.0128631591797,
      "loss": 0.6872,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.051678262650966644,
      "rewards/margins": 0.012700071558356285,
      "rewards/rejected": 0.038978200405836105,
      "step": 7610
    },
    {
      "epoch": 1.33,
      "learning_rate": 6.236775818639798e-08,
      "logits/chosen": -2.705895185470581,
      "logits/rejected": -2.704624891281128,
      "logps/chosen": -237.9052734375,
      "logps/rejected": -214.38229370117188,
      "loss": 0.6887,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0426960252225399,
      "rewards/margins": 0.01083569135516882,
      "rewards/rejected": 0.03186033293604851,
      "step": 7620
    },
    {
      "epoch": 1.34,
      "learning_rate": 6.231738035264483e-08,
      "logits/chosen": -2.694300889968872,
      "logits/rejected": -2.7320556640625,
      "logps/chosen": -244.7444305419922,
      "logps/rejected": -211.509521484375,
      "loss": 0.6882,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.04373529553413391,
      "rewards/margins": 0.009587273001670837,
      "rewards/rejected": 0.034148018807172775,
      "step": 7630
    },
    {
      "epoch": 1.34,
      "learning_rate": 6.226700251889169e-08,
      "logits/chosen": -2.657388687133789,
      "logits/rejected": -2.7366836071014404,
      "logps/chosen": -284.2294921875,
      "logps/rejected": -204.66275024414062,
      "loss": 0.6884,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.047968313097953796,
      "rewards/margins": 0.021980399265885353,
      "rewards/rejected": 0.025987911969423294,
      "step": 7640
    },
    {
      "epoch": 1.34,
      "learning_rate": 6.221662468513854e-08,
      "logits/chosen": -2.7871079444885254,
      "logits/rejected": -2.742713451385498,
      "logps/chosen": -217.3256072998047,
      "logps/rejected": -193.50462341308594,
      "loss": 0.6891,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.04595683142542839,
      "rewards/margins": 0.014400948770344257,
      "rewards/rejected": 0.03155588358640671,
      "step": 7650
    },
    {
      "epoch": 1.34,
      "learning_rate": 6.216624685138538e-08,
      "logits/chosen": -2.7406857013702393,
      "logits/rejected": -2.7888526916503906,
      "logps/chosen": -274.1748962402344,
      "logps/rejected": -234.21694946289062,
      "loss": 0.6902,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.04528680071234703,
      "rewards/margins": 0.00935632549226284,
      "rewards/rejected": 0.03593047335743904,
      "step": 7660
    },
    {
      "epoch": 1.34,
      "learning_rate": 6.211586901763223e-08,
      "logits/chosen": -2.7559306621551514,
      "logits/rejected": -2.737565755844116,
      "logps/chosen": -202.5470428466797,
      "logps/rejected": -195.29730224609375,
      "loss": 0.6876,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.04031279683113098,
      "rewards/margins": 0.015136400237679482,
      "rewards/rejected": 0.02517639473080635,
      "step": 7670
    },
    {
      "epoch": 1.35,
      "learning_rate": 6.206549118387909e-08,
      "logits/chosen": -2.7704107761383057,
      "logits/rejected": -2.7382752895355225,
      "logps/chosen": -218.0789031982422,
      "logps/rejected": -188.13656616210938,
      "loss": 0.6889,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.03927340358495712,
      "rewards/margins": 0.005905534606426954,
      "rewards/rejected": 0.03336786851286888,
      "step": 7680
    },
    {
      "epoch": 1.35,
      "learning_rate": 6.201511335012594e-08,
      "logits/chosen": -2.860550880432129,
      "logits/rejected": -2.737292528152466,
      "logps/chosen": -204.9586181640625,
      "logps/rejected": -200.66024780273438,
      "loss": 0.6898,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.03624052554368973,
      "rewards/margins": 0.00042196764843538404,
      "rewards/rejected": 0.03581856191158295,
      "step": 7690
    },
    {
      "epoch": 1.35,
      "learning_rate": 6.19647355163728e-08,
      "logits/chosen": -2.827122211456299,
      "logits/rejected": -2.780107021331787,
      "logps/chosen": -205.1537628173828,
      "logps/rejected": -179.7675018310547,
      "loss": 0.6877,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.03878549858927727,
      "rewards/margins": 0.011726005002856255,
      "rewards/rejected": 0.027059491723775864,
      "step": 7700
    },
    {
      "epoch": 1.35,
      "learning_rate": 6.191435768261965e-08,
      "logits/chosen": -2.7009811401367188,
      "logits/rejected": -2.671569347381592,
      "logps/chosen": -197.73291015625,
      "logps/rejected": -186.85595703125,
      "loss": 0.6911,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.025883939117193222,
      "rewards/margins": 0.004282969981431961,
      "rewards/rejected": 0.02160097099840641,
      "step": 7710
    },
    {
      "epoch": 1.35,
      "learning_rate": 6.18639798488665e-08,
      "logits/chosen": -2.832324266433716,
      "logits/rejected": -2.758772850036621,
      "logps/chosen": -247.593017578125,
      "logps/rejected": -217.5821533203125,
      "loss": 0.6867,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.05130542069673538,
      "rewards/margins": 0.017530951648950577,
      "rewards/rejected": 0.033774472773075104,
      "step": 7720
    },
    {
      "epoch": 1.35,
      "learning_rate": 6.181360201511334e-08,
      "logits/chosen": -2.813231945037842,
      "logits/rejected": -2.801682949066162,
      "logps/chosen": -224.65103149414062,
      "logps/rejected": -162.23861694335938,
      "loss": 0.6883,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.04042666405439377,
      "rewards/margins": 0.014801084995269775,
      "rewards/rejected": 0.02562558650970459,
      "step": 7730
    },
    {
      "epoch": 1.36,
      "learning_rate": 6.17632241813602e-08,
      "logits/chosen": -2.7659387588500977,
      "logits/rejected": -2.734480619430542,
      "logps/chosen": -200.85885620117188,
      "logps/rejected": -178.37416076660156,
      "loss": 0.6863,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.04493881016969681,
      "rewards/margins": 0.014324108138680458,
      "rewards/rejected": 0.030614707618951797,
      "step": 7740
    },
    {
      "epoch": 1.36,
      "learning_rate": 6.171284634760705e-08,
      "logits/chosen": -2.7255687713623047,
      "logits/rejected": -2.718503713607788,
      "logps/chosen": -214.28494262695312,
      "logps/rejected": -213.13113403320312,
      "loss": 0.6882,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.04565887898206711,
      "rewards/margins": 0.004408498294651508,
      "rewards/rejected": 0.041250377893447876,
      "step": 7750
    },
    {
      "epoch": 1.36,
      "learning_rate": 6.16624685138539e-08,
      "logits/chosen": -2.802783489227295,
      "logits/rejected": -2.802086591720581,
      "logps/chosen": -226.2981414794922,
      "logps/rejected": -178.6344757080078,
      "loss": 0.6887,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.03931064531207085,
      "rewards/margins": 0.010295582935214043,
      "rewards/rejected": 0.029015064239501953,
      "step": 7760
    },
    {
      "epoch": 1.36,
      "learning_rate": 6.161209068010074e-08,
      "logits/chosen": -2.771225690841675,
      "logits/rejected": -2.7833735942840576,
      "logps/chosen": -181.8852081298828,
      "logps/rejected": -168.8794708251953,
      "loss": 0.6892,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.02663729153573513,
      "rewards/margins": 0.00531434640288353,
      "rewards/rejected": 0.0213229488581419,
      "step": 7770
    },
    {
      "epoch": 1.36,
      "learning_rate": 6.15617128463476e-08,
      "logits/chosen": -2.755722761154175,
      "logits/rejected": -2.7322795391082764,
      "logps/chosen": -201.41317749023438,
      "logps/rejected": -173.2834014892578,
      "loss": 0.6873,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.041012976318597794,
      "rewards/margins": 0.010555104352533817,
      "rewards/rejected": 0.03045787289738655,
      "step": 7780
    },
    {
      "epoch": 1.36,
      "learning_rate": 6.151133501259445e-08,
      "logits/chosen": -2.802394390106201,
      "logits/rejected": -2.768019676208496,
      "logps/chosen": -194.97853088378906,
      "logps/rejected": -155.97921752929688,
      "loss": 0.688,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.03568575531244278,
      "rewards/margins": 0.010065983049571514,
      "rewards/rejected": 0.02561977133154869,
      "step": 7790
    },
    {
      "epoch": 1.37,
      "learning_rate": 6.14609571788413e-08,
      "logits/chosen": -2.8079912662506104,
      "logits/rejected": -2.8577840328216553,
      "logps/chosen": -259.79791259765625,
      "logps/rejected": -228.3444366455078,
      "loss": 0.6892,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.04476379603147507,
      "rewards/margins": 0.0020954939536750317,
      "rewards/rejected": 0.04266830533742905,
      "step": 7800
    },
    {
      "epoch": 1.37,
      "learning_rate": 6.141057934508816e-08,
      "logits/chosen": -2.6792571544647217,
      "logits/rejected": -2.672581195831299,
      "logps/chosen": -195.576416015625,
      "logps/rejected": -212.009033203125,
      "loss": 0.6878,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.031648848205804825,
      "rewards/margins": -0.004404547158628702,
      "rewards/rejected": 0.03605339676141739,
      "step": 7810
    },
    {
      "epoch": 1.37,
      "learning_rate": 6.136020151133501e-08,
      "logits/chosen": -2.752152681350708,
      "logits/rejected": -2.6936943531036377,
      "logps/chosen": -199.6970977783203,
      "logps/rejected": -192.76779174804688,
      "loss": 0.6888,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.03685902804136276,
      "rewards/margins": 0.012366501614451408,
      "rewards/rejected": 0.024492528289556503,
      "step": 7820
    },
    {
      "epoch": 1.37,
      "learning_rate": 6.130982367758187e-08,
      "logits/chosen": -2.7500405311584473,
      "logits/rejected": -2.7712624073028564,
      "logps/chosen": -225.90231323242188,
      "logps/rejected": -223.07925415039062,
      "loss": 0.6887,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.04552388936281204,
      "rewards/margins": 0.004638020880520344,
      "rewards/rejected": 0.04088587313890457,
      "step": 7830
    },
    {
      "epoch": 1.37,
      "learning_rate": 6.125944584382872e-08,
      "logits/chosen": -2.759676218032837,
      "logits/rejected": -2.7632689476013184,
      "logps/chosen": -265.64593505859375,
      "logps/rejected": -209.3302459716797,
      "loss": 0.6883,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.04995828866958618,
      "rewards/margins": 0.014445647597312927,
      "rewards/rejected": 0.035512641072273254,
      "step": 7840
    },
    {
      "epoch": 1.38,
      "learning_rate": 6.120906801007556e-08,
      "logits/chosen": -2.7992868423461914,
      "logits/rejected": -2.786892890930176,
      "logps/chosen": -246.7543182373047,
      "logps/rejected": -220.26803588867188,
      "loss": 0.69,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.04133310541510582,
      "rewards/margins": 0.005543719045817852,
      "rewards/rejected": 0.03578938916325569,
      "step": 7850
    },
    {
      "epoch": 1.38,
      "learning_rate": 6.115869017632241e-08,
      "logits/chosen": -2.7453935146331787,
      "logits/rejected": -2.8089358806610107,
      "logps/chosen": -205.07772827148438,
      "logps/rejected": -183.50558471679688,
      "loss": 0.6874,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.034324247390031815,
      "rewards/margins": 0.010133923031389713,
      "rewards/rejected": 0.024190323427319527,
      "step": 7860
    },
    {
      "epoch": 1.38,
      "learning_rate": 6.110831234256927e-08,
      "logits/chosen": -2.785832166671753,
      "logits/rejected": -2.8203999996185303,
      "logps/chosen": -229.1759033203125,
      "logps/rejected": -201.36029052734375,
      "loss": 0.6897,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.04147273674607277,
      "rewards/margins": 0.00905263889580965,
      "rewards/rejected": 0.032420095056295395,
      "step": 7870
    },
    {
      "epoch": 1.38,
      "learning_rate": 6.105793450881612e-08,
      "logits/chosen": -2.708287000656128,
      "logits/rejected": -2.7692911624908447,
      "logps/chosen": -221.4902801513672,
      "logps/rejected": -197.06146240234375,
      "loss": 0.688,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.04135482385754585,
      "rewards/margins": 0.010591486468911171,
      "rewards/rejected": 0.030763333663344383,
      "step": 7880
    },
    {
      "epoch": 1.38,
      "learning_rate": 6.100755667506296e-08,
      "logits/chosen": -2.724555253982544,
      "logits/rejected": -2.7219367027282715,
      "logps/chosen": -177.3289337158203,
      "logps/rejected": -170.86984252929688,
      "loss": 0.6896,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.031236102804541588,
      "rewards/margins": 0.005766207817941904,
      "rewards/rejected": 0.025469893589615822,
      "step": 7890
    },
    {
      "epoch": 1.38,
      "learning_rate": 6.095717884130982e-08,
      "logits/chosen": -2.713719367980957,
      "logits/rejected": -2.7233357429504395,
      "logps/chosen": -233.9386749267578,
      "logps/rejected": -172.72366333007812,
      "loss": 0.6905,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.045443419367074966,
      "rewards/margins": 0.016218410804867744,
      "rewards/rejected": 0.02922501601278782,
      "step": 7900
    },
    {
      "epoch": 1.39,
      "learning_rate": 6.090680100755667e-08,
      "logits/chosen": -2.775986433029175,
      "logits/rejected": -2.727430582046509,
      "logps/chosen": -224.6809844970703,
      "logps/rejected": -196.10922241210938,
      "loss": 0.6887,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.03934579715132713,
      "rewards/margins": 0.008914479985833168,
      "rewards/rejected": 0.030431320890784264,
      "step": 7910
    },
    {
      "epoch": 1.39,
      "learning_rate": 6.085642317380352e-08,
      "logits/chosen": -2.781262159347534,
      "logits/rejected": -2.7013962268829346,
      "logps/chosen": -210.90963745117188,
      "logps/rejected": -203.32460021972656,
      "loss": 0.6883,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.033894848078489304,
      "rewards/margins": 0.0063673751428723335,
      "rewards/rejected": 0.027527472004294395,
      "step": 7920
    },
    {
      "epoch": 1.39,
      "learning_rate": 6.080604534005038e-08,
      "logits/chosen": -2.781320095062256,
      "logits/rejected": -2.7733845710754395,
      "logps/chosen": -240.52676391601562,
      "logps/rejected": -226.217041015625,
      "loss": 0.6893,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.04577254876494408,
      "rewards/margins": 0.007031611166894436,
      "rewards/rejected": 0.038740936666727066,
      "step": 7930
    },
    {
      "epoch": 1.39,
      "learning_rate": 6.075566750629723e-08,
      "logits/chosen": -2.7862772941589355,
      "logits/rejected": -2.7734897136688232,
      "logps/chosen": -232.9326934814453,
      "logps/rejected": -199.85308837890625,
      "loss": 0.6887,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.037640467286109924,
      "rewards/margins": 0.00660460302606225,
      "rewards/rejected": 0.03103586472570896,
      "step": 7940
    },
    {
      "epoch": 1.39,
      "learning_rate": 6.070528967254408e-08,
      "logits/chosen": -2.747159481048584,
      "logits/rejected": -2.6515073776245117,
      "logps/chosen": -235.05618286132812,
      "logps/rejected": -231.2899932861328,
      "loss": 0.6873,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": 0.05150740593671799,
      "rewards/margins": 0.015792617574334145,
      "rewards/rejected": 0.03571479022502899,
      "step": 7950
    },
    {
      "epoch": 1.39,
      "learning_rate": 6.065491183879094e-08,
      "logits/chosen": -2.6942429542541504,
      "logits/rejected": -2.7198643684387207,
      "logps/chosen": -248.2423553466797,
      "logps/rejected": -197.20497131347656,
      "loss": 0.6894,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.04131951555609703,
      "rewards/margins": 0.014284191653132439,
      "rewards/rejected": 0.027035322040319443,
      "step": 7960
    },
    {
      "epoch": 1.4,
      "learning_rate": 6.060453400503778e-08,
      "logits/chosen": -2.7588915824890137,
      "logits/rejected": -2.7787914276123047,
      "logps/chosen": -242.1793975830078,
      "logps/rejected": -204.95382690429688,
      "loss": 0.6909,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.051094651222229004,
      "rewards/margins": 0.0132598876953125,
      "rewards/rejected": 0.037834759801626205,
      "step": 7970
    },
    {
      "epoch": 1.4,
      "learning_rate": 6.055415617128463e-08,
      "logits/chosen": -2.749837636947632,
      "logits/rejected": -2.7630608081817627,
      "logps/chosen": -258.34930419921875,
      "logps/rejected": -182.1297607421875,
      "loss": 0.6851,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": 0.04587540775537491,
      "rewards/margins": 0.02240714430809021,
      "rewards/rejected": 0.02346826158463955,
      "step": 7980
    },
    {
      "epoch": 1.4,
      "learning_rate": 6.050377833753148e-08,
      "logits/chosen": -2.76625919342041,
      "logits/rejected": -2.7197346687316895,
      "logps/chosen": -210.4891815185547,
      "logps/rejected": -174.40744018554688,
      "loss": 0.6865,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": 0.04220408946275711,
      "rewards/margins": 0.014574876055121422,
      "rewards/rejected": 0.027629217132925987,
      "step": 7990
    },
    {
      "epoch": 1.4,
      "learning_rate": 6.045340050377834e-08,
      "logits/chosen": -2.7272531986236572,
      "logits/rejected": -2.7292728424072266,
      "logps/chosen": -224.6431427001953,
      "logps/rejected": -181.2040557861328,
      "loss": 0.6889,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.037466295063495636,
      "rewards/margins": 0.013241834938526154,
      "rewards/rejected": 0.024224456399679184,
      "step": 8000
    },
    {
      "epoch": 1.4,
      "learning_rate": 6.040302267002518e-08,
      "logits/chosen": -2.7375807762145996,
      "logits/rejected": -2.7324814796447754,
      "logps/chosen": -239.2186279296875,
      "logps/rejected": -223.94253540039062,
      "loss": 0.6891,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.051830459386110306,
      "rewards/margins": 0.01332320086658001,
      "rewards/rejected": 0.03850725665688515,
      "step": 8010
    },
    {
      "epoch": 1.4,
      "learning_rate": 6.035264483627203e-08,
      "logits/chosen": -2.787344217300415,
      "logits/rejected": -2.757755756378174,
      "logps/chosen": -214.58969116210938,
      "logps/rejected": -212.26815795898438,
      "loss": 0.6884,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.03871138021349907,
      "rewards/margins": 0.001409774413332343,
      "rewards/rejected": 0.03730160370469093,
      "step": 8020
    },
    {
      "epoch": 1.41,
      "learning_rate": 6.030226700251889e-08,
      "logits/chosen": -2.788583755493164,
      "logits/rejected": -2.8225603103637695,
      "logps/chosen": -299.5536193847656,
      "logps/rejected": -207.89358520507812,
      "loss": 0.6885,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.050768353044986725,
      "rewards/margins": 0.012225328013300896,
      "rewards/rejected": 0.03854302689433098,
      "step": 8030
    },
    {
      "epoch": 1.41,
      "learning_rate": 6.025188916876574e-08,
      "logits/chosen": -2.7606899738311768,
      "logits/rejected": -2.760101079940796,
      "logps/chosen": -203.7574920654297,
      "logps/rejected": -188.63735961914062,
      "loss": 0.6875,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.03621258586645126,
      "rewards/margins": 0.00680736918002367,
      "rewards/rejected": 0.029405217617750168,
      "step": 8040
    },
    {
      "epoch": 1.41,
      "learning_rate": 6.020151133501259e-08,
      "logits/chosen": -2.7922425270080566,
      "logits/rejected": -2.77087664604187,
      "logps/chosen": -264.730712890625,
      "logps/rejected": -229.2702178955078,
      "loss": 0.6894,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.04078766703605652,
      "rewards/margins": 0.005166265182197094,
      "rewards/rejected": 0.03562140464782715,
      "step": 8050
    },
    {
      "epoch": 1.41,
      "learning_rate": 6.015113350125945e-08,
      "logits/chosen": -2.6650726795196533,
      "logits/rejected": -2.699510335922241,
      "logps/chosen": -241.74951171875,
      "logps/rejected": -217.79312133789062,
      "loss": 0.6895,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": 0.04188552126288414,
      "rewards/margins": -0.0013723727315664291,
      "rewards/rejected": 0.04325789585709572,
      "step": 8060
    },
    {
      "epoch": 1.41,
      "learning_rate": 6.01007556675063e-08,
      "logits/chosen": -2.6361894607543945,
      "logits/rejected": -2.804050922393799,
      "logps/chosen": -209.30386352539062,
      "logps/rejected": -223.2210693359375,
      "loss": 0.6911,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.039837904274463654,
      "rewards/margins": 0.003762881737202406,
      "rewards/rejected": 0.03607501834630966,
      "step": 8070
    },
    {
      "epoch": 1.42,
      "learning_rate": 6.005037783375314e-08,
      "logits/chosen": -2.715277671813965,
      "logits/rejected": -2.7054569721221924,
      "logps/chosen": -249.8915557861328,
      "logps/rejected": -224.6490020751953,
      "loss": 0.6887,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.042900118976831436,
      "rewards/margins": 0.008957276120781898,
      "rewards/rejected": 0.03394284099340439,
      "step": 8080
    },
    {
      "epoch": 1.42,
      "learning_rate": 6e-08,
      "logits/chosen": -2.749920606613159,
      "logits/rejected": -2.6681249141693115,
      "logps/chosen": -262.42718505859375,
      "logps/rejected": -270.71026611328125,
      "loss": 0.6896,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.05033283308148384,
      "rewards/margins": 0.003699377877637744,
      "rewards/rejected": 0.04663345217704773,
      "step": 8090
    },
    {
      "epoch": 1.42,
      "learning_rate": 5.994962216624685e-08,
      "logits/chosen": -2.699763536453247,
      "logits/rejected": -2.7738709449768066,
      "logps/chosen": -254.59536743164062,
      "logps/rejected": -225.08261108398438,
      "loss": 0.6882,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.050733782351017,
      "rewards/margins": 0.01060875691473484,
      "rewards/rejected": 0.04012501984834671,
      "step": 8100
    },
    {
      "epoch": 1.42,
      "learning_rate": 5.98992443324937e-08,
      "logits/chosen": -2.791287899017334,
      "logits/rejected": -2.747755765914917,
      "logps/chosen": -200.8659210205078,
      "logps/rejected": -184.97738647460938,
      "loss": 0.6883,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.03486446663737297,
      "rewards/margins": 0.002834696788340807,
      "rewards/rejected": 0.03202977031469345,
      "step": 8110
    },
    {
      "epoch": 1.42,
      "learning_rate": 5.984886649874054e-08,
      "logits/chosen": -2.6940653324127197,
      "logits/rejected": -2.760707139968872,
      "logps/chosen": -228.6831817626953,
      "logps/rejected": -221.67361450195312,
      "loss": 0.6887,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.04749389737844467,
      "rewards/margins": 0.011963879689574242,
      "rewards/rejected": 0.03553002327680588,
      "step": 8120
    },
    {
      "epoch": 1.42,
      "learning_rate": 5.97984886649874e-08,
      "logits/chosen": -2.7091734409332275,
      "logits/rejected": -2.727992534637451,
      "logps/chosen": -204.9541015625,
      "logps/rejected": -178.6293487548828,
      "loss": 0.6887,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.0264905896037817,
      "rewards/margins": 0.0013065729290246964,
      "rewards/rejected": 0.025184014812111855,
      "step": 8130
    },
    {
      "epoch": 1.43,
      "learning_rate": 5.974811083123425e-08,
      "logits/chosen": -2.7273199558258057,
      "logits/rejected": -2.7602429389953613,
      "logps/chosen": -236.5217742919922,
      "logps/rejected": -218.5023193359375,
      "loss": 0.6888,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.04743615910410881,
      "rewards/margins": 0.007979987189173698,
      "rewards/rejected": 0.03945617750287056,
      "step": 8140
    },
    {
      "epoch": 1.43,
      "learning_rate": 5.96977329974811e-08,
      "logits/chosen": -2.7114429473876953,
      "logits/rejected": -2.7514827251434326,
      "logps/chosen": -233.47323608398438,
      "logps/rejected": -183.81675720214844,
      "loss": 0.6869,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.046855200082063675,
      "rewards/margins": 0.016597609966993332,
      "rewards/rejected": 0.030257591977715492,
      "step": 8150
    },
    {
      "epoch": 1.43,
      "learning_rate": 5.964735516372796e-08,
      "logits/chosen": -2.7446465492248535,
      "logits/rejected": -2.6851656436920166,
      "logps/chosen": -206.99264526367188,
      "logps/rejected": -168.73605346679688,
      "loss": 0.6876,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.03506845980882645,
      "rewards/margins": 0.014280231669545174,
      "rewards/rejected": 0.02078823372721672,
      "step": 8160
    },
    {
      "epoch": 1.43,
      "learning_rate": 5.9596977329974804e-08,
      "logits/chosen": -2.819230794906616,
      "logits/rejected": -2.7767491340637207,
      "logps/chosen": -244.1392059326172,
      "logps/rejected": -208.48880004882812,
      "loss": 0.6895,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.057758230715990067,
      "rewards/margins": 0.0185552891343832,
      "rewards/rejected": 0.039202939718961716,
      "step": 8170
    },
    {
      "epoch": 1.43,
      "learning_rate": 5.9546599496221664e-08,
      "logits/chosen": -2.7350716590881348,
      "logits/rejected": -2.6791672706604004,
      "logps/chosen": -200.4189910888672,
      "logps/rejected": -196.80860900878906,
      "loss": 0.6887,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.034190356731414795,
      "rewards/margins": 0.00776415690779686,
      "rewards/rejected": 0.026426201686263084,
      "step": 8180
    },
    {
      "epoch": 1.43,
      "learning_rate": 5.949622166246852e-08,
      "logits/chosen": -2.7582621574401855,
      "logits/rejected": -2.7618229389190674,
      "logps/chosen": -180.63294982910156,
      "logps/rejected": -157.079833984375,
      "loss": 0.6901,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.03690091520547867,
      "rewards/margins": 0.011395210400223732,
      "rewards/rejected": 0.025505702942609787,
      "step": 8190
    },
    {
      "epoch": 1.44,
      "learning_rate": 5.944584382871536e-08,
      "logits/chosen": -2.731879472732544,
      "logits/rejected": -2.7573390007019043,
      "logps/chosen": -210.488525390625,
      "logps/rejected": -197.1478271484375,
      "loss": 0.6868,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.047597721219062805,
      "rewards/margins": 0.010359442792832851,
      "rewards/rejected": 0.03723827749490738,
      "step": 8200
    },
    {
      "epoch": 1.44,
      "learning_rate": 5.939546599496221e-08,
      "logits/chosen": -2.7964818477630615,
      "logits/rejected": -2.7716190814971924,
      "logps/chosen": -220.36572265625,
      "logps/rejected": -191.2345428466797,
      "loss": 0.69,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.048532258719205856,
      "rewards/margins": 0.012116949073970318,
      "rewards/rejected": 0.03641531243920326,
      "step": 8210
    },
    {
      "epoch": 1.44,
      "learning_rate": 5.9345088161209065e-08,
      "logits/chosen": -2.768618106842041,
      "logits/rejected": -2.745877265930176,
      "logps/chosen": -241.71826171875,
      "logps/rejected": -231.50302124023438,
      "loss": 0.6923,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.04764752835035324,
      "rewards/margins": 0.003467456204816699,
      "rewards/rejected": 0.04418007284402847,
      "step": 8220
    },
    {
      "epoch": 1.44,
      "learning_rate": 5.929471032745592e-08,
      "logits/chosen": -2.7671141624450684,
      "logits/rejected": -2.7717056274414062,
      "logps/chosen": -282.21734619140625,
      "logps/rejected": -230.12704467773438,
      "loss": 0.689,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.05740467458963394,
      "rewards/margins": 0.02093067206442356,
      "rewards/rejected": 0.03647400066256523,
      "step": 8230
    },
    {
      "epoch": 1.44,
      "learning_rate": 5.9244332493702766e-08,
      "logits/chosen": -2.7785863876342773,
      "logits/rejected": -2.7596054077148438,
      "logps/chosen": -269.4919128417969,
      "logps/rejected": -206.0177459716797,
      "loss": 0.6885,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.04548025503754616,
      "rewards/margins": 0.017482586205005646,
      "rewards/rejected": 0.02799767255783081,
      "step": 8240
    },
    {
      "epoch": 1.45,
      "learning_rate": 5.919395465994962e-08,
      "logits/chosen": -2.6811938285827637,
      "logits/rejected": -2.731386184692383,
      "logps/chosen": -189.69949340820312,
      "logps/rejected": -172.76312255859375,
      "loss": 0.687,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.038950253278017044,
      "rewards/margins": 0.009141791611909866,
      "rewards/rejected": 0.029808461666107178,
      "step": 8250
    },
    {
      "epoch": 1.45,
      "learning_rate": 5.9143576826196473e-08,
      "logits/chosen": -2.67179536819458,
      "logits/rejected": -2.7431387901306152,
      "logps/chosen": -238.79928588867188,
      "logps/rejected": -197.96249389648438,
      "loss": 0.6887,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.04331899806857109,
      "rewards/margins": 0.00855275709182024,
      "rewards/rejected": 0.03476623818278313,
      "step": 8260
    },
    {
      "epoch": 1.45,
      "learning_rate": 5.909319899244333e-08,
      "logits/chosen": -2.6874778270721436,
      "logits/rejected": -2.7527995109558105,
      "logps/chosen": -265.18646240234375,
      "logps/rejected": -254.72055053710938,
      "loss": 0.6902,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.04799005389213562,
      "rewards/margins": 0.008365727961063385,
      "rewards/rejected": 0.03962433338165283,
      "step": 8270
    },
    {
      "epoch": 1.45,
      "learning_rate": 5.904282115869017e-08,
      "logits/chosen": -2.775963068008423,
      "logits/rejected": -2.7594003677368164,
      "logps/chosen": -193.80345153808594,
      "logps/rejected": -182.29696655273438,
      "loss": 0.6904,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.03241405636072159,
      "rewards/margins": 0.0072534880600869656,
      "rewards/rejected": 0.025160569697618484,
      "step": 8280
    },
    {
      "epoch": 1.45,
      "learning_rate": 5.899244332493703e-08,
      "logits/chosen": -2.7540886402130127,
      "logits/rejected": -2.7759768962860107,
      "logps/chosen": -256.3172912597656,
      "logps/rejected": -248.9473114013672,
      "loss": 0.6896,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.04200480133295059,
      "rewards/margins": 0.005136861000210047,
      "rewards/rejected": 0.03686793893575668,
      "step": 8290
    },
    {
      "epoch": 1.45,
      "learning_rate": 5.894206549118388e-08,
      "logits/chosen": -2.7550156116485596,
      "logits/rejected": -2.8021984100341797,
      "logps/chosen": -229.73519897460938,
      "logps/rejected": -194.55474853515625,
      "loss": 0.6892,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.040971461683511734,
      "rewards/margins": 0.007404597010463476,
      "rewards/rejected": 0.03356685861945152,
      "step": 8300
    },
    {
      "epoch": 1.46,
      "learning_rate": 5.8891687657430735e-08,
      "logits/chosen": -2.7880303859710693,
      "logits/rejected": -2.7363808155059814,
      "logps/chosen": -173.42591857910156,
      "logps/rejected": -178.16490173339844,
      "loss": 0.6883,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.03242052346467972,
      "rewards/margins": 0.0012834550580009818,
      "rewards/rejected": 0.03113706409931183,
      "step": 8310
    },
    {
      "epoch": 1.46,
      "learning_rate": 5.8841309823677575e-08,
      "logits/chosen": -2.7852296829223633,
      "logits/rejected": -2.732570171356201,
      "logps/chosen": -205.2123260498047,
      "logps/rejected": -221.46542358398438,
      "loss": 0.6887,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.037264786660671234,
      "rewards/margins": -0.0015946425264701247,
      "rewards/rejected": 0.038859423249959946,
      "step": 8320
    },
    {
      "epoch": 1.46,
      "learning_rate": 5.879093198992443e-08,
      "logits/chosen": -2.7393336296081543,
      "logits/rejected": -2.750638484954834,
      "logps/chosen": -200.95550537109375,
      "logps/rejected": -182.22824096679688,
      "loss": 0.6876,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.03504057973623276,
      "rewards/margins": 0.0032831882126629353,
      "rewards/rejected": 0.03175738826394081,
      "step": 8330
    },
    {
      "epoch": 1.46,
      "learning_rate": 5.874055415617128e-08,
      "logits/chosen": -2.733309268951416,
      "logits/rejected": -2.7839436531066895,
      "logps/chosen": -209.6042022705078,
      "logps/rejected": -167.40438842773438,
      "loss": 0.6877,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.03753194957971573,
      "rewards/margins": 0.0077357119880616665,
      "rewards/rejected": 0.029796237125992775,
      "step": 8340
    },
    {
      "epoch": 1.46,
      "learning_rate": 5.869017632241813e-08,
      "logits/chosen": -2.8055219650268555,
      "logits/rejected": -2.817014694213867,
      "logps/chosen": -208.11178588867188,
      "logps/rejected": -204.49551391601562,
      "loss": 0.6887,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.036422837525606155,
      "rewards/margins": 0.003470013616606593,
      "rewards/rejected": 0.032952822744846344,
      "step": 8350
    },
    {
      "epoch": 1.46,
      "learning_rate": 5.863979848866498e-08,
      "logits/chosen": -2.8599841594696045,
      "logits/rejected": -2.8321750164031982,
      "logps/chosen": -245.27285766601562,
      "logps/rejected": -233.36593627929688,
      "loss": 0.689,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.045193519443273544,
      "rewards/margins": 0.0070549556985497475,
      "rewards/rejected": 0.03813857212662697,
      "step": 8360
    },
    {
      "epoch": 1.47,
      "learning_rate": 5.858942065491184e-08,
      "logits/chosen": -2.711230516433716,
      "logits/rejected": -2.7263026237487793,
      "logps/chosen": -247.5156707763672,
      "logps/rejected": -204.61376953125,
      "loss": 0.6859,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.06073460727930069,
      "rewards/margins": 0.02463601343333721,
      "rewards/rejected": 0.03609859198331833,
      "step": 8370
    },
    {
      "epoch": 1.47,
      "learning_rate": 5.853904282115869e-08,
      "logits/chosen": -2.771897315979004,
      "logits/rejected": -2.770329713821411,
      "logps/chosen": -261.06951904296875,
      "logps/rejected": -264.5949401855469,
      "loss": 0.6905,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.051393140107393265,
      "rewards/margins": 0.006638474762439728,
      "rewards/rejected": 0.04475466161966324,
      "step": 8380
    },
    {
      "epoch": 1.47,
      "learning_rate": 5.848866498740553e-08,
      "logits/chosen": -2.7597389221191406,
      "logits/rejected": -2.7436418533325195,
      "logps/chosen": -245.485595703125,
      "logps/rejected": -207.4766845703125,
      "loss": 0.6878,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.051292382180690765,
      "rewards/margins": 0.012479810044169426,
      "rewards/rejected": 0.03881257027387619,
      "step": 8390
    },
    {
      "epoch": 1.47,
      "learning_rate": 5.843828715365239e-08,
      "logits/chosen": -2.80478572845459,
      "logits/rejected": -2.771780490875244,
      "logps/chosen": -204.68588256835938,
      "logps/rejected": -193.27899169921875,
      "loss": 0.6904,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.04538872465491295,
      "rewards/margins": 0.008817224763333797,
      "rewards/rejected": 0.03657149896025658,
      "step": 8400
    },
    {
      "epoch": 1.47,
      "learning_rate": 5.8387909319899245e-08,
      "logits/chosen": -2.7240943908691406,
      "logits/rejected": -2.806950569152832,
      "logps/chosen": -256.45501708984375,
      "logps/rejected": -257.5248718261719,
      "loss": 0.6887,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.056347597390413284,
      "rewards/margins": 0.006222080439329147,
      "rewards/rejected": 0.05012550950050354,
      "step": 8410
    },
    {
      "epoch": 1.47,
      "learning_rate": 5.83375314861461e-08,
      "logits/chosen": -2.7173569202423096,
      "logits/rejected": -2.7189526557922363,
      "logps/chosen": -238.66342163085938,
      "logps/rejected": -196.93142700195312,
      "loss": 0.6873,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.046293385326862335,
      "rewards/margins": 0.011984572745859623,
      "rewards/rejected": 0.03430881351232529,
      "step": 8420
    },
    {
      "epoch": 1.48,
      "learning_rate": 5.828715365239294e-08,
      "logits/chosen": -2.821493148803711,
      "logits/rejected": -2.864586591720581,
      "logps/chosen": -195.20623779296875,
      "logps/rejected": -215.08755493164062,
      "loss": 0.6894,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.038646843284368515,
      "rewards/margins": 0.004198746755719185,
      "rewards/rejected": 0.03444809466600418,
      "step": 8430
    },
    {
      "epoch": 1.48,
      "learning_rate": 5.823677581863979e-08,
      "logits/chosen": -2.680703639984131,
      "logits/rejected": -2.7536604404449463,
      "logps/chosen": -219.0048828125,
      "logps/rejected": -202.65245056152344,
      "loss": 0.6898,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.042999859899282455,
      "rewards/margins": 0.009765975177288055,
      "rewards/rejected": 0.0332338847219944,
      "step": 8440
    },
    {
      "epoch": 1.48,
      "learning_rate": 5.8186397984886646e-08,
      "logits/chosen": -2.7437472343444824,
      "logits/rejected": -2.7347023487091064,
      "logps/chosen": -196.41883850097656,
      "logps/rejected": -166.17236328125,
      "loss": 0.6883,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.03196621313691139,
      "rewards/margins": 0.008594868704676628,
      "rewards/rejected": 0.023371344432234764,
      "step": 8450
    },
    {
      "epoch": 1.48,
      "learning_rate": 5.81360201511335e-08,
      "logits/chosen": -2.737215518951416,
      "logits/rejected": -2.731224536895752,
      "logps/chosen": -232.0885772705078,
      "logps/rejected": -181.93997192382812,
      "loss": 0.6881,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.041482407599687576,
      "rewards/margins": 0.011339427903294563,
      "rewards/rejected": 0.030142977833747864,
      "step": 8460
    },
    {
      "epoch": 1.48,
      "learning_rate": 5.808564231738035e-08,
      "logits/chosen": -2.743090867996216,
      "logits/rejected": -2.6777594089508057,
      "logps/chosen": -185.24954223632812,
      "logps/rejected": -203.14199829101562,
      "loss": 0.6897,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.02936091460287571,
      "rewards/margins": 0.0038882705848664045,
      "rewards/rejected": 0.025472644716501236,
      "step": 8470
    },
    {
      "epoch": 1.49,
      "learning_rate": 5.80352644836272e-08,
      "logits/chosen": -2.778669595718384,
      "logits/rejected": -2.7191758155822754,
      "logps/chosen": -235.8621063232422,
      "logps/rejected": -230.335205078125,
      "loss": 0.6885,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.03947572037577629,
      "rewards/margins": 0.003947444260120392,
      "rewards/rejected": 0.0355282798409462,
      "step": 8480
    },
    {
      "epoch": 1.49,
      "learning_rate": 5.7984886649874054e-08,
      "logits/chosen": -2.7726998329162598,
      "logits/rejected": -2.8148016929626465,
      "logps/chosen": -250.29202270507812,
      "logps/rejected": -219.3681182861328,
      "loss": 0.6879,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.05604994297027588,
      "rewards/margins": 0.015362827107310295,
      "rewards/rejected": 0.040687110275030136,
      "step": 8490
    },
    {
      "epoch": 1.49,
      "learning_rate": 5.793450881612091e-08,
      "logits/chosen": -2.778038740158081,
      "logits/rejected": -2.7753331661224365,
      "logps/chosen": -206.3338623046875,
      "logps/rejected": -184.36129760742188,
      "loss": 0.6886,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.03952028229832649,
      "rewards/margins": 0.009847410954535007,
      "rewards/rejected": 0.029672876000404358,
      "step": 8500
    },
    {
      "epoch": 1.49,
      "learning_rate": 5.7884130982367755e-08,
      "logits/chosen": -2.7196013927459717,
      "logits/rejected": -2.8112895488739014,
      "logps/chosen": -204.50921630859375,
      "logps/rejected": -174.06558227539062,
      "loss": 0.6882,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.04074891284108162,
      "rewards/margins": 0.0165697168558836,
      "rewards/rejected": 0.02417919412255287,
      "step": 8510
    },
    {
      "epoch": 1.49,
      "learning_rate": 5.783375314861461e-08,
      "logits/chosen": -2.7498068809509277,
      "logits/rejected": -2.757819414138794,
      "logps/chosen": -201.4228057861328,
      "logps/rejected": -219.12399291992188,
      "loss": 0.6885,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.037325937300920486,
      "rewards/margins": -0.0009258370846509933,
      "rewards/rejected": 0.03825177252292633,
      "step": 8520
    },
    {
      "epoch": 1.49,
      "learning_rate": 5.778337531486146e-08,
      "logits/chosen": -2.7941741943359375,
      "logits/rejected": -2.7413251399993896,
      "logps/chosen": -224.0157012939453,
      "logps/rejected": -202.8053436279297,
      "loss": 0.6884,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.04527192562818527,
      "rewards/margins": 0.014141691848635674,
      "rewards/rejected": 0.031130235642194748,
      "step": 8530
    },
    {
      "epoch": 1.5,
      "learning_rate": 5.7732997481108316e-08,
      "logits/chosen": -2.8099892139434814,
      "logits/rejected": -2.757983684539795,
      "logps/chosen": -243.79086303710938,
      "logps/rejected": -223.33932495117188,
      "loss": 0.6898,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.04493292048573494,
      "rewards/margins": 0.0019141886150464416,
      "rewards/rejected": 0.04301873594522476,
      "step": 8540
    },
    {
      "epoch": 1.5,
      "learning_rate": 5.7682619647355156e-08,
      "logits/chosen": -2.784567356109619,
      "logits/rejected": -2.7672345638275146,
      "logps/chosen": -235.2530059814453,
      "logps/rejected": -210.9793701171875,
      "loss": 0.6881,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.05156296491622925,
      "rewards/margins": 0.007609918713569641,
      "rewards/rejected": 0.043953049927949905,
      "step": 8550
    },
    {
      "epoch": 1.5,
      "learning_rate": 5.763224181360201e-08,
      "logits/chosen": -2.818497896194458,
      "logits/rejected": -2.760577440261841,
      "logps/chosen": -234.3348388671875,
      "logps/rejected": -193.94863891601562,
      "loss": 0.6871,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.05081092566251755,
      "rewards/margins": 0.01714308187365532,
      "rewards/rejected": 0.03366784378886223,
      "step": 8560
    },
    {
      "epoch": 1.5,
      "learning_rate": 5.758186397984886e-08,
      "logits/chosen": -2.7838993072509766,
      "logits/rejected": -2.7389278411865234,
      "logps/chosen": -220.52511596679688,
      "logps/rejected": -211.15255737304688,
      "loss": 0.6888,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.035474494099617004,
      "rewards/margins": -0.0008419624646194279,
      "rewards/rejected": 0.03631645813584328,
      "step": 8570
    },
    {
      "epoch": 1.5,
      "learning_rate": 5.753148614609572e-08,
      "logits/chosen": -2.788435459136963,
      "logits/rejected": -2.7799055576324463,
      "logps/chosen": -216.87448120117188,
      "logps/rejected": -198.84854125976562,
      "loss": 0.6883,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.038950562477111816,
      "rewards/margins": 0.015099948272109032,
      "rewards/rejected": 0.023850608617067337,
      "step": 8580
    },
    {
      "epoch": 1.5,
      "learning_rate": 5.7481108312342564e-08,
      "logits/chosen": -2.7052228450775146,
      "logits/rejected": -2.6766791343688965,
      "logps/chosen": -183.8777618408203,
      "logps/rejected": -182.99578857421875,
      "loss": 0.6892,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.04428397864103317,
      "rewards/margins": 0.010991683229804039,
      "rewards/rejected": 0.03329230099916458,
      "step": 8590
    },
    {
      "epoch": 1.51,
      "learning_rate": 5.743073047858942e-08,
      "logits/chosen": -2.815370559692383,
      "logits/rejected": -2.7483325004577637,
      "logps/chosen": -202.84475708007812,
      "logps/rejected": -165.16668701171875,
      "loss": 0.6854,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": 0.045902546495199203,
      "rewards/margins": 0.022786486893892288,
      "rewards/rejected": 0.023116057738661766,
      "step": 8600
    },
    {
      "epoch": 1.51,
      "learning_rate": 5.738035264483627e-08,
      "logits/chosen": -2.7344138622283936,
      "logits/rejected": -2.7439301013946533,
      "logps/chosen": -226.06161499023438,
      "logps/rejected": -174.56552124023438,
      "loss": 0.6868,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.039994433522224426,
      "rewards/margins": 0.020141465589404106,
      "rewards/rejected": 0.01985296607017517,
      "step": 8610
    },
    {
      "epoch": 1.51,
      "learning_rate": 5.7329974811083125e-08,
      "logits/chosen": -2.8340532779693604,
      "logits/rejected": -2.7644944190979004,
      "logps/chosen": -207.7313690185547,
      "logps/rejected": -182.41390991210938,
      "loss": 0.6868,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.03946467861533165,
      "rewards/margins": 0.01365002989768982,
      "rewards/rejected": 0.02581464685499668,
      "step": 8620
    },
    {
      "epoch": 1.51,
      "learning_rate": 5.727959697732997e-08,
      "logits/chosen": -2.798107624053955,
      "logits/rejected": -2.7637317180633545,
      "logps/chosen": -257.6904296875,
      "logps/rejected": -244.3838348388672,
      "loss": 0.6882,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.04575506970286369,
      "rewards/margins": 0.006212930660694838,
      "rewards/rejected": 0.03954214230179787,
      "step": 8630
    },
    {
      "epoch": 1.51,
      "learning_rate": 5.7229219143576825e-08,
      "logits/chosen": -2.7994701862335205,
      "logits/rejected": -2.774688959121704,
      "logps/chosen": -220.4487762451172,
      "logps/rejected": -172.90780639648438,
      "loss": 0.6893,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.043639928102493286,
      "rewards/margins": 0.01047286856919527,
      "rewards/rejected": 0.03316705673933029,
      "step": 8640
    },
    {
      "epoch": 1.52,
      "learning_rate": 5.717884130982368e-08,
      "logits/chosen": -2.764247417449951,
      "logits/rejected": -2.743495464324951,
      "logps/chosen": -239.54025268554688,
      "logps/rejected": -214.3405303955078,
      "loss": 0.6875,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.03702837601304054,
      "rewards/margins": 0.012995190918445587,
      "rewards/rejected": 0.024033186957240105,
      "step": 8650
    },
    {
      "epoch": 1.52,
      "learning_rate": 5.712846347607052e-08,
      "logits/chosen": -2.7130041122436523,
      "logits/rejected": -2.7699601650238037,
      "logps/chosen": -237.2503204345703,
      "logps/rejected": -185.75486755371094,
      "loss": 0.6877,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.04563732072710991,
      "rewards/margins": 0.015573574230074883,
      "rewards/rejected": 0.030063744634389877,
      "step": 8660
    },
    {
      "epoch": 1.52,
      "learning_rate": 5.707808564231737e-08,
      "logits/chosen": -2.774289608001709,
      "logits/rejected": -2.7651114463806152,
      "logps/chosen": -269.4395751953125,
      "logps/rejected": -211.79531860351562,
      "loss": 0.6894,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.05132768303155899,
      "rewards/margins": 0.011847235262393951,
      "rewards/rejected": 0.039480455219745636,
      "step": 8670
    },
    {
      "epoch": 1.52,
      "learning_rate": 5.702770780856423e-08,
      "logits/chosen": -2.7488410472869873,
      "logits/rejected": -2.811511516571045,
      "logps/chosen": -230.3901824951172,
      "logps/rejected": -180.4424591064453,
      "loss": 0.6856,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.05101857706904411,
      "rewards/margins": 0.01628846302628517,
      "rewards/rejected": 0.03473011404275894,
      "step": 8680
    },
    {
      "epoch": 1.52,
      "learning_rate": 5.697732997481108e-08,
      "logits/chosen": -2.715552568435669,
      "logits/rejected": -2.745255947113037,
      "logps/chosen": -223.7119903564453,
      "logps/rejected": -228.6637725830078,
      "loss": 0.6869,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.05288667604327202,
      "rewards/margins": 0.007302626967430115,
      "rewards/rejected": 0.045584045350551605,
      "step": 8690
    },
    {
      "epoch": 1.52,
      "learning_rate": 5.692695214105793e-08,
      "logits/chosen": -2.757916212081909,
      "logits/rejected": -2.6959335803985596,
      "logps/chosen": -219.01956176757812,
      "logps/rejected": -228.71627807617188,
      "loss": 0.6875,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.05256640911102295,
      "rewards/margins": 0.006362512707710266,
      "rewards/rejected": 0.046203892678022385,
      "step": 8700
    },
    {
      "epoch": 1.53,
      "learning_rate": 5.687657430730478e-08,
      "logits/chosen": -2.8467657566070557,
      "logits/rejected": -2.7943410873413086,
      "logps/chosen": -219.20028686523438,
      "logps/rejected": -235.470703125,
      "loss": 0.6871,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.045346882194280624,
      "rewards/margins": 0.001881635980680585,
      "rewards/rejected": 0.04346524551510811,
      "step": 8710
    },
    {
      "epoch": 1.53,
      "learning_rate": 5.6826196473551635e-08,
      "logits/chosen": -2.8179268836975098,
      "logits/rejected": -2.793691396713257,
      "logps/chosen": -269.15374755859375,
      "logps/rejected": -207.63232421875,
      "loss": 0.6889,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.053640447556972504,
      "rewards/margins": 0.012012636289000511,
      "rewards/rejected": 0.04162780940532684,
      "step": 8720
    },
    {
      "epoch": 1.53,
      "learning_rate": 5.677581863979849e-08,
      "logits/chosen": -2.76914381980896,
      "logits/rejected": -2.7731528282165527,
      "logps/chosen": -239.4773406982422,
      "logps/rejected": -220.02938842773438,
      "loss": 0.6883,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.035095784813165665,
      "rewards/margins": 0.004083072766661644,
      "rewards/rejected": 0.031012708321213722,
      "step": 8730
    },
    {
      "epoch": 1.53,
      "learning_rate": 5.6725440806045335e-08,
      "logits/chosen": -2.8200926780700684,
      "logits/rejected": -2.8051130771636963,
      "logps/chosen": -225.9854278564453,
      "logps/rejected": -199.87562561035156,
      "loss": 0.6884,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.04804544523358345,
      "rewards/margins": 0.0140976682305336,
      "rewards/rejected": 0.03394777700304985,
      "step": 8740
    },
    {
      "epoch": 1.53,
      "learning_rate": 5.667506297229219e-08,
      "logits/chosen": -2.771458625793457,
      "logits/rejected": -2.7316102981567383,
      "logps/chosen": -198.54049682617188,
      "logps/rejected": -189.3423614501953,
      "loss": 0.6905,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.0411757230758667,
      "rewards/margins": 0.005958369933068752,
      "rewards/rejected": 0.03521735221147537,
      "step": 8750
    },
    {
      "epoch": 1.53,
      "learning_rate": 5.662468513853904e-08,
      "logits/chosen": -2.7698636054992676,
      "logits/rejected": -2.7552199363708496,
      "logps/chosen": -229.97537231445312,
      "logps/rejected": -203.84976196289062,
      "loss": 0.6877,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.05870678275823593,
      "rewards/margins": 0.017172252759337425,
      "rewards/rejected": 0.04153453931212425,
      "step": 8760
    },
    {
      "epoch": 1.54,
      "learning_rate": 5.6574307304785896e-08,
      "logits/chosen": -2.6863961219787598,
      "logits/rejected": -2.6779088973999023,
      "logps/chosen": -234.57101440429688,
      "logps/rejected": -185.58840942382812,
      "loss": 0.6905,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.03976276144385338,
      "rewards/margins": 0.00548075046390295,
      "rewards/rejected": 0.034282006323337555,
      "step": 8770
    },
    {
      "epoch": 1.54,
      "learning_rate": 5.6523929471032737e-08,
      "logits/chosen": -2.739356517791748,
      "logits/rejected": -2.788348436355591,
      "logps/chosen": -209.69857788085938,
      "logps/rejected": -199.63473510742188,
      "loss": 0.6887,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.04317685216665268,
      "rewards/margins": 0.006445699837058783,
      "rewards/rejected": 0.03673115372657776,
      "step": 8780
    },
    {
      "epoch": 1.54,
      "learning_rate": 5.647355163727959e-08,
      "logits/chosen": -2.7598869800567627,
      "logits/rejected": -2.738487482070923,
      "logps/chosen": -249.5239715576172,
      "logps/rejected": -210.69021606445312,
      "loss": 0.6884,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.057052165269851685,
      "rewards/margins": 0.014407709240913391,
      "rewards/rejected": 0.04264445602893829,
      "step": 8790
    },
    {
      "epoch": 1.54,
      "learning_rate": 5.6423173803526444e-08,
      "logits/chosen": -2.7936174869537354,
      "logits/rejected": -2.7623283863067627,
      "logps/chosen": -206.39572143554688,
      "logps/rejected": -162.04994201660156,
      "loss": 0.6899,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.04691698029637337,
      "rewards/margins": 0.015475709922611713,
      "rewards/rejected": 0.03144126385450363,
      "step": 8800
    },
    {
      "epoch": 1.54,
      "learning_rate": 5.6372795969773304e-08,
      "logits/chosen": -2.7687063217163086,
      "logits/rejected": -2.7901759147644043,
      "logps/chosen": -271.50201416015625,
      "logps/rejected": -216.03878784179688,
      "loss": 0.6888,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.05467065051198006,
      "rewards/margins": 0.01723630167543888,
      "rewards/rejected": 0.03743434697389603,
      "step": 8810
    },
    {
      "epoch": 1.54,
      "learning_rate": 5.6322418136020145e-08,
      "logits/chosen": -2.794313430786133,
      "logits/rejected": -2.794102191925049,
      "logps/chosen": -233.6994171142578,
      "logps/rejected": -184.6134033203125,
      "loss": 0.6888,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.03926300257444382,
      "rewards/margins": 0.001541120931506157,
      "rewards/rejected": 0.03772187978029251,
      "step": 8820
    },
    {
      "epoch": 1.55,
      "learning_rate": 5.6272040302267e-08,
      "logits/chosen": -2.72084641456604,
      "logits/rejected": -2.8114678859710693,
      "logps/chosen": -229.1598663330078,
      "logps/rejected": -197.14385986328125,
      "loss": 0.6855,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.04723774269223213,
      "rewards/margins": 0.010595154948532581,
      "rewards/rejected": 0.036642588675022125,
      "step": 8830
    },
    {
      "epoch": 1.55,
      "learning_rate": 5.622166246851385e-08,
      "logits/chosen": -2.7159905433654785,
      "logits/rejected": -2.7547850608825684,
      "logps/chosen": -249.0121612548828,
      "logps/rejected": -196.9223175048828,
      "loss": 0.6863,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.05349539592862129,
      "rewards/margins": 0.01618610881268978,
      "rewards/rejected": 0.03730928525328636,
      "step": 8840
    },
    {
      "epoch": 1.55,
      "learning_rate": 5.6171284634760705e-08,
      "logits/chosen": -2.724531650543213,
      "logits/rejected": -2.7724790573120117,
      "logps/chosen": -231.1404571533203,
      "logps/rejected": -227.45590209960938,
      "loss": 0.689,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.04307236522436142,
      "rewards/margins": 0.007877699099481106,
      "rewards/rejected": 0.035194672644138336,
      "step": 8850
    },
    {
      "epoch": 1.55,
      "learning_rate": 5.612090680100755e-08,
      "logits/chosen": -2.754436731338501,
      "logits/rejected": -2.8007025718688965,
      "logps/chosen": -270.7969055175781,
      "logps/rejected": -242.35183715820312,
      "loss": 0.6872,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.05585699528455734,
      "rewards/margins": 0.009231283329427242,
      "rewards/rejected": 0.04662570729851723,
      "step": 8860
    },
    {
      "epoch": 1.55,
      "learning_rate": 5.6070528967254406e-08,
      "logits/chosen": -2.6838278770446777,
      "logits/rejected": -2.766242265701294,
      "logps/chosen": -255.3485565185547,
      "logps/rejected": -220.1199188232422,
      "loss": 0.6878,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.048295557498931885,
      "rewards/margins": 0.009508052840828896,
      "rewards/rejected": 0.03878750279545784,
      "step": 8870
    },
    {
      "epoch": 1.56,
      "learning_rate": 5.602015113350126e-08,
      "logits/chosen": -2.7382352352142334,
      "logits/rejected": -2.8044495582580566,
      "logps/chosen": -244.29153442382812,
      "logps/rejected": -218.97097778320312,
      "loss": 0.6876,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.05450066179037094,
      "rewards/margins": 0.008315160870552063,
      "rewards/rejected": 0.04618550091981888,
      "step": 8880
    },
    {
      "epoch": 1.56,
      "learning_rate": 5.5969773299748113e-08,
      "logits/chosen": -2.776350498199463,
      "logits/rejected": -2.752664566040039,
      "logps/chosen": -235.73233032226562,
      "logps/rejected": -208.52737426757812,
      "loss": 0.6874,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.04617000371217728,
      "rewards/margins": 0.0013287483016029,
      "rewards/rejected": 0.044841259717941284,
      "step": 8890
    },
    {
      "epoch": 1.56,
      "learning_rate": 5.5919395465994954e-08,
      "logits/chosen": -2.7392613887786865,
      "logits/rejected": -2.756448268890381,
      "logps/chosen": -219.9252166748047,
      "logps/rejected": -183.68324279785156,
      "loss": 0.6877,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.04604010656476021,
      "rewards/margins": 0.004281000234186649,
      "rewards/rejected": 0.041759099811315536,
      "step": 8900
    },
    {
      "epoch": 1.56,
      "learning_rate": 5.586901763224181e-08,
      "logits/chosen": -2.738739252090454,
      "logits/rejected": -2.8169057369232178,
      "logps/chosen": -252.9962158203125,
      "logps/rejected": -200.0865936279297,
      "loss": 0.6885,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.047353629022836685,
      "rewards/margins": 0.00408762414008379,
      "rewards/rejected": 0.04326600581407547,
      "step": 8910
    },
    {
      "epoch": 1.56,
      "learning_rate": 5.581863979848867e-08,
      "logits/chosen": -2.7649385929107666,
      "logits/rejected": -2.733037233352661,
      "logps/chosen": -203.55955505371094,
      "logps/rejected": -200.595458984375,
      "loss": 0.687,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.036483533680438995,
      "rewards/margins": 0.010326062329113483,
      "rewards/rejected": 0.026157474145293236,
      "step": 8920
    },
    {
      "epoch": 1.56,
      "learning_rate": 5.576826196473552e-08,
      "logits/chosen": -2.7592132091522217,
      "logits/rejected": -2.7713820934295654,
      "logps/chosen": -292.9085693359375,
      "logps/rejected": -248.684814453125,
      "loss": 0.6872,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.06435064971446991,
      "rewards/margins": 0.020021168515086174,
      "rewards/rejected": 0.04432948678731918,
      "step": 8930
    },
    {
      "epoch": 1.57,
      "learning_rate": 5.571788413098236e-08,
      "logits/chosen": -2.779628276824951,
      "logits/rejected": -2.7205421924591064,
      "logps/chosen": -201.10134887695312,
      "logps/rejected": -219.69430541992188,
      "loss": 0.6892,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.037483375519514084,
      "rewards/margins": 0.004479845520108938,
      "rewards/rejected": 0.03300352767109871,
      "step": 8940
    },
    {
      "epoch": 1.57,
      "learning_rate": 5.5667506297229215e-08,
      "logits/chosen": -2.7869956493377686,
      "logits/rejected": -2.8030505180358887,
      "logps/chosen": -224.43423461914062,
      "logps/rejected": -202.8345184326172,
      "loss": 0.6873,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.05292726680636406,
      "rewards/margins": 0.013693466782569885,
      "rewards/rejected": 0.039233800023794174,
      "step": 8950
    },
    {
      "epoch": 1.57,
      "learning_rate": 5.561712846347607e-08,
      "logits/chosen": -2.7221343517303467,
      "logits/rejected": -2.7979655265808105,
      "logps/chosen": -275.8337707519531,
      "logps/rejected": -228.05105590820312,
      "loss": 0.6862,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.05333739519119263,
      "rewards/margins": 0.012456092052161694,
      "rewards/rejected": 0.04088129848241806,
      "step": 8960
    },
    {
      "epoch": 1.57,
      "learning_rate": 5.556675062972292e-08,
      "logits/chosen": -2.7971296310424805,
      "logits/rejected": -2.7281224727630615,
      "logps/chosen": -217.47689819335938,
      "logps/rejected": -214.9822235107422,
      "loss": 0.6887,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.04712430015206337,
      "rewards/margins": 0.0025797581765800714,
      "rewards/rejected": 0.04454454034566879,
      "step": 8970
    },
    {
      "epoch": 1.57,
      "learning_rate": 5.551637279596977e-08,
      "logits/chosen": -2.710780382156372,
      "logits/rejected": -2.772583484649658,
      "logps/chosen": -215.26107788085938,
      "logps/rejected": -197.31336975097656,
      "loss": 0.6885,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.04553055018186569,
      "rewards/margins": 0.0024104660842567682,
      "rewards/rejected": 0.04312007874250412,
      "step": 8980
    },
    {
      "epoch": 1.57,
      "learning_rate": 5.546599496221662e-08,
      "logits/chosen": -2.8027522563934326,
      "logits/rejected": -2.8064560890197754,
      "logps/chosen": -245.5795135498047,
      "logps/rejected": -193.53579711914062,
      "loss": 0.6881,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.046005867421627045,
      "rewards/margins": 0.012948085553944111,
      "rewards/rejected": 0.03305777907371521,
      "step": 8990
    },
    {
      "epoch": 1.58,
      "learning_rate": 5.541561712846348e-08,
      "logits/chosen": -2.7577126026153564,
      "logits/rejected": -2.7995660305023193,
      "logps/chosen": -224.8982696533203,
      "logps/rejected": -200.3507843017578,
      "loss": 0.6894,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.050915975123643875,
      "rewards/margins": 0.012286419048905373,
      "rewards/rejected": 0.03862955421209335,
      "step": 9000
    },
    {
      "epoch": 1.58,
      "learning_rate": 5.536523929471032e-08,
      "logits/chosen": -2.725773334503174,
      "logits/rejected": -2.7094340324401855,
      "logps/chosen": -283.9950256347656,
      "logps/rejected": -243.35842895507812,
      "loss": 0.6876,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.06494607031345367,
      "rewards/margins": 0.016711626201868057,
      "rewards/rejected": 0.048234451562166214,
      "step": 9010
    },
    {
      "epoch": 1.58,
      "learning_rate": 5.531486146095717e-08,
      "logits/chosen": -2.817140579223633,
      "logits/rejected": -2.810319423675537,
      "logps/chosen": -226.22549438476562,
      "logps/rejected": -191.5854949951172,
      "loss": 0.6871,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.061543893069028854,
      "rewards/margins": 0.025458836928009987,
      "rewards/rejected": 0.03608505055308342,
      "step": 9020
    },
    {
      "epoch": 1.58,
      "learning_rate": 5.526448362720403e-08,
      "logits/chosen": -2.779113531112671,
      "logits/rejected": -2.7687220573425293,
      "logps/chosen": -229.80224609375,
      "logps/rejected": -204.94235229492188,
      "loss": 0.6867,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.05384296923875809,
      "rewards/margins": 0.02316504344344139,
      "rewards/rejected": 0.03067792020738125,
      "step": 9030
    },
    {
      "epoch": 1.58,
      "learning_rate": 5.5214105793450885e-08,
      "logits/chosen": -2.736494779586792,
      "logits/rejected": -2.705517053604126,
      "logps/chosen": -232.3329315185547,
      "logps/rejected": -209.12216186523438,
      "loss": 0.689,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.05568603798747063,
      "rewards/margins": 0.011279943399131298,
      "rewards/rejected": 0.04440609738230705,
      "step": 9040
    },
    {
      "epoch": 1.59,
      "learning_rate": 5.5163727959697725e-08,
      "logits/chosen": -2.800626039505005,
      "logits/rejected": -2.7749905586242676,
      "logps/chosen": -227.7764892578125,
      "logps/rejected": -180.96810913085938,
      "loss": 0.6876,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": 0.04726389795541763,
      "rewards/margins": 0.020383086055517197,
      "rewards/rejected": 0.02688080631196499,
      "step": 9050
    },
    {
      "epoch": 1.59,
      "learning_rate": 5.511335012594458e-08,
      "logits/chosen": -2.699848175048828,
      "logits/rejected": -2.845942974090576,
      "logps/chosen": -298.18463134765625,
      "logps/rejected": -216.5264129638672,
      "loss": 0.6855,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.058933716267347336,
      "rewards/margins": 0.012239878997206688,
      "rewards/rejected": 0.0466938316822052,
      "step": 9060
    },
    {
      "epoch": 1.59,
      "learning_rate": 5.506297229219143e-08,
      "logits/chosen": -2.8089985847473145,
      "logits/rejected": -2.822813034057617,
      "logps/chosen": -233.1894073486328,
      "logps/rejected": -217.60147094726562,
      "loss": 0.6879,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.05084647610783577,
      "rewards/margins": 0.004182029515504837,
      "rewards/rejected": 0.04666444659233093,
      "step": 9070
    },
    {
      "epoch": 1.59,
      "learning_rate": 5.5012594458438286e-08,
      "logits/chosen": -2.7646870613098145,
      "logits/rejected": -2.7673144340515137,
      "logps/chosen": -224.1854705810547,
      "logps/rejected": -191.7845001220703,
      "loss": 0.6873,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.0356200747191906,
      "rewards/margins": 0.013415751047432423,
      "rewards/rejected": 0.0222043227404356,
      "step": 9080
    },
    {
      "epoch": 1.59,
      "learning_rate": 5.496221662468513e-08,
      "logits/chosen": -2.754457950592041,
      "logits/rejected": -2.818115472793579,
      "logps/chosen": -299.34967041015625,
      "logps/rejected": -258.12750244140625,
      "loss": 0.6896,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.06361515820026398,
      "rewards/margins": 0.019738245755434036,
      "rewards/rejected": 0.04387691244482994,
      "step": 9090
    },
    {
      "epoch": 1.59,
      "learning_rate": 5.491183879093199e-08,
      "logits/chosen": -2.8182778358459473,
      "logits/rejected": -2.8445849418640137,
      "logps/chosen": -233.5463104248047,
      "logps/rejected": -215.1799774169922,
      "loss": 0.6886,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.043782927095890045,
      "rewards/margins": 0.010205776430666447,
      "rewards/rejected": 0.033577147871255875,
      "step": 9100
    },
    {
      "epoch": 1.6,
      "learning_rate": 5.486146095717884e-08,
      "logits/chosen": -2.8396553993225098,
      "logits/rejected": -2.7761013507843018,
      "logps/chosen": -244.1826934814453,
      "logps/rejected": -222.1802520751953,
      "loss": 0.6872,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.05905939266085625,
      "rewards/margins": 0.010740868747234344,
      "rewards/rejected": 0.0483185239136219,
      "step": 9110
    },
    {
      "epoch": 1.6,
      "learning_rate": 5.4811083123425694e-08,
      "logits/chosen": -2.8012256622314453,
      "logits/rejected": -2.7701072692871094,
      "logps/chosen": -203.14297485351562,
      "logps/rejected": -186.69833374023438,
      "loss": 0.6888,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.041209183633327484,
      "rewards/margins": 0.008670704439282417,
      "rewards/rejected": 0.032538481056690216,
      "step": 9120
    },
    {
      "epoch": 1.6,
      "learning_rate": 5.4760705289672534e-08,
      "logits/chosen": -2.8059725761413574,
      "logits/rejected": -2.7879891395568848,
      "logps/chosen": -237.21102905273438,
      "logps/rejected": -198.75558471679688,
      "loss": 0.6877,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.04212053492665291,
      "rewards/margins": 0.011806664988398552,
      "rewards/rejected": 0.03031386435031891,
      "step": 9130
    },
    {
      "epoch": 1.6,
      "learning_rate": 5.4710327455919395e-08,
      "logits/chosen": -2.696505069732666,
      "logits/rejected": -2.685786724090576,
      "logps/chosen": -177.89352416992188,
      "logps/rejected": -178.42393493652344,
      "loss": 0.6902,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.03737228736281395,
      "rewards/margins": 0.0012526328209787607,
      "rewards/rejected": 0.03611965477466583,
      "step": 9140
    },
    {
      "epoch": 1.6,
      "learning_rate": 5.465994962216625e-08,
      "logits/chosen": -2.761359691619873,
      "logits/rejected": -2.7841641902923584,
      "logps/chosen": -248.031982421875,
      "logps/rejected": -212.16357421875,
      "loss": 0.6866,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.05025426670908928,
      "rewards/margins": 0.013314127922058105,
      "rewards/rejected": 0.03694014251232147,
      "step": 9150
    },
    {
      "epoch": 1.6,
      "learning_rate": 5.46095717884131e-08,
      "logits/chosen": -2.7705512046813965,
      "logits/rejected": -2.7175521850585938,
      "logps/chosen": -218.7301483154297,
      "logps/rejected": -175.79981994628906,
      "loss": 0.6895,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.0510282889008522,
      "rewards/margins": 0.02606021985411644,
      "rewards/rejected": 0.024968067184090614,
      "step": 9160
    },
    {
      "epoch": 1.61,
      "learning_rate": 5.455919395465994e-08,
      "logits/chosen": -2.6632533073425293,
      "logits/rejected": -2.736161470413208,
      "logps/chosen": -239.3765411376953,
      "logps/rejected": -227.0360870361328,
      "loss": 0.6887,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": 0.03844603896141052,
      "rewards/margins": 0.0018884660676121712,
      "rewards/rejected": 0.036557577550411224,
      "step": 9170
    },
    {
      "epoch": 1.61,
      "learning_rate": 5.4508816120906796e-08,
      "logits/chosen": -2.732358455657959,
      "logits/rejected": -2.7481753826141357,
      "logps/chosen": -211.608154296875,
      "logps/rejected": -191.95108032226562,
      "loss": 0.6872,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.04012041911482811,
      "rewards/margins": 0.0042021796107292175,
      "rewards/rejected": 0.03591823950409889,
      "step": 9180
    },
    {
      "epoch": 1.61,
      "learning_rate": 5.445843828715365e-08,
      "logits/chosen": -2.7956507205963135,
      "logits/rejected": -2.782504081726074,
      "logps/chosen": -259.437744140625,
      "logps/rejected": -203.3538055419922,
      "loss": 0.6885,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.05651463195681572,
      "rewards/margins": 0.015020245686173439,
      "rewards/rejected": 0.04149438813328743,
      "step": 9190
    },
    {
      "epoch": 1.61,
      "learning_rate": 5.44080604534005e-08,
      "logits/chosen": -2.7223739624023438,
      "logits/rejected": -2.7335617542266846,
      "logps/chosen": -247.673828125,
      "logps/rejected": -217.86328125,
      "loss": 0.6876,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.05414513498544693,
      "rewards/margins": 0.009823627769947052,
      "rewards/rejected": 0.044321510940790176,
      "step": 9200
    },
    {
      "epoch": 1.61,
      "learning_rate": 5.435768261964735e-08,
      "logits/chosen": -2.748483419418335,
      "logits/rejected": -2.700045108795166,
      "logps/chosen": -252.93405151367188,
      "logps/rejected": -234.80014038085938,
      "loss": 0.6874,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.05770406126976013,
      "rewards/margins": 0.012363392859697342,
      "rewards/rejected": 0.04534066468477249,
      "step": 9210
    },
    {
      "epoch": 1.61,
      "learning_rate": 5.4307304785894204e-08,
      "logits/chosen": -2.8123254776000977,
      "logits/rejected": -2.8114287853240967,
      "logps/chosen": -220.3101348876953,
      "logps/rejected": -190.74180603027344,
      "loss": 0.6863,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.05203206464648247,
      "rewards/margins": 0.0164670217782259,
      "rewards/rejected": 0.03556504473090172,
      "step": 9220
    },
    {
      "epoch": 1.62,
      "learning_rate": 5.425692695214106e-08,
      "logits/chosen": -2.6993188858032227,
      "logits/rejected": -2.698683023452759,
      "logps/chosen": -224.6380157470703,
      "logps/rejected": -183.78054809570312,
      "loss": 0.6897,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.029758159071207047,
      "rewards/margins": 0.004012234043329954,
      "rewards/rejected": 0.025745924562215805,
      "step": 9230
    },
    {
      "epoch": 1.62,
      "learning_rate": 5.420654911838791e-08,
      "logits/chosen": -2.7783288955688477,
      "logits/rejected": -2.765300750732422,
      "logps/chosen": -257.1777038574219,
      "logps/rejected": -236.02853393554688,
      "loss": 0.6866,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.053491055965423584,
      "rewards/margins": 0.011988423764705658,
      "rewards/rejected": 0.041502632200717926,
      "step": 9240
    },
    {
      "epoch": 1.62,
      "learning_rate": 5.415617128463476e-08,
      "logits/chosen": -2.6956281661987305,
      "logits/rejected": -2.711439371109009,
      "logps/chosen": -221.18392944335938,
      "logps/rejected": -181.74325561523438,
      "loss": 0.6883,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.0465092733502388,
      "rewards/margins": 0.013767106458544731,
      "rewards/rejected": 0.03274216502904892,
      "step": 9250
    },
    {
      "epoch": 1.62,
      "learning_rate": 5.410579345088161e-08,
      "logits/chosen": -2.6863269805908203,
      "logits/rejected": -2.678377866744995,
      "logps/chosen": -221.8884735107422,
      "logps/rejected": -234.8877410888672,
      "loss": 0.687,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": 0.03930746018886566,
      "rewards/margins": -0.004805571865290403,
      "rewards/rejected": 0.04411303251981735,
      "step": 9260
    },
    {
      "epoch": 1.62,
      "learning_rate": 5.4055415617128465e-08,
      "logits/chosen": -2.704188585281372,
      "logits/rejected": -2.7121920585632324,
      "logps/chosen": -180.83108520507812,
      "logps/rejected": -161.4171600341797,
      "loss": 0.6877,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.030211174860596657,
      "rewards/margins": 0.0018332020845264196,
      "rewards/rejected": 0.028377970680594444,
      "step": 9270
    },
    {
      "epoch": 1.63,
      "learning_rate": 5.400503778337532e-08,
      "logits/chosen": -2.750490665435791,
      "logits/rejected": -2.709672451019287,
      "logps/chosen": -181.82015991210938,
      "logps/rejected": -176.56161499023438,
      "loss": 0.6884,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.0316244475543499,
      "rewards/margins": 0.004817810840904713,
      "rewards/rejected": 0.02680664137005806,
      "step": 9280
    },
    {
      "epoch": 1.63,
      "learning_rate": 5.395465994962216e-08,
      "logits/chosen": -2.706376075744629,
      "logits/rejected": -2.684630870819092,
      "logps/chosen": -205.58694458007812,
      "logps/rejected": -169.49627685546875,
      "loss": 0.6898,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.03486959636211395,
      "rewards/margins": 0.010895296931266785,
      "rewards/rejected": 0.023974299430847168,
      "step": 9290
    },
    {
      "epoch": 1.63,
      "learning_rate": 5.390428211586901e-08,
      "logits/chosen": -2.7642054557800293,
      "logits/rejected": -2.686594009399414,
      "logps/chosen": -209.44400024414062,
      "logps/rejected": -176.10757446289062,
      "loss": 0.6888,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.0430467426776886,
      "rewards/margins": 0.012011488899588585,
      "rewards/rejected": 0.031035255640745163,
      "step": 9300
    },
    {
      "epoch": 1.63,
      "learning_rate": 5.385390428211587e-08,
      "logits/chosen": -2.851130962371826,
      "logits/rejected": -2.8562734127044678,
      "logps/chosen": -215.0302734375,
      "logps/rejected": -205.55197143554688,
      "loss": 0.6896,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.047839634120464325,
      "rewards/margins": 0.004748721607029438,
      "rewards/rejected": 0.04309091344475746,
      "step": 9310
    },
    {
      "epoch": 1.63,
      "learning_rate": 5.3803526448362714e-08,
      "logits/chosen": -2.815729856491089,
      "logits/rejected": -2.8203251361846924,
      "logps/chosen": -228.56088256835938,
      "logps/rejected": -225.7352752685547,
      "loss": 0.6898,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.047780029475688934,
      "rewards/margins": 0.0014233911642804742,
      "rewards/rejected": 0.04635664075613022,
      "step": 9320
    },
    {
      "epoch": 1.63,
      "learning_rate": 5.375314861460957e-08,
      "logits/chosen": -2.782745838165283,
      "logits/rejected": -2.772181510925293,
      "logps/chosen": -271.4385070800781,
      "logps/rejected": -241.4732208251953,
      "loss": 0.6889,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.06021879240870476,
      "rewards/margins": 0.013006770983338356,
      "rewards/rejected": 0.04721202328801155,
      "step": 9330
    },
    {
      "epoch": 1.64,
      "learning_rate": 5.370277078085642e-08,
      "logits/chosen": -2.7766032218933105,
      "logits/rejected": -2.708247661590576,
      "logps/chosen": -223.17605590820312,
      "logps/rejected": -180.75387573242188,
      "loss": 0.688,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.04519111290574074,
      "rewards/margins": 0.013616492971777916,
      "rewards/rejected": 0.03157461807131767,
      "step": 9340
    },
    {
      "epoch": 1.64,
      "learning_rate": 5.3652392947103275e-08,
      "logits/chosen": -2.7174108028411865,
      "logits/rejected": -2.7053213119506836,
      "logps/chosen": -200.68690490722656,
      "logps/rejected": -192.92942810058594,
      "loss": 0.6858,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.03941025584936142,
      "rewards/margins": 0.0023084853310137987,
      "rewards/rejected": 0.03710177168250084,
      "step": 9350
    },
    {
      "epoch": 1.64,
      "learning_rate": 5.360201511335012e-08,
      "logits/chosen": -2.7744853496551514,
      "logits/rejected": -2.7775588035583496,
      "logps/chosen": -230.5467529296875,
      "logps/rejected": -145.92202758789062,
      "loss": 0.6877,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": 0.057361818850040436,
      "rewards/margins": 0.030046069994568825,
      "rewards/rejected": 0.02731575071811676,
      "step": 9360
    },
    {
      "epoch": 1.64,
      "learning_rate": 5.3551637279596975e-08,
      "logits/chosen": -2.803952693939209,
      "logits/rejected": -2.676877975463867,
      "logps/chosen": -189.181884765625,
      "logps/rejected": -245.7015838623047,
      "loss": 0.6888,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0401124581694603,
      "rewards/margins": -0.004306624177843332,
      "rewards/rejected": 0.04441908746957779,
      "step": 9370
    },
    {
      "epoch": 1.64,
      "learning_rate": 5.350125944584383e-08,
      "logits/chosen": -2.7858364582061768,
      "logits/rejected": -2.7707884311676025,
      "logps/chosen": -211.93972778320312,
      "logps/rejected": -190.19741821289062,
      "loss": 0.6889,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": 0.03734724968671799,
      "rewards/margins": 0.0010033950675278902,
      "rewards/rejected": 0.03634385019540787,
      "step": 9380
    },
    {
      "epoch": 1.64,
      "learning_rate": 5.345088161209068e-08,
      "logits/chosen": -2.7639200687408447,
      "logits/rejected": -2.776536703109741,
      "logps/chosen": -227.98196411132812,
      "logps/rejected": -218.516845703125,
      "loss": 0.6885,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.045007266104221344,
      "rewards/margins": 0.007175090257078409,
      "rewards/rejected": 0.03783217817544937,
      "step": 9390
    },
    {
      "epoch": 1.65,
      "learning_rate": 5.340050377833752e-08,
      "logits/chosen": -2.728909969329834,
      "logits/rejected": -2.7919864654541016,
      "logps/chosen": -263.0089416503906,
      "logps/rejected": -209.14541625976562,
      "loss": 0.6875,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.07431508600711823,
      "rewards/margins": 0.02915819361805916,
      "rewards/rejected": 0.045156896114349365,
      "step": 9400
    },
    {
      "epoch": 1.65,
      "learning_rate": 5.3350125944584377e-08,
      "logits/chosen": -2.732649564743042,
      "logits/rejected": -2.6923861503601074,
      "logps/chosen": -239.49832153320312,
      "logps/rejected": -213.4709014892578,
      "loss": 0.6869,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.05820096284151077,
      "rewards/margins": 0.018569031730294228,
      "rewards/rejected": 0.0396319255232811,
      "step": 9410
    },
    {
      "epoch": 1.65,
      "learning_rate": 5.329974811083123e-08,
      "logits/chosen": -2.7434606552124023,
      "logits/rejected": -2.7499537467956543,
      "logps/chosen": -261.1266784667969,
      "logps/rejected": -205.405029296875,
      "loss": 0.6876,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.06213949993252754,
      "rewards/margins": 0.024112531915307045,
      "rewards/rejected": 0.03802696242928505,
      "step": 9420
    },
    {
      "epoch": 1.65,
      "learning_rate": 5.324937027707809e-08,
      "logits/chosen": -2.7199864387512207,
      "logits/rejected": -2.798959970474243,
      "logps/chosen": -249.7239990234375,
      "logps/rejected": -202.08853149414062,
      "loss": 0.6867,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.0552804172039032,
      "rewards/margins": 0.010709071531891823,
      "rewards/rejected": 0.044571347534656525,
      "step": 9430
    },
    {
      "epoch": 1.65,
      "learning_rate": 5.319899244332493e-08,
      "logits/chosen": -2.7173819541931152,
      "logits/rejected": -2.7432703971862793,
      "logps/chosen": -216.4027557373047,
      "logps/rejected": -190.73458862304688,
      "loss": 0.691,
      "rewards/accuracies": 0.42500001192092896,
      "rewards/chosen": 0.04708380252122879,
      "rewards/margins": -0.0027190011460334063,
      "rewards/rejected": 0.049802809953689575,
      "step": 9440
    },
    {
      "epoch": 1.66,
      "learning_rate": 5.3148614609571785e-08,
      "logits/chosen": -2.7633492946624756,
      "logits/rejected": -2.704305410385132,
      "logps/chosen": -260.4012756347656,
      "logps/rejected": -246.67404174804688,
      "loss": 0.6874,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": 0.05863087251782417,
      "rewards/margins": 0.00220307894051075,
      "rewards/rejected": 0.05642779916524887,
      "step": 9450
    },
    {
      "epoch": 1.66,
      "learning_rate": 5.309823677581864e-08,
      "logits/chosen": -2.7123265266418457,
      "logits/rejected": -2.729973077774048,
      "logps/chosen": -227.2530975341797,
      "logps/rejected": -269.09375,
      "loss": 0.6884,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.04649816080927849,
      "rewards/margins": -0.0020359910558909178,
      "rewards/rejected": 0.048534151166677475,
      "step": 9460
    },
    {
      "epoch": 1.66,
      "learning_rate": 5.304785894206549e-08,
      "logits/chosen": -2.8059744834899902,
      "logits/rejected": -2.8446147441864014,
      "logps/chosen": -240.58425903320312,
      "logps/rejected": -197.65289306640625,
      "loss": 0.6862,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.059047769755125046,
      "rewards/margins": 0.01563527248799801,
      "rewards/rejected": 0.04341249540448189,
      "step": 9470
    },
    {
      "epoch": 1.66,
      "learning_rate": 5.299748110831234e-08,
      "logits/chosen": -2.7589809894561768,
      "logits/rejected": -2.745276927947998,
      "logps/chosen": -252.64474487304688,
      "logps/rejected": -227.1024169921875,
      "loss": 0.6877,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": 0.04754466563463211,
      "rewards/margins": 0.012337127700448036,
      "rewards/rejected": 0.035207539796829224,
      "step": 9480
    },
    {
      "epoch": 1.66,
      "learning_rate": 5.294710327455919e-08,
      "logits/chosen": -2.742103099822998,
      "logits/rejected": -2.6979806423187256,
      "logps/chosen": -221.4449462890625,
      "logps/rejected": -201.50653076171875,
      "loss": 0.6872,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": 0.0446447879076004,
      "rewards/margins": 0.01715948060154915,
      "rewards/rejected": 0.027485307306051254,
      "step": 9490
    },
    {
      "epoch": 1.66,
      "learning_rate": 5.2896725440806046e-08,
      "logits/chosen": -2.774909734725952,
      "logits/rejected": -2.7038991451263428,
      "logps/chosen": -258.15899658203125,
      "logps/rejected": -182.76722717285156,
      "loss": 0.6852,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": 0.056605808436870575,
      "rewards/margins": 0.020097170025110245,
      "rewards/rejected": 0.036508649587631226,
      "step": 9500
    }
  ],
  "logging_steps": 10,
  "max_steps": 20000,
  "num_train_epochs": 4,
  "save_steps": 500,
  "total_flos": 0.0,
  "trial_name": null,
  "trial_params": null
}