{
  "best_global_step": 500,
  "best_metric": 1.6298139371428988e-06,
  "best_model_checkpoint": "outputs/checkpoint-500",
  "epoch": 1.0,
  "eval_steps": 500,
  "global_step": 1487,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.003363605785401951,
      "grad_norm": 538.3883666992188,
      "learning_rate": 2.133333333333334e-06,
      "logits/chosen": -0.42334240674972534,
      "logits/rejected": -0.44424742460250854,
      "logps/chosen": -472.8944396972656,
      "logps/rejected": -542.6354370117188,
      "loss": 0.6807,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.006125260144472122,
      "rewards/margins": 0.028591837733983994,
      "rewards/rejected": -0.03471710532903671,
      "step": 5
    },
    {
      "epoch": 0.006727211570803902,
      "grad_norm": 318.2124328613281,
      "learning_rate": 4.800000000000001e-06,
      "logits/chosen": -0.38837987184524536,
      "logits/rejected": -0.40657514333724976,
      "logps/chosen": -469.99072265625,
      "logps/rejected": -530.008544921875,
      "loss": 0.5109,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.10788547992706299,
      "rewards/margins": 0.5291536450386047,
      "rewards/rejected": -0.637039065361023,
      "step": 10
    },
    {
      "epoch": 0.010090817356205853,
      "grad_norm": 160.6857147216797,
      "learning_rate": 7.4666666666666675e-06,
      "logits/chosen": -0.39634817838668823,
      "logits/rejected": -0.4210287928581238,
      "logps/chosen": -514.572265625,
      "logps/rejected": -551.3402099609375,
      "loss": 0.3733,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.3420025110244751,
      "rewards/margins": 1.2617156505584717,
      "rewards/rejected": -1.6037181615829468,
      "step": 15
    },
    {
      "epoch": 0.013454423141607804,
      "grad_norm": 360.12921142578125,
      "learning_rate": 1.0133333333333335e-05,
      "logits/chosen": -0.42768803238868713,
      "logits/rejected": -0.4297807812690735,
      "logps/chosen": -451.85107421875,
      "logps/rejected": -477.64691162109375,
      "loss": 0.3019,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.5818151235580444,
      "rewards/margins": 1.8074054718017578,
      "rewards/rejected": -2.389220714569092,
      "step": 20
    },
    {
      "epoch": 0.016818028927009756,
      "grad_norm": 80.35607147216797,
      "learning_rate": 1.2800000000000001e-05,
      "logits/chosen": -0.41607666015625,
      "logits/rejected": -0.4468047618865967,
      "logps/chosen": -482.618408203125,
      "logps/rejected": -533.0935668945312,
      "loss": 0.154,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -3.0908141136169434,
      "rewards/margins": 3.550386428833008,
      "rewards/rejected": -6.641201019287109,
      "step": 25
    },
    {
      "epoch": 0.020181634712411706,
      "grad_norm": 35.314720153808594,
      "learning_rate": 1.546666666666667e-05,
      "logits/chosen": -0.47216543555259705,
      "logits/rejected": -0.5090190172195435,
      "logps/chosen": -535.8880615234375,
      "logps/rejected": -653.056884765625,
      "loss": 0.0941,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -6.836358547210693,
      "rewards/margins": 6.413688659667969,
      "rewards/rejected": -13.25004768371582,
      "step": 30
    },
    {
      "epoch": 0.023545240497813657,
      "grad_norm": 82.30870819091797,
      "learning_rate": 1.8133333333333335e-05,
      "logits/chosen": -0.5336273312568665,
      "logits/rejected": -0.572300136089325,
      "logps/chosen": -597.3112182617188,
      "logps/rejected": -720.4039306640625,
      "loss": 0.1068,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -15.226017951965332,
      "rewards/margins": 8.14784049987793,
      "rewards/rejected": -23.373857498168945,
      "step": 35
    },
    {
      "epoch": 0.026908846283215607,
      "grad_norm": 123.47981262207031,
      "learning_rate": 2.08e-05,
      "logits/chosen": -0.5565825700759888,
      "logits/rejected": -0.590980589389801,
      "logps/chosen": -780.6314697265625,
      "logps/rejected": -913.3787231445312,
      "loss": 0.0462,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -27.703655242919922,
      "rewards/margins": 11.882033348083496,
      "rewards/rejected": -39.58568572998047,
      "step": 40
    },
    {
      "epoch": 0.030272452068617558,
      "grad_norm": 0.8892024755477905,
      "learning_rate": 2.346666666666667e-05,
      "logits/chosen": -0.5002325773239136,
      "logits/rejected": -0.5275143980979919,
      "logps/chosen": -994.3927612304688,
      "logps/rejected": -1203.075927734375,
      "loss": 0.044,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -48.51496124267578,
      "rewards/margins": 17.924285888671875,
      "rewards/rejected": -66.43924713134766,
      "step": 45
    },
    {
      "epoch": 0.03363605785401951,
      "grad_norm": 13.931035995483398,
      "learning_rate": 2.6133333333333336e-05,
      "logits/chosen": -0.4247140884399414,
      "logits/rejected": -0.4879741668701172,
      "logps/chosen": -1091.126708984375,
      "logps/rejected": -1341.5618896484375,
      "loss": 0.0072,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -64.32234191894531,
      "rewards/margins": 19.807674407958984,
      "rewards/rejected": -84.13002014160156,
      "step": 50
    },
    {
      "epoch": 0.03699966363942146,
      "grad_norm": 0.06871066987514496,
      "learning_rate": 2.8800000000000002e-05,
      "logits/chosen": -0.4350171983242035,
      "logits/rejected": -0.45743340253829956,
      "logps/chosen": -1356.8709716796875,
      "logps/rejected": -1587.9403076171875,
      "loss": 0.0006,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -90.3761978149414,
      "rewards/margins": 20.369304656982422,
      "rewards/rejected": -110.74549865722656,
      "step": 55
    },
    {
      "epoch": 0.04036326942482341,
      "grad_norm": 0.02559416927397251,
      "learning_rate": 3.146666666666667e-05,
      "logits/chosen": -0.4745863080024719,
      "logits/rejected": -0.5167993307113647,
      "logps/chosen": -1709.1048583984375,
      "logps/rejected": -1995.327392578125,
      "loss": 0.003,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -124.35623931884766,
      "rewards/margins": 25.103065490722656,
      "rewards/rejected": -149.45928955078125,
      "step": 60
    },
    {
      "epoch": 0.04372687521022536,
      "grad_norm": 25.904348373413086,
      "learning_rate": 3.413333333333334e-05,
      "logits/chosen": -0.6346669793128967,
      "logits/rejected": -0.6911542415618896,
      "logps/chosen": -2652.923095703125,
      "logps/rejected": -3064.0576171875,
      "loss": 0.0002,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -217.2215118408203,
      "rewards/margins": 36.975494384765625,
      "rewards/rejected": -254.197021484375,
      "step": 65
    },
    {
      "epoch": 0.047090480995627314,
      "grad_norm": 253.33956909179688,
      "learning_rate": 3.680000000000001e-05,
      "logits/chosen": -0.9056417346000671,
      "logits/rejected": -0.9392318725585938,
      "logps/chosen": -3372.73681640625,
      "logps/rejected": -3803.11376953125,
      "loss": 0.0236,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -293.10040283203125,
      "rewards/margins": 40.59074020385742,
      "rewards/rejected": -333.69110107421875,
      "step": 70
    },
    {
      "epoch": 0.050454086781029264,
      "grad_norm": 1.1117582289443817e-05,
      "learning_rate": 3.946666666666667e-05,
      "logits/chosen": -0.6963694095611572,
      "logits/rejected": -0.7381657361984253,
      "logps/chosen": -3417.608642578125,
      "logps/rejected": -3977.31787109375,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -300.21026611328125,
      "rewards/margins": 52.232086181640625,
      "rewards/rejected": -352.44232177734375,
      "step": 75
    },
    {
      "epoch": 0.053817692566431215,
      "grad_norm": 0.8030268549919128,
      "learning_rate": 3.988668555240793e-05,
      "logits/chosen": -0.4796960949897766,
      "logits/rejected": -0.5259932279586792,
      "logps/chosen": -4201.6181640625,
      "logps/rejected": -4850.1025390625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -375.77117919921875,
      "rewards/margins": 61.482765197753906,
      "rewards/rejected": -437.25396728515625,
      "step": 80
    },
    {
      "epoch": 0.057181298351833165,
      "grad_norm": 3.2619729833927025e-11,
      "learning_rate": 3.974504249291785e-05,
      "logits/chosen": -0.8354374766349792,
      "logits/rejected": -0.8719736933708191,
      "logps/chosen": -5894.9462890625,
      "logps/rejected": -6863.83349609375,
      "loss": 0.004,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -544.02880859375,
      "rewards/margins": 91.55522155761719,
      "rewards/rejected": -635.5841064453125,
      "step": 85
    },
    {
      "epoch": 0.060544904137235116,
      "grad_norm": 0.0013957361225038767,
      "learning_rate": 3.960339943342777e-05,
      "logits/chosen": -1.2153384685516357,
      "logits/rejected": -1.251381516456604,
      "logps/chosen": -4325.33837890625,
      "logps/rejected": -5033.6337890625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -386.16937255859375,
      "rewards/margins": 67.92083740234375,
      "rewards/rejected": -454.0901794433594,
      "step": 90
    },
    {
      "epoch": 0.06390850992263707,
      "grad_norm": 0.0014116838574409485,
      "learning_rate": 3.946175637393768e-05,
      "logits/chosen": -1.32501220703125,
      "logits/rejected": -1.3536356687545776,
      "logps/chosen": -3108.22119140625,
      "logps/rejected": -3648.40185546875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -269.1573791503906,
      "rewards/margins": 50.69862365722656,
      "rewards/rejected": -319.85601806640625,
      "step": 95
    },
    {
      "epoch": 0.06727211570803902,
      "grad_norm": 0.005626756697893143,
      "learning_rate": 3.93201133144476e-05,
      "logits/chosen": -1.4024055004119873,
      "logits/rejected": -1.4302337169647217,
      "logps/chosen": -3679.484375,
      "logps/rejected": -4275.189453125,
      "loss": 0.0002,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -320.5166931152344,
      "rewards/margins": 55.17875289916992,
      "rewards/rejected": -375.6954040527344,
      "step": 100
    },
    {
      "epoch": 0.07063572149344097,
      "grad_norm": 0.1722555309534073,
      "learning_rate": 3.9178470254957516e-05,
      "logits/chosen": -1.5305557250976562,
      "logits/rejected": -1.5439674854278564,
      "logps/chosen": -3515.41064453125,
      "logps/rejected": -3966.64453125,
      "loss": 0.0009,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -304.60272216796875,
      "rewards/margins": 42.39708709716797,
      "rewards/rejected": -346.99981689453125,
      "step": 105
    },
    {
      "epoch": 0.07399932727884292,
      "grad_norm": 141.29393005371094,
      "learning_rate": 3.903682719546742e-05,
      "logits/chosen": -2.0568065643310547,
      "logits/rejected": -2.0783262252807617,
      "logps/chosen": -2206.322509765625,
      "logps/rejected": -2439.143798828125,
      "loss": 0.4276,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -178.42588806152344,
      "rewards/margins": 21.017316818237305,
      "rewards/rejected": -199.44320678710938,
      "step": 110
    },
    {
      "epoch": 0.07736293306424487,
      "grad_norm": 35.6357307434082,
      "learning_rate": 3.889518413597734e-05,
      "logits/chosen": -2.6078736782073975,
      "logits/rejected": -2.6627509593963623,
      "logps/chosen": -1598.230712890625,
      "logps/rejected": -1927.600341796875,
      "loss": 0.035,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -115.0587387084961,
      "rewards/margins": 28.59347915649414,
      "rewards/rejected": -143.65220642089844,
      "step": 115
    },
    {
      "epoch": 0.08072653884964683,
      "grad_norm": 12.032210350036621,
      "learning_rate": 3.875354107648726e-05,
      "logits/chosen": -1.3765466213226318,
      "logits/rejected": -1.398712396621704,
      "logps/chosen": -3166.3935546875,
      "logps/rejected": -3802.62939453125,
      "loss": 0.1995,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -270.11798095703125,
      "rewards/margins": 59.735496520996094,
      "rewards/rejected": -329.85345458984375,
      "step": 120
    },
    {
      "epoch": 0.08409014463504877,
      "grad_norm": 21443.923828125,
      "learning_rate": 3.861189801699717e-05,
      "logits/chosen": -1.4013569355010986,
      "logits/rejected": -1.4180192947387695,
      "logps/chosen": -3985.53759765625,
      "logps/rejected": -4416.67236328125,
      "loss": 0.3474,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -350.1133117675781,
      "rewards/margins": 41.465476989746094,
      "rewards/rejected": -391.57879638671875,
      "step": 125
    },
    {
      "epoch": 0.08745375042045073,
      "grad_norm": 1.6643218921086333e-13,
      "learning_rate": 3.847025495750709e-05,
      "logits/chosen": -1.16879403591156,
      "logits/rejected": -1.1915514469146729,
      "logps/chosen": -6590.16796875,
      "logps/rejected": -7832.5087890625,
      "loss": 0.0248,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -608.3853759765625,
      "rewards/margins": 120.359375,
      "rewards/rejected": -728.7447509765625,
      "step": 130
    },
    {
      "epoch": 0.09081735620585267,
      "grad_norm": 290.0943298339844,
      "learning_rate": 3.8328611898017e-05,
      "logits/chosen": -1.2465391159057617,
      "logits/rejected": -1.2560391426086426,
      "logps/chosen": -8195.0400390625,
      "logps/rejected": -9282.3486328125,
      "loss": 0.0011,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -773.38427734375,
      "rewards/margins": 106.3785400390625,
      "rewards/rejected": -879.7628173828125,
      "step": 135
    },
    {
      "epoch": 0.09418096199125463,
      "grad_norm": 1.0540867734221138e-08,
      "learning_rate": 3.818696883852691e-05,
      "logits/chosen": -1.1860132217407227,
      "logits/rejected": -1.209563136100769,
      "logps/chosen": -6988.97802734375,
      "logps/rejected": -8128.85302734375,
      "loss": 0.0006,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -653.6453247070312,
      "rewards/margins": 111.12198638916016,
      "rewards/rejected": -764.7672729492188,
      "step": 140
    },
    {
      "epoch": 0.09754456777665657,
      "grad_norm": 0.001985086128115654,
      "learning_rate": 3.804532577903683e-05,
      "logits/chosen": -0.8574553728103638,
      "logits/rejected": -0.8877754211425781,
      "logps/chosen": -4358.93603515625,
      "logps/rejected": -5705.97119140625,
      "loss": 0.0013,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -398.50213623046875,
      "rewards/margins": 129.54965209960938,
      "rewards/rejected": -528.0518188476562,
      "step": 145
    },
    {
      "epoch": 0.10090817356205853,
      "grad_norm": 0.00240884255617857,
      "learning_rate": 3.790368271954675e-05,
      "logits/chosen": -0.6406459212303162,
      "logits/rejected": -0.6537853479385376,
      "logps/chosen": -5994.99658203125,
      "logps/rejected": -7005.546875,
      "loss": 0.0215,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -554.6009521484375,
      "rewards/margins": 96.48019409179688,
      "rewards/rejected": -651.0811767578125,
      "step": 150
    },
    {
      "epoch": 0.10427177934746047,
      "grad_norm": 0.0008545994060114026,
      "learning_rate": 3.776203966005666e-05,
      "logits/chosen": -0.6694740056991577,
      "logits/rejected": -0.6949743032455444,
      "logps/chosen": -5240.52880859375,
      "logps/rejected": -6943.98193359375,
      "loss": 0.0007,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -477.5779724121094,
      "rewards/margins": 162.48843383789062,
      "rewards/rejected": -640.0664672851562,
      "step": 155
    },
    {
      "epoch": 0.10763538513286243,
      "grad_norm": 0.008516278117895126,
      "learning_rate": 3.762039660056657e-05,
      "logits/chosen": -0.9882567524909973,
      "logits/rejected": -0.9881635904312134,
      "logps/chosen": -4937.580078125,
      "logps/rejected": -5833.3662109375,
      "loss": 0.0003,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -447.90191650390625,
      "rewards/margins": 84.32308197021484,
      "rewards/rejected": -532.2249755859375,
      "step": 160
    },
    {
      "epoch": 0.11099899091826437,
      "grad_norm": 0.0008959082770161331,
      "learning_rate": 3.747875354107649e-05,
      "logits/chosen": -1.070149540901184,
      "logits/rejected": -1.1271270513534546,
      "logps/chosen": -4075.26953125,
      "logps/rejected": -4888.1650390625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -362.55902099609375,
      "rewards/margins": 76.25186920166016,
      "rewards/rejected": -438.8108825683594,
      "step": 165
    },
    {
      "epoch": 0.11436259670366633,
      "grad_norm": 7.501885761485028e-07,
      "learning_rate": 3.73371104815864e-05,
      "logits/chosen": -1.1251145601272583,
      "logits/rejected": -1.1645164489746094,
      "logps/chosen": -3876.71728515625,
      "logps/rejected": -4718.76953125,
      "loss": 0.0072,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -340.0887756347656,
      "rewards/margins": 79.92701721191406,
      "rewards/rejected": -420.0157775878906,
      "step": 170
    },
    {
      "epoch": 0.11772620248906829,
      "grad_norm": 2.351641893386841,
      "learning_rate": 3.719546742209632e-05,
      "logits/chosen": -1.0339930057525635,
      "logits/rejected": -1.0601650476455688,
      "logps/chosen": -5286.54052734375,
      "logps/rejected": -6076.9833984375,
      "loss": 0.1049,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -480.7987365722656,
      "rewards/margins": 76.69218444824219,
      "rewards/rejected": -557.4909057617188,
      "step": 175
    },
    {
      "epoch": 0.12108980827447023,
      "grad_norm": 0.5039150714874268,
      "learning_rate": 3.705382436260624e-05,
      "logits/chosen": -1.036270022392273,
      "logits/rejected": -1.0916472673416138,
      "logps/chosen": -4608.01416015625,
      "logps/rejected": -5282.1044921875,
      "loss": 0.0241,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -414.0794982910156,
      "rewards/margins": 63.215118408203125,
      "rewards/rejected": -477.2945861816406,
      "step": 180
    },
    {
      "epoch": 0.12445341405987219,
      "grad_norm": 3.1063556171773143e-09,
      "learning_rate": 3.691218130311615e-05,
      "logits/chosen": -1.0299466848373413,
      "logits/rejected": -1.051079511642456,
      "logps/chosen": -5122.52197265625,
      "logps/rejected": -5743.392578125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -468.74560546875,
      "rewards/margins": 59.518798828125,
      "rewards/rejected": -528.264404296875,
      "step": 185
    },
    {
      "epoch": 0.12781701984527413,
      "grad_norm": 7.947054862976074,
      "learning_rate": 3.677053824362606e-05,
      "logits/chosen": -1.0401442050933838,
      "logits/rejected": -1.0674071311950684,
      "logps/chosen": -7001.37890625,
      "logps/rejected": -7921.5107421875,
      "loss": 0.0005,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -655.2426147460938,
      "rewards/margins": 89.79844665527344,
      "rewards/rejected": -745.0410766601562,
      "step": 190
    },
    {
      "epoch": 0.1311806256306761,
      "grad_norm": 0.00211973674595356,
      "learning_rate": 3.662889518413598e-05,
      "logits/chosen": -1.3397160768508911,
      "logits/rejected": -1.3663198947906494,
      "logps/chosen": -5249.849609375,
      "logps/rejected": -6379.10400390625,
      "loss": 0.0003,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -483.4076232910156,
      "rewards/margins": 109.14825439453125,
      "rewards/rejected": -592.5558471679688,
      "step": 195
    },
    {
      "epoch": 0.13454423141607805,
      "grad_norm": 0.020967978984117508,
      "learning_rate": 3.648725212464589e-05,
      "logits/chosen": -1.8219314813613892,
      "logits/rejected": -1.8746178150177002,
      "logps/chosen": -3764.813232421875,
      "logps/rejected": -4436.26171875,
      "loss": 0.0157,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -336.99127197265625,
      "rewards/margins": 61.78936004638672,
      "rewards/rejected": -398.7806091308594,
      "step": 200
    },
    {
      "epoch": 0.13790783720147998,
      "grad_norm": 0.01852581650018692,
      "learning_rate": 3.634560906515581e-05,
      "logits/chosen": -2.214688539505005,
      "logits/rejected": -2.2404356002807617,
      "logps/chosen": -4201.31201171875,
      "logps/rejected": -4747.4599609375,
      "loss": 0.0007,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -376.98419189453125,
      "rewards/margins": 51.9693717956543,
      "rewards/rejected": -428.9535217285156,
      "step": 205
    },
    {
      "epoch": 0.14127144298688193,
      "grad_norm": 0.08838436007499695,
      "learning_rate": 3.620396600566573e-05,
      "logits/chosen": -2.2639212608337402,
      "logits/rejected": -2.272895574569702,
      "logps/chosen": -4654.0302734375,
      "logps/rejected": -5493.9951171875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -421.83721923828125,
      "rewards/margins": 80.54930877685547,
      "rewards/rejected": -502.38653564453125,
      "step": 210
    },
    {
      "epoch": 0.1446350487722839,
      "grad_norm": 0.0011422558454796672,
      "learning_rate": 3.606232294617564e-05,
      "logits/chosen": -2.1935012340545654,
      "logits/rejected": -2.2115113735198975,
      "logps/chosen": -5708.86669921875,
      "logps/rejected": -6511.7685546875,
      "loss": 0.048,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -524.9622192382812,
      "rewards/margins": 76.56709289550781,
      "rewards/rejected": -601.5292358398438,
      "step": 215
    },
    {
      "epoch": 0.14799865455768585,
      "grad_norm": 0.07161225378513336,
      "learning_rate": 3.5920679886685554e-05,
      "logits/chosen": -2.0765814781188965,
      "logits/rejected": -2.0897815227508545,
      "logps/chosen": -5600.2666015625,
      "logps/rejected": -6602.2236328125,
      "loss": 0.094,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -516.849609375,
      "rewards/margins": 96.33836364746094,
      "rewards/rejected": -613.18798828125,
      "step": 220
    },
    {
      "epoch": 0.15136226034308778,
      "grad_norm": 0.0020164987072348595,
      "learning_rate": 3.577903682719547e-05,
      "logits/chosen": -2.091965436935425,
      "logits/rejected": -2.1032073497772217,
      "logps/chosen": -5732.93896484375,
      "logps/rejected": -6628.4755859375,
      "loss": 0.0004,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -530.3555908203125,
      "rewards/margins": 85.49518585205078,
      "rewards/rejected": -615.8507080078125,
      "step": 225
    },
    {
      "epoch": 0.15472586612848974,
      "grad_norm": 0.021852873265743256,
      "learning_rate": 3.5637393767705384e-05,
      "logits/chosen": -1.7631561756134033,
      "logits/rejected": -1.7782434225082397,
      "logps/chosen": -8286.4140625,
      "logps/rejected": -9887.568359375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -786.6765747070312,
      "rewards/margins": 156.62197875976562,
      "rewards/rejected": -943.2985229492188,
      "step": 230
    },
    {
      "epoch": 0.1580894719138917,
      "grad_norm": 6.509221194819434e-14,
      "learning_rate": 3.54957507082153e-05,
      "logits/chosen": -1.4768449068069458,
      "logits/rejected": -1.5156052112579346,
      "logps/chosen": -11658.5576171875,
      "logps/rejected": -13747.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1114.835693359375,
      "rewards/margins": 203.1212615966797,
      "rewards/rejected": -1317.956787109375,
      "step": 235
    },
    {
      "epoch": 0.16145307769929365,
      "grad_norm": 2.985531735231312e-15,
      "learning_rate": 3.5354107648725214e-05,
      "logits/chosen": -1.281465768814087,
      "logits/rejected": -1.317518949508667,
      "logps/chosen": -10860.865234375,
      "logps/rejected": -13446.638671875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1043.3338623046875,
      "rewards/margins": 251.22128295898438,
      "rewards/rejected": -1294.55517578125,
      "step": 240
    },
    {
      "epoch": 0.16481668348469558,
      "grad_norm": 0.370680034160614,
      "learning_rate": 3.5212464589235126e-05,
      "logits/chosen": -1.562699317932129,
      "logits/rejected": -1.5645049810409546,
      "logps/chosen": -9318.1484375,
      "logps/rejected": -10058.171875,
      "loss": 0.0004,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -888.1019287109375,
      "rewards/margins": 73.18414306640625,
      "rewards/rejected": -961.2861328125,
      "step": 245
    },
    {
      "epoch": 0.16818028927009754,
      "grad_norm": 3.930626392364502,
      "learning_rate": 3.5070821529745044e-05,
      "logits/chosen": -2.1242904663085938,
      "logits/rejected": -2.132848024368286,
      "logps/chosen": -4728.998046875,
      "logps/rejected": -5302.43359375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -428.2525939941406,
      "rewards/margins": 54.50465774536133,
      "rewards/rejected": -482.7572326660156,
      "step": 250
    },
    {
      "epoch": 0.1715438950554995,
      "grad_norm": 0.003015931462869048,
      "learning_rate": 3.492917847025496e-05,
      "logits/chosen": -2.1723647117614746,
      "logits/rejected": -2.1952521800994873,
      "logps/chosen": -4549.40576171875,
      "logps/rejected": -5108.9033203125,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -410.35614013671875,
      "rewards/margins": 53.43781280517578,
      "rewards/rejected": -463.7939453125,
      "step": 255
    },
    {
      "epoch": 0.17490750084090145,
      "grad_norm": 3.138783233325171e-09,
      "learning_rate": 3.4787535410764875e-05,
      "logits/chosen": -2.0758674144744873,
      "logits/rejected": -2.1039209365844727,
      "logps/chosen": -6597.8818359375,
      "logps/rejected": -7631.29052734375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -609.4803466796875,
      "rewards/margins": 96.97730255126953,
      "rewards/rejected": -706.457763671875,
      "step": 260
    },
    {
      "epoch": 0.1782711066263034,
      "grad_norm": 0.28145548701286316,
      "learning_rate": 3.464589235127479e-05,
      "logits/chosen": -1.9695711135864258,
      "logits/rejected": -2.0130746364593506,
      "logps/chosen": -5711.27099609375,
      "logps/rejected": -6486.2939453125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -531.5767822265625,
      "rewards/margins": 72.74842071533203,
      "rewards/rejected": -604.3253173828125,
      "step": 265
    },
    {
      "epoch": 0.18163471241170534,
      "grad_norm": 0.0009740713285282254,
      "learning_rate": 3.4504249291784705e-05,
      "logits/chosen": -1.875695824623108,
      "logits/rejected": -1.9033334255218506,
      "logps/chosen": -8268.619140625,
      "logps/rejected": -9688.5126953125,
      "loss": 0.0003,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -775.77783203125,
      "rewards/margins": 137.75735473632812,
      "rewards/rejected": -913.5352783203125,
      "step": 270
    },
    {
      "epoch": 0.1849983181971073,
      "grad_norm": 0.004479968920350075,
      "learning_rate": 3.436260623229462e-05,
      "logits/chosen": -1.820989966392517,
      "logits/rejected": -1.8426917791366577,
      "logps/chosen": -7759.6171875,
      "logps/rejected": -8459.2197265625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -728.7791748046875,
      "rewards/margins": 68.0088882446289,
      "rewards/rejected": -796.7881469726562,
      "step": 275
    },
    {
      "epoch": 0.18836192398250926,
      "grad_norm": 0.00394541397690773,
      "learning_rate": 3.4220963172804535e-05,
      "logits/chosen": -1.7857681512832642,
      "logits/rejected": -1.8148696422576904,
      "logps/chosen": -7218.51318359375,
      "logps/rejected": -8282.6103515625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -680.7989501953125,
      "rewards/margins": 103.92474365234375,
      "rewards/rejected": -784.7236328125,
      "step": 280
    },
    {
      "epoch": 0.1917255297679112,
      "grad_norm": 0.09004653245210648,
      "learning_rate": 3.4079320113314454e-05,
      "logits/chosen": -1.744394302368164,
      "logits/rejected": -1.7658741474151611,
      "logps/chosen": -6981.41943359375,
      "logps/rejected": -7962.33984375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -656.099365234375,
      "rewards/margins": 96.34953308105469,
      "rewards/rejected": -752.4488525390625,
      "step": 285
    },
    {
      "epoch": 0.19508913555331314,
      "grad_norm": 9.651357675295041e-15,
      "learning_rate": 3.3937677053824366e-05,
      "logits/chosen": -1.8263895511627197,
      "logits/rejected": -1.8450164794921875,
      "logps/chosen": -6877.29931640625,
      "logps/rejected": -7879.2001953125,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -646.1724853515625,
      "rewards/margins": 97.79473876953125,
      "rewards/rejected": -743.9672241210938,
      "step": 290
    },
    {
      "epoch": 0.1984527413387151,
      "grad_norm": 0.0002163769386243075,
      "learning_rate": 3.379603399433428e-05,
      "logits/chosen": -1.9514858722686768,
      "logits/rejected": -1.9794706106185913,
      "logps/chosen": -6847.56884765625,
      "logps/rejected": -7797.6484375,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -643.44580078125,
      "rewards/margins": 90.94596099853516,
      "rewards/rejected": -734.3917236328125,
      "step": 295
    },
    {
      "epoch": 0.20181634712411706,
      "grad_norm": 2.3778390281087303e-12,
      "learning_rate": 3.3654390934844196e-05,
      "logits/chosen": -2.0037875175476074,
      "logits/rejected": -2.0110690593719482,
      "logps/chosen": -6972.1328125,
      "logps/rejected": -7947.26025390625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -652.4072265625,
      "rewards/margins": 95.61768341064453,
      "rewards/rejected": -748.0250244140625,
      "step": 300
    },
    {
      "epoch": 0.20517995290951901,
      "grad_norm": 0.0008754780283197761,
      "learning_rate": 3.351274787535411e-05,
      "logits/chosen": -1.9802583456039429,
      "logits/rejected": -2.0262978076934814,
      "logps/chosen": -6556.4814453125,
      "logps/rejected": -8114.11865234375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -616.42919921875,
      "rewards/margins": 149.73580932617188,
      "rewards/rejected": -766.1649780273438,
      "step": 305
    },
    {
      "epoch": 0.20854355869492094,
      "grad_norm": 0.0017388317501172423,
      "learning_rate": 3.3371104815864026e-05,
      "logits/chosen": -1.988417387008667,
      "logits/rejected": -2.033423662185669,
      "logps/chosen": -6455.90478515625,
      "logps/rejected": -7415.35693359375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -606.9917602539062,
      "rewards/margins": 91.98776245117188,
      "rewards/rejected": -698.9794921875,
      "step": 310
    },
    {
      "epoch": 0.2119071644803229,
      "grad_norm": 0.0002731787972152233,
      "learning_rate": 3.3229461756373945e-05,
      "logits/chosen": -1.942177176475525,
      "logits/rejected": -1.9871822595596313,
      "logps/chosen": -6910.72802734375,
      "logps/rejected": -7918.3046875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -651.3721923828125,
      "rewards/margins": 97.24307250976562,
      "rewards/rejected": -748.615234375,
      "step": 315
    },
    {
      "epoch": 0.21527077026572486,
      "grad_norm": 0.000911675626412034,
      "learning_rate": 3.3087818696883856e-05,
      "logits/chosen": -1.9579578638076782,
      "logits/rejected": -1.983660101890564,
      "logps/chosen": -7442.2900390625,
      "logps/rejected": -8652.43359375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -702.0764770507812,
      "rewards/margins": 118.7767333984375,
      "rewards/rejected": -820.8532104492188,
      "step": 320
    },
    {
      "epoch": 0.21863437605112682,
      "grad_norm": 0.0036347596906125546,
      "learning_rate": 3.294617563739377e-05,
      "logits/chosen": -2.067349910736084,
      "logits/rejected": -2.1024138927459717,
      "logps/chosen": -7284.2822265625,
      "logps/rejected": -8511.47265625,
      "loss": 0.0002,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -689.1012573242188,
      "rewards/margins": 117.62461853027344,
      "rewards/rejected": -806.7258911132812,
      "step": 325
    },
    {
      "epoch": 0.22199798183652875,
      "grad_norm": 0.008414914831519127,
      "learning_rate": 3.2804532577903687e-05,
      "logits/chosen": -2.333470106124878,
      "logits/rejected": -2.3567776679992676,
      "logps/chosen": -8843.0986328125,
      "logps/rejected": -9845.935546875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -840.1085815429688,
      "rewards/margins": 99.18012237548828,
      "rewards/rejected": -939.2887573242188,
      "step": 330
    },
    {
      "epoch": 0.2253615876219307,
      "grad_norm": 0.006371296010911465,
      "learning_rate": 3.26628895184136e-05,
      "logits/chosen": -2.46016526222229,
      "logits/rejected": -2.496751308441162,
      "logps/chosen": -8491.3349609375,
      "logps/rejected": -10273.1689453125,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -801.6748657226562,
      "rewards/margins": 174.6243133544922,
      "rewards/rejected": -976.2990112304688,
      "step": 335
    },
    {
      "epoch": 0.22872519340733266,
      "grad_norm": 2.0056707859039307,
      "learning_rate": 3.252124645892352e-05,
      "logits/chosen": -2.5789637565612793,
      "logits/rejected": -2.626779556274414,
      "logps/chosen": -7815.66796875,
      "logps/rejected": -9229.810546875,
      "loss": 0.0002,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -742.4771118164062,
      "rewards/margins": 137.21392822265625,
      "rewards/rejected": -879.6911010742188,
      "step": 340
    },
    {
      "epoch": 0.23208879919273462,
      "grad_norm": 5.618482567214256e-13,
      "learning_rate": 3.2379603399433435e-05,
      "logits/chosen": -1.8890397548675537,
      "logits/rejected": -1.910965919494629,
      "logps/chosen": -7936.79296875,
      "logps/rejected": -8954.4404296875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -750.6145629882812,
      "rewards/margins": 99.30131530761719,
      "rewards/rejected": -849.9157104492188,
      "step": 345
    },
    {
      "epoch": 0.23545240497813658,
      "grad_norm": 3.17935848236084,
      "learning_rate": 3.223796033994335e-05,
      "logits/chosen": -1.2144317626953125,
      "logits/rejected": -1.2477210760116577,
      "logps/chosen": -6172.76318359375,
      "logps/rejected": -6893.64599609375,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -574.22412109375,
      "rewards/margins": 69.9994125366211,
      "rewards/rejected": -644.2235717773438,
      "step": 350
    },
    {
      "epoch": 0.2388160107635385,
      "grad_norm": 0.0002647149667609483,
      "learning_rate": 3.209631728045326e-05,
      "logits/chosen": -1.3102233409881592,
      "logits/rejected": -1.3423999547958374,
      "logps/chosen": -6331.74462890625,
      "logps/rejected": -7189.00244140625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -588.47607421875,
      "rewards/margins": 82.52430725097656,
      "rewards/rejected": -671.0003662109375,
      "step": 355
    },
    {
      "epoch": 0.24217961654894046,
      "grad_norm": 0.0056757694110274315,
      "learning_rate": 3.195467422096318e-05,
      "logits/chosen": -1.6168992519378662,
      "logits/rejected": -1.6366093158721924,
      "logps/chosen": -7692.0703125,
      "logps/rejected": -8702.49609375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -721.4474487304688,
      "rewards/margins": 97.52389526367188,
      "rewards/rejected": -818.9713134765625,
      "step": 360
    },
    {
      "epoch": 0.24554322233434242,
      "grad_norm": 3.361604744567259e-11,
      "learning_rate": 3.181303116147309e-05,
      "logits/chosen": -1.6607633829116821,
      "logits/rejected": -1.7014045715332031,
      "logps/chosen": -6473.5849609375,
      "logps/rejected": -7240.32177734375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -606.5350341796875,
      "rewards/margins": 75.29393005371094,
      "rewards/rejected": -681.8289794921875,
      "step": 365
    },
    {
      "epoch": 0.24890682811974438,
      "grad_norm": 0.00017852614109870046,
      "learning_rate": 3.167138810198301e-05,
      "logits/chosen": -1.7258447408676147,
      "logits/rejected": -1.7537481784820557,
      "logps/chosen": -7271.47021484375,
      "logps/rejected": -8451.4130859375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -682.031982421875,
      "rewards/margins": 116.30473327636719,
      "rewards/rejected": -798.3367919921875,
      "step": 370
    },
    {
      "epoch": 0.2522704339051463,
      "grad_norm": 0.04648948460817337,
      "learning_rate": 3.152974504249292e-05,
      "logits/chosen": -1.6939198970794678,
      "logits/rejected": -1.7449398040771484,
      "logps/chosen": -8333.658203125,
      "logps/rejected": -9769.0302734375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -787.0216064453125,
      "rewards/margins": 140.47116088867188,
      "rewards/rejected": -927.4927978515625,
      "step": 375
    },
    {
      "epoch": 0.25563403969054826,
      "grad_norm": 0.002139839343726635,
      "learning_rate": 3.138810198300283e-05,
      "logits/chosen": -1.6684097051620483,
      "logits/rejected": -1.714417815208435,
      "logps/chosen": -8569.6416015625,
      "logps/rejected": -9940.5146484375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -813.0410766601562,
      "rewards/margins": 132.70956420898438,
      "rewards/rejected": -945.7506713867188,
      "step": 380
    },
    {
      "epoch": 0.2589976454759502,
      "grad_norm": 0.00398358516395092,
      "learning_rate": 3.124645892351275e-05,
      "logits/chosen": -1.6359087228775024,
      "logits/rejected": -1.6910406351089478,
      "logps/chosen": -8893.5556640625,
      "logps/rejected": -10549.3408203125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -843.5509033203125,
      "rewards/margins": 161.89463806152344,
      "rewards/rejected": -1005.4454956054688,
      "step": 385
    },
    {
      "epoch": 0.2623612512613522,
      "grad_norm": 0.0061290874145925045,
      "learning_rate": 3.110481586402267e-05,
      "logits/chosen": -1.6922695636749268,
      "logits/rejected": -1.73398756980896,
      "logps/chosen": -9403.349609375,
      "logps/rejected": -10515.03125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -894.7874755859375,
      "rewards/margins": 107.76847839355469,
      "rewards/rejected": -1002.5558471679688,
      "step": 390
    },
    {
      "epoch": 0.26572485704675414,
      "grad_norm": 2.568659176284188e-15,
      "learning_rate": 3.096317280453258e-05,
      "logits/chosen": -1.72222900390625,
      "logits/rejected": -1.7957334518432617,
      "logps/chosen": -6630.6279296875,
      "logps/rejected": -8086.83203125,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -626.3182373046875,
      "rewards/margins": 142.21592712402344,
      "rewards/rejected": -768.5341796875,
      "step": 395
    },
    {
      "epoch": 0.2690884628321561,
      "grad_norm": 0.00672925217077136,
      "learning_rate": 3.082152974504249e-05,
      "logits/chosen": -1.824318528175354,
      "logits/rejected": -1.8836233615875244,
      "logps/chosen": -7641.08349609375,
      "logps/rejected": -9022.935546875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -723.184326171875,
      "rewards/margins": 135.42214965820312,
      "rewards/rejected": -858.6064453125,
      "step": 400
    },
    {
      "epoch": 0.272452068617558,
      "grad_norm": 0.00028440821915864944,
      "learning_rate": 3.067988668555241e-05,
      "logits/chosen": -1.9653116464614868,
      "logits/rejected": -2.002957582473755,
      "logps/chosen": -8171.77099609375,
      "logps/rejected": -9252.5947265625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -776.9030151367188,
      "rewards/margins": 106.26123046875,
      "rewards/rejected": -883.1643676757812,
      "step": 405
    },
    {
      "epoch": 0.27581567440295995,
      "grad_norm": 9.655363442107046e-08,
      "learning_rate": 3.053824362606232e-05,
      "logits/chosen": -2.089010238647461,
      "logits/rejected": -2.1260485649108887,
      "logps/chosen": -8759.2099609375,
      "logps/rejected": -10161.859375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -831.1525268554688,
      "rewards/margins": 136.78207397460938,
      "rewards/rejected": -967.9345703125,
      "step": 410
    },
    {
      "epoch": 0.2791792801883619,
      "grad_norm": 1.879391398063035e-08,
      "learning_rate": 3.039660056657224e-05,
      "logits/chosen": -2.0942583084106445,
      "logits/rejected": -2.1481876373291016,
      "logps/chosen": -8196.666015625,
      "logps/rejected": -9562.4521484375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -773.52001953125,
      "rewards/margins": 132.04148864746094,
      "rewards/rejected": -905.5614013671875,
      "step": 415
    },
    {
      "epoch": 0.28254288597376387,
      "grad_norm": 3.4815986538205834e-08,
      "learning_rate": 3.0254957507082156e-05,
      "logits/chosen": -2.152519702911377,
      "logits/rejected": -2.2138638496398926,
      "logps/chosen": -9034.111328125,
      "logps/rejected": -9963.8291015625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -859.9510498046875,
      "rewards/margins": 92.07875061035156,
      "rewards/rejected": -952.02978515625,
      "step": 420
    },
    {
      "epoch": 0.2859064917591658,
      "grad_norm": 0.001220289384946227,
      "learning_rate": 3.0113314447592074e-05,
      "logits/chosen": -2.16084361076355,
      "logits/rejected": -2.227583885192871,
      "logps/chosen": -8945.8642578125,
      "logps/rejected": -9989.822265625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -846.6091918945312,
      "rewards/margins": 103.22428131103516,
      "rewards/rejected": -949.8333740234375,
      "step": 425
    },
    {
      "epoch": 0.2892700975445678,
      "grad_norm": 0.00013460170885082334,
      "learning_rate": 2.9971671388101982e-05,
      "logits/chosen": -2.1689693927764893,
      "logits/rejected": -2.2217202186584473,
      "logps/chosen": -9981.978515625,
      "logps/rejected": -11387.2724609375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -946.5115966796875,
      "rewards/margins": 137.51290893554688,
      "rewards/rejected": -1084.024658203125,
      "step": 430
    },
    {
      "epoch": 0.29263370332996974,
      "grad_norm": 2.1656356060246714e-16,
      "learning_rate": 2.98300283286119e-05,
      "logits/chosen": -2.1561226844787598,
      "logits/rejected": -2.2153103351593018,
      "logps/chosen": -10188.7822265625,
      "logps/rejected": -12140.8515625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -966.1941528320312,
      "rewards/margins": 189.49853515625,
      "rewards/rejected": -1155.6927490234375,
      "step": 435
    },
    {
      "epoch": 0.2959973091153717,
      "grad_norm": 2.8926598665207984e-14,
      "learning_rate": 2.9688385269121816e-05,
      "logits/chosen": -2.1793198585510254,
      "logits/rejected": -2.224987506866455,
      "logps/chosen": -8446.521484375,
      "logps/rejected": -9817.201171875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -801.3267211914062,
      "rewards/margins": 133.46612548828125,
      "rewards/rejected": -934.7927856445312,
      "step": 440
    },
    {
      "epoch": 0.29936091490077366,
      "grad_norm": 3.003800088663411e-07,
      "learning_rate": 2.954674220963173e-05,
      "logits/chosen": -2.170809507369995,
      "logits/rejected": -2.2088828086853027,
      "logps/chosen": -9188.357421875,
      "logps/rejected": -10331.1875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -873.2960815429688,
      "rewards/margins": 110.52778625488281,
      "rewards/rejected": -983.8239135742188,
      "step": 445
    },
    {
      "epoch": 0.30272452068617556,
      "grad_norm": 0.0010642802808433771,
      "learning_rate": 2.9405099150141646e-05,
      "logits/chosen": -2.196560859680176,
      "logits/rejected": -2.23347806930542,
      "logps/chosen": -10534.2138671875,
      "logps/rejected": -11593.525390625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -999.6160888671875,
      "rewards/margins": 105.3785629272461,
      "rewards/rejected": -1104.9947509765625,
      "step": 450
    },
    {
      "epoch": 0.3060881264715775,
      "grad_norm": 4.0511680708732456e-05,
      "learning_rate": 2.9263456090651558e-05,
      "logits/chosen": -2.1799559593200684,
      "logits/rejected": -2.226578950881958,
      "logps/chosen": -8892.5283203125,
      "logps/rejected": -10149.953125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -841.8640747070312,
      "rewards/margins": 122.76402282714844,
      "rewards/rejected": -964.6281127929688,
      "step": 455
    },
    {
      "epoch": 0.3094517322569795,
      "grad_norm": 1.2883042188605032e-07,
      "learning_rate": 2.9121813031161473e-05,
      "logits/chosen": -2.182234287261963,
      "logits/rejected": -2.243534564971924,
      "logps/chosen": -7733.69921875,
      "logps/rejected": -9654.4697265625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -731.1583251953125,
      "rewards/margins": 187.7327117919922,
      "rewards/rejected": -918.8909912109375,
      "step": 460
    },
    {
      "epoch": 0.31281533804238143,
      "grad_norm": 0.0006883457535877824,
      "learning_rate": 2.8980169971671392e-05,
      "logits/chosen": -2.192173480987549,
      "logits/rejected": -2.2630774974823,
      "logps/chosen": -8683.0224609375,
      "logps/rejected": -9733.10546875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -824.6004638671875,
      "rewards/margins": 103.35011291503906,
      "rewards/rejected": -927.9505004882812,
      "step": 465
    },
    {
      "epoch": 0.3161789438277834,
      "grad_norm": 6.138899388830055e-10,
      "learning_rate": 2.8838526912181307e-05,
      "logits/chosen": -2.234004259109497,
      "logits/rejected": -2.2940242290496826,
      "logps/chosen": -9586.1328125,
      "logps/rejected": -12179.115234375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -911.7447509765625,
      "rewards/margins": 252.4266815185547,
      "rewards/rejected": -1164.1715087890625,
      "step": 470
    },
    {
      "epoch": 0.31954254961318534,
      "grad_norm": 8.097021009234595e-08,
      "learning_rate": 2.8696883852691222e-05,
      "logits/chosen": -2.245631694793701,
      "logits/rejected": -2.2898073196411133,
      "logps/chosen": -8785.8154296875,
      "logps/rejected": -10476.12109375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -835.0496826171875,
      "rewards/margins": 164.67672729492188,
      "rewards/rejected": -999.7264404296875,
      "step": 475
    },
    {
      "epoch": 0.3229061553985873,
      "grad_norm": 0.0024711284786462784,
      "learning_rate": 2.8555240793201134e-05,
      "logits/chosen": -2.233473539352417,
      "logits/rejected": -2.284325122833252,
      "logps/chosen": -9083.744140625,
      "logps/rejected": -10686.375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -863.83642578125,
      "rewards/margins": 156.17706298828125,
      "rewards/rejected": -1020.0134887695312,
      "step": 480
    },
    {
      "epoch": 0.32626976118398926,
      "grad_norm": 2.4892824512789957e-07,
      "learning_rate": 2.841359773371105e-05,
      "logits/chosen": -2.243166923522949,
      "logits/rejected": -2.305075168609619,
      "logps/chosen": -9265.416015625,
      "logps/rejected": -11269.51171875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -879.9576416015625,
      "rewards/margins": 195.82711791992188,
      "rewards/rejected": -1075.7847900390625,
      "step": 485
    },
    {
      "epoch": 0.32963336696939116,
      "grad_norm": 1.88933690914439e-09,
      "learning_rate": 2.8271954674220964e-05,
      "logits/chosen": -2.2389400005340576,
      "logits/rejected": -2.280388355255127,
      "logps/chosen": -8481.2861328125,
      "logps/rejected": -9816.94921875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -803.5653076171875,
      "rewards/margins": 130.07382202148438,
      "rewards/rejected": -933.6390380859375,
      "step": 490
    },
    {
      "epoch": 0.3329969727547931,
      "grad_norm": 1.4768192907726324e-13,
      "learning_rate": 2.813031161473088e-05,
      "logits/chosen": -2.2579054832458496,
      "logits/rejected": -2.3216917514801025,
      "logps/chosen": -8754.560546875,
      "logps/rejected": -10431.3818359375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -825.8941650390625,
      "rewards/margins": 165.5917205810547,
      "rewards/rejected": -991.48583984375,
      "step": 495
    },
    {
      "epoch": 0.3363605785401951,
      "grad_norm": 1.6377015299440245e-07,
      "learning_rate": 2.7988668555240798e-05,
      "logits/chosen": -2.23718523979187,
      "logits/rejected": -2.287094831466675,
      "logps/chosen": -8831.4521484375,
      "logps/rejected": -9914.525390625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -836.8912963867188,
      "rewards/margins": 105.77693176269531,
      "rewards/rejected": -942.6682739257812,
      "step": 500
    },
    {
      "epoch": 0.3363605785401951,
      "eval_logits/chosen": -2.250317096710205,
      "eval_logits/rejected": -2.308234930038452,
      "eval_logps/chosen": -8769.4951171875,
      "eval_logps/rejected": -10312.4423828125,
      "eval_loss": 1.6298139371428988e-06,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": -831.8590087890625,
      "eval_rewards/margins": 150.75428771972656,
      "eval_rewards/rejected": -982.6133422851562,
      "eval_runtime": 471.549,
      "eval_samples_per_second": 2.804,
      "eval_steps_per_second": 0.352,
      "step": 500
    },
    {
      "epoch": 0.33972418432559703,
      "grad_norm": 3.0497803891194053e-06,
      "learning_rate": 2.7847025495750713e-05,
      "logits/chosen": -2.2452359199523926,
      "logits/rejected": -2.3129477500915527,
      "logps/chosen": -9163.3642578125,
      "logps/rejected": -10374.771484375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -870.1813354492188,
      "rewards/margins": 118.91059875488281,
      "rewards/rejected": -989.0919189453125,
      "step": 505
    },
    {
      "epoch": 0.343087790110999,
      "grad_norm": 5.3246477449420127e-08,
      "learning_rate": 2.7705382436260625e-05,
      "logits/chosen": -2.2614550590515137,
      "logits/rejected": -2.3279013633728027,
      "logps/chosen": -9706.0498046875,
      "logps/rejected": -11408.1123046875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -921.0745239257812,
      "rewards/margins": 165.28619384765625,
      "rewards/rejected": -1086.360595703125,
      "step": 510
    },
    {
      "epoch": 0.34645139589640095,
      "grad_norm": 0.04736582934856415,
      "learning_rate": 2.756373937677054e-05,
      "logits/chosen": -2.262631893157959,
      "logits/rejected": -2.3286826610565186,
      "logps/chosen": -9356.8134765625,
      "logps/rejected": -10803.0888671875,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -891.3303833007812,
      "rewards/margins": 140.49685668945312,
      "rewards/rejected": -1031.827392578125,
      "step": 515
    },
    {
      "epoch": 0.3498150016818029,
      "grad_norm": 0.00028383397147990763,
      "learning_rate": 2.7422096317280455e-05,
      "logits/chosen": -2.3105111122131348,
      "logits/rejected": -2.3706812858581543,
      "logps/chosen": -7842.4501953125,
      "logps/rejected": -9131.822265625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -743.400390625,
      "rewards/margins": 125.19610595703125,
      "rewards/rejected": -868.5964965820312,
      "step": 520
    },
    {
      "epoch": 0.35317860746720486,
      "grad_norm": 0.009294699877500534,
      "learning_rate": 2.728045325779037e-05,
      "logits/chosen": -2.381293535232544,
      "logits/rejected": -2.450087547302246,
      "logps/chosen": -9598.525390625,
      "logps/rejected": -11597.609375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -913.5612182617188,
      "rewards/margins": 194.55950927734375,
      "rewards/rejected": -1108.1207275390625,
      "step": 525
    },
    {
      "epoch": 0.3565422132526068,
      "grad_norm": 0.0011821220396086574,
      "learning_rate": 2.713881019830029e-05,
      "logits/chosen": -2.373330593109131,
      "logits/rejected": -2.4507672786712646,
      "logps/chosen": -9063.4619140625,
      "logps/rejected": -10791.115234375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -862.4066162109375,
      "rewards/margins": 168.11550903320312,
      "rewards/rejected": -1030.522216796875,
      "step": 530
    },
    {
      "epoch": 0.3599058190380087,
      "grad_norm": 0.0015295963967218995,
      "learning_rate": 2.69971671388102e-05,
      "logits/chosen": -2.35516095161438,
      "logits/rejected": -2.424393892288208,
      "logps/chosen": -8334.900390625,
      "logps/rejected": -9726.677734375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -789.7471923828125,
      "rewards/margins": 135.3495635986328,
      "rewards/rejected": -925.0968017578125,
      "step": 535
    },
    {
      "epoch": 0.3632694248234107,
      "grad_norm": 2.462135626046802e-06,
      "learning_rate": 2.6855524079320115e-05,
      "logits/chosen": -2.380399703979492,
      "logits/rejected": -2.4510059356689453,
      "logps/chosen": -9531.154296875,
      "logps/rejected": -10873.814453125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -904.9728393554688,
      "rewards/margins": 131.51644897460938,
      "rewards/rejected": -1036.489501953125,
      "step": 540
    },
    {
      "epoch": 0.36663303060881264,
      "grad_norm": 0.0,
      "learning_rate": 2.671388101983003e-05,
      "logits/chosen": -2.3532772064208984,
      "logits/rejected": -2.4131760597229004,
      "logps/chosen": -9900.201171875,
      "logps/rejected": -11852.2529296875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -943.6917724609375,
      "rewards/margins": 193.18896484375,
      "rewards/rejected": -1136.880615234375,
      "step": 545
    },
    {
      "epoch": 0.3699966363942146,
      "grad_norm": 7.793628074717965e-14,
      "learning_rate": 2.6572237960339946e-05,
      "logits/chosen": -2.3581013679504395,
      "logits/rejected": -2.432398796081543,
      "logps/chosen": -9199.5439453125,
      "logps/rejected": -10811.568359375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -876.5926513671875,
      "rewards/margins": 157.53274536132812,
      "rewards/rejected": -1034.1253662109375,
      "step": 550
    },
    {
      "epoch": 0.37336024217961655,
      "grad_norm": 1.8348317656902147e-12,
      "learning_rate": 2.643059490084986e-05,
      "logits/chosen": -2.384094715118408,
      "logits/rejected": -2.4597606658935547,
      "logps/chosen": -9236.1572265625,
      "logps/rejected": -11326.6767578125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -878.9734497070312,
      "rewards/margins": 202.41941833496094,
      "rewards/rejected": -1081.392822265625,
      "step": 555
    },
    {
      "epoch": 0.3767238479650185,
      "grad_norm": 3.426003968343139e-05,
      "learning_rate": 2.628895184135978e-05,
      "logits/chosen": -2.372939348220825,
      "logits/rejected": -2.4145314693450928,
      "logps/chosen": -9478.986328125,
      "logps/rejected": -10691.611328125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -901.7972412109375,
      "rewards/margins": 119.78208923339844,
      "rewards/rejected": -1021.5794067382812,
      "step": 560
    },
    {
      "epoch": 0.38008745375042047,
      "grad_norm": 0.001000828924588859,
      "learning_rate": 2.6147308781869688e-05,
      "logits/chosen": -2.3587584495544434,
      "logits/rejected": -2.402035713195801,
      "logps/chosen": -9746.6875,
      "logps/rejected": -10661.720703125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -928.5568237304688,
      "rewards/margins": 90.29969787597656,
      "rewards/rejected": -1018.8565673828125,
      "step": 565
    },
    {
      "epoch": 0.3834510595358224,
      "grad_norm": 8.173354770768526e-10,
      "learning_rate": 2.6005665722379606e-05,
      "logits/chosen": -2.3608500957489014,
      "logits/rejected": -2.431461811065674,
      "logps/chosen": -9322.943359375,
      "logps/rejected": -11261.091796875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -885.10546875,
      "rewards/margins": 189.113037109375,
      "rewards/rejected": -1074.218505859375,
      "step": 570
    },
    {
      "epoch": 0.3868146653212243,
      "grad_norm": 2.6418536691608097e-08,
      "learning_rate": 2.586402266288952e-05,
      "logits/chosen": -2.3074240684509277,
      "logits/rejected": -2.3423027992248535,
      "logps/chosen": -8743.208984375,
      "logps/rejected": -9258.65234375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -832.7477416992188,
      "rewards/margins": 51.55024337768555,
      "rewards/rejected": -884.2979736328125,
      "step": 575
    },
    {
      "epoch": 0.3901782711066263,
      "grad_norm": 1.1764306151462733e-07,
      "learning_rate": 2.5722379603399436e-05,
      "logits/chosen": -2.346879243850708,
      "logits/rejected": -2.405113935470581,
      "logps/chosen": -8649.9736328125,
      "logps/rejected": -10530.8173828125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -820.7400512695312,
      "rewards/margins": 184.1744384765625,
      "rewards/rejected": -1004.9144287109375,
      "step": 580
    },
    {
      "epoch": 0.39354187689202824,
      "grad_norm": 1.4764815889009575e-12,
      "learning_rate": 2.558073654390935e-05,
      "logits/chosen": -2.3577311038970947,
      "logits/rejected": -2.418890953063965,
      "logps/chosen": -9643.0400390625,
      "logps/rejected": -11039.9638671875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -915.83642578125,
      "rewards/margins": 137.834716796875,
      "rewards/rejected": -1053.671142578125,
      "step": 585
    },
    {
      "epoch": 0.3969054826774302,
      "grad_norm": 2.290467016052844e-08,
      "learning_rate": 2.5439093484419263e-05,
      "logits/chosen": -2.320554256439209,
      "logits/rejected": -2.409496307373047,
      "logps/chosen": -7435.5625,
      "logps/rejected": -9336.154296875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -703.7423095703125,
      "rewards/margins": 183.55369567871094,
      "rewards/rejected": -887.2958984375,
      "step": 590
    },
    {
      "epoch": 0.40026908846283216,
      "grad_norm": 0.000951705442275852,
      "learning_rate": 2.529745042492918e-05,
      "logits/chosen": -2.340599298477173,
      "logits/rejected": -2.3895423412323,
      "logps/chosen": -8552.607421875,
      "logps/rejected": -9806.7978515625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -811.9674072265625,
      "rewards/margins": 123.87007904052734,
      "rewards/rejected": -935.8375244140625,
      "step": 595
    },
    {
      "epoch": 0.4036326942482341,
      "grad_norm": 6.395195484287797e-11,
      "learning_rate": 2.5155807365439097e-05,
      "logits/chosen": -2.344046115875244,
      "logits/rejected": -2.4036402702331543,
      "logps/chosen": -9658.517578125,
      "logps/rejected": -11503.2080078125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -921.5321044921875,
      "rewards/margins": 181.1289520263672,
      "rewards/rejected": -1102.6611328125,
      "step": 600
    },
    {
      "epoch": 0.40699630003363607,
      "grad_norm": 7.647595230697579e-12,
      "learning_rate": 2.5014164305949012e-05,
      "logits/chosen": -2.3500094413757324,
      "logits/rejected": -2.4067153930664062,
      "logps/chosen": -7853.10009765625,
      "logps/rejected": -9108.6982421875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -742.5648193359375,
      "rewards/margins": 121.40147399902344,
      "rewards/rejected": -863.9661865234375,
      "step": 605
    },
    {
      "epoch": 0.41035990581903803,
      "grad_norm": 3.01065961139102e-06,
      "learning_rate": 2.4872521246458927e-05,
      "logits/chosen": -2.36686635017395,
      "logits/rejected": -2.4282941818237305,
      "logps/chosen": -8357.40234375,
      "logps/rejected": -9278.1025390625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -794.9564819335938,
      "rewards/margins": 90.97221374511719,
      "rewards/rejected": -885.9287109375,
      "step": 610
    },
    {
      "epoch": 0.41372351160444,
      "grad_norm": 7.609626209159615e-07,
      "learning_rate": 2.473087818696884e-05,
      "logits/chosen": -2.3561949729919434,
      "logits/rejected": -2.401888847351074,
      "logps/chosen": -9303.798828125,
      "logps/rejected": -10313.6455078125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -885.8536987304688,
      "rewards/margins": 99.88475036621094,
      "rewards/rejected": -985.7384643554688,
      "step": 615
    },
    {
      "epoch": 0.4170871173898419,
      "grad_norm": 1.6553874902249487e-14,
      "learning_rate": 2.4589235127478754e-05,
      "logits/chosen": -2.3552451133728027,
      "logits/rejected": -2.429365873336792,
      "logps/chosen": -9315.1240234375,
      "logps/rejected": -11143.7333984375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -884.0511474609375,
      "rewards/margins": 177.26329040527344,
      "rewards/rejected": -1061.314453125,
      "step": 620
    },
    {
      "epoch": 0.42045072317524385,
      "grad_norm": 5.291678562002744e-08,
      "learning_rate": 2.444759206798867e-05,
      "logits/chosen": -2.353862762451172,
      "logits/rejected": -2.4107158184051514,
      "logps/chosen": -8397.6328125,
      "logps/rejected": -9575.841796875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -797.6805419921875,
      "rewards/margins": 115.51654052734375,
      "rewards/rejected": -913.1970825195312,
      "step": 625
    },
    {
      "epoch": 0.4238143289606458,
      "grad_norm": 1.835929884190507e-16,
      "learning_rate": 2.4305949008498588e-05,
      "logits/chosen": -2.3836007118225098,
      "logits/rejected": -2.4638640880584717,
      "logps/chosen": -8398.6923828125,
      "logps/rejected": -10677.0283203125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -795.8978271484375,
      "rewards/margins": 221.0430908203125,
      "rewards/rejected": -1016.9410400390625,
      "step": 630
    },
    {
      "epoch": 0.42717793474604776,
      "grad_norm": 4.621757688475121e-14,
      "learning_rate": 2.4164305949008503e-05,
      "logits/chosen": -2.3744118213653564,
      "logits/rejected": -2.4456794261932373,
      "logps/chosen": -9041.783203125,
      "logps/rejected": -10766.3876953125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -857.4827880859375,
      "rewards/margins": 167.5565185546875,
      "rewards/rejected": -1025.039306640625,
      "step": 635
    },
    {
      "epoch": 0.4305415405314497,
      "grad_norm": 0.003248578868806362,
      "learning_rate": 2.4022662889518418e-05,
      "logits/chosen": -2.3794054985046387,
      "logits/rejected": -2.4629693031311035,
      "logps/chosen": -8621.6552734375,
      "logps/rejected": -10413.462890625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -818.5867919921875,
      "rewards/margins": 174.3968048095703,
      "rewards/rejected": -992.9835205078125,
      "step": 640
    },
    {
      "epoch": 0.4339051463168517,
      "grad_norm": 1.451117555006931e-07,
      "learning_rate": 2.388101983002833e-05,
      "logits/chosen": -2.414998769760132,
      "logits/rejected": -2.473670482635498,
      "logps/chosen": -10356.787109375,
      "logps/rejected": -12423.662109375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -989.3740234375,
      "rewards/margins": 202.37220764160156,
      "rewards/rejected": -1191.746337890625,
      "step": 645
    },
    {
      "epoch": 0.43726875210225363,
      "grad_norm": 2.1352482448833143e-08,
      "learning_rate": 2.3739376770538245e-05,
      "logits/chosen": -2.3824939727783203,
      "logits/rejected": -2.426950454711914,
      "logps/chosen": -9070.0751953125,
      "logps/rejected": -10976.8486328125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -860.9542236328125,
      "rewards/margins": 185.2146759033203,
      "rewards/rejected": -1046.1689453125,
      "step": 650
    },
    {
      "epoch": 0.4406323578876556,
      "grad_norm": 0.004765070043504238,
      "learning_rate": 2.359773371104816e-05,
      "logits/chosen": -2.3830032348632812,
      "logits/rejected": -2.4377903938293457,
      "logps/chosen": -8619.86328125,
      "logps/rejected": -9881.05078125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -818.465576171875,
      "rewards/margins": 123.59765625,
      "rewards/rejected": -942.0631713867188,
      "step": 655
    },
    {
      "epoch": 0.4439959636730575,
      "grad_norm": 5.638601344599417e-15,
      "learning_rate": 2.3456090651558075e-05,
      "logits/chosen": -2.387718677520752,
      "logits/rejected": -2.4577600955963135,
      "logps/chosen": -9779.986328125,
      "logps/rejected": -12107.9716796875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -931.4807739257812,
      "rewards/margins": 226.39682006835938,
      "rewards/rejected": -1157.877685546875,
      "step": 660
    },
    {
      "epoch": 0.44735956945845945,
      "grad_norm": 0.03086795099079609,
      "learning_rate": 2.3314447592067994e-05,
      "logits/chosen": -2.405491352081299,
      "logits/rejected": -2.448850154876709,
      "logps/chosen": -9381.787109375,
      "logps/rejected": -10682.6689453125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -893.2320556640625,
      "rewards/margins": 127.97636413574219,
      "rewards/rejected": -1021.2083740234375,
      "step": 665
    },
    {
      "epoch": 0.4507231752438614,
      "grad_norm": 0.00036815201747231185,
      "learning_rate": 2.3172804532577905e-05,
      "logits/chosen": -2.376716136932373,
      "logits/rejected": -2.435704469680786,
      "logps/chosen": -9360.1796875,
      "logps/rejected": -10802.7275390625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -890.0862426757812,
      "rewards/margins": 142.4642333984375,
      "rewards/rejected": -1032.550537109375,
      "step": 670
    },
    {
      "epoch": 0.45408678102926336,
      "grad_norm": 1.1096083653683309e-05,
      "learning_rate": 2.303116147308782e-05,
      "logits/chosen": -2.3951449394226074,
      "logits/rejected": -2.443387031555176,
      "logps/chosen": -9257.7626953125,
      "logps/rejected": -10631.7158203125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -882.44384765625,
      "rewards/margins": 135.7457733154297,
      "rewards/rejected": -1018.1895751953125,
      "step": 675
    },
    {
      "epoch": 0.4574503868146653,
      "grad_norm": 5.129273064228765e-14,
      "learning_rate": 2.2889518413597736e-05,
      "logits/chosen": -2.382321834564209,
      "logits/rejected": -2.4462177753448486,
      "logps/chosen": -9793.5732421875,
      "logps/rejected": -11344.658203125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -935.3284301757812,
      "rewards/margins": 151.4036102294922,
      "rewards/rejected": -1086.73193359375,
      "step": 680
    },
    {
      "epoch": 0.4608139926000673,
      "grad_norm": 0.00023297348525375128,
      "learning_rate": 2.274787535410765e-05,
      "logits/chosen": -2.4033541679382324,
      "logits/rejected": -2.455732822418213,
      "logps/chosen": -8915.3173828125,
      "logps/rejected": -10221.861328125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -847.9646606445312,
      "rewards/margins": 127.8862075805664,
      "rewards/rejected": -975.8508911132812,
      "step": 685
    },
    {
      "epoch": 0.46417759838546924,
      "grad_norm": 6.369052420879262e-12,
      "learning_rate": 2.2606232294617566e-05,
      "logits/chosen": -2.407374620437622,
      "logits/rejected": -2.47847318649292,
      "logps/chosen": -8809.994140625,
      "logps/rejected": -10278.0595703125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -835.8258056640625,
      "rewards/margins": 144.3980255126953,
      "rewards/rejected": -980.2238159179688,
      "step": 690
    },
    {
      "epoch": 0.4675412041708712,
      "grad_norm": 3.460689868006739e-07,
      "learning_rate": 2.2464589235127478e-05,
      "logits/chosen": -2.3829731941223145,
      "logits/rejected": -2.4348952770233154,
      "logps/chosen": -9232.4091796875,
      "logps/rejected": -10810.36328125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -876.6639404296875,
      "rewards/margins": 154.1439208984375,
      "rewards/rejected": -1030.807861328125,
      "step": 695
    },
    {
      "epoch": 0.47090480995627315,
      "grad_norm": 0.0017172497464343905,
      "learning_rate": 2.2322946175637393e-05,
      "logits/chosen": -2.3799548149108887,
      "logits/rejected": -2.41900372505188,
      "logps/chosen": -9366.7138671875,
      "logps/rejected": -10292.5400390625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -892.8621826171875,
      "rewards/margins": 91.45470428466797,
      "rewards/rejected": -984.3170166015625,
      "step": 700
    },
    {
      "epoch": 0.47426841574167505,
      "grad_norm": 0.0008322283974848688,
      "learning_rate": 2.218130311614731e-05,
      "logits/chosen": -2.395921230316162,
      "logits/rejected": -2.466614007949829,
      "logps/chosen": -8541.337890625,
      "logps/rejected": -10370.1083984375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -809.6399536132812,
      "rewards/margins": 179.8800506591797,
      "rewards/rejected": -989.52001953125,
      "step": 705
    },
    {
      "epoch": 0.477632021527077,
      "grad_norm": 2.840573820606096e-08,
      "learning_rate": 2.2039660056657226e-05,
      "logits/chosen": -2.3547253608703613,
      "logits/rejected": -2.407517910003662,
      "logps/chosen": -8334.697265625,
      "logps/rejected": -10086.57421875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -790.6559448242188,
      "rewards/margins": 171.19837951660156,
      "rewards/rejected": -961.8543090820312,
      "step": 710
    },
    {
      "epoch": 0.48099562731247897,
      "grad_norm": 1.822071027390848e-07,
      "learning_rate": 2.189801699716714e-05,
      "logits/chosen": -2.3620803356170654,
      "logits/rejected": -2.4428133964538574,
      "logps/chosen": -7775.1455078125,
      "logps/rejected": -9572.26171875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -737.24169921875,
      "rewards/margins": 174.17442321777344,
      "rewards/rejected": -911.4161376953125,
      "step": 715
    },
    {
      "epoch": 0.4843592330978809,
      "grad_norm": 8.396268924570904e-08,
      "learning_rate": 2.1756373937677057e-05,
      "logits/chosen": -2.3242809772491455,
      "logits/rejected": -2.384099245071411,
      "logps/chosen": -9311.2919921875,
      "logps/rejected": -11442.0185546875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -885.0011596679688,
      "rewards/margins": 208.14755249023438,
      "rewards/rejected": -1093.1488037109375,
      "step": 720
    },
    {
      "epoch": 0.4877228388832829,
      "grad_norm": 2.04550021365435e-09,
      "learning_rate": 2.161473087818697e-05,
      "logits/chosen": -2.3817965984344482,
      "logits/rejected": -2.4505302906036377,
      "logps/chosen": -9729.5048828125,
      "logps/rejected": -11644.5498046875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -925.9532470703125,
      "rewards/margins": 188.06198120117188,
      "rewards/rejected": -1114.0150146484375,
      "step": 725
    },
    {
      "epoch": 0.49108644466868484,
      "grad_norm": 1.3809663482189372e-11,
      "learning_rate": 2.1473087818696884e-05,
      "logits/chosen": -2.513360023498535,
      "logits/rejected": -2.5747997760772705,
      "logps/chosen": -10130.029296875,
      "logps/rejected": -12416.2841796875,
      "loss": 0.2027,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -963.47802734375,
      "rewards/margins": 223.7931671142578,
      "rewards/rejected": -1187.271240234375,
      "step": 730
    },
    {
      "epoch": 0.4944500504540868,
      "grad_norm": 0.017272062599658966,
      "learning_rate": 2.1331444759206802e-05,
      "logits/chosen": -2.8644638061523438,
      "logits/rejected": -2.9070181846618652,
      "logps/chosen": -8297.6240234375,
      "logps/rejected": -9298.3291015625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -791.5880126953125,
      "rewards/margins": 97.62638092041016,
      "rewards/rejected": -889.21435546875,
      "step": 735
    },
    {
      "epoch": 0.49781365623948876,
      "grad_norm": 0.0011317819589748979,
      "learning_rate": 2.1189801699716717e-05,
      "logits/chosen": -3.066047430038452,
      "logits/rejected": -3.1139955520629883,
      "logps/chosen": -11239.828125,
      "logps/rejected": -13367.7265625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1079.0850830078125,
      "rewards/margins": 208.7364044189453,
      "rewards/rejected": -1287.8214111328125,
      "step": 740
    },
    {
      "epoch": 0.5011772620248907,
      "grad_norm": 0.000305292138364166,
      "learning_rate": 2.1048158640226632e-05,
      "logits/chosen": -3.1505126953125,
      "logits/rejected": -3.209517002105713,
      "logps/chosen": -10784.92578125,
      "logps/rejected": -12421.9921875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1034.216552734375,
      "rewards/margins": 161.51394653320312,
      "rewards/rejected": -1195.73046875,
      "step": 745
    },
    {
      "epoch": 0.5045408678102926,
      "grad_norm": 0.0027845939621329308,
      "learning_rate": 2.0906515580736544e-05,
      "logits/chosen": -3.2020633220672607,
      "logits/rejected": -3.2625765800476074,
      "logps/chosen": -10902.958984375,
      "logps/rejected": -12649.0859375,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1043.534423828125,
      "rewards/margins": 170.08108520507812,
      "rewards/rejected": -1213.615478515625,
      "step": 750
    },
    {
      "epoch": 0.5079044735956946,
      "grad_norm": 3.677980942029535e-07,
      "learning_rate": 2.076487252124646e-05,
      "logits/chosen": -3.0169548988342285,
      "logits/rejected": -3.0616793632507324,
      "logps/chosen": -11752.5546875,
      "logps/rejected": -12711.5029296875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1125.407470703125,
      "rewards/margins": 94.27081298828125,
      "rewards/rejected": -1219.67822265625,
      "step": 755
    },
    {
      "epoch": 0.5112680793810965,
      "grad_norm": 2.28953149417066e-06,
      "learning_rate": 2.0623229461756374e-05,
      "logits/chosen": -2.7937214374542236,
      "logits/rejected": -2.8539373874664307,
      "logps/chosen": -8987.6435546875,
      "logps/rejected": -11191.111328125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -857.9267578125,
      "rewards/margins": 215.7295684814453,
      "rewards/rejected": -1073.6563720703125,
      "step": 760
    },
    {
      "epoch": 0.5146316851664985,
      "grad_norm": 6.656330810983491e-07,
      "learning_rate": 2.0481586402266293e-05,
      "logits/chosen": -2.6380062103271484,
      "logits/rejected": -2.6904971599578857,
      "logps/chosen": -10492.505859375,
      "logps/rejected": -12141.62890625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1000.1650390625,
      "rewards/margins": 159.52828979492188,
      "rewards/rejected": -1159.6934814453125,
      "step": 765
    },
    {
      "epoch": 0.5179952909519004,
      "grad_norm": 0.0002926556917373091,
      "learning_rate": 2.0339943342776208e-05,
      "logits/chosen": -2.5359256267547607,
      "logits/rejected": -2.577279567718506,
      "logps/chosen": -8801.86328125,
      "logps/rejected": -9642.6640625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -842.5107421875,
      "rewards/margins": 82.14469909667969,
      "rewards/rejected": -924.6553955078125,
      "step": 770
    },
    {
      "epoch": 0.5213588967373024,
      "grad_norm": 0.01562671549618244,
      "learning_rate": 2.019830028328612e-05,
      "logits/chosen": -2.501791477203369,
      "logits/rejected": -2.5308659076690674,
      "logps/chosen": -8460.00390625,
      "logps/rejected": -9557.484375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -802.3045654296875,
      "rewards/margins": 105.98504638671875,
      "rewards/rejected": -908.2896728515625,
      "step": 775
    },
    {
      "epoch": 0.5247225025227044,
      "grad_norm": 7.185167874013132e-07,
      "learning_rate": 2.0056657223796035e-05,
      "logits/chosen": -2.4650492668151855,
      "logits/rejected": -2.5365099906921387,
      "logps/chosen": -7860.4619140625,
      "logps/rejected": -9544.0556640625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -743.4708251953125,
      "rewards/margins": 164.23150634765625,
      "rewards/rejected": -907.7023315429688,
      "step": 780
    },
    {
      "epoch": 0.5280861083081063,
      "grad_norm": 1.24436363990732e-10,
      "learning_rate": 1.991501416430595e-05,
      "logits/chosen": -2.4259071350097656,
      "logits/rejected": -2.4663820266723633,
      "logps/chosen": -8466.548828125,
      "logps/rejected": -9634.7822265625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -799.0520629882812,
      "rewards/margins": 115.73411560058594,
      "rewards/rejected": -914.7863159179688,
      "step": 785
    },
    {
      "epoch": 0.5314497140935083,
      "grad_norm": 0.00138977135065943,
      "learning_rate": 1.9773371104815865e-05,
      "logits/chosen": -2.4131436347961426,
      "logits/rejected": -2.4625144004821777,
      "logps/chosen": -7748.1669921875,
      "logps/rejected": -9190.7841796875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -729.1549072265625,
      "rewards/margins": 142.2391357421875,
      "rewards/rejected": -871.39404296875,
      "step": 790
    },
    {
      "epoch": 0.5348133198789102,
      "grad_norm": 0.02361525595188141,
      "learning_rate": 1.963172804532578e-05,
      "logits/chosen": -2.5722217559814453,
      "logits/rejected": -2.6563920974731445,
      "logps/chosen": -7934.11083984375,
      "logps/rejected": -10503.3095703125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -749.9426879882812,
      "rewards/margins": 251.44387817382812,
      "rewards/rejected": -1001.3865356445312,
      "step": 795
    },
    {
      "epoch": 0.5381769256643122,
      "grad_norm": 9.513410025030478e-12,
      "learning_rate": 1.9490084985835695e-05,
      "logits/chosen": -2.6857235431671143,
      "logits/rejected": -2.761955738067627,
      "logps/chosen": -10708.11328125,
      "logps/rejected": -12497.4248046875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1019.5017700195312,
      "rewards/margins": 175.0734100341797,
      "rewards/rejected": -1194.5753173828125,
      "step": 800
    },
    {
      "epoch": 0.5415405314497141,
      "grad_norm": 0.0003343418356962502,
      "learning_rate": 1.934844192634561e-05,
      "logits/chosen": -2.7842698097229004,
      "logits/rejected": -2.8386435508728027,
      "logps/chosen": -9550.51171875,
      "logps/rejected": -10974.5859375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -910.4201049804688,
      "rewards/margins": 138.74960327148438,
      "rewards/rejected": -1049.169677734375,
      "step": 805
    },
    {
      "epoch": 0.544904137235116,
      "grad_norm": 0.0010003831703215837,
      "learning_rate": 1.9206798866855526e-05,
      "logits/chosen": -2.832763195037842,
      "logits/rejected": -2.8846867084503174,
      "logps/chosen": -10057.1796875,
      "logps/rejected": -11729.03125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -961.0572509765625,
      "rewards/margins": 165.3815460205078,
      "rewards/rejected": -1126.438720703125,
      "step": 810
    },
    {
      "epoch": 0.548267743020518,
      "grad_norm": 0.006325883790850639,
      "learning_rate": 1.906515580736544e-05,
      "logits/chosen": -2.8355908393859863,
      "logits/rejected": -2.8983752727508545,
      "logps/chosen": -9889.0888671875,
      "logps/rejected": -12471.9814453125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -944.8602294921875,
      "rewards/margins": 254.2203826904297,
      "rewards/rejected": -1199.08056640625,
      "step": 815
    },
    {
      "epoch": 0.5516313488059199,
      "grad_norm": 0.00017795893654692918,
      "learning_rate": 1.8923512747875356e-05,
      "logits/chosen": -2.8413312435150146,
      "logits/rejected": -2.9433906078338623,
      "logps/chosen": -9068.4462890625,
      "logps/rejected": -11532.8935546875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -863.31884765625,
      "rewards/margins": 241.6269989013672,
      "rewards/rejected": -1104.946044921875,
      "step": 820
    },
    {
      "epoch": 0.5549949545913219,
      "grad_norm": 0.0031602182425558567,
      "learning_rate": 1.878186968838527e-05,
      "logits/chosen": -2.8462157249450684,
      "logits/rejected": -2.9127097129821777,
      "logps/chosen": -9300.34765625,
      "logps/rejected": -11338.3408203125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -885.9267578125,
      "rewards/margins": 199.34280395507812,
      "rewards/rejected": -1085.2696533203125,
      "step": 825
    },
    {
      "epoch": 0.5583585603767238,
      "grad_norm": 1.0807738362927921e-05,
      "learning_rate": 1.8640226628895186e-05,
      "logits/chosen": -2.8742213249206543,
      "logits/rejected": -2.920592784881592,
      "logps/chosen": -10750.509765625,
      "logps/rejected": -11702.9970703125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1027.3074951171875,
      "rewards/margins": 93.7032470703125,
      "rewards/rejected": -1121.0107421875,
      "step": 830
    },
    {
      "epoch": 0.5617221661621258,
      "grad_norm": 1.0340152734045205e-08,
      "learning_rate": 1.84985835694051e-05,
      "logits/chosen": -2.858445405960083,
      "logits/rejected": -2.9453184604644775,
      "logps/chosen": -10735.3330078125,
      "logps/rejected": -13205.9892578125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1025.2232666015625,
      "rewards/margins": 239.30795288085938,
      "rewards/rejected": -1264.53125,
      "step": 835
    },
    {
      "epoch": 0.5650857719475277,
      "grad_norm": 0.0005519635742530227,
      "learning_rate": 1.8356940509915016e-05,
      "logits/chosen": -2.8448379039764404,
      "logits/rejected": -2.8920960426330566,
      "logps/chosen": -10124.615234375,
      "logps/rejected": -11267.01953125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -967.572265625,
      "rewards/margins": 112.393310546875,
      "rewards/rejected": -1079.9654541015625,
      "step": 840
    },
    {
      "epoch": 0.5684493777329297,
      "grad_norm": 0.0028222929686307907,
      "learning_rate": 1.821529745042493e-05,
      "logits/chosen": -2.853471279144287,
      "logits/rejected": -2.935044765472412,
      "logps/chosen": -11404.908203125,
      "logps/rejected": -13688.501953125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1094.208984375,
      "rewards/margins": 224.4495391845703,
      "rewards/rejected": -1318.658447265625,
      "step": 845
    },
    {
      "epoch": 0.5718129835183317,
      "grad_norm": 0.0008201766759157181,
      "learning_rate": 1.8073654390934843e-05,
      "logits/chosen": -2.8505337238311768,
      "logits/rejected": -2.9212992191314697,
      "logps/chosen": -10643.669921875,
      "logps/rejected": -12836.9267578125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1016.7005615234375,
      "rewards/margins": 214.9639434814453,
      "rewards/rejected": -1231.66455078125,
      "step": 850
    },
    {
      "epoch": 0.5751765893037336,
      "grad_norm": 3.706893334388812e-19,
      "learning_rate": 1.7932011331444762e-05,
      "logits/chosen": -2.8723795413970947,
      "logits/rejected": -2.932310104370117,
      "logps/chosen": -10187.7978515625,
      "logps/rejected": -12462.943359375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -974.6251220703125,
      "rewards/margins": 223.3293914794922,
      "rewards/rejected": -1197.95458984375,
      "step": 855
    },
    {
      "epoch": 0.5785401950891356,
      "grad_norm": 2.5361642131006547e-17,
      "learning_rate": 1.7790368271954677e-05,
      "logits/chosen": -2.885969877243042,
      "logits/rejected": -2.922436475753784,
      "logps/chosen": -10790.6455078125,
      "logps/rejected": -12233.173828125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1030.8973388671875,
      "rewards/margins": 141.710693359375,
      "rewards/rejected": -1172.6080322265625,
      "step": 860
    },
    {
      "epoch": 0.5819038008745375,
      "grad_norm": 0.001409278716892004,
      "learning_rate": 1.764872521246459e-05,
      "logits/chosen": -2.820929765701294,
      "logits/rejected": -2.8934977054595947,
      "logps/chosen": -9601.783203125,
      "logps/rejected": -11597.865234375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -915.6341552734375,
      "rewards/margins": 194.81826782226562,
      "rewards/rejected": -1110.4525146484375,
      "step": 865
    },
    {
      "epoch": 0.5852674066599395,
      "grad_norm": 1.6339203210910536e-13,
      "learning_rate": 1.7507082152974507e-05,
      "logits/chosen": -2.807671308517456,
      "logits/rejected": -2.871948480606079,
      "logps/chosen": -9219.865234375,
      "logps/rejected": -10630.173828125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -877.0103759765625,
      "rewards/margins": 137.58877563476562,
      "rewards/rejected": -1014.5992431640625,
      "step": 870
    },
    {
      "epoch": 0.5886310124453414,
      "grad_norm": 0.00044560706010088325,
      "learning_rate": 1.736543909348442e-05,
      "logits/chosen": -2.8423473834991455,
      "logits/rejected": -2.8838841915130615,
      "logps/chosen": -9645.2783203125,
      "logps/rejected": -10977.7529296875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -920.4030151367188,
      "rewards/margins": 131.4100341796875,
      "rewards/rejected": -1051.8131103515625,
      "step": 875
    },
    {
      "epoch": 0.5919946182307434,
      "grad_norm": 6.125371874077246e-05,
      "learning_rate": 1.7223796033994334e-05,
      "logits/chosen": -2.839250087738037,
      "logits/rejected": -2.901228904724121,
      "logps/chosen": -8631.0888671875,
      "logps/rejected": -9897.578125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -823.7952270507812,
      "rewards/margins": 123.41202545166016,
      "rewards/rejected": -947.2072143554688,
      "step": 880
    },
    {
      "epoch": 0.5953582240161454,
      "grad_norm": 1.198594236484496e-05,
      "learning_rate": 1.7082152974504253e-05,
      "logits/chosen": -2.8220412731170654,
      "logits/rejected": -2.8963990211486816,
      "logps/chosen": -9232.4755859375,
      "logps/rejected": -10994.6064453125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -879.5452270507812,
      "rewards/margins": 170.85275268554688,
      "rewards/rejected": -1050.3980712890625,
      "step": 885
    },
    {
      "epoch": 0.5987218298015473,
      "grad_norm": 0.00013586942804977298,
      "learning_rate": 1.6940509915014164e-05,
      "logits/chosen": -2.8314290046691895,
      "logits/rejected": -2.9016964435577393,
      "logps/chosen": -11309.94921875,
      "logps/rejected": -12927.6171875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1081.023681640625,
      "rewards/margins": 158.16140747070312,
      "rewards/rejected": -1239.18505859375,
      "step": 890
    },
    {
      "epoch": 0.6020854355869492,
      "grad_norm": 0.0027604626957327127,
      "learning_rate": 1.679886685552408e-05,
      "logits/chosen": -2.8378920555114746,
      "logits/rejected": -2.8676486015319824,
      "logps/chosen": -9244.439453125,
      "logps/rejected": -10166.0966796875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -884.1746215820312,
      "rewards/margins": 90.19459533691406,
      "rewards/rejected": -974.369140625,
      "step": 895
    },
    {
      "epoch": 0.6054490413723511,
      "grad_norm": 0.006770930252969265,
      "learning_rate": 1.6657223796033998e-05,
      "logits/chosen": -2.8278746604919434,
      "logits/rejected": -2.8885953426361084,
      "logps/chosen": -9725.0,
      "logps/rejected": -11235.650390625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -929.3316650390625,
      "rewards/margins": 148.0374298095703,
      "rewards/rejected": -1077.369140625,
      "step": 900
    },
    {
      "epoch": 0.6088126471577531,
      "grad_norm": 1.377580503003628e-07,
      "learning_rate": 1.651558073654391e-05,
      "logits/chosen": -2.8259716033935547,
      "logits/rejected": -2.92206072807312,
      "logps/chosen": -8669.6826171875,
      "logps/rejected": -11386.427734375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -823.2698974609375,
      "rewards/margins": 266.0985107421875,
      "rewards/rejected": -1089.368408203125,
      "step": 905
    },
    {
      "epoch": 0.612176252943155,
      "grad_norm": 2.555495257183793e-06,
      "learning_rate": 1.6373937677053825e-05,
      "logits/chosen": -2.826153039932251,
      "logits/rejected": -2.8583192825317383,
      "logps/chosen": -10126.97265625,
      "logps/rejected": -12001.8251953125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -968.8478393554688,
      "rewards/margins": 184.26284790039062,
      "rewards/rejected": -1153.110595703125,
      "step": 910
    },
    {
      "epoch": 0.615539858728557,
      "grad_norm": 0.00025224272394552827,
      "learning_rate": 1.623229461756374e-05,
      "logits/chosen": -2.813370943069458,
      "logits/rejected": -2.889408588409424,
      "logps/chosen": -9784.34765625,
      "logps/rejected": -12053.3984375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -936.4768676757812,
      "rewards/margins": 222.1951141357422,
      "rewards/rejected": -1158.672119140625,
      "step": 915
    },
    {
      "epoch": 0.618903464513959,
      "grad_norm": 0.0003172833821736276,
      "learning_rate": 1.6090651558073655e-05,
      "logits/chosen": -2.798166036605835,
      "logits/rejected": -2.8649706840515137,
      "logps/chosen": -9360.8701171875,
      "logps/rejected": -11418.2431640625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -892.3360595703125,
      "rewards/margins": 199.59153747558594,
      "rewards/rejected": -1091.9276123046875,
      "step": 920
    },
    {
      "epoch": 0.6222670702993609,
      "grad_norm": 0.00348341534845531,
      "learning_rate": 1.594900849858357e-05,
      "logits/chosen": -2.8457531929016113,
      "logits/rejected": -2.9340806007385254,
      "logps/chosen": -9925.1494140625,
      "logps/rejected": -12067.138671875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -946.2434692382812,
      "rewards/margins": 208.6758575439453,
      "rewards/rejected": -1154.9195556640625,
      "step": 925
    },
    {
      "epoch": 0.6256306760847629,
      "grad_norm": 3.342704353264769e-16,
      "learning_rate": 1.5807365439093485e-05,
      "logits/chosen": -2.821262836456299,
      "logits/rejected": -2.8680968284606934,
      "logps/chosen": -9713.015625,
      "logps/rejected": -11146.162109375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -927.1214599609375,
      "rewards/margins": 139.81033325195312,
      "rewards/rejected": -1066.931884765625,
      "step": 930
    },
    {
      "epoch": 0.6289942818701648,
      "grad_norm": 1.7276519770348386e-07,
      "learning_rate": 1.56657223796034e-05,
      "logits/chosen": -2.804056167602539,
      "logits/rejected": -2.862974166870117,
      "logps/chosen": -9592.310546875,
      "logps/rejected": -10670.837890625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -916.8681640625,
      "rewards/margins": 106.45955657958984,
      "rewards/rejected": -1023.32763671875,
      "step": 935
    },
    {
      "epoch": 0.6323578876555668,
      "grad_norm": 0.00012656391481868923,
      "learning_rate": 1.5524079320113316e-05,
      "logits/chosen": -2.7862164974212646,
      "logits/rejected": -2.8613057136535645,
      "logps/chosen": -8957.962890625,
      "logps/rejected": -10950.5703125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -852.9378662109375,
      "rewards/margins": 195.1141357421875,
      "rewards/rejected": -1048.052001953125,
      "step": 940
    },
    {
      "epoch": 0.6357214934409687,
      "grad_norm": 6.961974463592924e-07,
      "learning_rate": 1.538243626062323e-05,
      "logits/chosen": -2.802111864089966,
      "logits/rejected": -2.900282144546509,
      "logps/chosen": -7927.17041015625,
      "logps/rejected": -9927.119140625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -755.3216552734375,
      "rewards/margins": 194.95626831054688,
      "rewards/rejected": -950.27783203125,
      "step": 945
    },
    {
      "epoch": 0.6390850992263707,
      "grad_norm": 0.0009280852391384542,
      "learning_rate": 1.5240793201133146e-05,
      "logits/chosen": -2.790306568145752,
      "logits/rejected": -2.8632864952087402,
      "logps/chosen": -9200.298828125,
      "logps/rejected": -11356.314453125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -875.5911254882812,
      "rewards/margins": 209.4214324951172,
      "rewards/rejected": -1085.0125732421875,
      "step": 950
    },
    {
      "epoch": 0.6424487050117726,
      "grad_norm": 3.5320075857103728e-12,
      "learning_rate": 1.509915014164306e-05,
      "logits/chosen": -2.8322410583496094,
      "logits/rejected": -2.878159999847412,
      "logps/chosen": -10465.3291015625,
      "logps/rejected": -11585.822265625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -999.44677734375,
      "rewards/margins": 110.87113952636719,
      "rewards/rejected": -1110.318115234375,
      "step": 955
    },
    {
      "epoch": 0.6458123107971746,
      "grad_norm": 0.00042570350342430174,
      "learning_rate": 1.4957507082152976e-05,
      "logits/chosen": -2.7900311946868896,
      "logits/rejected": -2.8491787910461426,
      "logps/chosen": -9756.724609375,
      "logps/rejected": -11449.685546875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -925.5272216796875,
      "rewards/margins": 165.4298858642578,
      "rewards/rejected": -1090.957275390625,
      "step": 960
    },
    {
      "epoch": 0.6491759165825766,
      "grad_norm": 1.1863468074579941e-07,
      "learning_rate": 1.4815864022662891e-05,
      "logits/chosen": -2.7810897827148438,
      "logits/rejected": -2.8927199840545654,
      "logps/chosen": -8894.904296875,
      "logps/rejected": -11299.54296875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -846.9240112304688,
      "rewards/margins": 236.0740203857422,
      "rewards/rejected": -1082.9981689453125,
      "step": 965
    },
    {
      "epoch": 0.6525395223679785,
      "grad_norm": 5.381330936415907e-08,
      "learning_rate": 1.4674220963172805e-05,
      "logits/chosen": -2.7986233234405518,
      "logits/rejected": -2.877079486846924,
      "logps/chosen": -9307.9892578125,
      "logps/rejected": -11374.0087890625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -886.8508911132812,
      "rewards/margins": 203.1802520751953,
      "rewards/rejected": -1090.031005859375,
      "step": 970
    },
    {
      "epoch": 0.6559031281533805,
      "grad_norm": 6.98414005455561e-05,
      "learning_rate": 1.453257790368272e-05,
      "logits/chosen": -2.808264970779419,
      "logits/rejected": -2.882354259490967,
      "logps/chosen": -10689.5947265625,
      "logps/rejected": -12864.92578125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1018.2445068359375,
      "rewards/margins": 213.23080444335938,
      "rewards/rejected": -1231.4754638671875,
      "step": 975
    },
    {
      "epoch": 0.6592667339387823,
      "grad_norm": 5.5831787329456196e-21,
      "learning_rate": 1.4390934844192637e-05,
      "logits/chosen": -2.779027223587036,
      "logits/rejected": -2.849564552307129,
      "logps/chosen": -9125.3408203125,
      "logps/rejected": -10919.26953125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -871.1730346679688,
      "rewards/margins": 174.49232482910156,
      "rewards/rejected": -1045.665283203125,
      "step": 980
    },
    {
      "epoch": 0.6626303397241843,
      "grad_norm": 0.00019887351663783193,
      "learning_rate": 1.424929178470255e-05,
      "logits/chosen": -2.8290863037109375,
      "logits/rejected": -2.874340534210205,
      "logps/chosen": -9025.892578125,
      "logps/rejected": -10362.625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -859.7750854492188,
      "rewards/margins": 131.11367797851562,
      "rewards/rejected": -990.8887939453125,
      "step": 985
    },
    {
      "epoch": 0.6659939455095862,
      "grad_norm": 0.017174849286675453,
      "learning_rate": 1.4107648725212465e-05,
      "logits/chosen": -2.818451404571533,
      "logits/rejected": -2.9163498878479004,
      "logps/chosen": -7560.68896484375,
      "logps/rejected": -9145.4326171875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -718.1976928710938,
      "rewards/margins": 153.8029022216797,
      "rewards/rejected": -872.0006103515625,
      "step": 990
    },
    {
      "epoch": 0.6693575512949882,
      "grad_norm": 0.0005373026360757649,
      "learning_rate": 1.396600566572238e-05,
      "logits/chosen": -2.7927427291870117,
      "logits/rejected": -2.829011917114258,
      "logps/chosen": -9582.50390625,
      "logps/rejected": -10262.5498046875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -915.3582153320312,
      "rewards/margins": 67.16543579101562,
      "rewards/rejected": -982.5235595703125,
      "step": 995
    },
    {
      "epoch": 0.6727211570803902,
      "grad_norm": 1.1687894584611058e-05,
      "learning_rate": 1.3824362606232296e-05,
      "logits/chosen": -2.7879626750946045,
      "logits/rejected": -2.841503381729126,
      "logps/chosen": -10726.580078125,
      "logps/rejected": -12842.58984375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1024.07958984375,
      "rewards/margins": 207.25393676757812,
      "rewards/rejected": -1231.333740234375,
      "step": 1000
    },
    {
      "epoch": 0.6727211570803902,
      "eval_logits/chosen": -2.7938742637634277,
      "eval_logits/rejected": -2.856015205383301,
      "eval_logps/chosen": -9602.14453125,
      "eval_logps/rejected": -11314.7001953125,
      "eval_loss": 3.090483005507849e-05,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": -915.1240844726562,
      "eval_rewards/margins": 167.71502685546875,
      "eval_rewards/rejected": -1082.8389892578125,
      "eval_runtime": 470.397,
      "eval_samples_per_second": 2.81,
      "eval_steps_per_second": 0.353,
      "step": 1000
    },
    {
      "epoch": 0.6760847628657921,
      "grad_norm": 2.390105692029465e-06,
      "learning_rate": 1.368271954674221e-05,
      "logits/chosen": -2.799574613571167,
      "logits/rejected": -2.846802234649658,
      "logps/chosen": -9061.2841796875,
      "logps/rejected": -10760.2138671875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -862.9988403320312,
      "rewards/margins": 165.59756469726562,
      "rewards/rejected": -1028.596435546875,
      "step": 1005
    },
    {
      "epoch": 0.6794483686511941,
      "grad_norm": 0.00015009417256806046,
      "learning_rate": 1.3541076487252124e-05,
      "logits/chosen": -2.7781386375427246,
      "logits/rejected": -2.8541760444641113,
      "logps/chosen": -8817.001953125,
      "logps/rejected": -10643.4072265625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -837.00439453125,
      "rewards/margins": 178.64309692382812,
      "rewards/rejected": -1015.6475830078125,
      "step": 1010
    },
    {
      "epoch": 0.682811974436596,
      "grad_norm": 1.0738487432604416e-08,
      "learning_rate": 1.3399433427762041e-05,
      "logits/chosen": -2.7793636322021484,
      "logits/rejected": -2.844918966293335,
      "logps/chosen": -10978.306640625,
      "logps/rejected": -13210.8466796875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1049.725341796875,
      "rewards/margins": 218.17404174804688,
      "rewards/rejected": -1267.8994140625,
      "step": 1015
    },
    {
      "epoch": 0.686175580221998,
      "grad_norm": 0.00012072798563167453,
      "learning_rate": 1.3257790368271956e-05,
      "logits/chosen": -2.7937846183776855,
      "logits/rejected": -2.847024440765381,
      "logps/chosen": -9231.228515625,
      "logps/rejected": -10854.28515625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -879.2262573242188,
      "rewards/margins": 158.9337921142578,
      "rewards/rejected": -1038.159912109375,
      "step": 1020
    },
    {
      "epoch": 0.6895391860073999,
      "grad_norm": 0.0007355925044976175,
      "learning_rate": 1.311614730878187e-05,
      "logits/chosen": -2.787400007247925,
      "logits/rejected": -2.8253772258758545,
      "logps/chosen": -10124.345703125,
      "logps/rejected": -11785.5302734375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -967.5921630859375,
      "rewards/margins": 163.0208740234375,
      "rewards/rejected": -1130.613037109375,
      "step": 1025
    },
    {
      "epoch": 0.6929027917928019,
      "grad_norm": 0.00026888636057265103,
      "learning_rate": 1.2974504249291786e-05,
      "logits/chosen": -2.8021836280822754,
      "logits/rejected": -2.8373055458068848,
      "logps/chosen": -10535.89453125,
      "logps/rejected": -12657.552734375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1006.5159912109375,
      "rewards/margins": 208.1797332763672,
      "rewards/rejected": -1214.69580078125,
      "step": 1030
    },
    {
      "epoch": 0.6962663975782039,
      "grad_norm": 4.5662272185609254e-08,
      "learning_rate": 1.28328611898017e-05,
      "logits/chosen": -2.757779836654663,
      "logits/rejected": -2.8093390464782715,
      "logps/chosen": -8730.380859375,
      "logps/rejected": -10336.0,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -830.5279541015625,
      "rewards/margins": 155.8953399658203,
      "rewards/rejected": -986.4232177734375,
      "step": 1035
    },
    {
      "epoch": 0.6996300033636058,
      "grad_norm": 8.862828129037293e-16,
      "learning_rate": 1.2691218130311615e-05,
      "logits/chosen": -2.7805848121643066,
      "logits/rejected": -2.8504841327667236,
      "logps/chosen": -9004.208984375,
      "logps/rejected": -10365.25,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -857.50244140625,
      "rewards/margins": 133.22036743164062,
      "rewards/rejected": -990.7228393554688,
      "step": 1040
    },
    {
      "epoch": 0.7029936091490078,
      "grad_norm": 9.825400047702715e-05,
      "learning_rate": 1.2549575070821532e-05,
      "logits/chosen": -2.778083086013794,
      "logits/rejected": -2.8238277435302734,
      "logps/chosen": -11010.4208984375,
      "logps/rejected": -12317.9169921875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1050.878662109375,
      "rewards/margins": 128.89404296875,
      "rewards/rejected": -1179.772705078125,
      "step": 1045
    },
    {
      "epoch": 0.7063572149344097,
      "grad_norm": 1.3385994179770933e-07,
      "learning_rate": 1.2407932011331445e-05,
      "logits/chosen": -2.7994015216827393,
      "logits/rejected": -2.8509275913238525,
      "logps/chosen": -10791.021484375,
      "logps/rejected": -12836.443359375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1031.8280029296875,
      "rewards/margins": 200.49099731445312,
      "rewards/rejected": -1232.31884765625,
      "step": 1050
    },
    {
      "epoch": 0.7097208207198117,
      "grad_norm": 2.0563353686497976e-08,
      "learning_rate": 1.226628895184136e-05,
      "logits/chosen": -2.792280673980713,
      "logits/rejected": -2.8346810340881348,
      "logps/chosen": -11597.57421875,
      "logps/rejected": -13500.5361328125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1112.801513671875,
      "rewards/margins": 186.8677520751953,
      "rewards/rejected": -1299.66943359375,
      "step": 1055
    },
    {
      "epoch": 0.7130844265052136,
      "grad_norm": 0.00024089723592624068,
      "learning_rate": 1.2124645892351277e-05,
      "logits/chosen": -2.798280715942383,
      "logits/rejected": -2.8641045093536377,
      "logps/chosen": -9915.234375,
      "logps/rejected": -11021.900390625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -946.5016479492188,
      "rewards/margins": 108.1216812133789,
      "rewards/rejected": -1054.623291015625,
      "step": 1060
    },
    {
      "epoch": 0.7164480322906155,
      "grad_norm": 8.393470922636592e-13,
      "learning_rate": 1.198300283286119e-05,
      "logits/chosen": -2.808591604232788,
      "logits/rejected": -2.845585584640503,
      "logps/chosen": -9722.740234375,
      "logps/rejected": -11271.9365234375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -925.7952880859375,
      "rewards/margins": 151.68478393554688,
      "rewards/rejected": -1077.4801025390625,
      "step": 1065
    },
    {
      "epoch": 0.7198116380760174,
      "grad_norm": 3.38553967083044e-08,
      "learning_rate": 1.1841359773371106e-05,
      "logits/chosen": -2.7805581092834473,
      "logits/rejected": -2.848875045776367,
      "logps/chosen": -8482.6083984375,
      "logps/rejected": -10266.2685546875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -810.305908203125,
      "rewards/margins": 175.25308227539062,
      "rewards/rejected": -985.5589599609375,
      "step": 1070
    },
    {
      "epoch": 0.7231752438614194,
      "grad_norm": 3.1674192086939e-06,
      "learning_rate": 1.169971671388102e-05,
      "logits/chosen": -2.776794195175171,
      "logits/rejected": -2.8481457233428955,
      "logps/chosen": -9793.3466796875,
      "logps/rejected": -11564.541015625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -938.1527099609375,
      "rewards/margins": 171.99630737304688,
      "rewards/rejected": -1110.1490478515625,
      "step": 1075
    },
    {
      "epoch": 0.7265388496468214,
      "grad_norm": 1.0635058167451916e-08,
      "learning_rate": 1.1558073654390936e-05,
      "logits/chosen": -2.8120133876800537,
      "logits/rejected": -2.862553834915161,
      "logps/chosen": -10794.6591796875,
      "logps/rejected": -12334.103515625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1030.221923828125,
      "rewards/margins": 149.95938110351562,
      "rewards/rejected": -1180.181396484375,
      "step": 1080
    },
    {
      "epoch": 0.7299024554322233,
      "grad_norm": 1.3493830497424142e-09,
      "learning_rate": 1.1416430594900851e-05,
      "logits/chosen": -2.7945902347564697,
      "logits/rejected": -2.864102363586426,
      "logps/chosen": -9389.095703125,
      "logps/rejected": -11166.21484375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -892.7608642578125,
      "rewards/margins": 173.32241821289062,
      "rewards/rejected": -1066.083251953125,
      "step": 1085
    },
    {
      "epoch": 0.7332660612176253,
      "grad_norm": 8.178382415735541e-10,
      "learning_rate": 1.1274787535410765e-05,
      "logits/chosen": -2.821089267730713,
      "logits/rejected": -2.9073050022125244,
      "logps/chosen": -10499.5244140625,
      "logps/rejected": -12829.2431640625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -999.1942138671875,
      "rewards/margins": 227.99282836914062,
      "rewards/rejected": -1227.1871337890625,
      "step": 1090
    },
    {
      "epoch": 0.7366296670030272,
      "grad_norm": 0.0006433354574255645,
      "learning_rate": 1.1133144475920681e-05,
      "logits/chosen": -2.810201644897461,
      "logits/rejected": -2.895108461380005,
      "logps/chosen": -9221.951171875,
      "logps/rejected": -11216.205078125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -879.5908203125,
      "rewards/margins": 193.7993621826172,
      "rewards/rejected": -1073.3902587890625,
      "step": 1095
    },
    {
      "epoch": 0.7399932727884292,
      "grad_norm": 1.2460844800443738e-06,
      "learning_rate": 1.0991501416430597e-05,
      "logits/chosen": -2.8140244483947754,
      "logits/rejected": -2.863375425338745,
      "logps/chosen": -11250.0498046875,
      "logps/rejected": -12878.349609375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1077.6595458984375,
      "rewards/margins": 159.76980590820312,
      "rewards/rejected": -1237.429443359375,
      "step": 1100
    },
    {
      "epoch": 0.7433568785738311,
      "grad_norm": 9.660657815402374e-05,
      "learning_rate": 1.084985835694051e-05,
      "logits/chosen": -2.7762808799743652,
      "logits/rejected": -2.8414618968963623,
      "logps/chosen": -8471.7265625,
      "logps/rejected": -9518.205078125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -807.1697998046875,
      "rewards/margins": 102.34693908691406,
      "rewards/rejected": -909.5167236328125,
      "step": 1105
    },
    {
      "epoch": 0.7467204843592331,
      "grad_norm": 5.022162685719422e-09,
      "learning_rate": 1.0708215297450427e-05,
      "logits/chosen": -2.8055579662323,
      "logits/rejected": -2.8668885231018066,
      "logps/chosen": -9358.171875,
      "logps/rejected": -11268.8857421875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -889.7477416992188,
      "rewards/margins": 186.5991668701172,
      "rewards/rejected": -1076.346923828125,
      "step": 1110
    },
    {
      "epoch": 0.7500840901446351,
      "grad_norm": 0.00030493695521727204,
      "learning_rate": 1.056657223796034e-05,
      "logits/chosen": -2.831561803817749,
      "logits/rejected": -2.8962368965148926,
      "logps/chosen": -8570.0712890625,
      "logps/rejected": -10562.6416015625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -815.5811767578125,
      "rewards/margins": 195.74131774902344,
      "rewards/rejected": -1011.3224487304688,
      "step": 1115
    },
    {
      "epoch": 0.753447695930037,
      "grad_norm": 2.6330010172159746e-08,
      "learning_rate": 1.0424929178470255e-05,
      "logits/chosen": -2.766068935394287,
      "logits/rejected": -2.831141233444214,
      "logps/chosen": -10283.7158203125,
      "logps/rejected": -11833.373046875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -981.8518676757812,
      "rewards/margins": 153.0648956298828,
      "rewards/rejected": -1134.916748046875,
      "step": 1120
    },
    {
      "epoch": 0.756811301715439,
      "grad_norm": 1.30602515469036e-07,
      "learning_rate": 1.028328611898017e-05,
      "logits/chosen": -2.763476848602295,
      "logits/rejected": -2.836503267288208,
      "logps/chosen": -8742.9453125,
      "logps/rejected": -10605.201171875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -830.77001953125,
      "rewards/margins": 183.00845336914062,
      "rewards/rejected": -1013.7785034179688,
      "step": 1125
    },
    {
      "epoch": 0.7601749075008409,
      "grad_norm": 0.008190412074327469,
      "learning_rate": 1.0141643059490086e-05,
      "logits/chosen": -2.8228518962860107,
      "logits/rejected": -2.877938985824585,
      "logps/chosen": -10602.185546875,
      "logps/rejected": -12101.6669921875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1014.99072265625,
      "rewards/margins": 145.71487426757812,
      "rewards/rejected": -1160.7056884765625,
      "step": 1130
    },
    {
      "epoch": 0.7635385132862429,
      "grad_norm": 1.1805859456744529e-15,
      "learning_rate": 1e-05,
      "logits/chosen": -2.8146510124206543,
      "logits/rejected": -2.898090362548828,
      "logps/chosen": -9464.1533203125,
      "logps/rejected": -11066.15625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -903.4000244140625,
      "rewards/margins": 155.8123779296875,
      "rewards/rejected": -1059.21240234375,
      "step": 1135
    },
    {
      "epoch": 0.7669021190716448,
      "grad_norm": 0.00013282893633004278,
      "learning_rate": 9.858356940509916e-06,
      "logits/chosen": -2.7848479747772217,
      "logits/rejected": -2.844801425933838,
      "logps/chosen": -9262.447265625,
      "logps/rejected": -11048.3369140625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -879.8966674804688,
      "rewards/margins": 175.1274871826172,
      "rewards/rejected": -1055.024169921875,
      "step": 1140
    },
    {
      "epoch": 0.7702657248570468,
      "grad_norm": 7.018149403847929e-07,
      "learning_rate": 9.716713881019831e-06,
      "logits/chosen": -2.793409824371338,
      "logits/rejected": -2.8511343002319336,
      "logps/chosen": -8987.6494140625,
      "logps/rejected": -10095.3203125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -856.1070556640625,
      "rewards/margins": 107.64167785644531,
      "rewards/rejected": -963.7487182617188,
      "step": 1145
    },
    {
      "epoch": 0.7736293306424487,
      "grad_norm": 3.45980466769525e-14,
      "learning_rate": 9.575070821529746e-06,
      "logits/chosen": -2.80358624458313,
      "logits/rejected": -2.8674545288085938,
      "logps/chosen": -9042.6474609375,
      "logps/rejected": -10385.384765625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -859.5359497070312,
      "rewards/margins": 131.424072265625,
      "rewards/rejected": -990.9599609375,
      "step": 1150
    },
    {
      "epoch": 0.7769929364278506,
      "grad_norm": 1.5055718411078386e-15,
      "learning_rate": 9.433427762039661e-06,
      "logits/chosen": -2.837970733642578,
      "logits/rejected": -2.866541862487793,
      "logps/chosen": -10442.5654296875,
      "logps/rejected": -11693.89453125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -996.1403198242188,
      "rewards/margins": 123.6158447265625,
      "rewards/rejected": -1119.756103515625,
      "step": 1155
    },
    {
      "epoch": 0.7803565422132526,
      "grad_norm": 2.773202979376114e-14,
      "learning_rate": 9.291784702549575e-06,
      "logits/chosen": -2.8178915977478027,
      "logits/rejected": -2.9039740562438965,
      "logps/chosen": -8575.626953125,
      "logps/rejected": -10910.5322265625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -815.9152221679688,
      "rewards/margins": 227.351318359375,
      "rewards/rejected": -1043.2666015625,
      "step": 1160
    },
    {
      "epoch": 0.7837201479986545,
      "grad_norm": 1.0363694494600062e-12,
      "learning_rate": 9.150141643059492e-06,
      "logits/chosen": -2.8260974884033203,
      "logits/rejected": -2.8643076419830322,
      "logps/chosen": -8642.7529296875,
      "logps/rejected": -9571.037109375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -824.1209106445312,
      "rewards/margins": 91.37593078613281,
      "rewards/rejected": -915.4967041015625,
      "step": 1165
    },
    {
      "epoch": 0.7870837537840565,
      "grad_norm": 8.04221815010886e-13,
      "learning_rate": 9.008498583569407e-06,
      "logits/chosen": -2.8220226764678955,
      "logits/rejected": -2.864751100540161,
      "logps/chosen": -11173.841796875,
      "logps/rejected": -12509.6787109375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1065.328125,
      "rewards/margins": 131.99929809570312,
      "rewards/rejected": -1197.327392578125,
      "step": 1170
    },
    {
      "epoch": 0.7904473595694584,
      "grad_norm": 0.0017975387163460255,
      "learning_rate": 8.86685552407932e-06,
      "logits/chosen": -2.8186745643615723,
      "logits/rejected": -2.8688082695007324,
      "logps/chosen": -9938.35546875,
      "logps/rejected": -11220.08984375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -948.02001953125,
      "rewards/margins": 126.10322570800781,
      "rewards/rejected": -1074.1231689453125,
      "step": 1175
    },
    {
      "epoch": 0.7938109653548604,
      "grad_norm": 0.010169821791350842,
      "learning_rate": 8.725212464589235e-06,
      "logits/chosen": -2.8241333961486816,
      "logits/rejected": -2.908398389816284,
      "logps/chosen": -9191.791015625,
      "logps/rejected": -10705.966796875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -877.0999145507812,
      "rewards/margins": 148.35140991210938,
      "rewards/rejected": -1025.4512939453125,
      "step": 1180
    },
    {
      "epoch": 0.7971745711402624,
      "grad_norm": 0.0010935779428109527,
      "learning_rate": 8.583569405099152e-06,
      "logits/chosen": -2.8121213912963867,
      "logits/rejected": -2.893502712249756,
      "logps/chosen": -8635.423828125,
      "logps/rejected": -10193.8017578125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -822.5247192382812,
      "rewards/margins": 152.88095092773438,
      "rewards/rejected": -975.4058837890625,
      "step": 1185
    },
    {
      "epoch": 0.8005381769256643,
      "grad_norm": 3.466755771341923e-08,
      "learning_rate": 8.441926345609066e-06,
      "logits/chosen": -2.8141236305236816,
      "logits/rejected": -2.8813865184783936,
      "logps/chosen": -11047.1328125,
      "logps/rejected": -12843.001953125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1054.4991455078125,
      "rewards/margins": 176.5262451171875,
      "rewards/rejected": -1231.025390625,
      "step": 1190
    },
    {
      "epoch": 0.8039017827110663,
      "grad_norm": 0.00012425007298588753,
      "learning_rate": 8.30028328611898e-06,
      "logits/chosen": -2.8468737602233887,
      "logits/rejected": -2.874774217605591,
      "logps/chosen": -9705.4404296875,
      "logps/rejected": -11098.9345703125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -925.8073120117188,
      "rewards/margins": 135.46636962890625,
      "rewards/rejected": -1061.2735595703125,
      "step": 1195
    },
    {
      "epoch": 0.8072653884964682,
      "grad_norm": 6.083288872638093e-10,
      "learning_rate": 8.158640226628896e-06,
      "logits/chosen": -2.796093463897705,
      "logits/rejected": -2.839709520339966,
      "logps/chosen": -9722.830078125,
      "logps/rejected": -11009.833984375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -929.0524291992188,
      "rewards/margins": 126.5392074584961,
      "rewards/rejected": -1055.591552734375,
      "step": 1200
    },
    {
      "epoch": 0.8106289942818702,
      "grad_norm": 0.0009486794006079435,
      "learning_rate": 8.016997167138811e-06,
      "logits/chosen": -2.8279972076416016,
      "logits/rejected": -2.8886404037475586,
      "logps/chosen": -10686.986328125,
      "logps/rejected": -12034.1845703125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1020.6011962890625,
      "rewards/margins": 132.75343322753906,
      "rewards/rejected": -1153.3544921875,
      "step": 1205
    },
    {
      "epoch": 0.8139926000672721,
      "grad_norm": 1.9851255230573282e-18,
      "learning_rate": 7.875354107648726e-06,
      "logits/chosen": -2.80661678314209,
      "logits/rejected": -2.869410991668701,
      "logps/chosen": -10065.533203125,
      "logps/rejected": -12266.775390625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -963.5203857421875,
      "rewards/margins": 217.26498413085938,
      "rewards/rejected": -1180.7852783203125,
      "step": 1210
    },
    {
      "epoch": 0.8173562058526741,
      "grad_norm": 1.1747106109030736e-12,
      "learning_rate": 7.733711048158641e-06,
      "logits/chosen": -2.8028969764709473,
      "logits/rejected": -2.8351283073425293,
      "logps/chosen": -10664.5146484375,
      "logps/rejected": -12032.5947265625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1018.87548828125,
      "rewards/margins": 135.18296813964844,
      "rewards/rejected": -1154.0584716796875,
      "step": 1215
    },
    {
      "epoch": 0.8207198116380761,
      "grad_norm": 0.00022655756038147956,
      "learning_rate": 7.5920679886685555e-06,
      "logits/chosen": -2.7991461753845215,
      "logits/rejected": -2.8422200679779053,
      "logps/chosen": -9296.318359375,
      "logps/rejected": -10694.9990234375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -885.9990234375,
      "rewards/margins": 138.09188842773438,
      "rewards/rejected": -1024.091064453125,
      "step": 1220
    },
    {
      "epoch": 0.824083417423478,
      "grad_norm": 5.949443715458358e-10,
      "learning_rate": 7.4504249291784715e-06,
      "logits/chosen": -2.778770923614502,
      "logits/rejected": -2.8441271781921387,
      "logps/chosen": -10293.416015625,
      "logps/rejected": -11809.3115234375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -978.853515625,
      "rewards/margins": 148.45840454101562,
      "rewards/rejected": -1127.31201171875,
      "step": 1225
    },
    {
      "epoch": 0.82744702320888,
      "grad_norm": 4.781293694300359e-10,
      "learning_rate": 7.308781869688386e-06,
      "logits/chosen": -2.802215576171875,
      "logits/rejected": -2.833371162414551,
      "logps/chosen": -10490.2109375,
      "logps/rejected": -11903.548828125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1002.0606689453125,
      "rewards/margins": 138.76223754882812,
      "rewards/rejected": -1140.8228759765625,
      "step": 1230
    },
    {
      "epoch": 0.8308106289942818,
      "grad_norm": 4.135162612328713e-08,
      "learning_rate": 7.167138810198301e-06,
      "logits/chosen": -2.829707622528076,
      "logits/rejected": -2.880465269088745,
      "logps/chosen": -10372.4638671875,
      "logps/rejected": -11888.443359375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -992.0382690429688,
      "rewards/margins": 147.89749145507812,
      "rewards/rejected": -1139.935791015625,
      "step": 1235
    },
    {
      "epoch": 0.8341742347796838,
      "grad_norm": 0.0008796412730589509,
      "learning_rate": 7.025495750708215e-06,
      "logits/chosen": -2.8305020332336426,
      "logits/rejected": -2.8851780891418457,
      "logps/chosen": -8346.248046875,
      "logps/rejected": -10430.5673828125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -795.0743408203125,
      "rewards/margins": 204.2864990234375,
      "rewards/rejected": -999.3607177734375,
      "step": 1240
    },
    {
      "epoch": 0.8375378405650857,
      "grad_norm": 4.176298995755938e-14,
      "learning_rate": 6.883852691218131e-06,
      "logits/chosen": -2.8175177574157715,
      "logits/rejected": -2.881476640701294,
      "logps/chosen": -11697.5888671875,
      "logps/rejected": -13179.8779296875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1117.081787109375,
      "rewards/margins": 146.4869384765625,
      "rewards/rejected": -1263.568603515625,
      "step": 1245
    },
    {
      "epoch": 0.8409014463504877,
      "grad_norm": 1.5769138883570122e-07,
      "learning_rate": 6.742209631728046e-06,
      "logits/chosen": -2.8091557025909424,
      "logits/rejected": -2.866306781768799,
      "logps/chosen": -8712.0107421875,
      "logps/rejected": -10711.115234375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -832.0964965820312,
      "rewards/margins": 195.47161865234375,
      "rewards/rejected": -1027.568115234375,
      "step": 1250
    },
    {
      "epoch": 0.8442650521358896,
      "grad_norm": 0.00013232213677838445,
      "learning_rate": 6.600566572237961e-06,
      "logits/chosen": -2.78804087638855,
      "logits/rejected": -2.8720898628234863,
      "logps/chosen": -8246.1826171875,
      "logps/rejected": -10737.1337890625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -784.9554443359375,
      "rewards/margins": 242.12026977539062,
      "rewards/rejected": -1027.0755615234375,
      "step": 1255
    },
    {
      "epoch": 0.8476286579212916,
      "grad_norm": 9.240195685423364e-14,
      "learning_rate": 6.458923512747876e-06,
      "logits/chosen": -2.8126938343048096,
      "logits/rejected": -2.8912158012390137,
      "logps/chosen": -10739.837890625,
      "logps/rejected": -13498.494140625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1021.3546752929688,
      "rewards/margins": 268.49468994140625,
      "rewards/rejected": -1289.849365234375,
      "step": 1260
    },
    {
      "epoch": 0.8509922637066936,
      "grad_norm": 1.3843155679919839e-13,
      "learning_rate": 6.317280453257792e-06,
      "logits/chosen": -2.802443265914917,
      "logits/rejected": -2.862860918045044,
      "logps/chosen": -10405.439453125,
      "logps/rejected": -12124.88671875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -993.5724487304688,
      "rewards/margins": 169.1416015625,
      "rewards/rejected": -1162.714111328125,
      "step": 1265
    },
    {
      "epoch": 0.8543558694920955,
      "grad_norm": 0.00035682509769685566,
      "learning_rate": 6.175637393767706e-06,
      "logits/chosen": -2.796865701675415,
      "logits/rejected": -2.8643102645874023,
      "logps/chosen": -10273.013671875,
      "logps/rejected": -11945.9775390625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -981.7595825195312,
      "rewards/margins": 163.26812744140625,
      "rewards/rejected": -1145.027587890625,
      "step": 1270
    },
    {
      "epoch": 0.8577194752774975,
      "grad_norm": 6.725296657350555e-07,
      "learning_rate": 6.033994334277621e-06,
      "logits/chosen": -2.7991766929626465,
      "logits/rejected": -2.846550226211548,
      "logps/chosen": -9824.337890625,
      "logps/rejected": -11339.392578125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -937.4664916992188,
      "rewards/margins": 149.6330108642578,
      "rewards/rejected": -1087.0994873046875,
      "step": 1275
    },
    {
      "epoch": 0.8610830810628994,
      "grad_norm": 3.1965289508661954e-06,
      "learning_rate": 5.892351274787535e-06,
      "logits/chosen": -2.815192222595215,
      "logits/rejected": -2.877464532852173,
      "logps/chosen": -10801.9501953125,
      "logps/rejected": -12576.736328125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1030.3582763671875,
      "rewards/margins": 173.4716339111328,
      "rewards/rejected": -1203.829833984375,
      "step": 1280
    },
    {
      "epoch": 0.8644466868483014,
      "grad_norm": 0.0025022514164447784,
      "learning_rate": 5.750708215297451e-06,
      "logits/chosen": -2.819690704345703,
      "logits/rejected": -2.8923349380493164,
      "logps/chosen": -9916.0732421875,
      "logps/rejected": -12737.7978515625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -943.8118286132812,
      "rewards/margins": 275.17303466796875,
      "rewards/rejected": -1218.98486328125,
      "step": 1285
    },
    {
      "epoch": 0.8678102926337034,
      "grad_norm": 7.309554348466918e-06,
      "learning_rate": 5.609065155807366e-06,
      "logits/chosen": -2.810753345489502,
      "logits/rejected": -2.899470567703247,
      "logps/chosen": -9528.119140625,
      "logps/rejected": -11743.994140625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -907.8414916992188,
      "rewards/margins": 216.2626495361328,
      "rewards/rejected": -1124.1041259765625,
      "step": 1290
    },
    {
      "epoch": 0.8711738984191053,
      "grad_norm": 0.005770276300609112,
      "learning_rate": 5.467422096317281e-06,
      "logits/chosen": -2.8202102184295654,
      "logits/rejected": -2.8867790699005127,
      "logps/chosen": -10290.908203125,
      "logps/rejected": -12813.1103515625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -979.2643432617188,
      "rewards/margins": 245.9068603515625,
      "rewards/rejected": -1225.171142578125,
      "step": 1295
    },
    {
      "epoch": 0.8745375042045073,
      "grad_norm": 1.1102901519862264e-15,
      "learning_rate": 5.325779036827196e-06,
      "logits/chosen": -2.789546489715576,
      "logits/rejected": -2.8249008655548096,
      "logps/chosen": -10103.6767578125,
      "logps/rejected": -11169.935546875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -964.3726806640625,
      "rewards/margins": 104.11744689941406,
      "rewards/rejected": -1068.489990234375,
      "step": 1300
    },
    {
      "epoch": 0.8779011099899092,
      "grad_norm": 0.006562657188624144,
      "learning_rate": 5.184135977337111e-06,
      "logits/chosen": -2.799182653427124,
      "logits/rejected": -2.8391506671905518,
      "logps/chosen": -10176.6904296875,
      "logps/rejected": -11213.810546875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -976.8902587890625,
      "rewards/margins": 100.76112365722656,
      "rewards/rejected": -1077.6512451171875,
      "step": 1305
    },
    {
      "epoch": 0.8812647157753112,
      "grad_norm": 3.911936963599261e-15,
      "learning_rate": 5.042492917847026e-06,
      "logits/chosen": -2.760122776031494,
      "logits/rejected": -2.828903913497925,
      "logps/chosen": -9423.291015625,
      "logps/rejected": -11266.4287109375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -898.5087890625,
      "rewards/margins": 179.74188232421875,
      "rewards/rejected": -1078.2508544921875,
      "step": 1310
    },
    {
      "epoch": 0.8846283215607131,
      "grad_norm": 2.5743313472048612e-06,
      "learning_rate": 4.9008498583569405e-06,
      "logits/chosen": -2.7582955360412598,
      "logits/rejected": -2.8061747550964355,
      "logps/chosen": -9637.4716796875,
      "logps/rejected": -11072.8046875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -919.1561279296875,
      "rewards/margins": 141.7895050048828,
      "rewards/rejected": -1060.945556640625,
      "step": 1315
    },
    {
      "epoch": 0.887991927346115,
      "grad_norm": 1.2985152864075644e-07,
      "learning_rate": 4.7592067988668565e-06,
      "logits/chosen": -2.786386013031006,
      "logits/rejected": -2.8414864540100098,
      "logps/chosen": -10035.974609375,
      "logps/rejected": -11432.9921875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -953.8132934570312,
      "rewards/margins": 137.33319091796875,
      "rewards/rejected": -1091.1463623046875,
      "step": 1320
    },
    {
      "epoch": 0.8913555331315169,
      "grad_norm": 8.316875209857244e-06,
      "learning_rate": 4.617563739376771e-06,
      "logits/chosen": -2.786102294921875,
      "logits/rejected": -2.8458876609802246,
      "logps/chosen": -8743.224609375,
      "logps/rejected": -10133.04296875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -832.9011840820312,
      "rewards/margins": 137.26849365234375,
      "rewards/rejected": -970.1695556640625,
      "step": 1325
    },
    {
      "epoch": 0.8947191389169189,
      "grad_norm": 0.0005092784413136542,
      "learning_rate": 4.475920679886686e-06,
      "logits/chosen": -2.8097424507141113,
      "logits/rejected": -2.8584444522857666,
      "logps/chosen": -9665.70703125,
      "logps/rejected": -11417.4326171875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -922.3358154296875,
      "rewards/margins": 170.23175048828125,
      "rewards/rejected": -1092.5675048828125,
      "step": 1330
    },
    {
      "epoch": 0.8980827447023209,
      "grad_norm": 6.248813451273719e-14,
      "learning_rate": 4.334277620396601e-06,
      "logits/chosen": -2.7994260787963867,
      "logits/rejected": -2.845296859741211,
      "logps/chosen": -11690.740234375,
      "logps/rejected": -13890.576171875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1115.7935791015625,
      "rewards/margins": 217.4487762451172,
      "rewards/rejected": -1333.2423095703125,
      "step": 1335
    },
    {
      "epoch": 0.9014463504877228,
      "grad_norm": 2.8636568671913665e-09,
      "learning_rate": 4.192634560906516e-06,
      "logits/chosen": -2.8310580253601074,
      "logits/rejected": -2.8783557415008545,
      "logps/chosen": -10738.3125,
      "logps/rejected": -12316.544921875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1021.0062255859375,
      "rewards/margins": 156.0279541015625,
      "rewards/rejected": -1177.0341796875,
      "step": 1340
    },
    {
      "epoch": 0.9048099562731248,
      "grad_norm": 1.0016544365498703e-05,
      "learning_rate": 4.0509915014164304e-06,
      "logits/chosen": -2.7939770221710205,
      "logits/rejected": -2.8541154861450195,
      "logps/chosen": -9718.306640625,
      "logps/rejected": -10962.98828125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -925.0177612304688,
      "rewards/margins": 122.42236328125,
      "rewards/rejected": -1047.440185546875,
      "step": 1345
    },
    {
      "epoch": 0.9081735620585267,
      "grad_norm": 0.0037028836086392403,
      "learning_rate": 3.909348441926346e-06,
      "logits/chosen": -2.7868614196777344,
      "logits/rejected": -2.8759703636169434,
      "logps/chosen": -8671.0009765625,
      "logps/rejected": -11556.412109375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -826.3351440429688,
      "rewards/margins": 280.611572265625,
      "rewards/rejected": -1106.9466552734375,
      "step": 1350
    },
    {
      "epoch": 0.9115371678439287,
      "grad_norm": 0.0001901549258036539,
      "learning_rate": 3.7677053824362607e-06,
      "logits/chosen": -2.7727763652801514,
      "logits/rejected": -2.817560911178589,
      "logps/chosen": -10751.6572265625,
      "logps/rejected": -12439.4921875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1022.61328125,
      "rewards/margins": 166.81524658203125,
      "rewards/rejected": -1189.428466796875,
      "step": 1355
    },
    {
      "epoch": 0.9149007736293306,
      "grad_norm": 1.1579720506310707e-14,
      "learning_rate": 3.6260623229461763e-06,
      "logits/chosen": -2.779843807220459,
      "logits/rejected": -2.8579468727111816,
      "logps/chosen": -9864.7158203125,
      "logps/rejected": -11656.6103515625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -940.4755859375,
      "rewards/margins": 173.31790161132812,
      "rewards/rejected": -1113.7935791015625,
      "step": 1360
    },
    {
      "epoch": 0.9182643794147326,
      "grad_norm": 1.903349357235129e-06,
      "learning_rate": 3.484419263456091e-06,
      "logits/chosen": -2.7661707401275635,
      "logits/rejected": -2.8108742237091064,
      "logps/chosen": -9664.5439453125,
      "logps/rejected": -11310.43359375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -919.25830078125,
      "rewards/margins": 161.37599182128906,
      "rewards/rejected": -1080.6343994140625,
      "step": 1365
    },
    {
      "epoch": 0.9216279852001346,
      "grad_norm": 0.0005056912777945399,
      "learning_rate": 3.342776203966006e-06,
      "logits/chosen": -2.7574992179870605,
      "logits/rejected": -2.83868670463562,
      "logps/chosen": -9033.677734375,
      "logps/rejected": -10860.384765625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -857.6072387695312,
      "rewards/margins": 177.92727661132812,
      "rewards/rejected": -1035.5345458984375,
      "step": 1370
    },
    {
      "epoch": 0.9249915909855365,
      "grad_norm": 4.149349479121156e-05,
      "learning_rate": 3.201133144475921e-06,
      "logits/chosen": -2.783395767211914,
      "logits/rejected": -2.8669962882995605,
      "logps/chosen": -9122.810546875,
      "logps/rejected": -10934.9248046875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -866.9713745117188,
      "rewards/margins": 175.8010711669922,
      "rewards/rejected": -1042.7724609375,
      "step": 1375
    },
    {
      "epoch": 0.9283551967709385,
      "grad_norm": 6.378389372230231e-08,
      "learning_rate": 3.059490084985836e-06,
      "logits/chosen": -2.805154323577881,
      "logits/rejected": -2.850851535797119,
      "logps/chosen": -8628.7265625,
      "logps/rejected": -10462.302734375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -822.8486328125,
      "rewards/margins": 178.14894104003906,
      "rewards/rejected": -1000.99755859375,
      "step": 1380
    },
    {
      "epoch": 0.9317188025563404,
      "grad_norm": 0.00025214088964276016,
      "learning_rate": 2.9178470254957506e-06,
      "logits/chosen": -2.756075382232666,
      "logits/rejected": -2.804140567779541,
      "logps/chosen": -9716.7939453125,
      "logps/rejected": -11073.6943359375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -926.9561767578125,
      "rewards/margins": 133.236328125,
      "rewards/rejected": -1060.1925048828125,
      "step": 1385
    },
    {
      "epoch": 0.9350824083417424,
      "grad_norm": 0.00032305694185197353,
      "learning_rate": 2.776203966005666e-06,
      "logits/chosen": -2.786590814590454,
      "logits/rejected": -2.872163772583008,
      "logps/chosen": -9273.45703125,
      "logps/rejected": -11041.5458984375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -881.0120849609375,
      "rewards/margins": 172.0091094970703,
      "rewards/rejected": -1053.021240234375,
      "step": 1390
    },
    {
      "epoch": 0.9384460141271443,
      "grad_norm": 2.3232505554915406e-05,
      "learning_rate": 2.634560906515581e-06,
      "logits/chosen": -2.7689390182495117,
      "logits/rejected": -2.817228317260742,
      "logps/chosen": -10809.998046875,
      "logps/rejected": -12414.248046875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1030.851318359375,
      "rewards/margins": 159.34378051757812,
      "rewards/rejected": -1190.1949462890625,
      "step": 1395
    },
    {
      "epoch": 0.9418096199125463,
      "grad_norm": 1.15345898166197e-07,
      "learning_rate": 2.4929178470254956e-06,
      "logits/chosen": -2.789632797241211,
      "logits/rejected": -2.8873038291931152,
      "logps/chosen": -9499.505859375,
      "logps/rejected": -11661.376953125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -907.5208740234375,
      "rewards/margins": 210.48178100585938,
      "rewards/rejected": -1118.002685546875,
      "step": 1400
    },
    {
      "epoch": 0.9451732256979481,
      "grad_norm": 2.530869096517563e-06,
      "learning_rate": 2.3512747875354108e-06,
      "logits/chosen": -2.7588977813720703,
      "logits/rejected": -2.823385715484619,
      "logps/chosen": -9105.587890625,
      "logps/rejected": -10396.720703125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -865.9644775390625,
      "rewards/margins": 126.78507232666016,
      "rewards/rejected": -992.74951171875,
      "step": 1405
    },
    {
      "epoch": 0.9485368314833501,
      "grad_norm": 1.471276959819079e-06,
      "learning_rate": 2.209631728045326e-06,
      "logits/chosen": -2.7888245582580566,
      "logits/rejected": -2.8254733085632324,
      "logps/chosen": -9374.6708984375,
      "logps/rejected": -10183.27734375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -895.3932495117188,
      "rewards/margins": 80.75559997558594,
      "rewards/rejected": -976.1487426757812,
      "step": 1410
    },
    {
      "epoch": 0.9519004372687521,
      "grad_norm": 0.0003817919350694865,
      "learning_rate": 2.067988668555241e-06,
      "logits/chosen": -2.791968822479248,
      "logits/rejected": -2.8503360748291016,
      "logps/chosen": -10258.4453125,
      "logps/rejected": -11736.912109375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -977.0296630859375,
      "rewards/margins": 144.40345764160156,
      "rewards/rejected": -1121.4332275390625,
      "step": 1415
    },
    {
      "epoch": 0.955264043054154,
      "grad_norm": 4.6956525039804076e-15,
      "learning_rate": 1.9263456090651557e-06,
      "logits/chosen": -2.770296812057495,
      "logits/rejected": -2.8440771102905273,
      "logps/chosen": -9079.8876953125,
      "logps/rejected": -10946.501953125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -863.3029174804688,
      "rewards/margins": 182.57151794433594,
      "rewards/rejected": -1045.8743896484375,
      "step": 1420
    },
    {
      "epoch": 0.958627648839556,
      "grad_norm": 4.823243584042847e-13,
      "learning_rate": 1.7847025495750709e-06,
      "logits/chosen": -2.774212121963501,
      "logits/rejected": -2.8170745372772217,
      "logps/chosen": -9913.640625,
      "logps/rejected": -11046.314453125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -946.7122192382812,
      "rewards/margins": 111.66419982910156,
      "rewards/rejected": -1058.37646484375,
      "step": 1425
    },
    {
      "epoch": 0.9619912546249579,
      "grad_norm": 0.00011467803415143862,
      "learning_rate": 1.6430594900849862e-06,
      "logits/chosen": -2.798743486404419,
      "logits/rejected": -2.865551710128784,
      "logps/chosen": -10278.037109375,
      "logps/rejected": -11942.447265625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -979.6921997070312,
      "rewards/margins": 162.4659881591797,
      "rewards/rejected": -1142.158447265625,
      "step": 1430
    },
    {
      "epoch": 0.9653548604103599,
      "grad_norm": 0.0002761161595117301,
      "learning_rate": 1.5014164305949011e-06,
      "logits/chosen": -2.7595863342285156,
      "logits/rejected": -2.8078465461730957,
      "logps/chosen": -7911.0615234375,
      "logps/rejected": -9055.5771484375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -754.9166259765625,
      "rewards/margins": 111.7831039428711,
      "rewards/rejected": -866.6998291015625,
      "step": 1435
    },
    {
      "epoch": 0.9687184661957619,
      "grad_norm": 0.010890788398683071,
      "learning_rate": 1.359773371104816e-06,
      "logits/chosen": -2.779064893722534,
      "logits/rejected": -2.815664529800415,
      "logps/chosen": -10334.6328125,
      "logps/rejected": -11284.044921875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -984.6209716796875,
      "rewards/margins": 93.98727416992188,
      "rewards/rejected": -1078.608154296875,
      "step": 1440
    },
    {
      "epoch": 0.9720820719811638,
      "grad_norm": 1.8010157980859892e-13,
      "learning_rate": 1.218130311614731e-06,
      "logits/chosen": -2.7915377616882324,
      "logits/rejected": -2.835366725921631,
      "logps/chosen": -10613.0771484375,
      "logps/rejected": -12232.240234375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1012.6925659179688,
      "rewards/margins": 160.3780517578125,
      "rewards/rejected": -1173.0706787109375,
      "step": 1445
    },
    {
      "epoch": 0.9754456777665658,
      "grad_norm": 2.0117573384936804e-08,
      "learning_rate": 1.0764872521246459e-06,
      "logits/chosen": -2.7724738121032715,
      "logits/rejected": -2.846680164337158,
      "logps/chosen": -9501.671875,
      "logps/rejected": -10645.544921875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -908.2498168945312,
      "rewards/margins": 111.72891998291016,
      "rewards/rejected": -1019.9786987304688,
      "step": 1450
    },
    {
      "epoch": 0.9788092835519677,
      "grad_norm": 9.917032472994691e-12,
      "learning_rate": 9.34844192634561e-07,
      "logits/chosen": -2.7602806091308594,
      "logits/rejected": -2.826261043548584,
      "logps/chosen": -11521.5029296875,
      "logps/rejected": -13138.5625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1100.6396484375,
      "rewards/margins": 159.37832641601562,
      "rewards/rejected": -1260.017822265625,
      "step": 1455
    },
    {
      "epoch": 0.9821728893373697,
      "grad_norm": 1.550924632081794e-15,
      "learning_rate": 7.932011331444759e-07,
      "logits/chosen": -2.8079962730407715,
      "logits/rejected": -2.863875150680542,
      "logps/chosen": -11043.158203125,
      "logps/rejected": -12635.833984375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1053.036376953125,
      "rewards/margins": 156.1077117919922,
      "rewards/rejected": -1209.1441650390625,
      "step": 1460
    },
    {
      "epoch": 0.9855364951227716,
      "grad_norm": 0.015953488647937775,
      "learning_rate": 6.51558073654391e-07,
      "logits/chosen": -2.780484914779663,
      "logits/rejected": -2.856376886367798,
      "logps/chosen": -9444.4609375,
      "logps/rejected": -10677.447265625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -901.3824462890625,
      "rewards/margins": 122.55001068115234,
      "rewards/rejected": -1023.9324951171875,
      "step": 1465
    },
    {
      "epoch": 0.9889001009081736,
      "grad_norm": 4.331354475084481e-08,
      "learning_rate": 5.09915014164306e-07,
      "logits/chosen": -2.8052170276641846,
      "logits/rejected": -2.8585524559020996,
      "logps/chosen": -10417.791015625,
      "logps/rejected": -11705.337890625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -989.41259765625,
      "rewards/margins": 128.55886840820312,
      "rewards/rejected": -1117.971435546875,
      "step": 1470
    },
    {
      "epoch": 0.9922637066935756,
      "grad_norm": 0.0001340260059805587,
      "learning_rate": 3.6827195467422096e-07,
      "logits/chosen": -2.7467541694641113,
      "logits/rejected": -2.8164498805999756,
      "logps/chosen": -8758.083984375,
      "logps/rejected": -10769.033203125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -833.5828247070312,
      "rewards/margins": 195.48825073242188,
      "rewards/rejected": -1029.071044921875,
      "step": 1475
    },
    {
      "epoch": 0.9956273124789775,
      "grad_norm": 0.00013555352052208036,
      "learning_rate": 2.2662889518413602e-07,
      "logits/chosen": -2.807600736618042,
      "logits/rejected": -2.8697259426116943,
      "logps/chosen": -9021.90625,
      "logps/rejected": -11001.6533203125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -858.5872802734375,
      "rewards/margins": 194.73312377929688,
      "rewards/rejected": -1053.3204345703125,
      "step": 1480
    },
    {
      "epoch": 0.9989909182643795,
      "grad_norm": 4.304754384065745e-06,
      "learning_rate": 8.4985835694051e-08,
      "logits/chosen": -2.7770111560821533,
      "logits/rejected": -2.8173835277557373,
      "logps/chosen": -10043.314453125,
      "logps/rejected": -11197.5400390625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -959.3912963867188,
      "rewards/margins": 114.22813415527344,
      "rewards/rejected": -1073.6192626953125,
      "step": 1485
    }
  ],
  "logging_steps": 5,
  "max_steps": 1487,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 500,
  "stateful_callbacks": {
    "EarlyStoppingCallback": [
      {
        "args": {
          "early_stopping_patience": 8,
          "early_stopping_threshold": 0.01
        },
        "attributes": {
          "early_stopping_patience_counter": 0
        }
      },
      {
        "args": {
          "early_stopping_patience": 8,
          "early_stopping_threshold": 0.01
        },
        "attributes": {
          "early_stopping_patience_counter": 0
        }
      },
      {
        "args": {
          "early_stopping_patience": 8,
          "early_stopping_threshold": 0.01
        },
        "attributes": {
          "early_stopping_patience_counter": 0
        }
      },
      {
        "args": {
          "early_stopping_patience": 8,
          "early_stopping_threshold": 0.01
        },
        "attributes": {
          "early_stopping_patience_counter": 0
        }
      },
      {
        "args": {
          "early_stopping_patience": 8,
          "early_stopping_threshold": 0.01
        },
        "attributes": {
          "early_stopping_patience_counter": 1
        }
      },
      {
        "args": {
          "early_stopping_patience": 8,
          "early_stopping_threshold": 0.01
        },
        "attributes": {
          "early_stopping_patience_counter": 1
        }
      },
      {
        "args": {
          "early_stopping_patience": 8,
          "early_stopping_threshold": 0.01
        },
        "attributes": {
          "early_stopping_patience_counter": 1
        }
      },
      {
        "args": {
          "early_stopping_patience": 8,
          "early_stopping_threshold": 0.01
        },
        "attributes": {
          "early_stopping_patience_counter": 1
        }
      }
    ],
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 2,
  "trial_name": null,
  "trial_params": null
}