{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 3.9977470228516254,
  "eval_steps": 500,
  "global_step": 6212,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.006437077566784679,
      "grad_norm": 120.16608464445075,
      "learning_rate": 9.98551191242756e-07,
      "logits/chosen": -0.5285888910293579,
      "logits/rejected": -0.550219714641571,
      "logps/chosen": -367.70001220703125,
      "logps/rejected": -355.82501220703125,
      "loss": 0.6736,
      "rewards/accuracies": 0.3812499940395355,
      "rewards/chosen": -0.07652588188648224,
      "rewards/margins": 0.06703948974609375,
      "rewards/rejected": -0.1435089111328125,
      "step": 10
    },
    {
      "epoch": 0.012874155133569359,
      "grad_norm": 135.5669189775458,
      "learning_rate": 9.96941403734707e-07,
      "logits/chosen": -0.5475219488143921,
      "logits/rejected": -0.556469738483429,
      "logps/chosen": -368.625,
      "logps/rejected": -344.0,
      "loss": 0.6127,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.3770507872104645,
      "rewards/margins": 0.3885040283203125,
      "rewards/rejected": -0.7652343511581421,
      "step": 20
    },
    {
      "epoch": 0.01931123270035404,
      "grad_norm": 137.23954540855013,
      "learning_rate": 9.953316162266581e-07,
      "logits/chosen": -0.4871459901332855,
      "logits/rejected": -0.5256897211074829,
      "logps/chosen": -347.4750061035156,
      "logps/rejected": -340.875,
      "loss": 0.5851,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.0062271119095385075,
      "rewards/margins": 0.5819091796875,
      "rewards/rejected": -0.5878967046737671,
      "step": 30
    },
    {
      "epoch": 0.025748310267138717,
      "grad_norm": 162.8825460193566,
      "learning_rate": 9.93721828718609e-07,
      "logits/chosen": -0.540789783000946,
      "logits/rejected": -0.5523315668106079,
      "logps/chosen": -392.82501220703125,
      "logps/rejected": -339.5874938964844,
      "loss": 0.6844,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.02431030198931694,
      "rewards/margins": 0.4530700743198395,
      "rewards/rejected": -0.4765075743198395,
      "step": 40
    },
    {
      "epoch": 0.0321853878339234,
      "grad_norm": 111.36198187326185,
      "learning_rate": 9.9211204121056e-07,
      "logits/chosen": -0.5651000738143921,
      "logits/rejected": -0.560577392578125,
      "logps/chosen": -364.45001220703125,
      "logps/rejected": -334.11248779296875,
      "loss": 0.5565,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.3750457763671875,
      "rewards/margins": 0.7763427495956421,
      "rewards/rejected": -0.401620477437973,
      "step": 50
    },
    {
      "epoch": 0.03862246540070808,
      "grad_norm": 99.12262240934288,
      "learning_rate": 9.905022537025112e-07,
      "logits/chosen": -0.4527038633823395,
      "logits/rejected": -0.4679809510707855,
      "logps/chosen": -359.25,
      "logps/rejected": -340.92498779296875,
      "loss": 0.6116,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": 0.714129626750946,
      "rewards/margins": 0.6304260492324829,
      "rewards/rejected": 0.08298339694738388,
      "step": 60
    },
    {
      "epoch": 0.04505954296749276,
      "grad_norm": 88.34508629839479,
      "learning_rate": 9.888924661944624e-07,
      "logits/chosen": -0.48187255859375,
      "logits/rejected": -0.49010008573532104,
      "logps/chosen": -346.2250061035156,
      "logps/rejected": -339.4624938964844,
      "loss": 0.5153,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": 0.822558581829071,
      "rewards/margins": 0.859326183795929,
      "rewards/rejected": -0.03675537183880806,
      "step": 70
    },
    {
      "epoch": 0.051496620534277435,
      "grad_norm": 111.01162050170656,
      "learning_rate": 9.872826786864134e-07,
      "logits/chosen": -0.545391857624054,
      "logits/rejected": -0.537060558795929,
      "logps/chosen": -408.15625,
      "logps/rejected": -402.45001220703125,
      "loss": 0.5767,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": 0.3031005859375,
      "rewards/margins": 0.9102783203125,
      "rewards/rejected": -0.6064819097518921,
      "step": 80
    },
    {
      "epoch": 0.057933698101062116,
      "grad_norm": 99.33785963112945,
      "learning_rate": 9.856728911783643e-07,
      "logits/chosen": -0.57373046875,
      "logits/rejected": -0.5748535394668579,
      "logps/chosen": -393.45001220703125,
      "logps/rejected": -360.95001220703125,
      "loss": 0.5902,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.21026000380516052,
      "rewards/margins": 0.7870117425918579,
      "rewards/rejected": -0.997509777545929,
      "step": 90
    },
    {
      "epoch": 0.0643707756678468,
      "grad_norm": 99.12075537368256,
      "learning_rate": 9.840631036703153e-07,
      "logits/chosen": -0.523236095905304,
      "logits/rejected": -0.5228271484375,
      "logps/chosen": -348.42498779296875,
      "logps/rejected": -317.54998779296875,
      "loss": 0.6495,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.176036074757576,
      "rewards/margins": 0.4732727110385895,
      "rewards/rejected": -0.6496826410293579,
      "step": 100
    },
    {
      "epoch": 0.07080785323463147,
      "grad_norm": 115.52409219899631,
      "learning_rate": 9.824533161622665e-07,
      "logits/chosen": -0.4966674745082855,
      "logits/rejected": -0.5160156488418579,
      "logps/chosen": -362.0249938964844,
      "logps/rejected": -325.75,
      "loss": 0.6544,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": 0.4094482362270355,
      "rewards/margins": 0.36219483613967896,
      "rewards/rejected": 0.0468292236328125,
      "step": 110
    },
    {
      "epoch": 0.07724493080141616,
      "grad_norm": 108.91370235062881,
      "learning_rate": 9.808435286542177e-07,
      "logits/chosen": -0.533123791217804,
      "logits/rejected": -0.561358630657196,
      "logps/chosen": -368.6499938964844,
      "logps/rejected": -338.875,
      "loss": 0.5637,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.9539794921875,
      "rewards/margins": 0.5897582769393921,
      "rewards/rejected": 0.3646179139614105,
      "step": 120
    },
    {
      "epoch": 0.08368200836820083,
      "grad_norm": 86.74166714430766,
      "learning_rate": 9.792337411461686e-07,
      "logits/chosen": -0.45283204317092896,
      "logits/rejected": -0.47824400663375854,
      "logps/chosen": -350.3999938964844,
      "logps/rejected": -337.0249938964844,
      "loss": 0.647,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": 0.527966320514679,
      "rewards/margins": 0.5209716558456421,
      "rewards/rejected": 0.0069335936568677425,
      "step": 130
    },
    {
      "epoch": 0.09011908593498552,
      "grad_norm": 103.64877138648835,
      "learning_rate": 9.776239536381196e-07,
      "logits/chosen": -0.5349365472793579,
      "logits/rejected": -0.564208984375,
      "logps/chosen": -401.625,
      "logps/rejected": -352.07501220703125,
      "loss": 0.5847,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.16360779106616974,
      "rewards/margins": 0.592480480670929,
      "rewards/rejected": -0.7553344964981079,
      "step": 140
    },
    {
      "epoch": 0.0965561635017702,
      "grad_norm": 116.65061603445305,
      "learning_rate": 9.760141661300708e-07,
      "logits/chosen": -0.4708251953125,
      "logits/rejected": -0.4907592833042145,
      "logps/chosen": -364.95001220703125,
      "logps/rejected": -327.5249938964844,
      "loss": 0.5924,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.10672607272863388,
      "rewards/margins": 0.5644897222518921,
      "rewards/rejected": -0.4580444395542145,
      "step": 150
    },
    {
      "epoch": 0.10299324106855487,
      "grad_norm": 81.12322692751495,
      "learning_rate": 9.744043786220218e-07,
      "logits/chosen": -0.5120483636856079,
      "logits/rejected": -0.5198394656181335,
      "logps/chosen": -359.8500061035156,
      "logps/rejected": -337.11248779296875,
      "loss": 0.5912,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": 0.36287230253219604,
      "rewards/margins": 0.609118640422821,
      "rewards/rejected": -0.24659423530101776,
      "step": 160
    },
    {
      "epoch": 0.10943031863533956,
      "grad_norm": 74.31516061519605,
      "learning_rate": 9.72794591113973e-07,
      "logits/chosen": -0.5851074457168579,
      "logits/rejected": -0.5960327386856079,
      "logps/chosen": -374.1000061035156,
      "logps/rejected": -360.8500061035156,
      "loss": 0.5686,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": 0.30054932832717896,
      "rewards/margins": 0.8222290277481079,
      "rewards/rejected": -0.5218139886856079,
      "step": 170
    },
    {
      "epoch": 0.11586739620212423,
      "grad_norm": 113.20782415215908,
      "learning_rate": 9.71184803605924e-07,
      "logits/chosen": -0.5738281011581421,
      "logits/rejected": -0.579742431640625,
      "logps/chosen": -387.3500061035156,
      "logps/rejected": -373.2250061035156,
      "loss": 0.6087,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.24147339165210724,
      "rewards/margins": 0.787463366985321,
      "rewards/rejected": -0.5462890863418579,
      "step": 180
    },
    {
      "epoch": 0.12230447376890892,
      "grad_norm": 130.1431697433103,
      "learning_rate": 9.69575016097875e-07,
      "logits/chosen": -0.583453357219696,
      "logits/rejected": -0.5979553461074829,
      "logps/chosen": -372.29998779296875,
      "logps/rejected": -363.6000061035156,
      "loss": 0.6054,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": 0.41649168729782104,
      "rewards/margins": 1.0996825695037842,
      "rewards/rejected": -0.683605968952179,
      "step": 190
    },
    {
      "epoch": 0.1287415513356936,
      "grad_norm": 91.70486267908973,
      "learning_rate": 9.67965228589826e-07,
      "logits/chosen": -0.5898193120956421,
      "logits/rejected": -0.587890625,
      "logps/chosen": -408.375,
      "logps/rejected": -347.3999938964844,
      "loss": 0.5409,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": 0.648663341999054,
      "rewards/margins": 1.29571533203125,
      "rewards/rejected": -0.64654541015625,
      "step": 200
    },
    {
      "epoch": 0.13517862890247828,
      "grad_norm": 130.11983466346078,
      "learning_rate": 9.663554410817772e-07,
      "logits/chosen": -0.4863647520542145,
      "logits/rejected": -0.539746105670929,
      "logps/chosen": -347.0375061035156,
      "logps/rejected": -338.32501220703125,
      "loss": 0.6107,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": 0.397308349609375,
      "rewards/margins": 0.7805420160293579,
      "rewards/rejected": -0.38372802734375,
      "step": 210
    },
    {
      "epoch": 0.14161570646926294,
      "grad_norm": 75.88348717635424,
      "learning_rate": 9.647456535737282e-07,
      "logits/chosen": -0.4582275450229645,
      "logits/rejected": -0.49046629667282104,
      "logps/chosen": -374.8500061035156,
      "logps/rejected": -359.04998779296875,
      "loss": 0.5886,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": 0.744860827922821,
      "rewards/margins": 0.8912719488143921,
      "rewards/rejected": -0.14583739638328552,
      "step": 220
    },
    {
      "epoch": 0.14805278403604763,
      "grad_norm": 72.21523489539429,
      "learning_rate": 9.631358660656794e-07,
      "logits/chosen": -0.4630493223667145,
      "logits/rejected": -0.4919677674770355,
      "logps/chosen": -342.1875,
      "logps/rejected": -331.42498779296875,
      "loss": 0.6393,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": 0.988513171672821,
      "rewards/margins": 0.6120818853378296,
      "rewards/rejected": 0.3760314881801605,
      "step": 230
    },
    {
      "epoch": 0.15448986160283232,
      "grad_norm": 96.90032768184547,
      "learning_rate": 9.615260785576303e-07,
      "logits/chosen": -0.5086669921875,
      "logits/rejected": -0.520275890827179,
      "logps/chosen": -369.5375061035156,
      "logps/rejected": -331.95001220703125,
      "loss": 0.6711,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": 0.80767822265625,
      "rewards/margins": 0.4611572325229645,
      "rewards/rejected": 0.3460754454135895,
      "step": 240
    },
    {
      "epoch": 0.160926939169617,
      "grad_norm": 117.88627280563217,
      "learning_rate": 9.599162910495813e-07,
      "logits/chosen": -0.578723132610321,
      "logits/rejected": -0.607922375202179,
      "logps/chosen": -410.42498779296875,
      "logps/rejected": -369.25,
      "loss": 0.6423,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.02398681640625,
      "rewards/margins": 0.622692883014679,
      "rewards/rejected": -0.646209716796875,
      "step": 250
    },
    {
      "epoch": 0.16736401673640167,
      "grad_norm": 114.49562778791112,
      "learning_rate": 9.583065035415325e-07,
      "logits/chosen": -0.546374499797821,
      "logits/rejected": -0.5821288824081421,
      "logps/chosen": -358.5625,
      "logps/rejected": -334.8125,
      "loss": 0.6349,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.6829162836074829,
      "rewards/margins": 0.758496105670929,
      "rewards/rejected": -1.4420654773712158,
      "step": 260
    },
    {
      "epoch": 0.17380109430318635,
      "grad_norm": 252.1668555560546,
      "learning_rate": 9.566967160334835e-07,
      "logits/chosen": -0.496002197265625,
      "logits/rejected": -0.529162585735321,
      "logps/chosen": -341.23748779296875,
      "logps/rejected": -344.5687561035156,
      "loss": 0.6088,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.3066039979457855,
      "rewards/margins": 0.724609375,
      "rewards/rejected": -1.030542016029358,
      "step": 270
    },
    {
      "epoch": 0.18023817186997104,
      "grad_norm": 102.98363396913689,
      "learning_rate": 9.550869285254346e-07,
      "logits/chosen": -0.546069324016571,
      "logits/rejected": -0.558239758014679,
      "logps/chosen": -370.57501220703125,
      "logps/rejected": -368.75,
      "loss": 0.5405,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": 0.058502197265625,
      "rewards/margins": 0.809436023235321,
      "rewards/rejected": -0.751208484172821,
      "step": 280
    },
    {
      "epoch": 0.1866752494367557,
      "grad_norm": 91.00041151449058,
      "learning_rate": 9.534771410173856e-07,
      "logits/chosen": -0.577075183391571,
      "logits/rejected": -0.5903075933456421,
      "logps/chosen": -375.0,
      "logps/rejected": -360.98748779296875,
      "loss": 0.6413,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": 0.1882476806640625,
      "rewards/margins": 0.7995971441268921,
      "rewards/rejected": -0.612011730670929,
      "step": 290
    },
    {
      "epoch": 0.1931123270035404,
      "grad_norm": 102.0109893516658,
      "learning_rate": 9.518673535093368e-07,
      "logits/chosen": -0.4876770079135895,
      "logits/rejected": -0.489013671875,
      "logps/chosen": -374.79998779296875,
      "logps/rejected": -336.17498779296875,
      "loss": 0.618,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.07204589992761612,
      "rewards/margins": 0.842114269733429,
      "rewards/rejected": -0.770495593547821,
      "step": 300
    },
    {
      "epoch": 0.19954940457032508,
      "grad_norm": 76.86741916960312,
      "learning_rate": 9.502575660012879e-07,
      "logits/chosen": -0.5677276849746704,
      "logits/rejected": -0.5572754144668579,
      "logps/chosen": -359.3999938964844,
      "logps/rejected": -367.92498779296875,
      "loss": 0.6237,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": 0.3591064512729645,
      "rewards/margins": 0.6754516363143921,
      "rewards/rejected": -0.3162597715854645,
      "step": 310
    },
    {
      "epoch": 0.20598648213710974,
      "grad_norm": 108.9030110579976,
      "learning_rate": 9.486477784932388e-07,
      "logits/chosen": -0.6708739995956421,
      "logits/rejected": -0.655590832233429,
      "logps/chosen": -376.125,
      "logps/rejected": -373.375,
      "loss": 0.5166,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": 0.2652526795864105,
      "rewards/margins": 1.1403319835662842,
      "rewards/rejected": -0.8749328851699829,
      "step": 320
    },
    {
      "epoch": 0.21242355970389443,
      "grad_norm": 107.07521441276505,
      "learning_rate": 9.470379909851899e-07,
      "logits/chosen": -0.5230712890625,
      "logits/rejected": -0.551342785358429,
      "logps/chosen": -373.09375,
      "logps/rejected": -354.7875061035156,
      "loss": 0.6214,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.1680450439453125,
      "rewards/margins": 0.9344116449356079,
      "rewards/rejected": -1.102380394935608,
      "step": 330
    },
    {
      "epoch": 0.21886063727067911,
      "grad_norm": 80.28388865404655,
      "learning_rate": 9.454282034771411e-07,
      "logits/chosen": -0.6387573480606079,
      "logits/rejected": -0.661419689655304,
      "logps/chosen": -365.1499938964844,
      "logps/rejected": -351.125,
      "loss": 0.5976,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.03964843600988388,
      "rewards/margins": 0.987072765827179,
      "rewards/rejected": -1.0261566638946533,
      "step": 340
    },
    {
      "epoch": 0.2252977148374638,
      "grad_norm": 66.301132685081,
      "learning_rate": 9.43818415969092e-07,
      "logits/chosen": -0.5279541015625,
      "logits/rejected": -0.5782226324081421,
      "logps/chosen": -373.9750061035156,
      "logps/rejected": -359.0249938964844,
      "loss": 0.5829,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.40840452909469604,
      "rewards/margins": 0.9593750238418579,
      "rewards/rejected": -0.552032470703125,
      "step": 350
    },
    {
      "epoch": 0.23173479240424846,
      "grad_norm": 133.60786340460766,
      "learning_rate": 9.422086284610431e-07,
      "logits/chosen": -0.4904952943325043,
      "logits/rejected": -0.4936676025390625,
      "logps/chosen": -351.54998779296875,
      "logps/rejected": -352.1875,
      "loss": 0.6358,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.3689941465854645,
      "rewards/margins": 0.801318347454071,
      "rewards/rejected": -0.43192750215530396,
      "step": 360
    },
    {
      "epoch": 0.23817186997103315,
      "grad_norm": 146.73254711353752,
      "learning_rate": 9.405988409529941e-07,
      "logits/chosen": -0.4536499083042145,
      "logits/rejected": -0.48334044218063354,
      "logps/chosen": -394.20001220703125,
      "logps/rejected": -344.3999938964844,
      "loss": 0.622,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.44520872831344604,
      "rewards/margins": 0.8761352300643921,
      "rewards/rejected": -0.43049317598342896,
      "step": 370
    },
    {
      "epoch": 0.24460894753781784,
      "grad_norm": 108.12091497250533,
      "learning_rate": 9.389890534449453e-07,
      "logits/chosen": -0.48052978515625,
      "logits/rejected": -0.48167723417282104,
      "logps/chosen": -388.875,
      "logps/rejected": -367.875,
      "loss": 0.5671,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.10392455756664276,
      "rewards/margins": 0.9621337652206421,
      "rewards/rejected": -0.858563244342804,
      "step": 380
    },
    {
      "epoch": 0.2510460251046025,
      "grad_norm": 85.66165714869449,
      "learning_rate": 9.373792659368963e-07,
      "logits/chosen": -0.504321277141571,
      "logits/rejected": -0.52301025390625,
      "logps/chosen": -362.92498779296875,
      "logps/rejected": -352.42498779296875,
      "loss": 0.5526,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.561572253704071,
      "rewards/margins": 1.221276879310608,
      "rewards/rejected": -1.7831909656524658,
      "step": 390
    },
    {
      "epoch": 0.2574831026713872,
      "grad_norm": 136.73196581533503,
      "learning_rate": 9.357694784288473e-07,
      "logits/chosen": -0.49071043729782104,
      "logits/rejected": -0.51934814453125,
      "logps/chosen": -366.36248779296875,
      "logps/rejected": -363.25,
      "loss": 0.6429,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.3850997984409332,
      "rewards/margins": 1.0957520008087158,
      "rewards/rejected": -1.481896996498108,
      "step": 400
    },
    {
      "epoch": 0.2639201802381719,
      "grad_norm": 62.68096976935343,
      "learning_rate": 9.341596909207984e-07,
      "logits/chosen": -0.4698120057582855,
      "logits/rejected": -0.49330443143844604,
      "logps/chosen": -373.75,
      "logps/rejected": -357.20001220703125,
      "loss": 0.554,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.1954345703125,
      "rewards/margins": 0.96142578125,
      "rewards/rejected": -1.1562988758087158,
      "step": 410
    },
    {
      "epoch": 0.27035725780495656,
      "grad_norm": 117.58664324623555,
      "learning_rate": 9.325499034127496e-07,
      "logits/chosen": -0.5307983160018921,
      "logits/rejected": -0.5357711911201477,
      "logps/chosen": -402.75,
      "logps/rejected": -393.67498779296875,
      "loss": 0.5572,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.15812988579273224,
      "rewards/margins": 1.0307495594024658,
      "rewards/rejected": -1.1893126964569092,
      "step": 420
    },
    {
      "epoch": 0.27679433537174125,
      "grad_norm": 93.75687305331235,
      "learning_rate": 9.309401159047005e-07,
      "logits/chosen": -0.5401611328125,
      "logits/rejected": -0.578295886516571,
      "logps/chosen": -353.67498779296875,
      "logps/rejected": -350.29998779296875,
      "loss": 0.697,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.4753173887729645,
      "rewards/margins": 0.94769287109375,
      "rewards/rejected": -1.422509789466858,
      "step": 430
    },
    {
      "epoch": 0.2832314129385259,
      "grad_norm": 95.5374818544099,
      "learning_rate": 9.293303283966516e-07,
      "logits/chosen": -0.619433581829071,
      "logits/rejected": -0.6208251714706421,
      "logps/chosen": -394.98748779296875,
      "logps/rejected": -389.11248779296875,
      "loss": 0.6189,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.571759045124054,
      "rewards/margins": 0.7771240472793579,
      "rewards/rejected": -1.349853515625,
      "step": 440
    },
    {
      "epoch": 0.28966849050531057,
      "grad_norm": 125.84423019681405,
      "learning_rate": 9.277205408886027e-07,
      "logits/chosen": -0.53460693359375,
      "logits/rejected": -0.530902087688446,
      "logps/chosen": -368.8999938964844,
      "logps/rejected": -367.3500061035156,
      "loss": 0.627,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.482666015625,
      "rewards/margins": 1.0159423351287842,
      "rewards/rejected": -1.497259497642517,
      "step": 450
    },
    {
      "epoch": 0.29610556807209526,
      "grad_norm": 119.6694420609397,
      "learning_rate": 9.261107533805537e-07,
      "logits/chosen": -0.5695999264717102,
      "logits/rejected": -0.574688732624054,
      "logps/chosen": -392.45001220703125,
      "logps/rejected": -376.875,
      "loss": 0.6691,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.489309698343277,
      "rewards/margins": 0.785595715045929,
      "rewards/rejected": -1.275183081626892,
      "step": 460
    },
    {
      "epoch": 0.30254264563887995,
      "grad_norm": 109.69479101002719,
      "learning_rate": 9.245009658725048e-07,
      "logits/chosen": -0.5947021245956421,
      "logits/rejected": -0.5928710699081421,
      "logps/chosen": -404.1625061035156,
      "logps/rejected": -355.2749938964844,
      "loss": 0.6002,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.16807861626148224,
      "rewards/margins": 0.9239562749862671,
      "rewards/rejected": -1.091485619544983,
      "step": 470
    },
    {
      "epoch": 0.30897972320566464,
      "grad_norm": 114.73014097944892,
      "learning_rate": 9.228911783644559e-07,
      "logits/chosen": -0.572558581829071,
      "logits/rejected": -0.602734386920929,
      "logps/chosen": -368.2749938964844,
      "logps/rejected": -358.1499938964844,
      "loss": 0.5264,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": 0.10758056491613388,
      "rewards/margins": 1.0086791515350342,
      "rewards/rejected": -0.9011077880859375,
      "step": 480
    },
    {
      "epoch": 0.3154168007724493,
      "grad_norm": 127.76637945406556,
      "learning_rate": 9.212813908564069e-07,
      "logits/chosen": -0.54168701171875,
      "logits/rejected": -0.570635974407196,
      "logps/chosen": -363.2875061035156,
      "logps/rejected": -343.17498779296875,
      "loss": 0.6542,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": 0.015155029483139515,
      "rewards/margins": 0.724200427532196,
      "rewards/rejected": -0.708392322063446,
      "step": 490
    },
    {
      "epoch": 0.321853878339234,
      "grad_norm": 92.4760176391222,
      "learning_rate": 9.19671603348358e-07,
      "logits/chosen": -0.5834716558456421,
      "logits/rejected": -0.608386218547821,
      "logps/chosen": -370.13751220703125,
      "logps/rejected": -359.7749938964844,
      "loss": 0.6958,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.02927246131002903,
      "rewards/margins": 0.681835949420929,
      "rewards/rejected": -0.7109962701797485,
      "step": 500
    },
    {
      "epoch": 0.32829095590601864,
      "grad_norm": 120.62329927967555,
      "learning_rate": 9.18061815840309e-07,
      "logits/chosen": -0.5348449945449829,
      "logits/rejected": -0.5554443597793579,
      "logps/chosen": -365.8999938964844,
      "logps/rejected": -356.29998779296875,
      "loss": 0.5751,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.2884887754917145,
      "rewards/margins": 0.861279308795929,
      "rewards/rejected": -0.5736328363418579,
      "step": 510
    },
    {
      "epoch": 0.33472803347280333,
      "grad_norm": 130.9763223613383,
      "learning_rate": 9.164520283322601e-07,
      "logits/chosen": -0.451071172952652,
      "logits/rejected": -0.48179322481155396,
      "logps/chosen": -363.6000061035156,
      "logps/rejected": -347.92498779296875,
      "loss": 0.6538,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": 0.3598876893520355,
      "rewards/margins": 0.71923828125,
      "rewards/rejected": -0.359466552734375,
      "step": 520
    },
    {
      "epoch": 0.341165111039588,
      "grad_norm": 109.62659240266296,
      "learning_rate": 9.148422408242112e-07,
      "logits/chosen": -0.4910888671875,
      "logits/rejected": -0.519116222858429,
      "logps/chosen": -358.5,
      "logps/rejected": -364.45001220703125,
      "loss": 0.6195,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.5476440191268921,
      "rewards/margins": 0.73748779296875,
      "rewards/rejected": -0.19072571396827698,
      "step": 530
    },
    {
      "epoch": 0.3476021886063727,
      "grad_norm": 109.68298542748705,
      "learning_rate": 9.132324533161622e-07,
      "logits/chosen": -0.4764648377895355,
      "logits/rejected": -0.502703845500946,
      "logps/chosen": -336.875,
      "logps/rejected": -318.25,
      "loss": 0.64,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": 0.613238513469696,
      "rewards/margins": 0.6423705816268921,
      "rewards/rejected": -0.02930908277630806,
      "step": 540
    },
    {
      "epoch": 0.3540392661731574,
      "grad_norm": 105.51467874045339,
      "learning_rate": 9.116226658081133e-07,
      "logits/chosen": -0.5284057855606079,
      "logits/rejected": -0.5428222417831421,
      "logps/chosen": -361.125,
      "logps/rejected": -332.3500061035156,
      "loss": 0.5463,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": 0.4479126036167145,
      "rewards/margins": 1.0054442882537842,
      "rewards/rejected": -0.558178722858429,
      "step": 550
    },
    {
      "epoch": 0.3604763437399421,
      "grad_norm": 99.67713382177797,
      "learning_rate": 9.100128783000644e-07,
      "logits/chosen": -0.5566619634628296,
      "logits/rejected": -0.5879882574081421,
      "logps/chosen": -348.375,
      "logps/rejected": -369.625,
      "loss": 0.5016,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.610461413860321,
      "rewards/margins": 1.308984398841858,
      "rewards/rejected": -0.698779284954071,
      "step": 560
    },
    {
      "epoch": 0.3669134213067268,
      "grad_norm": 131.25070143478592,
      "learning_rate": 9.084030907920153e-07,
      "logits/chosen": -0.5046325922012329,
      "logits/rejected": -0.545117199420929,
      "logps/chosen": -364.75,
      "logps/rejected": -372.5249938964844,
      "loss": 0.6631,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": 0.550830066204071,
      "rewards/margins": 0.9921020269393921,
      "rewards/rejected": -0.4401001036167145,
      "step": 570
    },
    {
      "epoch": 0.3733504988735114,
      "grad_norm": 109.14671818372386,
      "learning_rate": 9.067933032839665e-07,
      "logits/chosen": -0.4894042909145355,
      "logits/rejected": -0.4809204041957855,
      "logps/chosen": -340.4624938964844,
      "logps/rejected": -360.8500061035156,
      "loss": 0.6645,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.671185314655304,
      "rewards/margins": 0.7091064453125,
      "rewards/rejected": -0.03690185397863388,
      "step": 580
    },
    {
      "epoch": 0.3797875764402961,
      "grad_norm": 102.87805305261952,
      "learning_rate": 9.051835157759176e-07,
      "logits/chosen": -0.6076415777206421,
      "logits/rejected": -0.63128662109375,
      "logps/chosen": -394.92498779296875,
      "logps/rejected": -383.4375,
      "loss": 0.581,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": 0.48107606172561646,
      "rewards/margins": 1.0627930164337158,
      "rewards/rejected": -0.581591784954071,
      "step": 590
    },
    {
      "epoch": 0.3862246540070808,
      "grad_norm": 101.2501221806809,
      "learning_rate": 9.035737282678686e-07,
      "logits/chosen": -0.606689453125,
      "logits/rejected": -0.617114245891571,
      "logps/chosen": -325.23748779296875,
      "logps/rejected": -309.0874938964844,
      "loss": 0.599,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.31675416231155396,
      "rewards/margins": 0.9888916015625,
      "rewards/rejected": -0.671948254108429,
      "step": 600
    },
    {
      "epoch": 0.39266173157386547,
      "grad_norm": 112.02032310278221,
      "learning_rate": 9.019639407598196e-07,
      "logits/chosen": -0.5888427495956421,
      "logits/rejected": -0.611279308795929,
      "logps/chosen": -370.3500061035156,
      "logps/rejected": -363.8500061035156,
      "loss": 0.5333,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": 0.5602661371231079,
      "rewards/margins": 1.0866515636444092,
      "rewards/rejected": -0.5262695550918579,
      "step": 610
    },
    {
      "epoch": 0.39909880914065016,
      "grad_norm": 116.1836878867331,
      "learning_rate": 9.003541532517708e-07,
      "logits/chosen": -0.561572253704071,
      "logits/rejected": -0.577441394329071,
      "logps/chosen": -362.29998779296875,
      "logps/rejected": -352.7250061035156,
      "loss": 0.6513,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": 0.3332366943359375,
      "rewards/margins": 0.7449706792831421,
      "rewards/rejected": -0.4120422303676605,
      "step": 620
    },
    {
      "epoch": 0.40553588670743485,
      "grad_norm": 119.84221150272923,
      "learning_rate": 8.987443657437218e-07,
      "logits/chosen": -0.551226794719696,
      "logits/rejected": -0.5623534917831421,
      "logps/chosen": -386.70001220703125,
      "logps/rejected": -373.07501220703125,
      "loss": 0.6302,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.28680419921875,
      "rewards/margins": 0.9903320074081421,
      "rewards/rejected": -1.2776367664337158,
      "step": 630
    },
    {
      "epoch": 0.4119729642742195,
      "grad_norm": 119.20355705473106,
      "learning_rate": 8.971345782356729e-07,
      "logits/chosen": -0.5877685546875,
      "logits/rejected": -0.6196533441543579,
      "logps/chosen": -386.4750061035156,
      "logps/rejected": -392.2250061035156,
      "loss": 0.618,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.45903319120407104,
      "rewards/margins": 0.8375488519668579,
      "rewards/rejected": -1.2969481945037842,
      "step": 640
    },
    {
      "epoch": 0.41841004184100417,
      "grad_norm": 114.39134560983841,
      "learning_rate": 8.955247907276239e-07,
      "logits/chosen": -0.4747314453125,
      "logits/rejected": -0.520581066608429,
      "logps/chosen": -361.45001220703125,
      "logps/rejected": -340.42498779296875,
      "loss": 0.6872,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.09818115085363388,
      "rewards/margins": 0.6923888921737671,
      "rewards/rejected": -0.791015625,
      "step": 650
    },
    {
      "epoch": 0.42484711940778885,
      "grad_norm": 107.98675915515811,
      "learning_rate": 8.93915003219575e-07,
      "logits/chosen": -0.539172351360321,
      "logits/rejected": -0.5562499761581421,
      "logps/chosen": -381.54998779296875,
      "logps/rejected": -356.75,
      "loss": 0.551,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": 0.00932922400534153,
      "rewards/margins": 1.1506836414337158,
      "rewards/rejected": -1.142364501953125,
      "step": 660
    },
    {
      "epoch": 0.43128419697457354,
      "grad_norm": 103.2010559979854,
      "learning_rate": 8.923052157115261e-07,
      "logits/chosen": -0.528704822063446,
      "logits/rejected": -0.556774914264679,
      "logps/chosen": -388.6000061035156,
      "logps/rejected": -368.70001220703125,
      "loss": 0.5941,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.13016967475414276,
      "rewards/margins": 1.0416138172149658,
      "rewards/rejected": -1.172509789466858,
      "step": 670
    },
    {
      "epoch": 0.43772127454135823,
      "grad_norm": 123.08466573292604,
      "learning_rate": 8.90695428203477e-07,
      "logits/chosen": -0.5683318972587585,
      "logits/rejected": -0.575610339641571,
      "logps/chosen": -399.875,
      "logps/rejected": -361.45001220703125,
      "loss": 0.5778,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.6572631597518921,
      "rewards/margins": 1.0535399913787842,
      "rewards/rejected": -1.7103271484375,
      "step": 680
    },
    {
      "epoch": 0.4441583521081429,
      "grad_norm": 83.98215014096604,
      "learning_rate": 8.890856406954281e-07,
      "logits/chosen": -0.523547351360321,
      "logits/rejected": -0.5286010503768921,
      "logps/chosen": -384.7749938964844,
      "logps/rejected": -357.57501220703125,
      "loss": 0.5256,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.02643432654440403,
      "rewards/margins": 1.292199730873108,
      "rewards/rejected": -1.3194580078125,
      "step": 690
    },
    {
      "epoch": 0.4505954296749276,
      "grad_norm": 104.87203994034158,
      "learning_rate": 8.874758531873793e-07,
      "logits/chosen": -0.538464367389679,
      "logits/rejected": -0.5656677484512329,
      "logps/chosen": -364.5625,
      "logps/rejected": -369.8999938964844,
      "loss": 0.4687,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": 0.04945068433880806,
      "rewards/margins": 1.432836890220642,
      "rewards/rejected": -1.3836181163787842,
      "step": 700
    },
    {
      "epoch": 0.45703250724171224,
      "grad_norm": 57.76406203268148,
      "learning_rate": 8.858660656793303e-07,
      "logits/chosen": -0.512127697467804,
      "logits/rejected": -0.5491943359375,
      "logps/chosen": -405.0,
      "logps/rejected": -395.3374938964844,
      "loss": 0.5798,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.303070068359375,
      "rewards/margins": 1.1065673828125,
      "rewards/rejected": -0.8034912347793579,
      "step": 710
    },
    {
      "epoch": 0.4634695848084969,
      "grad_norm": 69.89352328895056,
      "learning_rate": 8.842562781712813e-07,
      "logits/chosen": -0.4332519471645355,
      "logits/rejected": -0.45560914278030396,
      "logps/chosen": -350.20001220703125,
      "logps/rejected": -314.125,
      "loss": 0.6324,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.3115478456020355,
      "rewards/margins": 0.9028564691543579,
      "rewards/rejected": -0.591540515422821,
      "step": 720
    },
    {
      "epoch": 0.4699066623752816,
      "grad_norm": 83.67294747788176,
      "learning_rate": 8.826464906632324e-07,
      "logits/chosen": -0.51568603515625,
      "logits/rejected": -0.5252624750137329,
      "logps/chosen": -381.92498779296875,
      "logps/rejected": -349.32501220703125,
      "loss": 0.6184,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.4420166015625,
      "rewards/margins": 0.99859619140625,
      "rewards/rejected": -0.556780993938446,
      "step": 730
    },
    {
      "epoch": 0.4763437399420663,
      "grad_norm": 104.60759299733223,
      "learning_rate": 8.810367031551835e-07,
      "logits/chosen": -0.42892152070999146,
      "logits/rejected": -0.4578002989292145,
      "logps/chosen": -409.29998779296875,
      "logps/rejected": -351.3500061035156,
      "loss": 0.5794,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": 0.088287353515625,
      "rewards/margins": 1.06494140625,
      "rewards/rejected": -0.9759521484375,
      "step": 740
    },
    {
      "epoch": 0.482780817508851,
      "grad_norm": 70.65265994679216,
      "learning_rate": 8.794269156471346e-07,
      "logits/chosen": -0.4439941346645355,
      "logits/rejected": -0.502819836139679,
      "logps/chosen": -358.51251220703125,
      "logps/rejected": -343.7250061035156,
      "loss": 0.603,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.317587286233902,
      "rewards/margins": 1.092810034751892,
      "rewards/rejected": -0.774609386920929,
      "step": 750
    },
    {
      "epoch": 0.4892178950756357,
      "grad_norm": 99.48553520919884,
      "learning_rate": 8.778171281390856e-07,
      "logits/chosen": -0.454559326171875,
      "logits/rejected": -0.46809083223342896,
      "logps/chosen": -387.20001220703125,
      "logps/rejected": -361.875,
      "loss": 0.591,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": 0.2822631895542145,
      "rewards/margins": 0.992480456829071,
      "rewards/rejected": -0.710858166217804,
      "step": 760
    },
    {
      "epoch": 0.49565497264242037,
      "grad_norm": 107.63172335214213,
      "learning_rate": 8.762073406310366e-07,
      "logits/chosen": -0.5004516839981079,
      "logits/rejected": -0.5167480707168579,
      "logps/chosen": -349.20001220703125,
      "logps/rejected": -351.1499938964844,
      "loss": 0.6649,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.29923707246780396,
      "rewards/margins": 0.953686535358429,
      "rewards/rejected": -1.253198266029358,
      "step": 770
    },
    {
      "epoch": 0.502092050209205,
      "grad_norm": 112.79451532298022,
      "learning_rate": 8.745975531229878e-07,
      "logits/chosen": -0.50775146484375,
      "logits/rejected": -0.5383545160293579,
      "logps/chosen": -372.2875061035156,
      "logps/rejected": -374.7749938964844,
      "loss": 0.6127,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.543988049030304,
      "rewards/margins": 1.2046020030975342,
      "rewards/rejected": -1.748632788658142,
      "step": 780
    },
    {
      "epoch": 0.5085291277759897,
      "grad_norm": 92.84930514639605,
      "learning_rate": 8.729877656149389e-07,
      "logits/chosen": -0.511303722858429,
      "logits/rejected": -0.52484130859375,
      "logps/chosen": -356.45001220703125,
      "logps/rejected": -347.0249938964844,
      "loss": 0.5365,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.19889526069164276,
      "rewards/margins": 1.256262183189392,
      "rewards/rejected": -1.454187035560608,
      "step": 790
    },
    {
      "epoch": 0.5149662053427744,
      "grad_norm": 118.08576455579058,
      "learning_rate": 8.713779781068898e-07,
      "logits/chosen": -0.526232898235321,
      "logits/rejected": -0.544995129108429,
      "logps/chosen": -344.3374938964844,
      "logps/rejected": -359.29998779296875,
      "loss": 0.567,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.2644104063510895,
      "rewards/margins": 1.2300536632537842,
      "rewards/rejected": -1.4940307140350342,
      "step": 800
    },
    {
      "epoch": 0.5214032829095591,
      "grad_norm": 149.5051687226972,
      "learning_rate": 8.697681905988409e-07,
      "logits/chosen": -0.533679187297821,
      "logits/rejected": -0.546582043170929,
      "logps/chosen": -416.1000061035156,
      "logps/rejected": -381.8500061035156,
      "loss": 0.6592,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.562670886516571,
      "rewards/margins": 0.9259033203125,
      "rewards/rejected": -1.48779296875,
      "step": 810
    },
    {
      "epoch": 0.5278403604763438,
      "grad_norm": 116.30993199121478,
      "learning_rate": 8.681584030907921e-07,
      "logits/chosen": -0.548510730266571,
      "logits/rejected": -0.5396972894668579,
      "logps/chosen": -415.82501220703125,
      "logps/rejected": -355.17498779296875,
      "loss": 0.604,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.35222166776657104,
      "rewards/margins": 0.830737292766571,
      "rewards/rejected": -1.182348608970642,
      "step": 820
    },
    {
      "epoch": 0.5342774380431284,
      "grad_norm": 88.88351113445877,
      "learning_rate": 8.66548615582743e-07,
      "logits/chosen": -0.41846925020217896,
      "logits/rejected": -0.44677734375,
      "logps/chosen": -355.38751220703125,
      "logps/rejected": -348.2749938964844,
      "loss": 0.7307,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.31745606660842896,
      "rewards/margins": 0.7986205816268921,
      "rewards/rejected": -1.1161620616912842,
      "step": 830
    },
    {
      "epoch": 0.5407145156099131,
      "grad_norm": 141.95886167076992,
      "learning_rate": 8.649388280746941e-07,
      "logits/chosen": -0.4906982481479645,
      "logits/rejected": -0.5586181879043579,
      "logps/chosen": -411.54998779296875,
      "logps/rejected": -367.82501220703125,
      "loss": 0.6348,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": 0.28123778104782104,
      "rewards/margins": 1.0207641124725342,
      "rewards/rejected": -0.7385803461074829,
      "step": 840
    },
    {
      "epoch": 0.5471515931766978,
      "grad_norm": 119.67858559796636,
      "learning_rate": 8.633290405666451e-07,
      "logits/chosen": -0.53289794921875,
      "logits/rejected": -0.526135265827179,
      "logps/chosen": -361.0,
      "logps/rejected": -353.1000061035156,
      "loss": 0.5787,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.18852539360523224,
      "rewards/margins": 1.079687476158142,
      "rewards/rejected": -1.2688720226287842,
      "step": 850
    },
    {
      "epoch": 0.5535886707434825,
      "grad_norm": 106.12475270901847,
      "learning_rate": 8.617192530585963e-07,
      "logits/chosen": -0.5385986566543579,
      "logits/rejected": -0.542651355266571,
      "logps/chosen": -427.5375061035156,
      "logps/rejected": -417.3999938964844,
      "loss": 0.6631,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.42586058378219604,
      "rewards/margins": 1.089013695716858,
      "rewards/rejected": -1.515173316001892,
      "step": 860
    },
    {
      "epoch": 0.5600257483102672,
      "grad_norm": 98.81134073733207,
      "learning_rate": 8.601094655505473e-07,
      "logits/chosen": -0.5274902582168579,
      "logits/rejected": -0.522229015827179,
      "logps/chosen": -392.29998779296875,
      "logps/rejected": -394.5249938964844,
      "loss": 0.613,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.3875488340854645,
      "rewards/margins": 1.0703125,
      "rewards/rejected": -1.4576904773712158,
      "step": 870
    },
    {
      "epoch": 0.5664628258770518,
      "grad_norm": 120.13198690851658,
      "learning_rate": 8.584996780424983e-07,
      "logits/chosen": -0.55718994140625,
      "logits/rejected": -0.553845226764679,
      "logps/chosen": -369.82501220703125,
      "logps/rejected": -343.07501220703125,
      "loss": 0.639,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.23508301377296448,
      "rewards/margins": 0.7032836675643921,
      "rewards/rejected": -0.939038097858429,
      "step": 880
    },
    {
      "epoch": 0.5728999034438365,
      "grad_norm": 62.94101570205876,
      "learning_rate": 8.568898905344494e-07,
      "logits/chosen": -0.4923339784145355,
      "logits/rejected": -0.5071777105331421,
      "logps/chosen": -350.5874938964844,
      "logps/rejected": -328.25,
      "loss": 0.602,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.03252258151769638,
      "rewards/margins": 0.9979614019393921,
      "rewards/rejected": -0.9653381109237671,
      "step": 890
    },
    {
      "epoch": 0.5793369810106211,
      "grad_norm": 112.45942278359949,
      "learning_rate": 8.552801030264006e-07,
      "logits/chosen": -0.5096191167831421,
      "logits/rejected": -0.4962158203125,
      "logps/chosen": -376.375,
      "logps/rejected": -356.29998779296875,
      "loss": 0.6255,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.581451416015625,
      "rewards/margins": 0.798583984375,
      "rewards/rejected": -1.37945556640625,
      "step": 900
    },
    {
      "epoch": 0.5857740585774058,
      "grad_norm": 108.90325211362334,
      "learning_rate": 8.536703155183515e-07,
      "logits/chosen": -0.552380383014679,
      "logits/rejected": -0.5654449462890625,
      "logps/chosen": -417.51251220703125,
      "logps/rejected": -411.625,
      "loss": 0.54,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.811279296875,
      "rewards/margins": 1.20068359375,
      "rewards/rejected": -2.0130858421325684,
      "step": 910
    },
    {
      "epoch": 0.5922111361441905,
      "grad_norm": 98.0959184069916,
      "learning_rate": 8.520605280103026e-07,
      "logits/chosen": -0.5830078125,
      "logits/rejected": -0.6092773675918579,
      "logps/chosen": -353.6625061035156,
      "logps/rejected": -364.67498779296875,
      "loss": 0.6542,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.9505707025527954,
      "rewards/margins": 0.778289794921875,
      "rewards/rejected": -1.729894995689392,
      "step": 920
    },
    {
      "epoch": 0.5986482137109752,
      "grad_norm": 99.99889212589564,
      "learning_rate": 8.504507405022537e-07,
      "logits/chosen": -0.47906494140625,
      "logits/rejected": -0.4897399842739105,
      "logps/chosen": -379.625,
      "logps/rejected": -377.2250061035156,
      "loss": 0.6211,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.747698962688446,
      "rewards/margins": 1.053491234779358,
      "rewards/rejected": -1.801123023033142,
      "step": 930
    },
    {
      "epoch": 0.6050852912777599,
      "grad_norm": 89.85940157465599,
      "learning_rate": 8.488409529942047e-07,
      "logits/chosen": -0.526043713092804,
      "logits/rejected": -0.553393542766571,
      "logps/chosen": -363.2250061035156,
      "logps/rejected": -347.8999938964844,
      "loss": 0.5999,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.332254022359848,
      "rewards/margins": 1.034509301185608,
      "rewards/rejected": -1.367010474205017,
      "step": 940
    },
    {
      "epoch": 0.6115223688445446,
      "grad_norm": 70.14280663264151,
      "learning_rate": 8.472311654861558e-07,
      "logits/chosen": -0.6128768920898438,
      "logits/rejected": -0.6567413210868835,
      "logps/chosen": -391.5,
      "logps/rejected": -375.8500061035156,
      "loss": 0.5634,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.06875000149011612,
      "rewards/margins": 1.0836913585662842,
      "rewards/rejected": -1.152441382408142,
      "step": 950
    },
    {
      "epoch": 0.6179594464113293,
      "grad_norm": 120.47345616509051,
      "learning_rate": 8.456213779781069e-07,
      "logits/chosen": -0.5240844488143921,
      "logits/rejected": -0.5285888910293579,
      "logps/chosen": -332.4125061035156,
      "logps/rejected": -343.63751220703125,
      "loss": 0.6299,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.15395812690258026,
      "rewards/margins": 1.132959008216858,
      "rewards/rejected": -1.2863037586212158,
      "step": 960
    },
    {
      "epoch": 0.624396523978114,
      "grad_norm": 92.26766562500985,
      "learning_rate": 8.440115904700579e-07,
      "logits/chosen": -0.578857421875,
      "logits/rejected": -0.598950207233429,
      "logps/chosen": -400.1000061035156,
      "logps/rejected": -380.45001220703125,
      "loss": 0.6269,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.084320068359375,
      "rewards/margins": 1.0315673351287842,
      "rewards/rejected": -0.947705090045929,
      "step": 970
    },
    {
      "epoch": 0.6308336015448986,
      "grad_norm": 85.69917047655541,
      "learning_rate": 8.42401802962009e-07,
      "logits/chosen": -0.6307373046875,
      "logits/rejected": -0.6424804925918579,
      "logps/chosen": -399.7749938964844,
      "logps/rejected": -380.07501220703125,
      "loss": 0.5451,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.20753173530101776,
      "rewards/margins": 1.097631812095642,
      "rewards/rejected": -1.3062012195587158,
      "step": 980
    },
    {
      "epoch": 0.6372706791116833,
      "grad_norm": 99.91368279665215,
      "learning_rate": 8.4079201545396e-07,
      "logits/chosen": -0.53326416015625,
      "logits/rejected": -0.5157104730606079,
      "logps/chosen": -370.75,
      "logps/rejected": -344.92498779296875,
      "loss": 0.6636,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.07122802734375,
      "rewards/margins": 0.891162097454071,
      "rewards/rejected": -0.9619506597518921,
      "step": 990
    },
    {
      "epoch": 0.643707756678468,
      "grad_norm": 111.09429678604513,
      "learning_rate": 8.391822279459111e-07,
      "logits/chosen": -0.5557830929756165,
      "logits/rejected": -0.578845202922821,
      "logps/chosen": -386.07501220703125,
      "logps/rejected": -370.2250061035156,
      "loss": 0.5953,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.06235351413488388,
      "rewards/margins": 1.00732421875,
      "rewards/rejected": -1.069116234779358,
      "step": 1000
    },
    {
      "epoch": 0.6501448342452526,
      "grad_norm": 108.22353551131003,
      "learning_rate": 8.375724404378622e-07,
      "logits/chosen": -0.5925537347793579,
      "logits/rejected": -0.598681628704071,
      "logps/chosen": -346.75,
      "logps/rejected": -322.8999938964844,
      "loss": 0.6265,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.20738525688648224,
      "rewards/margins": 0.9044555425643921,
      "rewards/rejected": -1.1110718250274658,
      "step": 1010
    },
    {
      "epoch": 0.6565819118120373,
      "grad_norm": 84.63983202639838,
      "learning_rate": 8.359626529298132e-07,
      "logits/chosen": -0.501538097858429,
      "logits/rejected": -0.47550660371780396,
      "logps/chosen": -374.3500061035156,
      "logps/rejected": -349.63751220703125,
      "loss": 0.5603,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.20573119819164276,
      "rewards/margins": 0.952099621295929,
      "rewards/rejected": -1.15771484375,
      "step": 1020
    },
    {
      "epoch": 0.663018989378822,
      "grad_norm": 94.59002794744202,
      "learning_rate": 8.343528654217643e-07,
      "logits/chosen": -0.45048826932907104,
      "logits/rejected": -0.44732666015625,
      "logps/chosen": -379.25,
      "logps/rejected": -379.625,
      "loss": 0.6549,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.4843383729457855,
      "rewards/margins": 0.846728503704071,
      "rewards/rejected": -1.330957055091858,
      "step": 1030
    },
    {
      "epoch": 0.6694560669456067,
      "grad_norm": 128.4884473761372,
      "learning_rate": 8.327430779137154e-07,
      "logits/chosen": -0.5034545660018921,
      "logits/rejected": -0.524914562702179,
      "logps/chosen": -385.2749938964844,
      "logps/rejected": -350.29998779296875,
      "loss": 0.5829,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.407562255859375,
      "rewards/margins": 1.02783203125,
      "rewards/rejected": -1.435888648033142,
      "step": 1040
    },
    {
      "epoch": 0.6758931445123914,
      "grad_norm": 107.25803443764735,
      "learning_rate": 8.311332904056663e-07,
      "logits/chosen": -0.46173095703125,
      "logits/rejected": -0.5187743902206421,
      "logps/chosen": -372.2749938964844,
      "logps/rejected": -341.04998779296875,
      "loss": 0.6023,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.3631958067417145,
      "rewards/margins": 1.004797339439392,
      "rewards/rejected": -1.368017554283142,
      "step": 1050
    },
    {
      "epoch": 0.682330222079176,
      "grad_norm": 126.80827854824096,
      "learning_rate": 8.295235028976175e-07,
      "logits/chosen": -0.47755128145217896,
      "logits/rejected": -0.509960949420929,
      "logps/chosen": -350.42498779296875,
      "logps/rejected": -357.9375,
      "loss": 0.6958,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.19310760498046875,
      "rewards/margins": 0.7228027582168579,
      "rewards/rejected": -0.9160064458847046,
      "step": 1060
    },
    {
      "epoch": 0.6887672996459607,
      "grad_norm": 81.10373132613064,
      "learning_rate": 8.279137153895686e-07,
      "logits/chosen": -0.42036741971969604,
      "logits/rejected": -0.4339141845703125,
      "logps/chosen": -385.1499938964844,
      "logps/rejected": -373.45001220703125,
      "loss": 0.6191,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.156494140625,
      "rewards/margins": 0.8669067621231079,
      "rewards/rejected": -1.024011254310608,
      "step": 1070
    },
    {
      "epoch": 0.6952043772127454,
      "grad_norm": 54.410337891839184,
      "learning_rate": 8.263039278815196e-07,
      "logits/chosen": -0.47700196504592896,
      "logits/rejected": -0.5096435546875,
      "logps/chosen": -364.7749938964844,
      "logps/rejected": -334.29998779296875,
      "loss": 0.4926,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": 0.12125243991613388,
      "rewards/margins": 1.276452660560608,
      "rewards/rejected": -1.155175805091858,
      "step": 1080
    },
    {
      "epoch": 0.7016414547795301,
      "grad_norm": 103.66262035258725,
      "learning_rate": 8.246941403734706e-07,
      "logits/chosen": -0.45051270723342896,
      "logits/rejected": -0.4598144590854645,
      "logps/chosen": -361.4750061035156,
      "logps/rejected": -396.45001220703125,
      "loss": 0.6205,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.45460206270217896,
      "rewards/margins": 1.1227295398712158,
      "rewards/rejected": -1.5778076648712158,
      "step": 1090
    },
    {
      "epoch": 0.7080785323463148,
      "grad_norm": 110.31202907587142,
      "learning_rate": 8.230843528654218e-07,
      "logits/chosen": -0.36857908964157104,
      "logits/rejected": -0.4070068299770355,
      "logps/chosen": -347.8500061035156,
      "logps/rejected": -325.8999938964844,
      "loss": 0.6092,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.662280261516571,
      "rewards/margins": 1.011010766029358,
      "rewards/rejected": -1.6741943359375,
      "step": 1100
    },
    {
      "epoch": 0.7145156099130995,
      "grad_norm": 113.5610108186264,
      "learning_rate": 8.214745653573728e-07,
      "logits/chosen": -0.45977783203125,
      "logits/rejected": -0.4886474609375,
      "logps/chosen": -422.75,
      "logps/rejected": -394.92498779296875,
      "loss": 0.5894,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.7705932855606079,
      "rewards/margins": 1.0309569835662842,
      "rewards/rejected": -1.8014647960662842,
      "step": 1110
    },
    {
      "epoch": 0.7209526874798842,
      "grad_norm": 123.78507873787207,
      "learning_rate": 8.198647778493239e-07,
      "logits/chosen": -0.48577880859375,
      "logits/rejected": -0.5226074457168579,
      "logps/chosen": -393.75,
      "logps/rejected": -369.07501220703125,
      "loss": 0.5765,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.38194578886032104,
      "rewards/margins": 1.10162353515625,
      "rewards/rejected": -1.484594702720642,
      "step": 1120
    },
    {
      "epoch": 0.7273897650466689,
      "grad_norm": 127.99687246016022,
      "learning_rate": 8.182549903412748e-07,
      "logits/chosen": -0.4909118711948395,
      "logits/rejected": -0.4707275331020355,
      "logps/chosen": -353.5249938964844,
      "logps/rejected": -350.3500061035156,
      "loss": 0.6522,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.3532470762729645,
      "rewards/margins": 0.9039306640625,
      "rewards/rejected": -1.257135033607483,
      "step": 1130
    },
    {
      "epoch": 0.7338268426134535,
      "grad_norm": 100.74894669506055,
      "learning_rate": 8.16645202833226e-07,
      "logits/chosen": -0.4802612364292145,
      "logits/rejected": -0.5090576410293579,
      "logps/chosen": -360.29998779296875,
      "logps/rejected": -378.67498779296875,
      "loss": 0.6234,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.32078856229782104,
      "rewards/margins": 1.0515868663787842,
      "rewards/rejected": -1.372583031654358,
      "step": 1140
    },
    {
      "epoch": 0.7402639201802381,
      "grad_norm": 124.88195018416995,
      "learning_rate": 8.150354153251771e-07,
      "logits/chosen": -0.46788328886032104,
      "logits/rejected": -0.4967285096645355,
      "logps/chosen": -366.98748779296875,
      "logps/rejected": -367.8999938964844,
      "loss": 0.6054,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.23341064155101776,
      "rewards/margins": 1.050317406654358,
      "rewards/rejected": -0.817883312702179,
      "step": 1150
    },
    {
      "epoch": 0.7467009977470228,
      "grad_norm": 115.48073006138456,
      "learning_rate": 8.13425627817128e-07,
      "logits/chosen": -0.48234254121780396,
      "logits/rejected": -0.49266356229782104,
      "logps/chosen": -372.6499938964844,
      "logps/rejected": -340.3999938964844,
      "loss": 0.538,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.525585949420929,
      "rewards/margins": 1.1814696788787842,
      "rewards/rejected": -0.655688464641571,
      "step": 1160
    },
    {
      "epoch": 0.7531380753138075,
      "grad_norm": 85.01864055690064,
      "learning_rate": 8.118158403090791e-07,
      "logits/chosen": -0.3582519590854645,
      "logits/rejected": -0.3791137635707855,
      "logps/chosen": -382.1000061035156,
      "logps/rejected": -345.29998779296875,
      "loss": 0.6241,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": 0.05527343600988388,
      "rewards/margins": 1.0609130859375,
      "rewards/rejected": -1.0050170421600342,
      "step": 1170
    },
    {
      "epoch": 0.7595751528805922,
      "grad_norm": 114.9441718662302,
      "learning_rate": 8.102060528010303e-07,
      "logits/chosen": -0.46600341796875,
      "logits/rejected": -0.4788574278354645,
      "logps/chosen": -399.51251220703125,
      "logps/rejected": -397.3500061035156,
      "loss": 0.5901,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.261474609375,
      "rewards/margins": 1.3340332508087158,
      "rewards/rejected": -1.596215844154358,
      "step": 1180
    },
    {
      "epoch": 0.7660122304473769,
      "grad_norm": 108.13993143362673,
      "learning_rate": 8.085962652929813e-07,
      "logits/chosen": -0.45205992460250854,
      "logits/rejected": -0.4634765684604645,
      "logps/chosen": -386.98748779296875,
      "logps/rejected": -358.625,
      "loss": 0.6667,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.49949342012405396,
      "rewards/margins": 1.317773461341858,
      "rewards/rejected": -1.817724585533142,
      "step": 1190
    },
    {
      "epoch": 0.7724493080141616,
      "grad_norm": 147.08462386802262,
      "learning_rate": 8.069864777849323e-07,
      "logits/chosen": -0.49937134981155396,
      "logits/rejected": -0.505419909954071,
      "logps/chosen": -369.0625,
      "logps/rejected": -382.4750061035156,
      "loss": 0.7564,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.2877746522426605,
      "rewards/margins": 0.7278076410293579,
      "rewards/rejected": -1.015435814857483,
      "step": 1200
    },
    {
      "epoch": 0.7788863855809462,
      "grad_norm": 115.55623695277468,
      "learning_rate": 8.053766902768834e-07,
      "logits/chosen": -0.4474243223667145,
      "logits/rejected": -0.46574705839157104,
      "logps/chosen": -355.86248779296875,
      "logps/rejected": -345.07501220703125,
      "loss": 0.569,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.0020507811568677425,
      "rewards/margins": 0.920788586139679,
      "rewards/rejected": -0.917816162109375,
      "step": 1210
    },
    {
      "epoch": 0.7853234631477309,
      "grad_norm": 98.71710653719403,
      "learning_rate": 8.037669027688345e-07,
      "logits/chosen": -0.484619140625,
      "logits/rejected": -0.47932130098342896,
      "logps/chosen": -372.7250061035156,
      "logps/rejected": -371.20001220703125,
      "loss": 0.5893,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.21046753227710724,
      "rewards/margins": 0.9305664300918579,
      "rewards/rejected": -1.142114281654358,
      "step": 1220
    },
    {
      "epoch": 0.7917605407145156,
      "grad_norm": 116.58967696349914,
      "learning_rate": 8.021571152607856e-07,
      "logits/chosen": -0.41608887910842896,
      "logits/rejected": -0.4575439393520355,
      "logps/chosen": -374.63751220703125,
      "logps/rejected": -356.92498779296875,
      "loss": 0.6804,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.17164306342601776,
      "rewards/margins": 0.873791515827179,
      "rewards/rejected": -1.045166015625,
      "step": 1230
    },
    {
      "epoch": 0.7981976182813003,
      "grad_norm": 89.90212458778099,
      "learning_rate": 8.005473277527366e-07,
      "logits/chosen": -0.47612303495407104,
      "logits/rejected": -0.46992188692092896,
      "logps/chosen": -382.98748779296875,
      "logps/rejected": -376.42498779296875,
      "loss": 0.6204,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.11334228515625,
      "rewards/margins": 1.0497925281524658,
      "rewards/rejected": -1.1633727550506592,
      "step": 1240
    },
    {
      "epoch": 0.804634695848085,
      "grad_norm": 113.94650452614209,
      "learning_rate": 7.989375402446876e-07,
      "logits/chosen": -0.4763244688510895,
      "logits/rejected": -0.49517822265625,
      "logps/chosen": -347.23748779296875,
      "logps/rejected": -325.0249938964844,
      "loss": 0.623,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.11918945610523224,
      "rewards/margins": 0.8921142816543579,
      "rewards/rejected": -1.0122039318084717,
      "step": 1250
    },
    {
      "epoch": 0.8110717734148697,
      "grad_norm": 98.14429996042087,
      "learning_rate": 7.973277527366388e-07,
      "logits/chosen": -0.4635772705078125,
      "logits/rejected": -0.4755493104457855,
      "logps/chosen": -373.70001220703125,
      "logps/rejected": -357.25,
      "loss": 0.6548,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.548840343952179,
      "rewards/margins": 0.88629150390625,
      "rewards/rejected": -1.435449242591858,
      "step": 1260
    },
    {
      "epoch": 0.8175088509816544,
      "grad_norm": 97.0864968634607,
      "learning_rate": 7.957179652285899e-07,
      "logits/chosen": -0.525805652141571,
      "logits/rejected": -0.510266125202179,
      "logps/chosen": -354.6499938964844,
      "logps/rejected": -351.95001220703125,
      "loss": 0.5806,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.37434083223342896,
      "rewards/margins": 1.059057593345642,
      "rewards/rejected": -1.4339721202850342,
      "step": 1270
    },
    {
      "epoch": 0.823945928548439,
      "grad_norm": 81.72733031375124,
      "learning_rate": 7.941081777205408e-07,
      "logits/chosen": -0.47092896699905396,
      "logits/rejected": -0.47998350858688354,
      "logps/chosen": -387.8374938964844,
      "logps/rejected": -374.79998779296875,
      "loss": 0.5909,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.31041258573532104,
      "rewards/margins": 1.3404357433319092,
      "rewards/rejected": -1.651647925376892,
      "step": 1280
    },
    {
      "epoch": 0.8303830061152236,
      "grad_norm": 61.78251081973873,
      "learning_rate": 7.924983902124919e-07,
      "logits/chosen": -0.3810485899448395,
      "logits/rejected": -0.41566163301467896,
      "logps/chosen": -376.1000061035156,
      "logps/rejected": -358.54998779296875,
      "loss": 0.5402,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.06690673530101776,
      "rewards/margins": 1.2426269054412842,
      "rewards/rejected": -1.310052514076233,
      "step": 1290
    },
    {
      "epoch": 0.8368200836820083,
      "grad_norm": 109.83631710755957,
      "learning_rate": 7.908886027044431e-07,
      "logits/chosen": -0.42792969942092896,
      "logits/rejected": -0.45692139863967896,
      "logps/chosen": -384.875,
      "logps/rejected": -376.32501220703125,
      "loss": 0.6567,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.03769531100988388,
      "rewards/margins": 1.0447266101837158,
      "rewards/rejected": -1.082800269126892,
      "step": 1300
    },
    {
      "epoch": 0.843257161248793,
      "grad_norm": 91.84896650553513,
      "learning_rate": 7.89278815196394e-07,
      "logits/chosen": -0.36790770292282104,
      "logits/rejected": -0.3922485411167145,
      "logps/chosen": -359.7749938964844,
      "logps/rejected": -365.0249938964844,
      "loss": 0.5082,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.12186889350414276,
      "rewards/margins": 1.2075378894805908,
      "rewards/rejected": -1.328283667564392,
      "step": 1310
    },
    {
      "epoch": 0.8496942388155777,
      "grad_norm": 123.08939858015702,
      "learning_rate": 7.876690276883451e-07,
      "logits/chosen": -0.4756835997104645,
      "logits/rejected": -0.5056854486465454,
      "logps/chosen": -392.625,
      "logps/rejected": -380.5249938964844,
      "loss": 0.6656,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.29423826932907104,
      "rewards/margins": 0.930346667766571,
      "rewards/rejected": -1.225061058998108,
      "step": 1320
    },
    {
      "epoch": 0.8561313163823624,
      "grad_norm": 89.38220487310218,
      "learning_rate": 7.860592401802961e-07,
      "logits/chosen": -0.5153290033340454,
      "logits/rejected": -0.5448852777481079,
      "logps/chosen": -406.92498779296875,
      "logps/rejected": -377.625,
      "loss": 0.6152,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.2519470155239105,
      "rewards/margins": 0.8592529296875,
      "rewards/rejected": -1.1110961437225342,
      "step": 1330
    },
    {
      "epoch": 0.8625683939491471,
      "grad_norm": 138.69932522724545,
      "learning_rate": 7.844494526722473e-07,
      "logits/chosen": -0.43743896484375,
      "logits/rejected": -0.45673829317092896,
      "logps/chosen": -381.0375061035156,
      "logps/rejected": -372.75,
      "loss": 0.6838,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.09254150092601776,
      "rewards/margins": 0.789379894733429,
      "rewards/rejected": -0.882275402545929,
      "step": 1340
    },
    {
      "epoch": 0.8690054715159318,
      "grad_norm": 101.36114737984545,
      "learning_rate": 7.828396651641983e-07,
      "logits/chosen": -0.5053650140762329,
      "logits/rejected": -0.511737048625946,
      "logps/chosen": -386.0874938964844,
      "logps/rejected": -340.79376220703125,
      "loss": 0.6082,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": 0.2793136537075043,
      "rewards/margins": 1.0363891124725342,
      "rewards/rejected": -0.7570556402206421,
      "step": 1350
    },
    {
      "epoch": 0.8754425490827165,
      "grad_norm": 56.94113918190888,
      "learning_rate": 7.812298776561493e-07,
      "logits/chosen": -0.45692139863967896,
      "logits/rejected": -0.47791749238967896,
      "logps/chosen": -400.7250061035156,
      "logps/rejected": -383.375,
      "loss": 0.5721,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.06065673753619194,
      "rewards/margins": 1.01776123046875,
      "rewards/rejected": -0.9569336175918579,
      "step": 1360
    },
    {
      "epoch": 0.8818796266495011,
      "grad_norm": 87.21788738477099,
      "learning_rate": 7.796200901481004e-07,
      "logits/chosen": -0.43765562772750854,
      "logits/rejected": -0.4749755859375,
      "logps/chosen": -340.1000061035156,
      "logps/rejected": -339.7250061035156,
      "loss": 0.7079,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.2620178163051605,
      "rewards/margins": 0.7551513910293579,
      "rewards/rejected": -1.0172851085662842,
      "step": 1370
    },
    {
      "epoch": 0.8883167042162858,
      "grad_norm": 116.26399959827697,
      "learning_rate": 7.780103026400516e-07,
      "logits/chosen": -0.457510381937027,
      "logits/rejected": -0.48211669921875,
      "logps/chosen": -376.32501220703125,
      "logps/rejected": -366.95001220703125,
      "loss": 0.7661,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.46033018827438354,
      "rewards/margins": 0.6608642339706421,
      "rewards/rejected": -1.121118187904358,
      "step": 1380
    },
    {
      "epoch": 0.8947537817830705,
      "grad_norm": 70.21186430940142,
      "learning_rate": 7.764005151320025e-07,
      "logits/chosen": -0.47960203886032104,
      "logits/rejected": -0.517138659954071,
      "logps/chosen": -344.79998779296875,
      "logps/rejected": -343.3999938964844,
      "loss": 0.4856,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.19971923530101776,
      "rewards/margins": 1.256591796875,
      "rewards/rejected": -1.4572632312774658,
      "step": 1390
    },
    {
      "epoch": 0.9011908593498552,
      "grad_norm": 112.5823484060804,
      "learning_rate": 7.747907276239536e-07,
      "logits/chosen": -0.500781238079071,
      "logits/rejected": -0.5128418207168579,
      "logps/chosen": -392.375,
      "logps/rejected": -373.1000061035156,
      "loss": 0.5976,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.542877197265625,
      "rewards/margins": 1.1980102062225342,
      "rewards/rejected": -1.740942358970642,
      "step": 1400
    },
    {
      "epoch": 0.9076279369166399,
      "grad_norm": 108.00708309053341,
      "learning_rate": 7.731809401159047e-07,
      "logits/chosen": -0.44998472929000854,
      "logits/rejected": -0.4686279296875,
      "logps/chosen": -384.8374938964844,
      "logps/rejected": -384.95001220703125,
      "loss": 0.5577,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.6136230230331421,
      "rewards/margins": 1.4021484851837158,
      "rewards/rejected": -2.0157883167266846,
      "step": 1410
    },
    {
      "epoch": 0.9140650144834245,
      "grad_norm": 75.63651943592622,
      "learning_rate": 7.715711526078557e-07,
      "logits/chosen": -0.4461120665073395,
      "logits/rejected": -0.46266478300094604,
      "logps/chosen": -357.75,
      "logps/rejected": -340.36248779296875,
      "loss": 0.6517,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.07795409858226776,
      "rewards/margins": 1.0103881359100342,
      "rewards/rejected": -1.087805151939392,
      "step": 1420
    },
    {
      "epoch": 0.9205020920502092,
      "grad_norm": 82.65401725045969,
      "learning_rate": 7.699613650998068e-07,
      "logits/chosen": -0.4739746153354645,
      "logits/rejected": -0.5115722417831421,
      "logps/chosen": -378.875,
      "logps/rejected": -375.20001220703125,
      "loss": 0.6256,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": 0.321298211812973,
      "rewards/margins": 0.9541260004043579,
      "rewards/rejected": -0.6323302984237671,
      "step": 1430
    },
    {
      "epoch": 0.9269391696169939,
      "grad_norm": 112.68383111556054,
      "learning_rate": 7.683515775917579e-07,
      "logits/chosen": -0.5020507574081421,
      "logits/rejected": -0.510693371295929,
      "logps/chosen": -360.61248779296875,
      "logps/rejected": -361.67498779296875,
      "loss": 0.6268,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.434814453125,
      "rewards/margins": 0.9604736566543579,
      "rewards/rejected": -0.5249465703964233,
      "step": 1440
    },
    {
      "epoch": 0.9333762471837785,
      "grad_norm": 121.03410484944297,
      "learning_rate": 7.667417900837089e-07,
      "logits/chosen": -0.44097900390625,
      "logits/rejected": -0.45458984375,
      "logps/chosen": -374.5625,
      "logps/rejected": -340.5249938964844,
      "loss": 0.6559,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": 0.13691405951976776,
      "rewards/margins": 1.165869116783142,
      "rewards/rejected": -1.029052734375,
      "step": 1450
    },
    {
      "epoch": 0.9398133247505632,
      "grad_norm": 99.05584268644827,
      "learning_rate": 7.6513200257566e-07,
      "logits/chosen": -0.5095580816268921,
      "logits/rejected": -0.5365966558456421,
      "logps/chosen": -380.42498779296875,
      "logps/rejected": -349.1499938964844,
      "loss": 0.59,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.10870361328125,
      "rewards/margins": 1.219384789466858,
      "rewards/rejected": -1.110510230064392,
      "step": 1460
    },
    {
      "epoch": 0.9462504023173479,
      "grad_norm": 130.78950082280264,
      "learning_rate": 7.63522215067611e-07,
      "logits/chosen": -0.45726317167282104,
      "logits/rejected": -0.4390853941440582,
      "logps/chosen": -384.7124938964844,
      "logps/rejected": -380.2749938964844,
      "loss": 0.7047,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.13157348334789276,
      "rewards/margins": 0.8795715570449829,
      "rewards/rejected": -1.01123046875,
      "step": 1470
    },
    {
      "epoch": 0.9526874798841326,
      "grad_norm": 100.85870827827803,
      "learning_rate": 7.619124275595621e-07,
      "logits/chosen": -0.4605956971645355,
      "logits/rejected": -0.5074218511581421,
      "logps/chosen": -406.5249938964844,
      "logps/rejected": -398.3500061035156,
      "loss": 0.5323,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": 0.14664916694164276,
      "rewards/margins": 1.08514404296875,
      "rewards/rejected": -0.939013659954071,
      "step": 1480
    },
    {
      "epoch": 0.9591245574509173,
      "grad_norm": 99.51512777249033,
      "learning_rate": 7.603026400515131e-07,
      "logits/chosen": -0.519457995891571,
      "logits/rejected": -0.5551391839981079,
      "logps/chosen": -412.1499938964844,
      "logps/rejected": -398.42498779296875,
      "loss": 0.5587,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.12549439072608948,
      "rewards/margins": 1.107519507408142,
      "rewards/rejected": -1.232666015625,
      "step": 1490
    },
    {
      "epoch": 0.965561635017702,
      "grad_norm": 118.0541567845769,
      "learning_rate": 7.586928525434642e-07,
      "logits/chosen": -0.43101805448532104,
      "logits/rejected": -0.40216064453125,
      "logps/chosen": -390.5249938964844,
      "logps/rejected": -370.0,
      "loss": 0.6822,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.705487072467804,
      "rewards/margins": 1.0155029296875,
      "rewards/rejected": -1.7212402820587158,
      "step": 1500
    },
    {
      "epoch": 0.9719987125844867,
      "grad_norm": 98.22598118527986,
      "learning_rate": 7.570830650354153e-07,
      "logits/chosen": -0.46629029512405396,
      "logits/rejected": -0.47856444120407104,
      "logps/chosen": -374.36248779296875,
      "logps/rejected": -351.8999938964844,
      "loss": 0.5244,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.17139282822608948,
      "rewards/margins": 1.374121069908142,
      "rewards/rejected": -1.545922875404358,
      "step": 1510
    },
    {
      "epoch": 0.9784357901512714,
      "grad_norm": 117.92656420297726,
      "learning_rate": 7.554732775273664e-07,
      "logits/chosen": -0.486328125,
      "logits/rejected": -0.4902099668979645,
      "logps/chosen": -353.3500061035156,
      "logps/rejected": -372.4750061035156,
      "loss": 0.6914,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.07288818061351776,
      "rewards/margins": 0.8248291015625,
      "rewards/rejected": -0.752392590045929,
      "step": 1520
    },
    {
      "epoch": 0.984872867718056,
      "grad_norm": 103.7352972935465,
      "learning_rate": 7.538634900193173e-07,
      "logits/chosen": -0.44585877656936646,
      "logits/rejected": -0.4797119200229645,
      "logps/chosen": -364.51251220703125,
      "logps/rejected": -334.07501220703125,
      "loss": 0.5836,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": 0.17331238090991974,
      "rewards/margins": 1.103173851966858,
      "rewards/rejected": -0.929003894329071,
      "step": 1530
    },
    {
      "epoch": 0.9913099452848407,
      "grad_norm": 92.0120508509047,
      "learning_rate": 7.522537025112685e-07,
      "logits/chosen": -0.432577520608902,
      "logits/rejected": -0.44459229707717896,
      "logps/chosen": -374.3999938964844,
      "logps/rejected": -374.625,
      "loss": 0.5247,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": 0.04023437574505806,
      "rewards/margins": 1.244116187095642,
      "rewards/rejected": -1.2042725086212158,
      "step": 1540
    },
    {
      "epoch": 0.9977470228516253,
      "grad_norm": 113.60262425936246,
      "learning_rate": 7.506439150032196e-07,
      "logits/chosen": -0.44282227754592896,
      "logits/rejected": -0.4719482362270355,
      "logps/chosen": -399.1499938964844,
      "logps/rejected": -388.2749938964844,
      "loss": 0.5858,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.48509520292282104,
      "rewards/margins": 1.2941710948944092,
      "rewards/rejected": -1.7789428234100342,
      "step": 1550
    },
    {
      "epoch": 1.0038622465400708,
      "grad_norm": 28.701645450043788,
      "learning_rate": 7.490341274951706e-07,
      "logits/chosen": -0.5120528340339661,
      "logits/rejected": -0.5163445472717285,
      "logps/chosen": -368.6842041015625,
      "logps/rejected": -373.47369384765625,
      "loss": 0.2804,
      "rewards/accuracies": 0.8223684430122375,
      "rewards/chosen": 0.9294819235801697,
      "rewards/margins": 4.5301833152771,
      "rewards/rejected": -3.5959086418151855,
      "step": 1560
    },
    {
      "epoch": 1.0102993241068554,
      "grad_norm": 60.32074871596137,
      "learning_rate": 7.474243399871216e-07,
      "logits/chosen": -0.4825439453125,
      "logits/rejected": -0.52020263671875,
      "logps/chosen": -371.92498779296875,
      "logps/rejected": -425.45001220703125,
      "loss": 0.0487,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 2.0789122581481934,
      "rewards/margins": 7.958593845367432,
      "rewards/rejected": -5.882421970367432,
      "step": 1570
    },
    {
      "epoch": 1.0167364016736402,
      "grad_norm": 71.52396938669607,
      "learning_rate": 7.458145524790728e-07,
      "logits/chosen": -0.6259857416152954,
      "logits/rejected": -0.646166980266571,
      "logps/chosen": -352.3999938964844,
      "logps/rejected": -411.70001220703125,
      "loss": 0.0354,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": 1.0871460437774658,
      "rewards/margins": 8.010156631469727,
      "rewards/rejected": -6.9296875,
      "step": 1580
    },
    {
      "epoch": 1.0231734792404248,
      "grad_norm": 9.724575140553721,
      "learning_rate": 7.442047649710238e-07,
      "logits/chosen": -0.687255859375,
      "logits/rejected": -0.711743175983429,
      "logps/chosen": -335.79998779296875,
      "logps/rejected": -400.25,
      "loss": 0.052,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.412109375,
      "rewards/margins": 8.798828125,
      "rewards/rejected": -9.207422256469727,
      "step": 1590
    },
    {
      "epoch": 1.0296105568072096,
      "grad_norm": 42.3472252623567,
      "learning_rate": 7.425949774629749e-07,
      "logits/chosen": -0.783618152141571,
      "logits/rejected": -0.7865371704101562,
      "logps/chosen": -406.625,
      "logps/rejected": -462.3999938964844,
      "loss": 0.0677,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.48292237520217896,
      "rewards/margins": 10.419140815734863,
      "rewards/rejected": -10.904687881469727,
      "step": 1600
    },
    {
      "epoch": 1.0360476343739942,
      "grad_norm": 44.590992711597345,
      "learning_rate": 7.409851899549258e-07,
      "logits/chosen": -0.7693115472793579,
      "logits/rejected": -0.7798522710800171,
      "logps/chosen": -367.6499938964844,
      "logps/rejected": -441.5625,
      "loss": 0.0596,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.7735351324081421,
      "rewards/margins": 9.705469131469727,
      "rewards/rejected": -10.483593940734863,
      "step": 1610
    },
    {
      "epoch": 1.042484711940779,
      "grad_norm": 12.122547274948735,
      "learning_rate": 7.39375402446877e-07,
      "logits/chosen": -0.7435302734375,
      "logits/rejected": -0.745318591594696,
      "logps/chosen": -353.125,
      "logps/rejected": -426.17498779296875,
      "loss": 0.0561,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.11784668266773224,
      "rewards/margins": 9.178125381469727,
      "rewards/rejected": -9.298437118530273,
      "step": 1620
    },
    {
      "epoch": 1.0489217895075635,
      "grad_norm": 4.7060888899695135,
      "learning_rate": 7.377656149388281e-07,
      "logits/chosen": -0.799609363079071,
      "logits/rejected": -0.780871570110321,
      "logps/chosen": -393.625,
      "logps/rejected": -469.5249938964844,
      "loss": 0.0357,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.3866333067417145,
      "rewards/margins": 9.78515625,
      "rewards/rejected": -10.168749809265137,
      "step": 1630
    },
    {
      "epoch": 1.0553588670743483,
      "grad_norm": 19.42538895152891,
      "learning_rate": 7.36155827430779e-07,
      "logits/chosen": -0.8111816644668579,
      "logits/rejected": -0.814648449420929,
      "logps/chosen": -377.04998779296875,
      "logps/rejected": -452.11248779296875,
      "loss": 0.0331,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.6400299072265625,
      "rewards/margins": 9.115625381469727,
      "rewards/rejected": -9.759374618530273,
      "step": 1640
    },
    {
      "epoch": 1.061795944641133,
      "grad_norm": 1.1011234013173983,
      "learning_rate": 7.345460399227301e-07,
      "logits/chosen": -0.773486316204071,
      "logits/rejected": -0.772021472454071,
      "logps/chosen": -405.7749938964844,
      "logps/rejected": -478.2749938964844,
      "loss": 0.0311,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -0.60272216796875,
      "rewards/margins": 10.611719131469727,
      "rewards/rejected": -11.21875,
      "step": 1650
    },
    {
      "epoch": 1.0682330222079175,
      "grad_norm": 26.178562770306236,
      "learning_rate": 7.329362524146813e-07,
      "logits/chosen": -0.712841808795929,
      "logits/rejected": -0.7065674066543579,
      "logps/chosen": -353.73748779296875,
      "logps/rejected": -436.2749938964844,
      "loss": 0.0538,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.7074829339981079,
      "rewards/margins": 8.717187881469727,
      "rewards/rejected": -9.428906440734863,
      "step": 1660
    },
    {
      "epoch": 1.0746700997747023,
      "grad_norm": 1.209371501717102,
      "learning_rate": 7.313264649066323e-07,
      "logits/chosen": -0.7484130859375,
      "logits/rejected": -0.774340808391571,
      "logps/chosen": -378.5625,
      "logps/rejected": -436.29998779296875,
      "loss": 0.0313,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": 0.3489746153354645,
      "rewards/margins": 8.931640625,
      "rewards/rejected": -8.578906059265137,
      "step": 1670
    },
    {
      "epoch": 1.0811071773414869,
      "grad_norm": 5.4457811328971815,
      "learning_rate": 7.297166773985833e-07,
      "logits/chosen": -0.726611316204071,
      "logits/rejected": -0.738110363483429,
      "logps/chosen": -353.79998779296875,
      "logps/rejected": -463.6499938964844,
      "loss": 0.0396,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -0.09562988579273224,
      "rewards/margins": 10.096094131469727,
      "rewards/rejected": -10.197656631469727,
      "step": 1680
    },
    {
      "epoch": 1.0875442549082717,
      "grad_norm": 78.32697539506492,
      "learning_rate": 7.281068898905344e-07,
      "logits/chosen": -0.871044933795929,
      "logits/rejected": -0.8653320074081421,
      "logps/chosen": -401.8500061035156,
      "logps/rejected": -505.54998779296875,
      "loss": 0.0434,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.7566101551055908,
      "rewards/margins": 10.932031631469727,
      "rewards/rejected": -12.690625190734863,
      "step": 1690
    },
    {
      "epoch": 1.0939813324750562,
      "grad_norm": 30.05286319924308,
      "learning_rate": 7.264971023824855e-07,
      "logits/chosen": -0.869873046875,
      "logits/rejected": -0.87158203125,
      "logps/chosen": -365.3374938964844,
      "logps/rejected": -441.5,
      "loss": 0.0598,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -2.2253174781799316,
      "rewards/margins": 10.394922256469727,
      "rewards/rejected": -12.62890625,
      "step": 1700
    },
    {
      "epoch": 1.100418410041841,
      "grad_norm": 14.131277242386574,
      "learning_rate": 7.248873148744366e-07,
      "logits/chosen": -0.841992199420929,
      "logits/rejected": -0.883532702922821,
      "logps/chosen": -409.67498779296875,
      "logps/rejected": -448.1000061035156,
      "loss": 0.0498,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -2.0384764671325684,
      "rewards/margins": 9.805468559265137,
      "rewards/rejected": -11.846094131469727,
      "step": 1710
    },
    {
      "epoch": 1.1068554876086256,
      "grad_norm": 7.132116530747616,
      "learning_rate": 7.232775273663876e-07,
      "logits/chosen": -0.8307861089706421,
      "logits/rejected": -0.877148449420929,
      "logps/chosen": -413.26251220703125,
      "logps/rejected": -495.6499938964844,
      "loss": 0.0771,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.5871948003768921,
      "rewards/margins": 11.547656059265137,
      "rewards/rejected": -12.12109375,
      "step": 1720
    },
    {
      "epoch": 1.1132925651754104,
      "grad_norm": 0.47230522302878464,
      "learning_rate": 7.216677398583386e-07,
      "logits/chosen": -0.8125,
      "logits/rejected": -0.8301757574081421,
      "logps/chosen": -352.57501220703125,
      "logps/rejected": -443.875,
      "loss": 0.0518,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.527294933795929,
      "rewards/margins": 10.287500381469727,
      "rewards/rejected": -10.814844131469727,
      "step": 1730
    },
    {
      "epoch": 1.119729642742195,
      "grad_norm": 37.48581782587474,
      "learning_rate": 7.200579523502898e-07,
      "logits/chosen": -0.687573254108429,
      "logits/rejected": -0.688122570514679,
      "logps/chosen": -382.375,
      "logps/rejected": -495.54998779296875,
      "loss": 0.0648,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.15927734971046448,
      "rewards/margins": 11.528905868530273,
      "rewards/rejected": -11.6953125,
      "step": 1740
    },
    {
      "epoch": 1.1261667203089798,
      "grad_norm": 41.26402499082609,
      "learning_rate": 7.184481648422408e-07,
      "logits/chosen": -0.878955066204071,
      "logits/rejected": -0.896044909954071,
      "logps/chosen": -364.6000061035156,
      "logps/rejected": -447.7250061035156,
      "loss": 0.0548,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.19716796278953552,
      "rewards/margins": 10.053515434265137,
      "rewards/rejected": -10.253125190734863,
      "step": 1750
    },
    {
      "epoch": 1.1326037978757644,
      "grad_norm": 3.6822102341100567,
      "learning_rate": 7.168383773341918e-07,
      "logits/chosen": -0.8634277582168579,
      "logits/rejected": -0.873974621295929,
      "logps/chosen": -392.20001220703125,
      "logps/rejected": -458.20001220703125,
      "loss": 0.0257,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -0.2974609434604645,
      "rewards/margins": 10.384374618530273,
      "rewards/rejected": -10.675000190734863,
      "step": 1760
    },
    {
      "epoch": 1.1390408754425492,
      "grad_norm": 4.0760843716589426,
      "learning_rate": 7.152285898261429e-07,
      "logits/chosen": -0.8376098871231079,
      "logits/rejected": -0.8130859136581421,
      "logps/chosen": -376.8999938964844,
      "logps/rejected": -478.8999938964844,
      "loss": 0.0287,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -1.418371558189392,
      "rewards/margins": 11.157031059265137,
      "rewards/rejected": -12.575780868530273,
      "step": 1770
    },
    {
      "epoch": 1.1454779530093337,
      "grad_norm": 17.081009996286852,
      "learning_rate": 7.136188023180941e-07,
      "logits/chosen": -0.81988525390625,
      "logits/rejected": -0.8177734613418579,
      "logps/chosen": -406.5,
      "logps/rejected": -492.4750061035156,
      "loss": 0.0404,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.221630811691284,
      "rewards/margins": 11.858593940734863,
      "rewards/rejected": -14.083593368530273,
      "step": 1780
    },
    {
      "epoch": 1.1519150305761183,
      "grad_norm": 16.670165572478858,
      "learning_rate": 7.12009014810045e-07,
      "logits/chosen": -0.878173828125,
      "logits/rejected": -0.886474609375,
      "logps/chosen": -406.82501220703125,
      "logps/rejected": -475.7250061035156,
      "loss": 0.0203,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.0299072265625,
      "rewards/margins": 11.866796493530273,
      "rewards/rejected": -13.893750190734863,
      "step": 1790
    },
    {
      "epoch": 1.1583521081429031,
      "grad_norm": 14.789491416020818,
      "learning_rate": 7.103992273019961e-07,
      "logits/chosen": -0.789306640625,
      "logits/rejected": -0.776171863079071,
      "logps/chosen": -414.42498779296875,
      "logps/rejected": -475.95001220703125,
      "loss": 0.034,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -1.1190063953399658,
      "rewards/margins": 10.380859375,
      "rewards/rejected": -11.503125190734863,
      "step": 1800
    },
    {
      "epoch": 1.164789185709688,
      "grad_norm": 13.586311507009619,
      "learning_rate": 7.087894397939471e-07,
      "logits/chosen": -0.8396972417831421,
      "logits/rejected": -0.8243652582168579,
      "logps/chosen": -399.6499938964844,
      "logps/rejected": -458.79998779296875,
      "loss": 0.0374,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.6400878429412842,
      "rewards/margins": 10.001562118530273,
      "rewards/rejected": -11.642187118530273,
      "step": 1810
    },
    {
      "epoch": 1.1712262632764725,
      "grad_norm": 55.57830649329583,
      "learning_rate": 7.071796522858983e-07,
      "logits/chosen": -0.861035168170929,
      "logits/rejected": -0.846386730670929,
      "logps/chosen": -402.2250061035156,
      "logps/rejected": -496.1000061035156,
      "loss": 0.0441,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.8700439929962158,
      "rewards/margins": 11.387499809265137,
      "rewards/rejected": -13.250781059265137,
      "step": 1820
    },
    {
      "epoch": 1.177663340843257,
      "grad_norm": 18.828474480482623,
      "learning_rate": 7.055698647778493e-07,
      "logits/chosen": -0.851696789264679,
      "logits/rejected": -0.8456054925918579,
      "logps/chosen": -372.8999938964844,
      "logps/rejected": -439.6000061035156,
      "loss": 0.066,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -3.3226075172424316,
      "rewards/margins": 10.33984375,
      "rewards/rejected": -13.662500381469727,
      "step": 1830
    },
    {
      "epoch": 1.1841004184100419,
      "grad_norm": 0.9764059185254257,
      "learning_rate": 7.039600772698003e-07,
      "logits/chosen": -0.82470703125,
      "logits/rejected": -0.806689441204071,
      "logps/chosen": -388.01251220703125,
      "logps/rejected": -486.29998779296875,
      "loss": 0.0703,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -2.5329222679138184,
      "rewards/margins": 10.471094131469727,
      "rewards/rejected": -13.009374618530273,
      "step": 1840
    },
    {
      "epoch": 1.1905374959768265,
      "grad_norm": 3.7355014099628647,
      "learning_rate": 7.023502897617514e-07,
      "logits/chosen": -0.7955566644668579,
      "logits/rejected": -0.8161865472793579,
      "logps/chosen": -420.95001220703125,
      "logps/rejected": -477.20001220703125,
      "loss": 0.0414,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.4476196765899658,
      "rewards/margins": 10.553515434265137,
      "rewards/rejected": -12.003125190734863,
      "step": 1850
    },
    {
      "epoch": 1.1969745735436113,
      "grad_norm": 23.271976132825916,
      "learning_rate": 7.007405022537025e-07,
      "logits/chosen": -0.7967590093612671,
      "logits/rejected": -0.8336181640625,
      "logps/chosen": -372.2749938964844,
      "logps/rejected": -485.0249938964844,
      "loss": 0.0653,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -3.135058641433716,
      "rewards/margins": 11.382031440734863,
      "rewards/rejected": -14.517969131469727,
      "step": 1860
    },
    {
      "epoch": 1.2034116511103958,
      "grad_norm": 65.18811151029988,
      "learning_rate": 6.991307147456535e-07,
      "logits/chosen": -0.8287597894668579,
      "logits/rejected": -0.823779284954071,
      "logps/chosen": -387.79998779296875,
      "logps/rejected": -462.4750061035156,
      "loss": 0.035,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -2.011474609375,
      "rewards/margins": 11.100781440734863,
      "rewards/rejected": -13.106249809265137,
      "step": 1870
    },
    {
      "epoch": 1.2098487286771806,
      "grad_norm": 10.973771958166278,
      "learning_rate": 6.975209272376046e-07,
      "logits/chosen": -0.7869873046875,
      "logits/rejected": -0.7986816167831421,
      "logps/chosen": -360.5249938964844,
      "logps/rejected": -439.20001220703125,
      "loss": 0.038,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -1.812768578529358,
      "rewards/margins": 10.350781440734863,
      "rewards/rejected": -12.167187690734863,
      "step": 1880
    },
    {
      "epoch": 1.2162858062439652,
      "grad_norm": 0.1795200411379023,
      "learning_rate": 6.959111397295557e-07,
      "logits/chosen": -0.8089843988418579,
      "logits/rejected": -0.7916015386581421,
      "logps/chosen": -327.3500061035156,
      "logps/rejected": -447.54998779296875,
      "loss": 0.0326,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.839111328125,
      "rewards/margins": 10.127344131469727,
      "rewards/rejected": -10.97265625,
      "step": 1890
    },
    {
      "epoch": 1.22272288381075,
      "grad_norm": 6.31069489977895,
      "learning_rate": 6.943013522215067e-07,
      "logits/chosen": -0.7904297113418579,
      "logits/rejected": -0.789257824420929,
      "logps/chosen": -380.79998779296875,
      "logps/rejected": -457.5,
      "loss": 0.0564,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.0034423829056322575,
      "rewards/margins": 10.767187118530273,
      "rewards/rejected": -10.768750190734863,
      "step": 1900
    },
    {
      "epoch": 1.2291599613775346,
      "grad_norm": 1.890302975072155,
      "learning_rate": 6.926915647134578e-07,
      "logits/chosen": -0.8453521728515625,
      "logits/rejected": -0.8412109613418579,
      "logps/chosen": -400.70001220703125,
      "logps/rejected": -479.8500061035156,
      "loss": 0.0241,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -0.521453857421875,
      "rewards/margins": 10.838281631469727,
      "rewards/rejected": -11.364062309265137,
      "step": 1910
    },
    {
      "epoch": 1.2355970389443192,
      "grad_norm": 5.2209876862192015,
      "learning_rate": 6.910817772054089e-07,
      "logits/chosen": -0.8232421875,
      "logits/rejected": -0.831591784954071,
      "logps/chosen": -363.8500061035156,
      "logps/rejected": -451.125,
      "loss": 0.0463,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.1357421875,
      "rewards/margins": 10.284375190734863,
      "rewards/rejected": -11.41796875,
      "step": 1920
    },
    {
      "epoch": 1.242034116511104,
      "grad_norm": 8.784840887843162,
      "learning_rate": 6.894719896973599e-07,
      "logits/chosen": -0.854248046875,
      "logits/rejected": -0.857421875,
      "logps/chosen": -385.2250061035156,
      "logps/rejected": -471.3999938964844,
      "loss": 0.0269,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.4132934510707855,
      "rewards/margins": 10.916406631469727,
      "rewards/rejected": -11.321093559265137,
      "step": 1930
    },
    {
      "epoch": 1.2484711940778888,
      "grad_norm": 11.499308969293939,
      "learning_rate": 6.87862202189311e-07,
      "logits/chosen": -0.8663085699081421,
      "logits/rejected": -0.8819824457168579,
      "logps/chosen": -367.20001220703125,
      "logps/rejected": -482.4750061035156,
      "loss": 0.0393,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.7445312738418579,
      "rewards/margins": 10.687891006469727,
      "rewards/rejected": -11.436718940734863,
      "step": 1940
    },
    {
      "epoch": 1.2549082716446733,
      "grad_norm": 5.476246704655694,
      "learning_rate": 6.86252414681262e-07,
      "logits/chosen": -0.785839855670929,
      "logits/rejected": -0.802783191204071,
      "logps/chosen": -393.25,
      "logps/rejected": -481.1499938964844,
      "loss": 0.07,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -1.304406762123108,
      "rewards/margins": 10.49609375,
      "rewards/rejected": -11.80078125,
      "step": 1950
    },
    {
      "epoch": 1.261345349211458,
      "grad_norm": 1.8068433654986311,
      "learning_rate": 6.846426271732131e-07,
      "logits/chosen": -0.8782958984375,
      "logits/rejected": -0.878369152545929,
      "logps/chosen": -366.95001220703125,
      "logps/rejected": -481.4750061035156,
      "loss": 0.0281,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -1.360253930091858,
      "rewards/margins": 11.563281059265137,
      "rewards/rejected": -12.9296875,
      "step": 1960
    },
    {
      "epoch": 1.2677824267782427,
      "grad_norm": 1.5992649831411958,
      "learning_rate": 6.830328396651641e-07,
      "logits/chosen": -0.916015625,
      "logits/rejected": -0.910888671875,
      "logps/chosen": -351.2749938964844,
      "logps/rejected": -440.92498779296875,
      "loss": 0.0428,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -1.484375,
      "rewards/margins": 10.620312690734863,
      "rewards/rejected": -12.095312118530273,
      "step": 1970
    },
    {
      "epoch": 1.2742195043450273,
      "grad_norm": 11.932348554376217,
      "learning_rate": 6.814230521571152e-07,
      "logits/chosen": -0.7955566644668579,
      "logits/rejected": -0.812182605266571,
      "logps/chosen": -405.0249938964844,
      "logps/rejected": -501.5,
      "loss": 0.0258,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -1.031854271888733,
      "rewards/margins": 11.592187881469727,
      "rewards/rejected": -12.624218940734863,
      "step": 1980
    },
    {
      "epoch": 1.280656581911812,
      "grad_norm": 0.7284651571104193,
      "learning_rate": 6.798132646490663e-07,
      "logits/chosen": -0.687243640422821,
      "logits/rejected": -0.7085205316543579,
      "logps/chosen": -370.9125061035156,
      "logps/rejected": -477.25,
      "loss": 0.0594,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.3655884265899658,
      "rewards/margins": 11.599218368530273,
      "rewards/rejected": -12.961718559265137,
      "step": 1990
    },
    {
      "epoch": 1.2870936594785967,
      "grad_norm": 0.1726195121834098,
      "learning_rate": 6.782034771410174e-07,
      "logits/chosen": -0.886474609375,
      "logits/rejected": -0.9327148199081421,
      "logps/chosen": -374.79998779296875,
      "logps/rejected": -464.75,
      "loss": 0.0157,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -1.9635009765625,
      "rewards/margins": 10.838281631469727,
      "rewards/rejected": -12.809374809265137,
      "step": 2000
    },
    {
      "epoch": 1.2935307370453815,
      "grad_norm": 4.855214989312065,
      "learning_rate": 6.765936896329683e-07,
      "logits/chosen": -0.7554687261581421,
      "logits/rejected": -0.762524425983429,
      "logps/chosen": -353.7749938964844,
      "logps/rejected": -453.79998779296875,
      "loss": 0.0237,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -2.166912794113159,
      "rewards/margins": 10.567187309265137,
      "rewards/rejected": -12.739062309265137,
      "step": 2010
    },
    {
      "epoch": 1.299967814612166,
      "grad_norm": 3.805160643591438,
      "learning_rate": 6.749839021249195e-07,
      "logits/chosen": -0.774462878704071,
      "logits/rejected": -0.8001464605331421,
      "logps/chosen": -431.25,
      "logps/rejected": -509.375,
      "loss": 0.0564,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -2.2074341773986816,
      "rewards/margins": 11.928125381469727,
      "rewards/rejected": -14.134374618530273,
      "step": 2020
    },
    {
      "epoch": 1.3064048921789508,
      "grad_norm": 12.782556298742978,
      "learning_rate": 6.733741146168706e-07,
      "logits/chosen": -0.6968749761581421,
      "logits/rejected": -0.719860851764679,
      "logps/chosen": -363.04998779296875,
      "logps/rejected": -473.6499938964844,
      "loss": 0.03,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -1.2151978015899658,
      "rewards/margins": 12.274218559265137,
      "rewards/rejected": -13.489062309265137,
      "step": 2030
    },
    {
      "epoch": 1.3128419697457354,
      "grad_norm": 90.75357097442223,
      "learning_rate": 6.717643271088216e-07,
      "logits/chosen": -0.7369629144668579,
      "logits/rejected": -0.760327160358429,
      "logps/chosen": -395.13751220703125,
      "logps/rejected": -462.79998779296875,
      "loss": 0.0882,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -2.4952149391174316,
      "rewards/margins": 10.8828125,
      "rewards/rejected": -13.3671875,
      "step": 2040
    },
    {
      "epoch": 1.31927904731252,
      "grad_norm": 0.5115947981333193,
      "learning_rate": 6.701545396007726e-07,
      "logits/chosen": -0.807690441608429,
      "logits/rejected": -0.8475097417831421,
      "logps/chosen": -365.8999938964844,
      "logps/rejected": -443.20001220703125,
      "loss": 0.0529,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -2.2901368141174316,
      "rewards/margins": 10.971875190734863,
      "rewards/rejected": -13.258593559265137,
      "step": 2050
    },
    {
      "epoch": 1.3257161248793048,
      "grad_norm": 0.18940957690475052,
      "learning_rate": 6.685447520927238e-07,
      "logits/chosen": -0.775683581829071,
      "logits/rejected": -0.818310558795929,
      "logps/chosen": -360.25,
      "logps/rejected": -440.8999938964844,
      "loss": 0.0328,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -1.39141845703125,
      "rewards/margins": 10.676562309265137,
      "rewards/rejected": -12.060155868530273,
      "step": 2060
    },
    {
      "epoch": 1.3321532024460896,
      "grad_norm": 6.818009967898228,
      "learning_rate": 6.669349645846748e-07,
      "logits/chosen": -0.794970691204071,
      "logits/rejected": -0.8131347894668579,
      "logps/chosen": -396.70001220703125,
      "logps/rejected": -451.1499938964844,
      "loss": 0.036,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.8651977777481079,
      "rewards/margins": 10.630468368530273,
      "rewards/rejected": -11.493749618530273,
      "step": 2070
    },
    {
      "epoch": 1.3385902800128742,
      "grad_norm": 20.477227570556575,
      "learning_rate": 6.653251770766258e-07,
      "logits/chosen": -0.772412121295929,
      "logits/rejected": -0.787646472454071,
      "logps/chosen": -381.73748779296875,
      "logps/rejected": -452.6499938964844,
      "loss": 0.046,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.25780028104782104,
      "rewards/margins": 10.538281440734863,
      "rewards/rejected": -10.794530868530273,
      "step": 2080
    },
    {
      "epoch": 1.3450273575796587,
      "grad_norm": 1.8958489445333633,
      "learning_rate": 6.637153895685768e-07,
      "logits/chosen": -0.758129894733429,
      "logits/rejected": -0.7566162347793579,
      "logps/chosen": -376.875,
      "logps/rejected": -473.07501220703125,
      "loss": 0.0434,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.41459959745407104,
      "rewards/margins": 10.59375,
      "rewards/rejected": -11.004687309265137,
      "step": 2090
    },
    {
      "epoch": 1.3514644351464435,
      "grad_norm": 15.797070454042899,
      "learning_rate": 6.62105602060528e-07,
      "logits/chosen": -0.7794433832168579,
      "logits/rejected": -0.765332043170929,
      "logps/chosen": -375.7749938964844,
      "logps/rejected": -448.20001220703125,
      "loss": 0.0274,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -0.5860840082168579,
      "rewards/margins": 10.074609756469727,
      "rewards/rejected": -10.665234565734863,
      "step": 2100
    },
    {
      "epoch": 1.3579015127132281,
      "grad_norm": 17.274563010286307,
      "learning_rate": 6.604958145524791e-07,
      "logits/chosen": -0.7362338900566101,
      "logits/rejected": -0.7653137445449829,
      "logps/chosen": -395.25,
      "logps/rejected": -454.4750061035156,
      "loss": 0.0504,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.1345703601837158,
      "rewards/margins": 9.571093559265137,
      "rewards/rejected": -10.703906059265137,
      "step": 2110
    },
    {
      "epoch": 1.364338590280013,
      "grad_norm": 34.1120810421438,
      "learning_rate": 6.5888602704443e-07,
      "logits/chosen": -0.7216430902481079,
      "logits/rejected": -0.7294555902481079,
      "logps/chosen": -366.0375061035156,
      "logps/rejected": -448.3999938964844,
      "loss": 0.04,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.01531982421875,
      "rewards/margins": 10.12109375,
      "rewards/rejected": -10.103906631469727,
      "step": 2120
    },
    {
      "epoch": 1.3707756678467975,
      "grad_norm": 35.51436144182908,
      "learning_rate": 6.572762395363811e-07,
      "logits/chosen": -0.7086426019668579,
      "logits/rejected": -0.7312988042831421,
      "logps/chosen": -378.67498779296875,
      "logps/rejected": -473.5375061035156,
      "loss": 0.0406,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.3042968809604645,
      "rewards/margins": 10.831250190734863,
      "rewards/rejected": -10.525781631469727,
      "step": 2130
    },
    {
      "epoch": 1.3772127454135823,
      "grad_norm": 7.163516950663505,
      "learning_rate": 6.556664520283323e-07,
      "logits/chosen": -0.7219482660293579,
      "logits/rejected": -0.7713867425918579,
      "logps/chosen": -375.95001220703125,
      "logps/rejected": -441.20001220703125,
      "loss": 0.0507,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.01722412183880806,
      "rewards/margins": 10.337499618530273,
      "rewards/rejected": -10.358593940734863,
      "step": 2140
    },
    {
      "epoch": 1.3836498229803669,
      "grad_norm": 5.920479235836306,
      "learning_rate": 6.540566645202833e-07,
      "logits/chosen": -0.726318359375,
      "logits/rejected": -0.734570324420929,
      "logps/chosen": -371.0,
      "logps/rejected": -441.1000061035156,
      "loss": 0.0288,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -0.1243896484375,
      "rewards/margins": 10.610937118530273,
      "rewards/rejected": -10.733593940734863,
      "step": 2150
    },
    {
      "epoch": 1.3900869005471517,
      "grad_norm": 8.789538295836175,
      "learning_rate": 6.524468770122343e-07,
      "logits/chosen": -0.649243175983429,
      "logits/rejected": -0.679980456829071,
      "logps/chosen": -353.42498779296875,
      "logps/rejected": -464.07501220703125,
      "loss": 0.0424,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.13173827528953552,
      "rewards/margins": 11.746874809265137,
      "rewards/rejected": -11.876562118530273,
      "step": 2160
    },
    {
      "epoch": 1.3965239781139362,
      "grad_norm": 10.025462859350501,
      "learning_rate": 6.508370895041854e-07,
      "logits/chosen": -0.73486328125,
      "logits/rejected": -0.7211669683456421,
      "logps/chosen": -390.8500061035156,
      "logps/rejected": -462.9750061035156,
      "loss": 0.0531,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.454126000404358,
      "rewards/margins": 11.36328125,
      "rewards/rejected": -12.815625190734863,
      "step": 2170
    },
    {
      "epoch": 1.4029610556807208,
      "grad_norm": 11.342140558916348,
      "learning_rate": 6.492273019961365e-07,
      "logits/chosen": -0.7691650390625,
      "logits/rejected": -0.7724243402481079,
      "logps/chosen": -381.4750061035156,
      "logps/rejected": -460.6499938964844,
      "loss": 0.0549,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -2.3446898460388184,
      "rewards/margins": 10.916406631469727,
      "rewards/rejected": -13.258593559265137,
      "step": 2180
    },
    {
      "epoch": 1.4093981332475056,
      "grad_norm": 0.7950552155099938,
      "learning_rate": 6.476175144880875e-07,
      "logits/chosen": -0.7981933355331421,
      "logits/rejected": -0.8179931640625,
      "logps/chosen": -403.0249938964844,
      "logps/rejected": -469.625,
      "loss": 0.0397,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -1.8957030773162842,
      "rewards/margins": 11.323827743530273,
      "rewards/rejected": -13.215624809265137,
      "step": 2190
    },
    {
      "epoch": 1.4158352108142904,
      "grad_norm": 20.523500260808675,
      "learning_rate": 6.460077269800386e-07,
      "logits/chosen": -0.7896484136581421,
      "logits/rejected": -0.8050537109375,
      "logps/chosen": -353.57501220703125,
      "logps/rejected": -488.6499938964844,
      "loss": 0.0333,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.691693127155304,
      "rewards/margins": 11.971094131469727,
      "rewards/rejected": -12.664843559265137,
      "step": 2200
    },
    {
      "epoch": 1.422272288381075,
      "grad_norm": 4.165148973749865,
      "learning_rate": 6.443979394719896e-07,
      "logits/chosen": -0.7835693359375,
      "logits/rejected": -0.8207031488418579,
      "logps/chosen": -404.6499938964844,
      "logps/rejected": -483.82501220703125,
      "loss": 0.0284,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -1.223669409751892,
      "rewards/margins": 11.237500190734863,
      "rewards/rejected": -12.465624809265137,
      "step": 2210
    },
    {
      "epoch": 1.4287093659478596,
      "grad_norm": 5.180688078431531,
      "learning_rate": 6.427881519639408e-07,
      "logits/chosen": -0.684216320514679,
      "logits/rejected": -0.7363525629043579,
      "logps/chosen": -403.5249938964844,
      "logps/rejected": -505.1000061035156,
      "loss": 0.0265,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -1.9525146484375,
      "rewards/margins": 12.81640625,
      "rewards/rejected": -14.774999618530273,
      "step": 2220
    },
    {
      "epoch": 1.4351464435146444,
      "grad_norm": 14.212893761212056,
      "learning_rate": 6.411783644558918e-07,
      "logits/chosen": -0.8042968511581421,
      "logits/rejected": -0.794909656047821,
      "logps/chosen": -403.8999938964844,
      "logps/rejected": -481.2250061035156,
      "loss": 0.0624,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.993322730064392,
      "rewards/margins": 11.608593940734863,
      "rewards/rejected": -13.594141006469727,
      "step": 2230
    },
    {
      "epoch": 1.441583521081429,
      "grad_norm": 6.871037130676326,
      "learning_rate": 6.395685769478428e-07,
      "logits/chosen": -0.700634777545929,
      "logits/rejected": -0.715405285358429,
      "logps/chosen": -352.38751220703125,
      "logps/rejected": -474.79998779296875,
      "loss": 0.0557,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.9448608160018921,
      "rewards/margins": 10.33984375,
      "rewards/rejected": -11.283594131469727,
      "step": 2240
    },
    {
      "epoch": 1.4480205986482138,
      "grad_norm": 0.18746076155577607,
      "learning_rate": 6.379587894397939e-07,
      "logits/chosen": -0.752368152141571,
      "logits/rejected": -0.7897704839706421,
      "logps/chosen": -397.29998779296875,
      "logps/rejected": -457.3500061035156,
      "loss": 0.0314,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.0015380382537842,
      "rewards/margins": 10.325780868530273,
      "rewards/rejected": -11.323437690734863,
      "step": 2250
    },
    {
      "epoch": 1.4544576762149983,
      "grad_norm": 2.5507421968165973,
      "learning_rate": 6.363490019317451e-07,
      "logits/chosen": -0.742602527141571,
      "logits/rejected": -0.7437683343887329,
      "logps/chosen": -383.07501220703125,
      "logps/rejected": -486.375,
      "loss": 0.0506,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.1386229991912842,
      "rewards/margins": 12.111719131469727,
      "rewards/rejected": -13.255468368530273,
      "step": 2260
    },
    {
      "epoch": 1.4608947537817831,
      "grad_norm": 2.2763823295013963,
      "learning_rate": 6.34739214423696e-07,
      "logits/chosen": -0.7447265386581421,
      "logits/rejected": -0.748046875,
      "logps/chosen": -410.625,
      "logps/rejected": -481.70001220703125,
      "loss": 0.0478,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.0862915515899658,
      "rewards/margins": 10.287500381469727,
      "rewards/rejected": -11.372655868530273,
      "step": 2270
    },
    {
      "epoch": 1.4673318313485677,
      "grad_norm": 0.4260998042645073,
      "learning_rate": 6.331294269156471e-07,
      "logits/chosen": -0.845996081829071,
      "logits/rejected": -0.8616698980331421,
      "logps/chosen": -393.95001220703125,
      "logps/rejected": -484.67498779296875,
      "loss": 0.0348,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -1.0209228992462158,
      "rewards/margins": 11.013280868530273,
      "rewards/rejected": -12.045312881469727,
      "step": 2280
    },
    {
      "epoch": 1.4737689089153525,
      "grad_norm": 123.17291450113709,
      "learning_rate": 6.315196394075981e-07,
      "logits/chosen": -0.721881091594696,
      "logits/rejected": -0.744799792766571,
      "logps/chosen": -368.5874938964844,
      "logps/rejected": -469.8999938964844,
      "loss": 0.0443,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.0027587413787842,
      "rewards/margins": 10.260156631469727,
      "rewards/rejected": -11.270703315734863,
      "step": 2290
    },
    {
      "epoch": 1.480205986482137,
      "grad_norm": 17.53638862046089,
      "learning_rate": 6.299098518995493e-07,
      "logits/chosen": -0.7756592035293579,
      "logits/rejected": -0.777020275592804,
      "logps/chosen": -386.3999938964844,
      "logps/rejected": -475.2749938964844,
      "loss": 0.0551,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.0754516124725342,
      "rewards/margins": 10.524999618530273,
      "rewards/rejected": -11.599218368530273,
      "step": 2300
    },
    {
      "epoch": 1.4866430640489217,
      "grad_norm": 40.43102058317517,
      "learning_rate": 6.283000643915003e-07,
      "logits/chosen": -0.7683349847793579,
      "logits/rejected": -0.8065429925918579,
      "logps/chosen": -340.1499938964844,
      "logps/rejected": -432.79998779296875,
      "loss": 0.0536,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.523364245891571,
      "rewards/margins": 10.595312118530273,
      "rewards/rejected": -11.111719131469727,
      "step": 2310
    },
    {
      "epoch": 1.4930801416157065,
      "grad_norm": 0.9213782840626744,
      "learning_rate": 6.266902768834513e-07,
      "logits/chosen": -0.7571777105331421,
      "logits/rejected": -0.7656280398368835,
      "logps/chosen": -394.17498779296875,
      "logps/rejected": -493.625,
      "loss": 0.0195,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -0.618359386920929,
      "rewards/margins": 11.130468368530273,
      "rewards/rejected": -11.751562118530273,
      "step": 2320
    },
    {
      "epoch": 1.4995172191824913,
      "grad_norm": 22.104939943938994,
      "learning_rate": 6.250804893754024e-07,
      "logits/chosen": -0.784497082233429,
      "logits/rejected": -0.773754894733429,
      "logps/chosen": -378.79998779296875,
      "logps/rejected": -431.25,
      "loss": 0.0639,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.8636840581893921,
      "rewards/margins": 10.059765815734863,
      "rewards/rejected": -10.924219131469727,
      "step": 2330
    },
    {
      "epoch": 1.5059542967492758,
      "grad_norm": 1.6915245555680156,
      "learning_rate": 6.234707018673535e-07,
      "logits/chosen": -0.834716796875,
      "logits/rejected": -0.8809570074081421,
      "logps/chosen": -370.54998779296875,
      "logps/rejected": -439.8500061035156,
      "loss": 0.052,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -1.103784203529358,
      "rewards/margins": 10.172656059265137,
      "rewards/rejected": -11.285937309265137,
      "step": 2340
    },
    {
      "epoch": 1.5123913743160604,
      "grad_norm": 17.859220853952976,
      "learning_rate": 6.218609143593045e-07,
      "logits/chosen": -0.827197253704071,
      "logits/rejected": -0.8333495855331421,
      "logps/chosen": -385.57501220703125,
      "logps/rejected": -463.6000061035156,
      "loss": 0.0792,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -1.35723876953125,
      "rewards/margins": 10.078125,
      "rewards/rejected": -11.44140625,
      "step": 2350
    },
    {
      "epoch": 1.5188284518828452,
      "grad_norm": 0.2607261691968034,
      "learning_rate": 6.202511268512556e-07,
      "logits/chosen": -0.760913074016571,
      "logits/rejected": -0.776318371295929,
      "logps/chosen": -389.17498779296875,
      "logps/rejected": -465.5249938964844,
      "loss": 0.0401,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.0231444835662842,
      "rewards/margins": 9.904296875,
      "rewards/rejected": -10.931640625,
      "step": 2360
    },
    {
      "epoch": 1.52526552944963,
      "grad_norm": 15.593710610614462,
      "learning_rate": 6.186413393432067e-07,
      "logits/chosen": -0.8382812738418579,
      "logits/rejected": -0.856738269329071,
      "logps/chosen": -407.2250061035156,
      "logps/rejected": -464.125,
      "loss": 0.0363,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.1426513195037842,
      "rewards/margins": 10.119531631469727,
      "rewards/rejected": -11.264062881469727,
      "step": 2370
    },
    {
      "epoch": 1.5317026070164146,
      "grad_norm": 7.034407955917834,
      "learning_rate": 6.170315518351577e-07,
      "logits/chosen": -0.7542724609375,
      "logits/rejected": -0.766064465045929,
      "logps/chosen": -378.70001220703125,
      "logps/rejected": -481.57501220703125,
      "loss": 0.0612,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.373419165611267,
      "rewards/margins": 10.766406059265137,
      "rewards/rejected": -12.145312309265137,
      "step": 2380
    },
    {
      "epoch": 1.5381396845831992,
      "grad_norm": 6.582834920072221,
      "learning_rate": 6.154217643271088e-07,
      "logits/chosen": -0.723925769329071,
      "logits/rejected": -0.734448254108429,
      "logps/chosen": -404.125,
      "logps/rejected": -502.75,
      "loss": 0.0557,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.978906273841858,
      "rewards/margins": 10.76171875,
      "rewards/rejected": -12.744531631469727,
      "step": 2390
    },
    {
      "epoch": 1.544576762149984,
      "grad_norm": 0.4422958879108554,
      "learning_rate": 6.138119768190599e-07,
      "logits/chosen": -0.8257080316543579,
      "logits/rejected": -0.862231433391571,
      "logps/chosen": -370.45001220703125,
      "logps/rejected": -459.9750061035156,
      "loss": 0.047,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -2.163891553878784,
      "rewards/margins": 11.420312881469727,
      "rewards/rejected": -13.585156440734863,
      "step": 2400
    },
    {
      "epoch": 1.5510138397167685,
      "grad_norm": 23.621743283386632,
      "learning_rate": 6.122021893110108e-07,
      "logits/chosen": -0.7707275152206421,
      "logits/rejected": -0.782275378704071,
      "logps/chosen": -374.75,
      "logps/rejected": -467.54998779296875,
      "loss": 0.0745,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.935156226158142,
      "rewards/margins": 10.9296875,
      "rewards/rejected": -12.871874809265137,
      "step": 2410
    },
    {
      "epoch": 1.5574509172835533,
      "grad_norm": 174.52576579494084,
      "learning_rate": 6.10592401802962e-07,
      "logits/chosen": -0.825268566608429,
      "logits/rejected": -0.830151379108429,
      "logps/chosen": -368.95001220703125,
      "logps/rejected": -481.8500061035156,
      "loss": 0.1028,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.627960205078125,
      "rewards/margins": 10.973047256469727,
      "rewards/rejected": -12.61328125,
      "step": 2420
    },
    {
      "epoch": 1.563887994850338,
      "grad_norm": 6.353329449967956,
      "learning_rate": 6.08982614294913e-07,
      "logits/chosen": -0.8233642578125,
      "logits/rejected": -0.8514159917831421,
      "logps/chosen": -370.625,
      "logps/rejected": -465.70001220703125,
      "loss": 0.0247,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -1.6372802257537842,
      "rewards/margins": 10.8671875,
      "rewards/rejected": -12.513280868530273,
      "step": 2430
    },
    {
      "epoch": 1.5703250724171225,
      "grad_norm": 42.632038258106874,
      "learning_rate": 6.073728267868641e-07,
      "logits/chosen": -0.7555176019668579,
      "logits/rejected": -0.775775134563446,
      "logps/chosen": -403.6499938964844,
      "logps/rejected": -472.1499938964844,
      "loss": 0.0496,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.56494140625,
      "rewards/margins": 10.884374618530273,
      "rewards/rejected": -11.444531440734863,
      "step": 2440
    },
    {
      "epoch": 1.5767621499839073,
      "grad_norm": 4.234950034854178,
      "learning_rate": 6.057630392788152e-07,
      "logits/chosen": -0.7583984136581421,
      "logits/rejected": -0.771484375,
      "logps/chosen": -392.73748779296875,
      "logps/rejected": -478.0249938964844,
      "loss": 0.0933,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.113104224205017,
      "rewards/margins": 10.796093940734863,
      "rewards/rejected": -11.916406631469727,
      "step": 2450
    },
    {
      "epoch": 1.583199227550692,
      "grad_norm": 3.9163310479581566,
      "learning_rate": 6.041532517707662e-07,
      "logits/chosen": -0.8863281011581421,
      "logits/rejected": -0.895947277545929,
      "logps/chosen": -395.3500061035156,
      "logps/rejected": -483.54998779296875,
      "loss": 0.0371,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.408911108970642,
      "rewards/margins": 11.666406631469727,
      "rewards/rejected": -13.063281059265137,
      "step": 2460
    },
    {
      "epoch": 1.5896363051174767,
      "grad_norm": 62.02613513340239,
      "learning_rate": 6.025434642627173e-07,
      "logits/chosen": -0.8659423589706421,
      "logits/rejected": -0.879638671875,
      "logps/chosen": -402.1499938964844,
      "logps/rejected": -507.375,
      "loss": 0.039,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -1.2672119140625,
      "rewards/margins": 11.375781059265137,
      "rewards/rejected": -12.642969131469727,
      "step": 2470
    },
    {
      "epoch": 1.5960733826842612,
      "grad_norm": 3.3628306955247598,
      "learning_rate": 6.009336767546684e-07,
      "logits/chosen": -0.891796886920929,
      "logits/rejected": -0.920849621295929,
      "logps/chosen": -405.875,
      "logps/rejected": -493.67498779296875,
      "loss": 0.0499,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.6985229253768921,
      "rewards/margins": 11.516406059265137,
      "rewards/rejected": -12.217968940734863,
      "step": 2480
    },
    {
      "epoch": 1.602510460251046,
      "grad_norm": 6.944832508288456,
      "learning_rate": 5.993238892466194e-07,
      "logits/chosen": -0.8244873285293579,
      "logits/rejected": -0.838452160358429,
      "logps/chosen": -371.3999938964844,
      "logps/rejected": -458.82501220703125,
      "loss": 0.0703,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.8302001953125,
      "rewards/margins": 10.130468368530273,
      "rewards/rejected": -10.963281631469727,
      "step": 2490
    },
    {
      "epoch": 1.6089475378178308,
      "grad_norm": 16.431404128160846,
      "learning_rate": 5.977141017385705e-07,
      "logits/chosen": -1.0031249523162842,
      "logits/rejected": -1.000390648841858,
      "logps/chosen": -377.54998779296875,
      "logps/rejected": -472.5,
      "loss": 0.0395,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.730297863483429,
      "rewards/margins": 10.501562118530273,
      "rewards/rejected": -11.234375,
      "step": 2500
    },
    {
      "epoch": 1.6153846153846154,
      "grad_norm": 0.9146958952965157,
      "learning_rate": 5.961043142305216e-07,
      "logits/chosen": -0.8604491949081421,
      "logits/rejected": -0.87451171875,
      "logps/chosen": -382.51251220703125,
      "logps/rejected": -439.375,
      "loss": 0.052,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.5503753423690796,
      "rewards/margins": 9.967187881469727,
      "rewards/rejected": -10.515625,
      "step": 2510
    },
    {
      "epoch": 1.6218216929514,
      "grad_norm": 1.2281780912032512,
      "learning_rate": 5.944945267224725e-07,
      "logits/chosen": -0.8853515386581421,
      "logits/rejected": -0.891796886920929,
      "logps/chosen": -390.8500061035156,
      "logps/rejected": -468.1499938964844,
      "loss": 0.04,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.706024169921875,
      "rewards/margins": 11.03125,
      "rewards/rejected": -11.733593940734863,
      "step": 2520
    },
    {
      "epoch": 1.6282587705181848,
      "grad_norm": 0.9436650570997343,
      "learning_rate": 5.928847392144237e-07,
      "logits/chosen": -0.8895508050918579,
      "logits/rejected": -0.8848632574081421,
      "logps/chosen": -392.96875,
      "logps/rejected": -470.4750061035156,
      "loss": 0.0295,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.9778808355331421,
      "rewards/margins": 10.90625,
      "rewards/rejected": -11.883593559265137,
      "step": 2530
    },
    {
      "epoch": 1.6346958480849694,
      "grad_norm": 0.2775703409910018,
      "learning_rate": 5.912749517063748e-07,
      "logits/chosen": -0.940356433391571,
      "logits/rejected": -0.9769531488418579,
      "logps/chosen": -396.0,
      "logps/rejected": -452.0249938964844,
      "loss": 0.0129,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -1.201135277748108,
      "rewards/margins": 11.169530868530273,
      "rewards/rejected": -12.374218940734863,
      "step": 2540
    },
    {
      "epoch": 1.6411329256517542,
      "grad_norm": 31.914598423294287,
      "learning_rate": 5.896651641983258e-07,
      "logits/chosen": -1.011376976966858,
      "logits/rejected": -1.037695288658142,
      "logps/chosen": -381.79998779296875,
      "logps/rejected": -435.6499938964844,
      "loss": 0.0321,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -1.6757323741912842,
      "rewards/margins": 10.783594131469727,
      "rewards/rejected": -12.4609375,
      "step": 2550
    },
    {
      "epoch": 1.6475700032185387,
      "grad_norm": 72.59157572433088,
      "learning_rate": 5.880553766902768e-07,
      "logits/chosen": -0.977001965045929,
      "logits/rejected": -0.993701159954071,
      "logps/chosen": -403.92498779296875,
      "logps/rejected": -496.8999938964844,
      "loss": 0.0418,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.3961181640625,
      "rewards/margins": 11.057031631469727,
      "rewards/rejected": -12.449999809265137,
      "step": 2560
    },
    {
      "epoch": 1.6540070807853233,
      "grad_norm": 14.184682824537003,
      "learning_rate": 5.86445589182228e-07,
      "logits/chosen": -0.903369128704071,
      "logits/rejected": -0.904467761516571,
      "logps/chosen": -392.32501220703125,
      "logps/rejected": -475.57501220703125,
      "loss": 0.0652,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.341552734375,
      "rewards/margins": 12.142969131469727,
      "rewards/rejected": -13.478124618530273,
      "step": 2570
    },
    {
      "epoch": 1.6604441583521081,
      "grad_norm": 5.25218753944226,
      "learning_rate": 5.84835801674179e-07,
      "logits/chosen": -0.955322265625,
      "logits/rejected": -0.977343738079071,
      "logps/chosen": -375.13751220703125,
      "logps/rejected": -466.1499938964844,
      "loss": 0.0314,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -1.451562523841858,
      "rewards/margins": 11.546093940734863,
      "rewards/rejected": -12.996874809265137,
      "step": 2580
    },
    {
      "epoch": 1.666881235918893,
      "grad_norm": 0.25832510910108303,
      "learning_rate": 5.832260141661301e-07,
      "logits/chosen": -0.9334472417831421,
      "logits/rejected": -0.955920398235321,
      "logps/chosen": -417.75,
      "logps/rejected": -496.07501220703125,
      "loss": 0.0242,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -1.4179565906524658,
      "rewards/margins": 11.3984375,
      "rewards/rejected": -12.81640625,
      "step": 2590
    },
    {
      "epoch": 1.6733183134856775,
      "grad_norm": 8.01847196423977,
      "learning_rate": 5.81616226658081e-07,
      "logits/chosen": -0.925707995891571,
      "logits/rejected": -0.940673828125,
      "logps/chosen": -397.67498779296875,
      "logps/rejected": -461.79998779296875,
      "loss": 0.0398,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -1.860571265220642,
      "rewards/margins": 11.041406631469727,
      "rewards/rejected": -12.903905868530273,
      "step": 2600
    },
    {
      "epoch": 1.679755391052462,
      "grad_norm": 8.250100146015557,
      "learning_rate": 5.800064391500322e-07,
      "logits/chosen": -0.935986340045929,
      "logits/rejected": -0.957763671875,
      "logps/chosen": -415.67498779296875,
      "logps/rejected": -511.6000061035156,
      "loss": 0.1153,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -2.994311571121216,
      "rewards/margins": 11.036718368530273,
      "rewards/rejected": -14.035937309265137,
      "step": 2610
    },
    {
      "epoch": 1.6861924686192469,
      "grad_norm": 13.134105792412328,
      "learning_rate": 5.783966516419833e-07,
      "logits/chosen": -0.983447253704071,
      "logits/rejected": -0.974560558795929,
      "logps/chosen": -386.82501220703125,
      "logps/rejected": -476.75,
      "loss": 0.0335,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -2.349414110183716,
      "rewards/margins": 11.560155868530273,
      "rewards/rejected": -13.899999618530273,
      "step": 2620
    },
    {
      "epoch": 1.6926295461860317,
      "grad_norm": 116.36227986157657,
      "learning_rate": 5.767868641339343e-07,
      "logits/chosen": -0.8326416015625,
      "logits/rejected": -0.829663097858429,
      "logps/chosen": -353.1499938964844,
      "logps/rejected": -462.7749938964844,
      "loss": 0.0935,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -2.1159729957580566,
      "rewards/margins": 11.43359375,
      "rewards/rejected": -13.545312881469727,
      "step": 2630
    },
    {
      "epoch": 1.6990666237528163,
      "grad_norm": 4.5619674703943085,
      "learning_rate": 5.751770766258853e-07,
      "logits/chosen": -0.8194214105606079,
      "logits/rejected": -0.84722900390625,
      "logps/chosen": -422.07501220703125,
      "logps/rejected": -498.375,
      "loss": 0.0954,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -1.7252624034881592,
      "rewards/margins": 10.96875,
      "rewards/rejected": -12.690625190734863,
      "step": 2640
    },
    {
      "epoch": 1.7055037013196008,
      "grad_norm": 3.006491121324236,
      "learning_rate": 5.735672891178365e-07,
      "logits/chosen": -0.9326721429824829,
      "logits/rejected": -0.929638683795929,
      "logps/chosen": -418.79998779296875,
      "logps/rejected": -483.8999938964844,
      "loss": 0.0323,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -2.345507860183716,
      "rewards/margins": 11.989062309265137,
      "rewards/rejected": -14.334375381469727,
      "step": 2650
    },
    {
      "epoch": 1.7119407788863856,
      "grad_norm": 4.13341575346933,
      "learning_rate": 5.719575016097875e-07,
      "logits/chosen": -0.828857421875,
      "logits/rejected": -0.8162841796875,
      "logps/chosen": -356.2250061035156,
      "logps/rejected": -526.7000122070312,
      "loss": 0.045,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -3.123608350753784,
      "rewards/margins": 13.339062690734863,
      "rewards/rejected": -16.462499618530273,
      "step": 2660
    },
    {
      "epoch": 1.7183778564531704,
      "grad_norm": 22.117373309483987,
      "learning_rate": 5.703477141017385e-07,
      "logits/chosen": -0.844775378704071,
      "logits/rejected": -0.8448730707168579,
      "logps/chosen": -404.32501220703125,
      "logps/rejected": -502.25,
      "loss": 0.0476,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -3.886608839035034,
      "rewards/margins": 12.126562118530273,
      "rewards/rejected": -16.014062881469727,
      "step": 2670
    },
    {
      "epoch": 1.724814934019955,
      "grad_norm": 22.82723612005747,
      "learning_rate": 5.687379265936896e-07,
      "logits/chosen": -0.9837402105331421,
      "logits/rejected": -1.027099609375,
      "logps/chosen": -404.8999938964844,
      "logps/rejected": -503.8999938964844,
      "loss": 0.0874,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -5.081250190734863,
      "rewards/margins": 12.1015625,
      "rewards/rejected": -17.173437118530273,
      "step": 2680
    },
    {
      "epoch": 1.7312520115867396,
      "grad_norm": 10.704380536262521,
      "learning_rate": 5.671281390856407e-07,
      "logits/chosen": -0.9532226324081421,
      "logits/rejected": -0.9735351800918579,
      "logps/chosen": -422.7749938964844,
      "logps/rejected": -521.2999877929688,
      "loss": 0.0372,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -4.213476657867432,
      "rewards/margins": 12.93359375,
      "rewards/rejected": -17.1484375,
      "step": 2690
    },
    {
      "epoch": 1.7376890891535242,
      "grad_norm": 2.73829439018618,
      "learning_rate": 5.655183515775918e-07,
      "logits/chosen": -0.94189453125,
      "logits/rejected": -0.97607421875,
      "logps/chosen": -440.29998779296875,
      "logps/rejected": -522.8499755859375,
      "loss": 0.0201,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.729840040206909,
      "rewards/margins": 12.442187309265137,
      "rewards/rejected": -15.1640625,
      "step": 2700
    },
    {
      "epoch": 1.744126166720309,
      "grad_norm": 36.282664030023824,
      "learning_rate": 5.639085640695428e-07,
      "logits/chosen": -0.8478027582168579,
      "logits/rejected": -0.8516601324081421,
      "logps/chosen": -416.2250061035156,
      "logps/rejected": -503.7749938964844,
      "loss": 0.151,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -2.7637696266174316,
      "rewards/margins": 11.846875190734863,
      "rewards/rejected": -14.606249809265137,
      "step": 2710
    },
    {
      "epoch": 1.7505632442870938,
      "grad_norm": 2.1283637906112407,
      "learning_rate": 5.622987765614938e-07,
      "logits/chosen": -0.9247070550918579,
      "logits/rejected": -0.9312499761581421,
      "logps/chosen": -409.7749938964844,
      "logps/rejected": -511.5,
      "loss": 0.0602,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -2.046765089035034,
      "rewards/margins": 12.314062118530273,
      "rewards/rejected": -14.3515625,
      "step": 2720
    },
    {
      "epoch": 1.7570003218538783,
      "grad_norm": 134.5608375076651,
      "learning_rate": 5.60688989053445e-07,
      "logits/chosen": -0.8810790777206421,
      "logits/rejected": -0.922985851764679,
      "logps/chosen": -412.5,
      "logps/rejected": -479.42498779296875,
      "loss": 0.083,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -2.50537109375,
      "rewards/margins": 11.076562881469727,
      "rewards/rejected": -13.574999809265137,
      "step": 2730
    },
    {
      "epoch": 1.763437399420663,
      "grad_norm": 44.62636520944944,
      "learning_rate": 5.590792015453961e-07,
      "logits/chosen": -0.8690429925918579,
      "logits/rejected": -0.862597644329071,
      "logps/chosen": -439.42498779296875,
      "logps/rejected": -518.0999755859375,
      "loss": 0.0385,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -2.424877882003784,
      "rewards/margins": 12.205469131469727,
      "rewards/rejected": -14.629687309265137,
      "step": 2740
    },
    {
      "epoch": 1.7698744769874477,
      "grad_norm": 11.000258418597038,
      "learning_rate": 5.57469414037347e-07,
      "logits/chosen": -0.8440917730331421,
      "logits/rejected": -0.877392590045929,
      "logps/chosen": -431.0,
      "logps/rejected": -490.1000061035156,
      "loss": 0.0655,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -2.1365966796875,
      "rewards/margins": 11.603124618530273,
      "rewards/rejected": -13.740625381469727,
      "step": 2750
    },
    {
      "epoch": 1.7763115545542325,
      "grad_norm": 24.728950785009513,
      "learning_rate": 5.558596265292981e-07,
      "logits/chosen": -0.793408215045929,
      "logits/rejected": -0.82470703125,
      "logps/chosen": -384.2250061035156,
      "logps/rejected": -455.75,
      "loss": 0.0423,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -1.9411132335662842,
      "rewards/margins": 11.362500190734863,
      "rewards/rejected": -13.303125381469727,
      "step": 2760
    },
    {
      "epoch": 1.782748632121017,
      "grad_norm": 18.185794833969,
      "learning_rate": 5.542498390212492e-07,
      "logits/chosen": -0.8042999505996704,
      "logits/rejected": -0.7938781976699829,
      "logps/chosen": -376.0,
      "logps/rejected": -467.70001220703125,
      "loss": 0.0949,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.829443335533142,
      "rewards/margins": 10.978906631469727,
      "rewards/rejected": -12.80078125,
      "step": 2770
    },
    {
      "epoch": 1.7891857096878017,
      "grad_norm": 36.77834630320014,
      "learning_rate": 5.526400515132002e-07,
      "logits/chosen": -0.898144543170929,
      "logits/rejected": -0.8753417730331421,
      "logps/chosen": -413.8999938964844,
      "logps/rejected": -524.0999755859375,
      "loss": 0.0878,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -2.239550828933716,
      "rewards/margins": 11.630468368530273,
      "rewards/rejected": -13.878125190734863,
      "step": 2780
    },
    {
      "epoch": 1.7956227872545865,
      "grad_norm": 50.535377745071834,
      "learning_rate": 5.510302640051513e-07,
      "logits/chosen": -0.871777355670929,
      "logits/rejected": -0.909472644329071,
      "logps/chosen": -396.1000061035156,
      "logps/rejected": -475.0,
      "loss": 0.037,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -1.981176733970642,
      "rewards/margins": 11.844531059265137,
      "rewards/rejected": -13.824999809265137,
      "step": 2790
    },
    {
      "epoch": 1.8020598648213713,
      "grad_norm": 187.81436429316912,
      "learning_rate": 5.494204764971023e-07,
      "logits/chosen": -0.8958984613418579,
      "logits/rejected": -0.9166015386581421,
      "logps/chosen": -413.04998779296875,
      "logps/rejected": -503.125,
      "loss": 0.0652,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -2.3756103515625,
      "rewards/margins": 11.768750190734863,
      "rewards/rejected": -14.146875381469727,
      "step": 2800
    },
    {
      "epoch": 1.8084969423881558,
      "grad_norm": 2.46028333934951,
      "learning_rate": 5.478106889890535e-07,
      "logits/chosen": -0.9292968511581421,
      "logits/rejected": -0.9300781488418579,
      "logps/chosen": -367.375,
      "logps/rejected": -495.1000061035156,
      "loss": 0.0362,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.463037133216858,
      "rewards/margins": 12.038281440734863,
      "rewards/rejected": -13.5078125,
      "step": 2810
    },
    {
      "epoch": 1.8149340199549404,
      "grad_norm": 231.08328656997156,
      "learning_rate": 5.462009014810045e-07,
      "logits/chosen": -0.952929675579071,
      "logits/rejected": -0.947460949420929,
      "logps/chosen": -433.07501220703125,
      "logps/rejected": -511.25,
      "loss": 0.0422,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.267065405845642,
      "rewards/margins": 13.215624809265137,
      "rewards/rejected": -14.4765625,
      "step": 2820
    },
    {
      "epoch": 1.821371097521725,
      "grad_norm": 20.163208476823606,
      "learning_rate": 5.445911139729555e-07,
      "logits/chosen": -0.944793701171875,
      "logits/rejected": -0.9900878667831421,
      "logps/chosen": -394.07501220703125,
      "logps/rejected": -501.0,
      "loss": 0.0365,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -2.1129150390625,
      "rewards/margins": 12.45703125,
      "rewards/rejected": -14.5625,
      "step": 2830
    },
    {
      "epoch": 1.8278081750885098,
      "grad_norm": 144.33944891843873,
      "learning_rate": 5.429813264649066e-07,
      "logits/chosen": -0.917529284954071,
      "logits/rejected": -0.927441418170929,
      "logps/chosen": -393.42498779296875,
      "logps/rejected": -512.9500122070312,
      "loss": 0.0597,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -2.2644286155700684,
      "rewards/margins": 12.955469131469727,
      "rewards/rejected": -15.217187881469727,
      "step": 2840
    },
    {
      "epoch": 1.8342452526552946,
      "grad_norm": 21.552588183350537,
      "learning_rate": 5.413715389568578e-07,
      "logits/chosen": -0.94049072265625,
      "logits/rejected": -0.9557861089706421,
      "logps/chosen": -371.3374938964844,
      "logps/rejected": -473.95001220703125,
      "loss": 0.0413,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.8835632801055908,
      "rewards/margins": 12.410937309265137,
      "rewards/rejected": -14.28515625,
      "step": 2850
    },
    {
      "epoch": 1.8406823302220792,
      "grad_norm": 0.33607759158894557,
      "learning_rate": 5.397617514488087e-07,
      "logits/chosen": -1.03466796875,
      "logits/rejected": -1.0185546875,
      "logps/chosen": -397.7749938964844,
      "logps/rejected": -503.3500061035156,
      "loss": 0.1239,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -2.576586961746216,
      "rewards/margins": 11.790624618530273,
      "rewards/rejected": -14.370312690734863,
      "step": 2860
    },
    {
      "epoch": 1.8471194077888637,
      "grad_norm": 7.301426102324497,
      "learning_rate": 5.381519639407598e-07,
      "logits/chosen": -0.9674316644668579,
      "logits/rejected": -0.9830077886581421,
      "logps/chosen": -387.2250061035156,
      "logps/rejected": -525.6500244140625,
      "loss": 0.0404,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -3.5799803733825684,
      "rewards/margins": 12.387499809265137,
      "rewards/rejected": -15.962499618530273,
      "step": 2870
    },
    {
      "epoch": 1.8535564853556485,
      "grad_norm": 8.160121310799461,
      "learning_rate": 5.365421764327109e-07,
      "logits/chosen": -1.011450171470642,
      "logits/rejected": -1.022363305091858,
      "logps/chosen": -412.375,
      "logps/rejected": -507.95001220703125,
      "loss": 0.1486,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -3.354541063308716,
      "rewards/margins": 11.653124809265137,
      "rewards/rejected": -15.006250381469727,
      "step": 2880
    },
    {
      "epoch": 1.8599935629224333,
      "grad_norm": 5.827434791236949,
      "learning_rate": 5.34932388924662e-07,
      "logits/chosen": -0.955517590045929,
      "logits/rejected": -0.963061511516571,
      "logps/chosen": -429.04998779296875,
      "logps/rejected": -512.7125244140625,
      "loss": 0.1118,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -3.6991209983825684,
      "rewards/margins": 12.186718940734863,
      "rewards/rejected": -15.900781631469727,
      "step": 2890
    },
    {
      "epoch": 1.866430640489218,
      "grad_norm": 4.154458697921168,
      "learning_rate": 5.33322601416613e-07,
      "logits/chosen": -0.95458984375,
      "logits/rejected": -0.9620605707168579,
      "logps/chosen": -408.5249938964844,
      "logps/rejected": -510.5,
      "loss": 0.056,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -2.6628174781799316,
      "rewards/margins": 11.815625190734863,
      "rewards/rejected": -14.484375,
      "step": 2900
    },
    {
      "epoch": 1.8728677180560025,
      "grad_norm": 25.750618944342897,
      "learning_rate": 5.31712813908564e-07,
      "logits/chosen": -0.978076159954071,
      "logits/rejected": -0.968994140625,
      "logps/chosen": -376.70001220703125,
      "logps/rejected": -513.2000122070312,
      "loss": 0.0537,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.340625047683716,
      "rewards/margins": 11.696874618530273,
      "rewards/rejected": -14.035937309265137,
      "step": 2910
    },
    {
      "epoch": 1.8793047956227873,
      "grad_norm": 34.482062477676116,
      "learning_rate": 5.301030264005151e-07,
      "logits/chosen": -0.887988269329071,
      "logits/rejected": -0.9131835699081421,
      "logps/chosen": -393.8999938964844,
      "logps/rejected": -460.1499938964844,
      "loss": 0.034,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -1.0007812976837158,
      "rewards/margins": 11.300000190734863,
      "rewards/rejected": -12.290624618530273,
      "step": 2920
    },
    {
      "epoch": 1.885741873189572,
      "grad_norm": 0.6449608149453013,
      "learning_rate": 5.284932388924662e-07,
      "logits/chosen": -0.908447265625,
      "logits/rejected": -0.928955078125,
      "logps/chosen": -366.4624938964844,
      "logps/rejected": -453.07501220703125,
      "loss": 0.0462,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.143225073814392,
      "rewards/margins": 11.33984375,
      "rewards/rejected": -12.48046875,
      "step": 2930
    },
    {
      "epoch": 1.8921789507563567,
      "grad_norm": 10.761664351455913,
      "learning_rate": 5.268834513844172e-07,
      "logits/chosen": -0.9051758050918579,
      "logits/rejected": -0.942919909954071,
      "logps/chosen": -378.25,
      "logps/rejected": -482.6000061035156,
      "loss": 0.0644,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.2742919921875,
      "rewards/margins": 10.936327934265137,
      "rewards/rejected": -12.215624809265137,
      "step": 2940
    },
    {
      "epoch": 1.8986160283231412,
      "grad_norm": 91.23334834373519,
      "learning_rate": 5.252736638763683e-07,
      "logits/chosen": -0.938720703125,
      "logits/rejected": -0.9503418207168579,
      "logps/chosen": -413.625,
      "logps/rejected": -475.70001220703125,
      "loss": 0.0455,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.732348620891571,
      "rewards/margins": 11.340624809265137,
      "rewards/rejected": -12.078125,
      "step": 2950
    },
    {
      "epoch": 1.9050531058899258,
      "grad_norm": 12.915735407500012,
      "learning_rate": 5.236638763683194e-07,
      "logits/chosen": -0.85382080078125,
      "logits/rejected": -0.871350109577179,
      "logps/chosen": -366.95001220703125,
      "logps/rejected": -493.75,
      "loss": 0.0473,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.8136841058731079,
      "rewards/margins": 11.69140625,
      "rewards/rejected": -12.508593559265137,
      "step": 2960
    },
    {
      "epoch": 1.9114901834567106,
      "grad_norm": 227.12309414417433,
      "learning_rate": 5.220540888602704e-07,
      "logits/chosen": -0.9459472894668579,
      "logits/rejected": -0.935351550579071,
      "logps/chosen": -412.7749938964844,
      "logps/rejected": -502.0,
      "loss": 0.0448,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.43291014432907104,
      "rewards/margins": 11.698437690734863,
      "rewards/rejected": -12.13671875,
      "step": 2970
    },
    {
      "epoch": 1.9179272610234954,
      "grad_norm": 0.196421800385388,
      "learning_rate": 5.204443013522215e-07,
      "logits/chosen": -0.9383789300918579,
      "logits/rejected": -0.953320324420929,
      "logps/chosen": -379.6000061035156,
      "logps/rejected": -474.6499938964844,
      "loss": 0.0569,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.2946655750274658,
      "rewards/margins": 11.255078315734863,
      "rewards/rejected": -12.55078125,
      "step": 2980
    },
    {
      "epoch": 1.92436433859028,
      "grad_norm": 1.346626934480126,
      "learning_rate": 5.188345138441726e-07,
      "logits/chosen": -0.924121081829071,
      "logits/rejected": -0.9498046636581421,
      "logps/chosen": -392.54998779296875,
      "logps/rejected": -464.4750061035156,
      "loss": 0.0398,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.9569946527481079,
      "rewards/margins": 11.33203125,
      "rewards/rejected": -12.289843559265137,
      "step": 2990
    },
    {
      "epoch": 1.9308014161570646,
      "grad_norm": 8.375416288346715,
      "learning_rate": 5.172247263361235e-07,
      "logits/chosen": -1.010498046875,
      "logits/rejected": -1.031152367591858,
      "logps/chosen": -442.7250061035156,
      "logps/rejected": -524.5999755859375,
      "loss": 0.031,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -1.4194824695587158,
      "rewards/margins": 12.033594131469727,
      "rewards/rejected": -13.460156440734863,
      "step": 3000
    },
    {
      "epoch": 1.9372384937238494,
      "grad_norm": 12.910753832152915,
      "learning_rate": 5.156149388280747e-07,
      "logits/chosen": -0.9345703125,
      "logits/rejected": -0.956250011920929,
      "logps/chosen": -392.625,
      "logps/rejected": -460.6499938964844,
      "loss": 0.0398,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -2.1406006813049316,
      "rewards/margins": 12.096094131469727,
      "rewards/rejected": -14.235937118530273,
      "step": 3010
    },
    {
      "epoch": 1.9436755712906342,
      "grad_norm": 1.0305047534231448,
      "learning_rate": 5.140051513200258e-07,
      "logits/chosen": -0.94677734375,
      "logits/rejected": -0.9766601324081421,
      "logps/chosen": -381.4750061035156,
      "logps/rejected": -468.8500061035156,
      "loss": 0.0257,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -1.2906372547149658,
      "rewards/margins": 11.268750190734863,
      "rewards/rejected": -12.560155868530273,
      "step": 3020
    },
    {
      "epoch": 1.9501126488574188,
      "grad_norm": 0.36209350367976334,
      "learning_rate": 5.123953638119768e-07,
      "logits/chosen": -1.013085961341858,
      "logits/rejected": -1.033300757408142,
      "logps/chosen": -421.5249938964844,
      "logps/rejected": -502.75,
      "loss": 0.0626,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.6013672351837158,
      "rewards/margins": 11.564844131469727,
      "rewards/rejected": -13.159375190734863,
      "step": 3030
    },
    {
      "epoch": 1.9565497264242033,
      "grad_norm": 11.479780504467316,
      "learning_rate": 5.107855763039278e-07,
      "logits/chosen": -1.0580322742462158,
      "logits/rejected": -1.06884765625,
      "logps/chosen": -393.7749938964844,
      "logps/rejected": -477.07501220703125,
      "loss": 0.0234,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -1.379449486732483,
      "rewards/margins": 11.125781059265137,
      "rewards/rejected": -12.494531631469727,
      "step": 3040
    },
    {
      "epoch": 1.9629868039909881,
      "grad_norm": 18.87941406876472,
      "learning_rate": 5.09175788795879e-07,
      "logits/chosen": -0.984814465045929,
      "logits/rejected": -1.028466820716858,
      "logps/chosen": -381.54998779296875,
      "logps/rejected": -484.4750061035156,
      "loss": 0.0733,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -1.355200171470642,
      "rewards/margins": 12.080469131469727,
      "rewards/rejected": -13.435937881469727,
      "step": 3050
    },
    {
      "epoch": 1.969423881557773,
      "grad_norm": 46.197107715130024,
      "learning_rate": 5.0756600128783e-07,
      "logits/chosen": -1.0607421398162842,
      "logits/rejected": -1.0695312023162842,
      "logps/chosen": -407.5249938964844,
      "logps/rejected": -496.1000061035156,
      "loss": 0.0459,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -2.9796385765075684,
      "rewards/margins": 12.356249809265137,
      "rewards/rejected": -15.340624809265137,
      "step": 3060
    },
    {
      "epoch": 1.9758609591245575,
      "grad_norm": 2.624768811992857,
      "learning_rate": 5.059562137797811e-07,
      "logits/chosen": -1.1554687023162842,
      "logits/rejected": -1.15771484375,
      "logps/chosen": -403.79998779296875,
      "logps/rejected": -472.04998779296875,
      "loss": 0.0206,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -3.441601514816284,
      "rewards/margins": 12.533594131469727,
      "rewards/rejected": -15.979687690734863,
      "step": 3070
    },
    {
      "epoch": 1.982298036691342,
      "grad_norm": 1.3093063165499863,
      "learning_rate": 5.04346426271732e-07,
      "logits/chosen": -1.0871093273162842,
      "logits/rejected": -1.099609375,
      "logps/chosen": -447.0249938964844,
      "logps/rejected": -526.1500244140625,
      "loss": 0.0173,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -3.60986328125,
      "rewards/margins": 13.119531631469727,
      "rewards/rejected": -16.731250762939453,
      "step": 3080
    },
    {
      "epoch": 1.9887351142581267,
      "grad_norm": 0.12450097571649546,
      "learning_rate": 5.027366387636832e-07,
      "logits/chosen": -1.007080078125,
      "logits/rejected": -1.0055663585662842,
      "logps/chosen": -383.2749938964844,
      "logps/rejected": -498.3500061035156,
      "loss": 0.0646,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -4.04296875,
      "rewards/margins": 12.362500190734863,
      "rewards/rejected": -16.412500381469727,
      "step": 3090
    },
    {
      "epoch": 1.9951721918249115,
      "grad_norm": 1.3344752910311428,
      "learning_rate": 5.011268512556343e-07,
      "logits/chosen": -1.055419921875,
      "logits/rejected": -1.0906250476837158,
      "logps/chosen": -408.9750061035156,
      "logps/rejected": -504.7250061035156,
      "loss": 0.0326,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -3.272106885910034,
      "rewards/margins": 12.796093940734863,
      "rewards/rejected": -16.071094512939453,
      "step": 3100
    },
    {
      "epoch": 2.001287415513357,
      "grad_norm": 0.1077285501528772,
      "learning_rate": 4.995170637475852e-07,
      "logits/chosen": -1.0288857221603394,
      "logits/rejected": -1.0637335777282715,
      "logps/chosen": -362.5789489746094,
      "logps/rejected": -500.15789794921875,
      "loss": 0.0212,
      "rewards/accuracies": 0.9934210777282715,
      "rewards/chosen": -3.1022307872772217,
      "rewards/margins": 13.444901466369629,
      "rewards/rejected": -16.542762756347656,
      "step": 3110
    },
    {
      "epoch": 2.0077244930801417,
      "grad_norm": 0.09265850851102228,
      "learning_rate": 4.979072762395364e-07,
      "logits/chosen": -1.086035132408142,
      "logits/rejected": -1.103613257408142,
      "logps/chosen": -406.2749938964844,
      "logps/rejected": -530.4000244140625,
      "loss": 0.0227,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -1.9188964366912842,
      "rewards/margins": 15.073437690734863,
      "rewards/rejected": -16.998437881469727,
      "step": 3120
    },
    {
      "epoch": 2.0141615706469262,
      "grad_norm": 3.06968476113232,
      "learning_rate": 4.962974887314874e-07,
      "logits/chosen": -1.0173828601837158,
      "logits/rejected": -1.0482666492462158,
      "logps/chosen": -374.79998779296875,
      "logps/rejected": -495.29998779296875,
      "loss": 0.0458,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -2.3733763694763184,
      "rewards/margins": 15.339062690734863,
      "rewards/rejected": -17.7109375,
      "step": 3130
    },
    {
      "epoch": 2.020598648213711,
      "grad_norm": 6.148575080941695,
      "learning_rate": 4.946877012234385e-07,
      "logits/chosen": -1.033300757408142,
      "logits/rejected": -1.0410645008087158,
      "logps/chosen": -366.11248779296875,
      "logps/rejected": -507.20001220703125,
      "loss": 0.0245,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -2.705615282058716,
      "rewards/margins": 14.09765625,
      "rewards/rejected": -16.806249618530273,
      "step": 3140
    },
    {
      "epoch": 2.027035725780496,
      "grad_norm": 3.966203474050368,
      "learning_rate": 4.930779137153895e-07,
      "logits/chosen": -0.929516613483429,
      "logits/rejected": -0.9665282964706421,
      "logps/chosen": -374.4750061035156,
      "logps/rejected": -498.2749938964844,
      "loss": 0.0152,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.3712525367736816,
      "rewards/margins": 14.732030868530273,
      "rewards/rejected": -17.109375,
      "step": 3150
    },
    {
      "epoch": 2.0334728033472804,
      "grad_norm": 0.1594348793038751,
      "learning_rate": 4.914681262073406e-07,
      "logits/chosen": -1.0707519054412842,
      "logits/rejected": -1.0604248046875,
      "logps/chosen": -389.5,
      "logps/rejected": -529.5,
      "loss": 0.0136,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.8618407249450684,
      "rewards/margins": 15.59375,
      "rewards/rejected": -18.470312118530273,
      "step": 3160
    },
    {
      "epoch": 2.039909880914065,
      "grad_norm": 0.014532573399047911,
      "learning_rate": 4.898583386992917e-07,
      "logits/chosen": -1.038476586341858,
      "logits/rejected": -1.043554663658142,
      "logps/chosen": -366.57501220703125,
      "logps/rejected": -519.25,
      "loss": 0.0406,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -2.771923780441284,
      "rewards/margins": 15.8203125,
      "rewards/rejected": -18.603124618530273,
      "step": 3170
    },
    {
      "epoch": 2.0463469584808496,
      "grad_norm": 0.7015845235202134,
      "learning_rate": 4.882485511912428e-07,
      "logits/chosen": -1.0251953601837158,
      "logits/rejected": -1.0403563976287842,
      "logps/chosen": -433.5375061035156,
      "logps/rejected": -579.0,
      "loss": 0.0147,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -3.81591796875,
      "rewards/margins": 16.274999618530273,
      "rewards/rejected": -20.09375,
      "step": 3180
    },
    {
      "epoch": 2.052784036047634,
      "grad_norm": 0.11338404332547947,
      "learning_rate": 4.866387636831938e-07,
      "logits/chosen": -1.0687987804412842,
      "logits/rejected": -1.0724608898162842,
      "logps/chosen": -381.3999938964844,
      "logps/rejected": -527.0499877929688,
      "loss": 0.0854,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -4.091454982757568,
      "rewards/margins": 15.41796875,
      "rewards/rejected": -19.514062881469727,
      "step": 3190
    },
    {
      "epoch": 2.059221113614419,
      "grad_norm": 0.07307065029187323,
      "learning_rate": 4.850289761751449e-07,
      "logits/chosen": -1.0838134288787842,
      "logits/rejected": -1.1128661632537842,
      "logps/chosen": -390.20001220703125,
      "logps/rejected": -526.5499877929688,
      "loss": 0.0145,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -3.879467725753784,
      "rewards/margins": 15.40625,
      "rewards/rejected": -19.292186737060547,
      "step": 3200
    },
    {
      "epoch": 2.0656581911812038,
      "grad_norm": 0.4252136551579939,
      "learning_rate": 4.834191886670959e-07,
      "logits/chosen": -1.02734375,
      "logits/rejected": -1.0543944835662842,
      "logps/chosen": -434.95001220703125,
      "logps/rejected": -551.9500122070312,
      "loss": 0.0057,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -3.806933641433716,
      "rewards/margins": 15.7109375,
      "rewards/rejected": -19.514062881469727,
      "step": 3210
    },
    {
      "epoch": 2.0720952687479883,
      "grad_norm": 0.22065842973577401,
      "learning_rate": 4.818094011590471e-07,
      "logits/chosen": -1.009033203125,
      "logits/rejected": -1.04412841796875,
      "logps/chosen": -389.54998779296875,
      "logps/rejected": -543.875,
      "loss": 0.0228,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -3.7743163108825684,
      "rewards/margins": 16.923437118530273,
      "rewards/rejected": -20.692188262939453,
      "step": 3220
    },
    {
      "epoch": 2.078532346314773,
      "grad_norm": 3.366783353939681,
      "learning_rate": 4.80199613650998e-07,
      "logits/chosen": -1.0969727039337158,
      "logits/rejected": -1.1025390625,
      "logps/chosen": -449.0249938964844,
      "logps/rejected": -566.5499877929688,
      "loss": 0.0137,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -3.6866211891174316,
      "rewards/margins": 16.260936737060547,
      "rewards/rejected": -19.940624237060547,
      "step": 3230
    },
    {
      "epoch": 2.084969423881558,
      "grad_norm": 1.3351611741915037,
      "learning_rate": 4.785898261429491e-07,
      "logits/chosen": -1.1100585460662842,
      "logits/rejected": -1.1037108898162842,
      "logps/chosen": -400.375,
      "logps/rejected": -533.0999755859375,
      "loss": 0.0096,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -4.844922065734863,
      "rewards/margins": 15.753125190734863,
      "rewards/rejected": -20.606250762939453,
      "step": 3240
    },
    {
      "epoch": 2.0914065014483425,
      "grad_norm": 0.671875093679778,
      "learning_rate": 4.769800386349002e-07,
      "logits/chosen": -1.1155273914337158,
      "logits/rejected": -1.1248047351837158,
      "logps/chosen": -423.29998779296875,
      "logps/rejected": -563.4500122070312,
      "loss": 0.0046,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -4.465918064117432,
      "rewards/margins": 15.721875190734863,
      "rewards/rejected": -20.184375762939453,
      "step": 3250
    },
    {
      "epoch": 2.097843579015127,
      "grad_norm": 12.140898346072861,
      "learning_rate": 4.7537025112685124e-07,
      "logits/chosen": -1.0279541015625,
      "logits/rejected": -1.0253509283065796,
      "logps/chosen": -385.79998779296875,
      "logps/rejected": -513.7249755859375,
      "loss": 0.0232,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -3.7276854515075684,
      "rewards/margins": 14.610937118530273,
      "rewards/rejected": -18.337499618530273,
      "step": 3260
    },
    {
      "epoch": 2.1042806565819117,
      "grad_norm": 2.623763638066555,
      "learning_rate": 4.7376046361880226e-07,
      "logits/chosen": -1.017480492591858,
      "logits/rejected": -1.009667992591858,
      "logps/chosen": -428.0,
      "logps/rejected": -552.2249755859375,
      "loss": 0.0279,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -3.2283692359924316,
      "rewards/margins": 16.339061737060547,
      "rewards/rejected": -19.5703125,
      "step": 3270
    },
    {
      "epoch": 2.1107177341486967,
      "grad_norm": 0.4497581563597138,
      "learning_rate": 4.721506761107534e-07,
      "logits/chosen": -1.0849609375,
      "logits/rejected": -1.1357421875,
      "logps/chosen": -423.75,
      "logps/rejected": -557.0,
      "loss": 0.0014,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.2232909202575684,
      "rewards/margins": 16.860937118530273,
      "rewards/rejected": -20.0859375,
      "step": 3280
    },
    {
      "epoch": 2.1171548117154813,
      "grad_norm": 1.3339774174292212,
      "learning_rate": 4.705408886027044e-07,
      "logits/chosen": -0.9937744140625,
      "logits/rejected": -0.982617199420929,
      "logps/chosen": -368.25,
      "logps/rejected": -534.9000244140625,
      "loss": 0.0594,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -3.929394483566284,
      "rewards/margins": 16.626562118530273,
      "rewards/rejected": -20.557811737060547,
      "step": 3290
    },
    {
      "epoch": 2.123591889282266,
      "grad_norm": 0.0051571726531345384,
      "learning_rate": 4.689311010946555e-07,
      "logits/chosen": -1.063818335533142,
      "logits/rejected": -1.061132788658142,
      "logps/chosen": -383.9750061035156,
      "logps/rejected": -543.3499755859375,
      "loss": 0.0181,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -3.5445799827575684,
      "rewards/margins": 16.131250381469727,
      "rewards/rejected": -19.682811737060547,
      "step": 3300
    },
    {
      "epoch": 2.1300289668490504,
      "grad_norm": 0.059582082026372385,
      "learning_rate": 4.6732131358660656e-07,
      "logits/chosen": -0.9920898675918579,
      "logits/rejected": -1.022851586341858,
      "logps/chosen": -393.1499938964844,
      "logps/rejected": -518.4000244140625,
      "loss": 0.0238,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -3.5086913108825684,
      "rewards/margins": 14.434374809265137,
      "rewards/rejected": -17.932811737060547,
      "step": 3310
    },
    {
      "epoch": 2.136466044415835,
      "grad_norm": 0.05020904902019842,
      "learning_rate": 4.6571152607855763e-07,
      "logits/chosen": -0.925341784954071,
      "logits/rejected": -0.948437511920929,
      "logps/chosen": -405.0,
      "logps/rejected": -542.6500244140625,
      "loss": 0.025,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.8140380382537842,
      "rewards/margins": 16.903125762939453,
      "rewards/rejected": -18.720312118530273,
      "step": 3320
    },
    {
      "epoch": 2.14290312198262,
      "grad_norm": 0.9021303309972339,
      "learning_rate": 4.6410173857050865e-07,
      "logits/chosen": -1.002783179283142,
      "logits/rejected": -0.99658203125,
      "logps/chosen": -420.1000061035156,
      "logps/rejected": -559.9500122070312,
      "loss": 0.0169,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -4.581494331359863,
      "rewards/margins": 15.056249618530273,
      "rewards/rejected": -19.6328125,
      "step": 3330
    },
    {
      "epoch": 2.1493401995494046,
      "grad_norm": 0.3791121350351295,
      "learning_rate": 4.624919510624598e-07,
      "logits/chosen": -1.060644507408142,
      "logits/rejected": -1.098242163658142,
      "logps/chosen": -393.42498779296875,
      "logps/rejected": -543.2999877929688,
      "loss": 0.0144,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -3.459033250808716,
      "rewards/margins": 16.667186737060547,
      "rewards/rejected": -20.120311737060547,
      "step": 3340
    },
    {
      "epoch": 2.155777277116189,
      "grad_norm": 0.045148022238358086,
      "learning_rate": 4.608821635544108e-07,
      "logits/chosen": -0.9743896722793579,
      "logits/rejected": -1.025634765625,
      "logps/chosen": -433.61248779296875,
      "logps/rejected": -542.5499877929688,
      "loss": 0.0166,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.9928221702575684,
      "rewards/margins": 16.131250381469727,
      "rewards/rejected": -19.1328125,
      "step": 3350
    },
    {
      "epoch": 2.1622143546829737,
      "grad_norm": 0.7861734963114936,
      "learning_rate": 4.5927237604636187e-07,
      "logits/chosen": -0.998339831829071,
      "logits/rejected": -1.03173828125,
      "logps/chosen": -432.07501220703125,
      "logps/rejected": -545.2999877929688,
      "loss": 0.0151,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -3.386035203933716,
      "rewards/margins": 15.671875,
      "rewards/rejected": -19.059375762939453,
      "step": 3360
    },
    {
      "epoch": 2.1686514322497588,
      "grad_norm": 0.25553750983202145,
      "learning_rate": 4.576625885383129e-07,
      "logits/chosen": -0.948168933391571,
      "logits/rejected": -0.9716796875,
      "logps/chosen": -445.82501220703125,
      "logps/rejected": -577.1500244140625,
      "loss": 0.0047,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -3.176953077316284,
      "rewards/margins": 16.501562118530273,
      "rewards/rejected": -19.682811737060547,
      "step": 3370
    },
    {
      "epoch": 2.1750885098165433,
      "grad_norm": 0.29127717157768923,
      "learning_rate": 4.56052801030264e-07,
      "logits/chosen": -1.062585473060608,
      "logits/rejected": -1.0586426258087158,
      "logps/chosen": -405.9750061035156,
      "logps/rejected": -540.9249877929688,
      "loss": 0.0088,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -3.174121141433716,
      "rewards/margins": 15.868749618530273,
      "rewards/rejected": -19.057811737060547,
      "step": 3380
    },
    {
      "epoch": 2.181525587383328,
      "grad_norm": 0.10089211561220655,
      "learning_rate": 4.5444301352221504e-07,
      "logits/chosen": -1.073828101158142,
      "logits/rejected": -1.073632836341858,
      "logps/chosen": -457.9750061035156,
      "logps/rejected": -594.4000244140625,
      "loss": 0.0132,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -4.7056884765625,
      "rewards/margins": 15.8515625,
      "rewards/rejected": -20.556249618530273,
      "step": 3390
    },
    {
      "epoch": 2.1879626649501125,
      "grad_norm": 0.0006958724241457409,
      "learning_rate": 4.528332260141661e-07,
      "logits/chosen": -1.022985816001892,
      "logits/rejected": -1.043212890625,
      "logps/chosen": -411.5,
      "logps/rejected": -548.3250122070312,
      "loss": 0.0101,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -4.134961128234863,
      "rewards/margins": 16.254688262939453,
      "rewards/rejected": -20.389062881469727,
      "step": 3400
    },
    {
      "epoch": 2.1943997425168975,
      "grad_norm": 14.135716304337999,
      "learning_rate": 4.512234385061172e-07,
      "logits/chosen": -0.994384765625,
      "logits/rejected": -1.0382568836212158,
      "logps/chosen": -464.2250061035156,
      "logps/rejected": -574.7000122070312,
      "loss": 0.0188,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -4.747363090515137,
      "rewards/margins": 16.634374618530273,
      "rewards/rejected": -21.364063262939453,
      "step": 3410
    },
    {
      "epoch": 2.200836820083682,
      "grad_norm": 0.31310709472957954,
      "learning_rate": 4.4961365099806826e-07,
      "logits/chosen": -1.076269507408142,
      "logits/rejected": -1.083398461341858,
      "logps/chosen": -407.2250061035156,
      "logps/rejected": -520.4500122070312,
      "loss": 0.0011,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.8101563453674316,
      "rewards/margins": 17.2265625,
      "rewards/rejected": -21.035938262939453,
      "step": 3420
    },
    {
      "epoch": 2.2072738976504667,
      "grad_norm": 0.008193914417783646,
      "learning_rate": 4.480038634900193e-07,
      "logits/chosen": -0.981396496295929,
      "logits/rejected": -0.9964355230331421,
      "logps/chosen": -392.75,
      "logps/rejected": -537.625,
      "loss": 0.0184,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -4.210669040679932,
      "rewards/margins": 16.301563262939453,
      "rewards/rejected": -20.514842987060547,
      "step": 3430
    },
    {
      "epoch": 2.2137109752172512,
      "grad_norm": 0.5881690669947562,
      "learning_rate": 4.4639407598197035e-07,
      "logits/chosen": -1.028222680091858,
      "logits/rejected": -1.021386742591858,
      "logps/chosen": -438.875,
      "logps/rejected": -579.25,
      "loss": 0.0069,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -4.130139350891113,
      "rewards/margins": 16.584375381469727,
      "rewards/rejected": -20.701562881469727,
      "step": 3440
    },
    {
      "epoch": 2.220148052784036,
      "grad_norm": 1.8005519369536847,
      "learning_rate": 4.447842884739214e-07,
      "logits/chosen": -0.9946044683456421,
      "logits/rejected": -1.0242187976837158,
      "logps/chosen": -400.92498779296875,
      "logps/rejected": -537.0499877929688,
      "loss": 0.0187,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -4.555712699890137,
      "rewards/margins": 16.268749237060547,
      "rewards/rejected": -20.821874618530273,
      "step": 3450
    },
    {
      "epoch": 2.226585130350821,
      "grad_norm": 0.03380963376951087,
      "learning_rate": 4.431745009658725e-07,
      "logits/chosen": -0.999707043170929,
      "logits/rejected": -1.041748046875,
      "logps/chosen": -408.6000061035156,
      "logps/rejected": -551.0750122070312,
      "loss": 0.0045,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -4.134863376617432,
      "rewards/margins": 16.989063262939453,
      "rewards/rejected": -21.118749618530273,
      "step": 3460
    },
    {
      "epoch": 2.2330222079176054,
      "grad_norm": 0.033513908815007994,
      "learning_rate": 4.415647134578235e-07,
      "logits/chosen": -1.010839819908142,
      "logits/rejected": -1.040429711341858,
      "logps/chosen": -424.95001220703125,
      "logps/rejected": -571.0,
      "loss": 0.0094,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -4.512225151062012,
      "rewards/margins": 17.228124618530273,
      "rewards/rejected": -21.732812881469727,
      "step": 3470
    },
    {
      "epoch": 2.23945928548439,
      "grad_norm": 0.2740000677105209,
      "learning_rate": 4.3995492594977465e-07,
      "logits/chosen": -0.97900390625,
      "logits/rejected": -0.9859619140625,
      "logps/chosen": -396.0249938964844,
      "logps/rejected": -535.1749877929688,
      "loss": 0.0206,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -5.21337890625,
      "rewards/margins": 15.759374618530273,
      "rewards/rejected": -20.957813262939453,
      "step": 3480
    },
    {
      "epoch": 2.2458963630511746,
      "grad_norm": 0.15223632079148156,
      "learning_rate": 4.3834513844172567e-07,
      "logits/chosen": -1.014062523841858,
      "logits/rejected": -1.0511353015899658,
      "logps/chosen": -442.7250061035156,
      "logps/rejected": -571.4749755859375,
      "loss": 0.0295,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -5.354589939117432,
      "rewards/margins": 15.748437881469727,
      "rewards/rejected": -21.092187881469727,
      "step": 3490
    },
    {
      "epoch": 2.2523334406179596,
      "grad_norm": 0.13728561648140786,
      "learning_rate": 4.3673535093367674e-07,
      "logits/chosen": -1.0329101085662842,
      "logits/rejected": -1.036962866783142,
      "logps/chosen": -382.4750061035156,
      "logps/rejected": -534.5,
      "loss": 0.005,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -5.963476657867432,
      "rewards/margins": 16.146875381469727,
      "rewards/rejected": -22.104686737060547,
      "step": 3500
    },
    {
      "epoch": 2.258770518184744,
      "grad_norm": 10.709637400608113,
      "learning_rate": 4.3512556342562776e-07,
      "logits/chosen": -1.0541503429412842,
      "logits/rejected": -1.050878882408142,
      "logps/chosen": -443.9375,
      "logps/rejected": -584.8499755859375,
      "loss": 0.017,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -5.727734565734863,
      "rewards/margins": 17.151561737060547,
      "rewards/rejected": -22.8671875,
      "step": 3510
    },
    {
      "epoch": 2.2652075957515287,
      "grad_norm": 0.7174296831368239,
      "learning_rate": 4.335157759175789e-07,
      "logits/chosen": -1.0232422351837158,
      "logits/rejected": -1.032568335533142,
      "logps/chosen": -406.875,
      "logps/rejected": -583.6500244140625,
      "loss": 0.0365,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -5.3134765625,
      "rewards/margins": 17.565624237060547,
      "rewards/rejected": -22.865625381469727,
      "step": 3520
    },
    {
      "epoch": 2.2716446733183133,
      "grad_norm": 0.6260293290211867,
      "learning_rate": 4.319059884095299e-07,
      "logits/chosen": -1.082421898841858,
      "logits/rejected": -1.1144530773162842,
      "logps/chosen": -413.70001220703125,
      "logps/rejected": -555.6500244140625,
      "loss": 0.0217,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -5.38232421875,
      "rewards/margins": 16.2734375,
      "rewards/rejected": -21.674999237060547,
      "step": 3530
    },
    {
      "epoch": 2.2780817508850983,
      "grad_norm": 1.703213885575144,
      "learning_rate": 4.30296200901481e-07,
      "logits/chosen": -0.9996337890625,
      "logits/rejected": -1.014892578125,
      "logps/chosen": -439.04998779296875,
      "logps/rejected": -576.0750122070312,
      "loss": 0.0138,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -4.720019340515137,
      "rewards/margins": 16.753124237060547,
      "rewards/rejected": -21.4765625,
      "step": 3540
    },
    {
      "epoch": 2.284518828451883,
      "grad_norm": 2.632339736606739,
      "learning_rate": 4.2868641339343205e-07,
      "logits/chosen": -1.0037109851837158,
      "logits/rejected": -1.0343017578125,
      "logps/chosen": -425.125,
      "logps/rejected": -536.75,
      "loss": 0.0159,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -4.025781154632568,
      "rewards/margins": 16.020313262939453,
      "rewards/rejected": -20.042186737060547,
      "step": 3550
    },
    {
      "epoch": 2.2909559060186675,
      "grad_norm": 0.20391483750374004,
      "learning_rate": 4.2707662588538313e-07,
      "logits/chosen": -1.028173804283142,
      "logits/rejected": -1.041967749595642,
      "logps/chosen": -420.92498779296875,
      "logps/rejected": -558.2000122070312,
      "loss": 0.0198,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -4.070458889007568,
      "rewards/margins": 15.535937309265137,
      "rewards/rejected": -19.610937118530273,
      "step": 3560
    },
    {
      "epoch": 2.297392983585452,
      "grad_norm": 0.22150018868741062,
      "learning_rate": 4.2546683837733415e-07,
      "logits/chosen": -1.0495116710662842,
      "logits/rejected": -1.0852539539337158,
      "logps/chosen": -424.1499938964844,
      "logps/rejected": -548.6500244140625,
      "loss": 0.0221,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -3.4621949195861816,
      "rewards/margins": 16.325000762939453,
      "rewards/rejected": -19.792186737060547,
      "step": 3570
    },
    {
      "epoch": 2.3038300611522367,
      "grad_norm": 0.09769741421519501,
      "learning_rate": 4.238570508692853e-07,
      "logits/chosen": -1.0507323741912842,
      "logits/rejected": -1.040673851966858,
      "logps/chosen": -417.2749938964844,
      "logps/rejected": -568.0499877929688,
      "loss": 0.0132,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -4.569872856140137,
      "rewards/margins": 16.069530487060547,
      "rewards/rejected": -20.625,
      "step": 3580
    },
    {
      "epoch": 2.3102671387190217,
      "grad_norm": 0.3834551065281765,
      "learning_rate": 4.222472633612363e-07,
      "logits/chosen": -1.0048736333847046,
      "logits/rejected": -1.031103491783142,
      "logps/chosen": -425.29998779296875,
      "logps/rejected": -565.8499755859375,
      "loss": 0.0092,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -3.7234864234924316,
      "rewards/margins": 16.870311737060547,
      "rewards/rejected": -20.598438262939453,
      "step": 3590
    },
    {
      "epoch": 2.3167042162858063,
      "grad_norm": 1.9144721824557707,
      "learning_rate": 4.2063747585318737e-07,
      "logits/chosen": -1.0072753429412842,
      "logits/rejected": -1.0269043445587158,
      "logps/chosen": -381.07501220703125,
      "logps/rejected": -536.6500244140625,
      "loss": 0.005,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -4.131249904632568,
      "rewards/margins": 16.9375,
      "rewards/rejected": -21.082813262939453,
      "step": 3600
    },
    {
      "epoch": 2.323141293852591,
      "grad_norm": 36.214552892647944,
      "learning_rate": 4.190276883451384e-07,
      "logits/chosen": -1.02734375,
      "logits/rejected": -1.035888671875,
      "logps/chosen": -441.3374938964844,
      "logps/rejected": -592.0499877929688,
      "loss": 0.0068,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -4.705712795257568,
      "rewards/margins": 17.370311737060547,
      "rewards/rejected": -22.082813262939453,
      "step": 3610
    },
    {
      "epoch": 2.329578371419376,
      "grad_norm": 0.047373315431620606,
      "learning_rate": 4.174179008370895e-07,
      "logits/chosen": -1.1228516101837158,
      "logits/rejected": -1.125585913658142,
      "logps/chosen": -416.7749938964844,
      "logps/rejected": -573.9000244140625,
      "loss": 0.0159,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -4.840966701507568,
      "rewards/margins": 16.671875,
      "rewards/rejected": -21.510936737060547,
      "step": 3620
    },
    {
      "epoch": 2.3360154489861604,
      "grad_norm": 0.5143692709122827,
      "learning_rate": 4.1580811332904054e-07,
      "logits/chosen": -1.047265648841858,
      "logits/rejected": -1.072656273841858,
      "logps/chosen": -401.3999938964844,
      "logps/rejected": -503.2250061035156,
      "loss": 0.0226,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -4.970654487609863,
      "rewards/margins": 15.118749618530273,
      "rewards/rejected": -20.075000762939453,
      "step": 3630
    },
    {
      "epoch": 2.342452526552945,
      "grad_norm": 0.13432767152874814,
      "learning_rate": 4.141983258209916e-07,
      "logits/chosen": -1.0247802734375,
      "logits/rejected": -1.006445288658142,
      "logps/chosen": -397.3125,
      "logps/rejected": -522.9000244140625,
      "loss": 0.0371,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -3.4481444358825684,
      "rewards/margins": 15.620312690734863,
      "rewards/rejected": -19.079687118530273,
      "step": 3640
    },
    {
      "epoch": 2.3488896041197296,
      "grad_norm": 0.34195251874543514,
      "learning_rate": 4.125885383129427e-07,
      "logits/chosen": -0.9477783441543579,
      "logits/rejected": -0.9656982421875,
      "logps/chosen": -404.51251220703125,
      "logps/rejected": -527.25,
      "loss": 0.0182,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -4.066552639007568,
      "rewards/margins": 15.949999809265137,
      "rewards/rejected": -20.014062881469727,
      "step": 3650
    },
    {
      "epoch": 2.355326681686514,
      "grad_norm": 2.526519532504119,
      "learning_rate": 4.1097875080489376e-07,
      "logits/chosen": -1.093359351158142,
      "logits/rejected": -1.110498070716858,
      "logps/chosen": -408.98748779296875,
      "logps/rejected": -553.0499877929688,
      "loss": 0.0089,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -4.961523532867432,
      "rewards/margins": 16.207813262939453,
      "rewards/rejected": -21.168750762939453,
      "step": 3660
    },
    {
      "epoch": 2.361763759253299,
      "grad_norm": 0.06085996531440162,
      "learning_rate": 4.093689632968448e-07,
      "logits/chosen": -1.125634789466858,
      "logits/rejected": -1.1251952648162842,
      "logps/chosen": -434.2250061035156,
      "logps/rejected": -596.0499877929688,
      "loss": 0.005,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -4.989697456359863,
      "rewards/margins": 17.685937881469727,
      "rewards/rejected": -22.674999237060547,
      "step": 3670
    },
    {
      "epoch": 2.3682008368200838,
      "grad_norm": 1.0978779971202857,
      "learning_rate": 4.0775917578879585e-07,
      "logits/chosen": -1.00048828125,
      "logits/rejected": -1.0075194835662842,
      "logps/chosen": -383.17498779296875,
      "logps/rejected": -517.9500122070312,
      "loss": 0.0307,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -4.374267578125,
      "rewards/margins": 15.589062690734863,
      "rewards/rejected": -19.954687118530273,
      "step": 3680
    },
    {
      "epoch": 2.3746379143868683,
      "grad_norm": 0.004285537440066397,
      "learning_rate": 4.061493882807469e-07,
      "logits/chosen": -0.9914916753768921,
      "logits/rejected": -1.009521484375,
      "logps/chosen": -420.8374938964844,
      "logps/rejected": -559.1500244140625,
      "loss": 0.0128,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -5.005566596984863,
      "rewards/margins": 17.623437881469727,
      "rewards/rejected": -22.639062881469727,
      "step": 3690
    },
    {
      "epoch": 2.381074991953653,
      "grad_norm": 0.0753736597073573,
      "learning_rate": 4.04539600772698e-07,
      "logits/chosen": -0.992919921875,
      "logits/rejected": -1.01214599609375,
      "logps/chosen": -429.8500061035156,
      "logps/rejected": -572.2000122070312,
      "loss": 0.0165,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -4.004248142242432,
      "rewards/margins": 17.706249237060547,
      "rewards/rejected": -21.701562881469727,
      "step": 3700
    },
    {
      "epoch": 2.3875120695204375,
      "grad_norm": 12.902251613669751,
      "learning_rate": 4.02929813264649e-07,
      "logits/chosen": -1.042822241783142,
      "logits/rejected": -1.052832007408142,
      "logps/chosen": -420.6000061035156,
      "logps/rejected": -559.1500244140625,
      "loss": 0.0239,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -5.941210746765137,
      "rewards/margins": 15.0078125,
      "rewards/rejected": -20.9375,
      "step": 3710
    },
    {
      "epoch": 2.3939491470872225,
      "grad_norm": 4.529963145208864,
      "learning_rate": 4.0132002575660014e-07,
      "logits/chosen": -0.9861816167831421,
      "logits/rejected": -1.039648413658142,
      "logps/chosen": -417.32501220703125,
      "logps/rejected": -582.5499877929688,
      "loss": 0.0046,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.405077934265137,
      "rewards/margins": 17.334375381469727,
      "rewards/rejected": -21.7265625,
      "step": 3720
    },
    {
      "epoch": 2.400386224654007,
      "grad_norm": 0.9629898122683673,
      "learning_rate": 3.9971023824855116e-07,
      "logits/chosen": -1.044580101966858,
      "logits/rejected": -1.0702636241912842,
      "logps/chosen": -394.17498779296875,
      "logps/rejected": -539.5999755859375,
      "loss": 0.0088,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -3.7095704078674316,
      "rewards/margins": 17.260936737060547,
      "rewards/rejected": -20.965625762939453,
      "step": 3730
    },
    {
      "epoch": 2.4068233022207917,
      "grad_norm": 0.20698119594926714,
      "learning_rate": 3.9810045074050224e-07,
      "logits/chosen": -1.0583007335662842,
      "logits/rejected": -1.0828125476837158,
      "logps/chosen": -473.20001220703125,
      "logps/rejected": -547.3499755859375,
      "loss": 0.0101,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -3.494311571121216,
      "rewards/margins": 16.2109375,
      "rewards/rejected": -19.709375381469727,
      "step": 3740
    },
    {
      "epoch": 2.4132603797875767,
      "grad_norm": 0.013493617041070298,
      "learning_rate": 3.9649066323245326e-07,
      "logits/chosen": -0.940869152545929,
      "logits/rejected": -0.9414306879043579,
      "logps/chosen": -391.5,
      "logps/rejected": -520.6749877929688,
      "loss": 0.0205,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -3.1650633811950684,
      "rewards/margins": 16.353124618530273,
      "rewards/rejected": -19.526561737060547,
      "step": 3750
    },
    {
      "epoch": 2.4196974573543613,
      "grad_norm": 0.013139894005101382,
      "learning_rate": 3.948808757244044e-07,
      "logits/chosen": -0.9778076410293579,
      "logits/rejected": -0.998120129108429,
      "logps/chosen": -427.25,
      "logps/rejected": -550.0499877929688,
      "loss": 0.0311,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -3.592041015625,
      "rewards/margins": 15.556249618530273,
      "rewards/rejected": -19.140625,
      "step": 3760
    },
    {
      "epoch": 2.426134534921146,
      "grad_norm": 7.327332077965629,
      "learning_rate": 3.932710882163554e-07,
      "logits/chosen": -1.0476562976837158,
      "logits/rejected": -1.07373046875,
      "logps/chosen": -432.92498779296875,
      "logps/rejected": -563.9500122070312,
      "loss": 0.0118,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -3.391796827316284,
      "rewards/margins": 17.470312118530273,
      "rewards/rejected": -20.870311737060547,
      "step": 3770
    },
    {
      "epoch": 2.4325716124879304,
      "grad_norm": 0.1810775822374288,
      "learning_rate": 3.916613007083065e-07,
      "logits/chosen": -1.052490234375,
      "logits/rejected": -1.092626929283142,
      "logps/chosen": -379.625,
      "logps/rejected": -523.9500122070312,
      "loss": 0.0189,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -3.958789110183716,
      "rewards/margins": 16.422657012939453,
      "rewards/rejected": -20.373437881469727,
      "step": 3780
    },
    {
      "epoch": 2.439008690054715,
      "grad_norm": 0.23327394750489913,
      "learning_rate": 3.9005151320025755e-07,
      "logits/chosen": -1.0048339366912842,
      "logits/rejected": -1.0261719226837158,
      "logps/chosen": -396.11248779296875,
      "logps/rejected": -539.2000122070312,
      "loss": 0.0096,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -3.868115186691284,
      "rewards/margins": 16.415624618530273,
      "rewards/rejected": -20.299999237060547,
      "step": 3790
    },
    {
      "epoch": 2.4454457676215,
      "grad_norm": 0.02558078884725733,
      "learning_rate": 3.884417256922086e-07,
      "logits/chosen": -0.972582995891571,
      "logits/rejected": -0.9895995855331421,
      "logps/chosen": -400.0249938964844,
      "logps/rejected": -556.4000244140625,
      "loss": 0.0014,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.6697998046875,
      "rewards/margins": 16.653125762939453,
      "rewards/rejected": -20.310937881469727,
      "step": 3800
    },
    {
      "epoch": 2.4518828451882846,
      "grad_norm": 0.05959869141952533,
      "learning_rate": 3.8683193818415965e-07,
      "logits/chosen": -0.9869140386581421,
      "logits/rejected": -0.977246105670929,
      "logps/chosen": -407.04998779296875,
      "logps/rejected": -573.4000244140625,
      "loss": 0.0133,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -4.500256538391113,
      "rewards/margins": 15.440625190734863,
      "rewards/rejected": -19.932811737060547,
      "step": 3810
    },
    {
      "epoch": 2.458319922755069,
      "grad_norm": 0.08842132782631504,
      "learning_rate": 3.8522215067611077e-07,
      "logits/chosen": -1.037939429283142,
      "logits/rejected": -1.053808569908142,
      "logps/chosen": -423.07501220703125,
      "logps/rejected": -584.8499755859375,
      "loss": 0.0145,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -4.993945121765137,
      "rewards/margins": 17.1484375,
      "rewards/rejected": -22.149999618530273,
      "step": 3820
    },
    {
      "epoch": 2.4647570003218537,
      "grad_norm": 30.947385597831875,
      "learning_rate": 3.836123631680618e-07,
      "logits/chosen": -1.028906226158142,
      "logits/rejected": -1.0302002429962158,
      "logps/chosen": -372.0,
      "logps/rejected": -546.8250122070312,
      "loss": 0.0212,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -4.22998046875,
      "rewards/margins": 16.628124237060547,
      "rewards/rejected": -20.865625381469727,
      "step": 3830
    },
    {
      "epoch": 2.4711940778886383,
      "grad_norm": 0.05211605695940069,
      "learning_rate": 3.8200257566001287e-07,
      "logits/chosen": -1.055566430091858,
      "logits/rejected": -1.0647461414337158,
      "logps/chosen": -409.04998779296875,
      "logps/rejected": -544.0499877929688,
      "loss": 0.0048,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -3.8119139671325684,
      "rewards/margins": 18.073436737060547,
      "rewards/rejected": -21.878124237060547,
      "step": 3840
    },
    {
      "epoch": 2.4776311554554233,
      "grad_norm": 9.213720198102742,
      "learning_rate": 3.803927881519639e-07,
      "logits/chosen": -1.0117919445037842,
      "logits/rejected": -1.0427124500274658,
      "logps/chosen": -433.5249938964844,
      "logps/rejected": -587.8499755859375,
      "loss": 0.0304,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -4.554394721984863,
      "rewards/margins": 16.564062118530273,
      "rewards/rejected": -21.112499237060547,
      "step": 3850
    },
    {
      "epoch": 2.484068233022208,
      "grad_norm": 0.24431172158453063,
      "learning_rate": 3.78783000643915e-07,
      "logits/chosen": -1.0658690929412842,
      "logits/rejected": -1.073632836341858,
      "logps/chosen": -367.375,
      "logps/rejected": -512.0999755859375,
      "loss": 0.0288,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -3.274951219558716,
      "rewards/margins": 15.74609375,
      "rewards/rejected": -19.03125,
      "step": 3860
    },
    {
      "epoch": 2.4905053105889925,
      "grad_norm": 71.25388858956305,
      "learning_rate": 3.7717321313586603e-07,
      "logits/chosen": -1.061425805091858,
      "logits/rejected": -1.0839354991912842,
      "logps/chosen": -455.8374938964844,
      "logps/rejected": -562.2000122070312,
      "loss": 0.0104,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -4.843505859375,
      "rewards/margins": 16.064062118530273,
      "rewards/rejected": -20.912500381469727,
      "step": 3870
    },
    {
      "epoch": 2.4969423881557775,
      "grad_norm": 1.8350700859136448,
      "learning_rate": 3.755634256278171e-07,
      "logits/chosen": -0.9950927495956421,
      "logits/rejected": -1.032861351966858,
      "logps/chosen": -421.9750061035156,
      "logps/rejected": -555.4000244140625,
      "loss": 0.0139,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -3.4852051734924316,
      "rewards/margins": 16.821874618530273,
      "rewards/rejected": -20.3125,
      "step": 3880
    },
    {
      "epoch": 2.503379465722562,
      "grad_norm": 5.050750304416249,
      "learning_rate": 3.739536381197682e-07,
      "logits/chosen": -1.1130859851837158,
      "logits/rejected": -1.1386229991912842,
      "logps/chosen": -406.8500061035156,
      "logps/rejected": -538.0499877929688,
      "loss": 0.0127,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -5.451952934265137,
      "rewards/margins": 16.712499618530273,
      "rewards/rejected": -22.167186737060547,
      "step": 3890
    },
    {
      "epoch": 2.5098165432893467,
      "grad_norm": 5.355053391341112,
      "learning_rate": 3.7234385061171925e-07,
      "logits/chosen": -0.9847412109375,
      "logits/rejected": -1.0299804210662842,
      "logps/chosen": -412.7250061035156,
      "logps/rejected": -550.6500244140625,
      "loss": 0.017,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -4.985693454742432,
      "rewards/margins": 18.120311737060547,
      "rewards/rejected": -23.103124618530273,
      "step": 3900
    },
    {
      "epoch": 2.5162536208561312,
      "grad_norm": 0.0193165295031974,
      "learning_rate": 3.707340631036703e-07,
      "logits/chosen": -1.0419189929962158,
      "logits/rejected": -1.063818335533142,
      "logps/chosen": -406.9624938964844,
      "logps/rejected": -541.1500244140625,
      "loss": 0.0132,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -5.443945407867432,
      "rewards/margins": 17.647655487060547,
      "rewards/rejected": -23.082813262939453,
      "step": 3910
    },
    {
      "epoch": 2.522690698422916,
      "grad_norm": 1.277209930482019,
      "learning_rate": 3.6912427559562135e-07,
      "logits/chosen": -1.0840332508087158,
      "logits/rejected": -1.095361351966858,
      "logps/chosen": -430.375,
      "logps/rejected": -571.4000244140625,
      "loss": 0.0064,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -5.085156440734863,
      "rewards/margins": 16.314062118530273,
      "rewards/rejected": -21.404687881469727,
      "step": 3920
    },
    {
      "epoch": 2.529127775989701,
      "grad_norm": 14.484108215530707,
      "learning_rate": 3.675144880875724e-07,
      "logits/chosen": -1.103124976158142,
      "logits/rejected": -1.1331055164337158,
      "logps/chosen": -451.17498779296875,
      "logps/rejected": -567.4000244140625,
      "loss": 0.0092,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -4.98681640625,
      "rewards/margins": 17.245311737060547,
      "rewards/rejected": -22.225000381469727,
      "step": 3930
    },
    {
      "epoch": 2.5355648535564854,
      "grad_norm": 0.08431049236655122,
      "learning_rate": 3.659047005795235e-07,
      "logits/chosen": -1.0732421875,
      "logits/rejected": -1.0867431163787842,
      "logps/chosen": -447.67498779296875,
      "logps/rejected": -586.1500244140625,
      "loss": 0.0087,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -4.617505073547363,
      "rewards/margins": 17.545312881469727,
      "rewards/rejected": -22.178125381469727,
      "step": 3940
    },
    {
      "epoch": 2.54200193112327,
      "grad_norm": 0.001819636969881986,
      "learning_rate": 3.642949130714745e-07,
      "logits/chosen": -1.0378296375274658,
      "logits/rejected": -1.06689453125,
      "logps/chosen": -378.20001220703125,
      "logps/rejected": -521.8499755859375,
      "loss": 0.0131,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -4.829297065734863,
      "rewards/margins": 16.543750762939453,
      "rewards/rejected": -21.359375,
      "step": 3950
    },
    {
      "epoch": 2.5484390086900546,
      "grad_norm": 9.099365929589844,
      "learning_rate": 3.6268512556342564e-07,
      "logits/chosen": -1.011962890625,
      "logits/rejected": -1.035668969154358,
      "logps/chosen": -410.3500061035156,
      "logps/rejected": -571.75,
      "loss": 0.0182,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -4.998046875,
      "rewards/margins": 17.016407012939453,
      "rewards/rejected": -22.017187118530273,
      "step": 3960
    },
    {
      "epoch": 2.554876086256839,
      "grad_norm": 0.0459123345549701,
      "learning_rate": 3.6107533805537666e-07,
      "logits/chosen": -1.1208984851837158,
      "logits/rejected": -1.1435546875,
      "logps/chosen": -439.8999938964844,
      "logps/rejected": -591.9500122070312,
      "loss": 0.0131,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -4.621191501617432,
      "rewards/margins": 18.442188262939453,
      "rewards/rejected": -23.067188262939453,
      "step": 3970
    },
    {
      "epoch": 2.561313163823624,
      "grad_norm": 1.9349974401487566,
      "learning_rate": 3.5946555054732774e-07,
      "logits/chosen": -1.0195801258087158,
      "logits/rejected": -1.0270507335662842,
      "logps/chosen": -426.95001220703125,
      "logps/rejected": -560.0,
      "loss": 0.0157,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -5.405713081359863,
      "rewards/margins": 17.421875,
      "rewards/rejected": -22.818750381469727,
      "step": 3980
    },
    {
      "epoch": 2.5677502413904087,
      "grad_norm": 0.4891014195678684,
      "learning_rate": 3.5785576303927876e-07,
      "logits/chosen": -1.0330078601837158,
      "logits/rejected": -1.0408203601837158,
      "logps/chosen": -429.7749938964844,
      "logps/rejected": -601.8499755859375,
      "loss": 0.0099,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -5.31982421875,
      "rewards/margins": 17.131250381469727,
      "rewards/rejected": -22.443750381469727,
      "step": 3990
    },
    {
      "epoch": 2.5741873189571933,
      "grad_norm": 1.5627512684455094,
      "learning_rate": 3.562459755312299e-07,
      "logits/chosen": -1.0496826171875,
      "logits/rejected": -1.080175757408142,
      "logps/chosen": -415.54998779296875,
      "logps/rejected": -572.1500244140625,
      "loss": 0.0004,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.251953125,
      "rewards/margins": 17.899999618530273,
      "rewards/rejected": -23.146875381469727,
      "step": 4000
    },
    {
      "epoch": 2.5806243965239783,
      "grad_norm": 0.08739818731065581,
      "learning_rate": 3.546361880231809e-07,
      "logits/chosen": -1.082617163658142,
      "logits/rejected": -1.087744116783142,
      "logps/chosen": -473.4750061035156,
      "logps/rejected": -611.75,
      "loss": 0.0101,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -6.598681449890137,
      "rewards/margins": 17.556249618530273,
      "rewards/rejected": -24.149999618530273,
      "step": 4010
    },
    {
      "epoch": 2.587061474090763,
      "grad_norm": 0.3471564851888695,
      "learning_rate": 3.53026400515132e-07,
      "logits/chosen": -1.062597632408142,
      "logits/rejected": -1.064550757408142,
      "logps/chosen": -414.20001220703125,
      "logps/rejected": -553.9000244140625,
      "loss": 0.0095,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -5.682898044586182,
      "rewards/margins": 16.796875,
      "rewards/rejected": -22.484375,
      "step": 4020
    },
    {
      "epoch": 2.5934985516575475,
      "grad_norm": 0.014063778025355996,
      "learning_rate": 3.5141661300708305e-07,
      "logits/chosen": -1.0272216796875,
      "logits/rejected": -1.03466796875,
      "logps/chosen": -370.1875,
      "logps/rejected": -538.1500244140625,
      "loss": 0.0206,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -4.160009860992432,
      "rewards/margins": 17.0546875,
      "rewards/rejected": -21.209375381469727,
      "step": 4030
    },
    {
      "epoch": 2.599935629224332,
      "grad_norm": 192.87824451729486,
      "learning_rate": 3.498068254990341e-07,
      "logits/chosen": -1.0537598133087158,
      "logits/rejected": -1.0952637195587158,
      "logps/chosen": -421.1499938964844,
      "logps/rejected": -559.3499755859375,
      "loss": 0.0109,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -4.37060546875,
      "rewards/margins": 16.501562118530273,
      "rewards/rejected": -20.875,
      "step": 4040
    },
    {
      "epoch": 2.6063727067911167,
      "grad_norm": 0.08960042129229139,
      "learning_rate": 3.4819703799098514e-07,
      "logits/chosen": -1.115136742591858,
      "logits/rejected": -1.1196777820587158,
      "logps/chosen": -430.95001220703125,
      "logps/rejected": -589.5999755859375,
      "loss": 0.0109,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -5.207763671875,
      "rewards/margins": 16.924999237060547,
      "rewards/rejected": -22.129688262939453,
      "step": 4050
    },
    {
      "epoch": 2.6128097843579017,
      "grad_norm": 0.20842717683750414,
      "learning_rate": 3.4658725048293627e-07,
      "logits/chosen": -1.0168945789337158,
      "logits/rejected": -1.046289086341858,
      "logps/chosen": -443.2250061035156,
      "logps/rejected": -565.0999755859375,
      "loss": 0.013,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -4.522119045257568,
      "rewards/margins": 16.446874618530273,
      "rewards/rejected": -20.96875,
      "step": 4060
    },
    {
      "epoch": 2.6192468619246863,
      "grad_norm": 0.25064627796735495,
      "learning_rate": 3.449774629748873e-07,
      "logits/chosen": -1.0129883289337158,
      "logits/rejected": -1.024804711341858,
      "logps/chosen": -379.375,
      "logps/rejected": -528.3499755859375,
      "loss": 0.0263,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -3.7671875953674316,
      "rewards/margins": 16.249217987060547,
      "rewards/rejected": -20.014062881469727,
      "step": 4070
    },
    {
      "epoch": 2.625683939491471,
      "grad_norm": 0.08946523961309825,
      "learning_rate": 3.4336767546683836e-07,
      "logits/chosen": -1.0355956554412842,
      "logits/rejected": -1.0556151866912842,
      "logps/chosen": -433.6499938964844,
      "logps/rejected": -589.7000122070312,
      "loss": 0.0141,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -5.182519435882568,
      "rewards/margins": 15.75,
      "rewards/rejected": -20.9296875,
      "step": 4080
    },
    {
      "epoch": 2.6321210170582554,
      "grad_norm": 0.009005467084888181,
      "learning_rate": 3.417578879587894e-07,
      "logits/chosen": -1.0455322265625,
      "logits/rejected": -1.050805687904358,
      "logps/chosen": -472.875,
      "logps/rejected": -595.9500122070312,
      "loss": 0.0027,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.389208793640137,
      "rewards/margins": 17.415624618530273,
      "rewards/rejected": -21.810937881469727,
      "step": 4090
    },
    {
      "epoch": 2.63855809462504,
      "grad_norm": 0.03113362210332774,
      "learning_rate": 3.401481004507405e-07,
      "logits/chosen": -1.061279296875,
      "logits/rejected": -1.085205078125,
      "logps/chosen": -399.92498779296875,
      "logps/rejected": -553.8499755859375,
      "loss": 0.0112,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -4.193945407867432,
      "rewards/margins": 16.964061737060547,
      "rewards/rejected": -21.159374237060547,
      "step": 4100
    },
    {
      "epoch": 2.644995172191825,
      "grad_norm": 0.37326898026603983,
      "learning_rate": 3.3853831294269153e-07,
      "logits/chosen": -1.027929663658142,
      "logits/rejected": -1.041015625,
      "logps/chosen": -418.70001220703125,
      "logps/rejected": -548.0,
      "loss": 0.0094,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -4.916943550109863,
      "rewards/margins": 16.417186737060547,
      "rewards/rejected": -21.334375381469727,
      "step": 4110
    },
    {
      "epoch": 2.6514322497586096,
      "grad_norm": 7.897992961962735,
      "learning_rate": 3.369285254346426e-07,
      "logits/chosen": -0.9991455078125,
      "logits/rejected": -1.0133056640625,
      "logps/chosen": -395.0249938964844,
      "logps/rejected": -551.0499877929688,
      "loss": 0.0139,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -3.975878953933716,
      "rewards/margins": 17.159374237060547,
      "rewards/rejected": -21.131250381469727,
      "step": 4120
    },
    {
      "epoch": 2.657869327325394,
      "grad_norm": 0.07509522718083096,
      "learning_rate": 3.353187379265937e-07,
      "logits/chosen": -1.073095679283142,
      "logits/rejected": -1.070459008216858,
      "logps/chosen": -396.95001220703125,
      "logps/rejected": -549.5,
      "loss": 0.0196,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -4.819872856140137,
      "rewards/margins": 17.485937118530273,
      "rewards/rejected": -22.306249618530273,
      "step": 4130
    },
    {
      "epoch": 2.664306404892179,
      "grad_norm": 0.5200990215076599,
      "learning_rate": 3.3370895041854475e-07,
      "logits/chosen": -1.039453148841858,
      "logits/rejected": -1.0915038585662842,
      "logps/chosen": -453.75,
      "logps/rejected": -574.2999877929688,
      "loss": 0.0225,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -4.282422065734863,
      "rewards/margins": 15.921875,
      "rewards/rejected": -20.217187881469727,
      "step": 4140
    },
    {
      "epoch": 2.6707434824589638,
      "grad_norm": 0.25225569399148123,
      "learning_rate": 3.3209916291049577e-07,
      "logits/chosen": -1.0086181163787842,
      "logits/rejected": -1.0000488758087158,
      "logps/chosen": -372.42498779296875,
      "logps/rejected": -509.29998779296875,
      "loss": 0.0469,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -4.168701171875,
      "rewards/margins": 14.98828125,
      "rewards/rejected": -19.173437118530273,
      "step": 4150
    },
    {
      "epoch": 2.6771805600257483,
      "grad_norm": 0.08673995336101631,
      "learning_rate": 3.3048937540244685e-07,
      "logits/chosen": -1.0426757335662842,
      "logits/rejected": -1.098046898841858,
      "logps/chosen": -460.25,
      "logps/rejected": -575.3499755859375,
      "loss": 0.0059,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -3.84619140625,
      "rewards/margins": 16.293750762939453,
      "rewards/rejected": -20.1328125,
      "step": 4160
    },
    {
      "epoch": 2.683617637592533,
      "grad_norm": 13.353206676041454,
      "learning_rate": 3.288795878943979e-07,
      "logits/chosen": -1.021875023841858,
      "logits/rejected": -1.0466797351837158,
      "logps/chosen": -418.9375,
      "logps/rejected": -592.9000244140625,
      "loss": 0.005,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -3.449902296066284,
      "rewards/margins": 17.214061737060547,
      "rewards/rejected": -20.654687881469727,
      "step": 4170
    },
    {
      "epoch": 2.6900547151593175,
      "grad_norm": 0.09935680046220585,
      "learning_rate": 3.27269800386349e-07,
      "logits/chosen": -0.990771472454071,
      "logits/rejected": -1.0391356945037842,
      "logps/chosen": -396.1499938964844,
      "logps/rejected": -495.29998779296875,
      "loss": 0.0223,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -3.1318116188049316,
      "rewards/margins": 15.576562881469727,
      "rewards/rejected": -18.704687118530273,
      "step": 4180
    },
    {
      "epoch": 2.6964917927261025,
      "grad_norm": 2.336769551114649,
      "learning_rate": 3.256600128783e-07,
      "logits/chosen": -1.0742676258087158,
      "logits/rejected": -1.09033203125,
      "logps/chosen": -416.04998779296875,
      "logps/rejected": -533.1749877929688,
      "loss": 0.0215,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -3.793750047683716,
      "rewards/margins": 14.542187690734863,
      "rewards/rejected": -18.3359375,
      "step": 4190
    },
    {
      "epoch": 2.702928870292887,
      "grad_norm": 1.6019642715844329,
      "learning_rate": 3.2405022537025114e-07,
      "logits/chosen": -1.0363037586212158,
      "logits/rejected": -1.076416015625,
      "logps/chosen": -403.375,
      "logps/rejected": -547.3499755859375,
      "loss": 0.011,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -3.212695360183716,
      "rewards/margins": 15.584375381469727,
      "rewards/rejected": -18.792186737060547,
      "step": 4200
    },
    {
      "epoch": 2.7093659478596717,
      "grad_norm": 2.140234939910703,
      "learning_rate": 3.2244043786220216e-07,
      "logits/chosen": -1.002539038658142,
      "logits/rejected": -1.024999976158142,
      "logps/chosen": -464.86248779296875,
      "logps/rejected": -558.5,
      "loss": 0.0084,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.7227540016174316,
      "rewards/margins": 14.568750381469727,
      "rewards/rejected": -17.287500381469727,
      "step": 4210
    },
    {
      "epoch": 2.7158030254264562,
      "grad_norm": 11.833136190216157,
      "learning_rate": 3.2083065035415323e-07,
      "logits/chosen": -0.994433581829071,
      "logits/rejected": -1.042333960533142,
      "logps/chosen": -406.4624938964844,
      "logps/rejected": -500.2749938964844,
      "loss": 0.0137,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.553906202316284,
      "rewards/margins": 15.292187690734863,
      "rewards/rejected": -17.853124618530273,
      "step": 4220
    },
    {
      "epoch": 2.722240102993241,
      "grad_norm": 0.20643996913297855,
      "learning_rate": 3.1922086284610425e-07,
      "logits/chosen": -0.989501953125,
      "logits/rejected": -1.02734375,
      "logps/chosen": -407.3500061035156,
      "logps/rejected": -531.7000122070312,
      "loss": 0.0192,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -3.6068358421325684,
      "rewards/margins": 14.211718559265137,
      "rewards/rejected": -17.814062118530273,
      "step": 4230
    },
    {
      "epoch": 2.728677180560026,
      "grad_norm": 0.05435070454747595,
      "learning_rate": 3.176110753380554e-07,
      "logits/chosen": -0.961718738079071,
      "logits/rejected": -0.965283215045929,
      "logps/chosen": -399.92498779296875,
      "logps/rejected": -552.5,
      "loss": 0.014,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -3.569287061691284,
      "rewards/margins": 16.609375,
      "rewards/rejected": -20.1796875,
      "step": 4240
    },
    {
      "epoch": 2.7351142581268104,
      "grad_norm": 18.859101722505248,
      "learning_rate": 3.160012878300064e-07,
      "logits/chosen": -0.983837902545929,
      "logits/rejected": -0.985034167766571,
      "logps/chosen": -400.67498779296875,
      "logps/rejected": -515.5,
      "loss": 0.0057,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.3128905296325684,
      "rewards/margins": 15.3515625,
      "rewards/rejected": -18.6640625,
      "step": 4250
    },
    {
      "epoch": 2.741551335693595,
      "grad_norm": 0.19188923573777983,
      "learning_rate": 3.143915003219575e-07,
      "logits/chosen": -1.030126929283142,
      "logits/rejected": -1.056542992591858,
      "logps/chosen": -402.17498779296875,
      "logps/rejected": -529.5999755859375,
      "loss": 0.0133,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.809277296066284,
      "rewards/margins": 16.501562118530273,
      "rewards/rejected": -19.318750381469727,
      "step": 4260
    },
    {
      "epoch": 2.74798841326038,
      "grad_norm": 0.30196685444916815,
      "learning_rate": 3.1278171281390855e-07,
      "logits/chosen": -0.997265636920929,
      "logits/rejected": -1.048925757408142,
      "logps/chosen": -402.3999938964844,
      "logps/rejected": -580.0,
      "loss": 0.0344,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -3.306884765625,
      "rewards/margins": 17.953125,
      "rewards/rejected": -21.268749237060547,
      "step": 4270
    },
    {
      "epoch": 2.7544254908271646,
      "grad_norm": 0.10959414872036556,
      "learning_rate": 3.111719253058596e-07,
      "logits/chosen": -1.031945824623108,
      "logits/rejected": -1.056616187095642,
      "logps/chosen": -427.4624938964844,
      "logps/rejected": -535.9000244140625,
      "loss": 0.0092,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -4.003564357757568,
      "rewards/margins": 16.3671875,
      "rewards/rejected": -20.381250381469727,
      "step": 4280
    },
    {
      "epoch": 2.760862568393949,
      "grad_norm": 1.1898548530459272,
      "learning_rate": 3.0956213779781064e-07,
      "logits/chosen": -0.935986340045929,
      "logits/rejected": -0.9755859375,
      "logps/chosen": -395.4750061035156,
      "logps/rejected": -536.5,
      "loss": 0.0402,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -4.188818454742432,
      "rewards/margins": 15.578906059265137,
      "rewards/rejected": -19.765625,
      "step": 4290
    },
    {
      "epoch": 2.7672996459607337,
      "grad_norm": 0.0349341778282771,
      "learning_rate": 3.0795235028976177e-07,
      "logits/chosen": -1.085205078125,
      "logits/rejected": -1.1107909679412842,
      "logps/chosen": -442.7250061035156,
      "logps/rejected": -600.7000122070312,
      "loss": 0.0097,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -3.6946043968200684,
      "rewards/margins": 17.035938262939453,
      "rewards/rejected": -20.739063262939453,
      "step": 4300
    },
    {
      "epoch": 2.7737367235275183,
      "grad_norm": 0.022601355069047334,
      "learning_rate": 3.063425627817128e-07,
      "logits/chosen": -1.023046851158142,
      "logits/rejected": -0.9951171875,
      "logps/chosen": -410.1000061035156,
      "logps/rejected": -555.0,
      "loss": 0.0253,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -3.625805616378784,
      "rewards/margins": 16.006250381469727,
      "rewards/rejected": -19.629688262939453,
      "step": 4310
    },
    {
      "epoch": 2.7801738010943033,
      "grad_norm": 0.4877006768131671,
      "learning_rate": 3.0473277527366386e-07,
      "logits/chosen": -1.080957055091858,
      "logits/rejected": -1.0974609851837158,
      "logps/chosen": -389.0249938964844,
      "logps/rejected": -538.25,
      "loss": 0.0089,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -3.261798143386841,
      "rewards/margins": 16.251562118530273,
      "rewards/rejected": -19.510936737060547,
      "step": 4320
    },
    {
      "epoch": 2.786610878661088,
      "grad_norm": 0.03266239825168514,
      "learning_rate": 3.031229877656149e-07,
      "logits/chosen": -1.054443359375,
      "logits/rejected": -1.0895507335662842,
      "logps/chosen": -429.67498779296875,
      "logps/rejected": -539.75,
      "loss": 0.0094,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -4.355712890625,
      "rewards/margins": 15.524999618530273,
      "rewards/rejected": -19.889062881469727,
      "step": 4330
    },
    {
      "epoch": 2.7930479562278725,
      "grad_norm": 3.742203170068705,
      "learning_rate": 3.01513200257566e-07,
      "logits/chosen": -1.047509789466858,
      "logits/rejected": -1.061767578125,
      "logps/chosen": -392.1499938964844,
      "logps/rejected": -558.5750122070312,
      "loss": 0.0141,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -3.4574217796325684,
      "rewards/margins": 16.379688262939453,
      "rewards/rejected": -19.839061737060547,
      "step": 4340
    },
    {
      "epoch": 2.799485033794657,
      "grad_norm": 1.1471591775678358,
      "learning_rate": 2.9990341274951703e-07,
      "logits/chosen": -1.0753600597381592,
      "logits/rejected": -1.1260833740234375,
      "logps/chosen": -400.875,
      "logps/rejected": -516.4000244140625,
      "loss": 0.0219,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -3.001171827316284,
      "rewards/margins": 15.435937881469727,
      "rewards/rejected": -18.435937881469727,
      "step": 4350
    },
    {
      "epoch": 2.8059221113614417,
      "grad_norm": 0.16781605556701026,
      "learning_rate": 2.982936252414681e-07,
      "logits/chosen": -1.0686767101287842,
      "logits/rejected": -1.083740234375,
      "logps/chosen": -399.0,
      "logps/rejected": -552.8499755859375,
      "loss": 0.0274,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -3.817089796066284,
      "rewards/margins": 15.465624809265137,
      "rewards/rejected": -19.284374237060547,
      "step": 4360
    },
    {
      "epoch": 2.8123591889282267,
      "grad_norm": 1.9416319834531985,
      "learning_rate": 2.966838377334192e-07,
      "logits/chosen": -1.076025366783142,
      "logits/rejected": -1.080078125,
      "logps/chosen": -406.5249938964844,
      "logps/rejected": -547.0499877929688,
      "loss": 0.022,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -2.145703077316284,
      "rewards/margins": 16.671875,
      "rewards/rejected": -18.817188262939453,
      "step": 4370
    },
    {
      "epoch": 2.8187962664950112,
      "grad_norm": 5.762013159949488,
      "learning_rate": 2.9507405022537025e-07,
      "logits/chosen": -1.105371117591858,
      "logits/rejected": -1.121679663658142,
      "logps/chosen": -432.67498779296875,
      "logps/rejected": -557.6500244140625,
      "loss": 0.0176,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -3.159130811691284,
      "rewards/margins": 16.162500381469727,
      "rewards/rejected": -19.323436737060547,
      "step": 4380
    },
    {
      "epoch": 2.825233344061796,
      "grad_norm": 0.9620630047817021,
      "learning_rate": 2.9346426271732127e-07,
      "logits/chosen": -1.098535180091858,
      "logits/rejected": -1.11181640625,
      "logps/chosen": -383.42498779296875,
      "logps/rejected": -552.375,
      "loss": 0.014,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -3.7589354515075684,
      "rewards/margins": 16.293750762939453,
      "rewards/rejected": -20.051563262939453,
      "step": 4390
    },
    {
      "epoch": 2.831670421628581,
      "grad_norm": 0.06912227797655064,
      "learning_rate": 2.9185447520927234e-07,
      "logits/chosen": -1.1637694835662842,
      "logits/rejected": -1.1516602039337158,
      "logps/chosen": -392.45001220703125,
      "logps/rejected": -568.0999755859375,
      "loss": 0.012,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -3.525561571121216,
      "rewards/margins": 16.5859375,
      "rewards/rejected": -20.115625381469727,
      "step": 4400
    },
    {
      "epoch": 2.8381074991953654,
      "grad_norm": 0.23191462260970216,
      "learning_rate": 2.902446877012234e-07,
      "logits/chosen": -1.08929443359375,
      "logits/rejected": -1.105126976966858,
      "logps/chosen": -389.67498779296875,
      "logps/rejected": -540.0499877929688,
      "loss": 0.0138,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.943164110183716,
      "rewards/margins": 15.912500381469727,
      "rewards/rejected": -18.854686737060547,
      "step": 4410
    },
    {
      "epoch": 2.84454457676215,
      "grad_norm": 0.22733204817053218,
      "learning_rate": 2.886349001931745e-07,
      "logits/chosen": -1.051367163658142,
      "logits/rejected": -1.085546851158142,
      "logps/chosen": -441.1000061035156,
      "logps/rejected": -564.75,
      "loss": 0.0176,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -2.992919921875,
      "rewards/margins": 14.8984375,
      "rewards/rejected": -17.885936737060547,
      "step": 4420
    },
    {
      "epoch": 2.8509816543289346,
      "grad_norm": 0.35599187838621965,
      "learning_rate": 2.870251126851255e-07,
      "logits/chosen": -1.1135742664337158,
      "logits/rejected": -1.129785180091858,
      "logps/chosen": -402.0249938964844,
      "logps/rejected": -500.67498779296875,
      "loss": 0.0251,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -2.205517530441284,
      "rewards/margins": 15.65625,
      "rewards/rejected": -17.860937118530273,
      "step": 4430
    },
    {
      "epoch": 2.857418731895719,
      "grad_norm": 0.5999684700633137,
      "learning_rate": 2.8541532517707664e-07,
      "logits/chosen": -1.177343726158142,
      "logits/rejected": -1.2083008289337158,
      "logps/chosen": -423.8999938964844,
      "logps/rejected": -576.5999755859375,
      "loss": 0.0089,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.318981885910034,
      "rewards/margins": 16.435937881469727,
      "rewards/rejected": -18.759374618530273,
      "step": 4440
    },
    {
      "epoch": 2.863855809462504,
      "grad_norm": 0.41307829335787627,
      "learning_rate": 2.8380553766902766e-07,
      "logits/chosen": -1.1286132335662842,
      "logits/rejected": -1.140625,
      "logps/chosen": -418.79998779296875,
      "logps/rejected": -560.75,
      "loss": 0.0131,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.341552734375,
      "rewards/margins": 16.698436737060547,
      "rewards/rejected": -19.03125,
      "step": 4450
    },
    {
      "epoch": 2.8702928870292888,
      "grad_norm": 0.06329392464426446,
      "learning_rate": 2.8219575016097873e-07,
      "logits/chosen": -1.0729491710662842,
      "logits/rejected": -1.0994141101837158,
      "logps/chosen": -405.7749938964844,
      "logps/rejected": -532.4500122070312,
      "loss": 0.0218,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -2.7913575172424316,
      "rewards/margins": 15.798437118530273,
      "rewards/rejected": -18.600000381469727,
      "step": 4460
    },
    {
      "epoch": 2.8767299645960733,
      "grad_norm": 0.5929940057123108,
      "learning_rate": 2.8058596265292975e-07,
      "logits/chosen": -1.12939453125,
      "logits/rejected": -1.158447265625,
      "logps/chosen": -425.6000061035156,
      "logps/rejected": -554.8499755859375,
      "loss": 0.009,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.7857298851013184,
      "rewards/margins": 16.956249237060547,
      "rewards/rejected": -19.728124618530273,
      "step": 4470
    },
    {
      "epoch": 2.883167042162858,
      "grad_norm": 10.781761622532624,
      "learning_rate": 2.789761751448809e-07,
      "logits/chosen": -1.1557128429412842,
      "logits/rejected": -1.17626953125,
      "logps/chosen": -387.67498779296875,
      "logps/rejected": -530.3499755859375,
      "loss": 0.0136,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -3.205273389816284,
      "rewards/margins": 15.399999618530273,
      "rewards/rejected": -18.606250762939453,
      "step": 4480
    },
    {
      "epoch": 2.8896041197296425,
      "grad_norm": 5.678623060839604,
      "learning_rate": 2.773663876368319e-07,
      "logits/chosen": -1.122949242591858,
      "logits/rejected": -1.1169922351837158,
      "logps/chosen": -412.875,
      "logps/rejected": -559.7000122070312,
      "loss": 0.0155,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -4.047656059265137,
      "rewards/margins": 15.893750190734863,
      "rewards/rejected": -19.956249237060547,
      "step": 4490
    },
    {
      "epoch": 2.8960411972964275,
      "grad_norm": 0.12843353367463994,
      "learning_rate": 2.7575660012878297e-07,
      "logits/chosen": -1.0998046398162842,
      "logits/rejected": -1.0850098133087158,
      "logps/chosen": -403.29998779296875,
      "logps/rejected": -578.4000244140625,
      "loss": 0.0046,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -4.0653076171875,
      "rewards/margins": 15.801562309265137,
      "rewards/rejected": -19.857812881469727,
      "step": 4500
    },
    {
      "epoch": 2.902478274863212,
      "grad_norm": 0.18293130358900542,
      "learning_rate": 2.7414681262073405e-07,
      "logits/chosen": -1.146093726158142,
      "logits/rejected": -1.1618163585662842,
      "logps/chosen": -414.625,
      "logps/rejected": -538.0999755859375,
      "loss": 0.0088,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -3.832763671875,
      "rewards/margins": 16.057811737060547,
      "rewards/rejected": -19.892187118530273,
      "step": 4510
    },
    {
      "epoch": 2.9089153524299967,
      "grad_norm": 15.929814025771526,
      "learning_rate": 2.725370251126851e-07,
      "logits/chosen": -1.20361328125,
      "logits/rejected": -1.2047851085662842,
      "logps/chosen": -441.79998779296875,
      "logps/rejected": -596.75,
      "loss": 0.0151,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -3.934033155441284,
      "rewards/margins": 17.510936737060547,
      "rewards/rejected": -21.459375381469727,
      "step": 4520
    },
    {
      "epoch": 2.9153524299967817,
      "grad_norm": 26.74681492633243,
      "learning_rate": 2.7092723760463614e-07,
      "logits/chosen": -1.161865234375,
      "logits/rejected": -1.198339819908142,
      "logps/chosen": -409.6000061035156,
      "logps/rejected": -548.0999755859375,
      "loss": 0.0184,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -3.522265672683716,
      "rewards/margins": 16.2265625,
      "rewards/rejected": -19.748437881469727,
      "step": 4530
    },
    {
      "epoch": 2.9217895075635663,
      "grad_norm": 0.18148041344029991,
      "learning_rate": 2.6931745009658727e-07,
      "logits/chosen": -1.149511694908142,
      "logits/rejected": -1.160742163658142,
      "logps/chosen": -404.625,
      "logps/rejected": -598.25,
      "loss": 0.0044,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -4.6748046875,
      "rewards/margins": 17.774999618530273,
      "rewards/rejected": -22.448436737060547,
      "step": 4540
    },
    {
      "epoch": 2.928226585130351,
      "grad_norm": 0.09689001421228971,
      "learning_rate": 2.677076625885383e-07,
      "logits/chosen": -1.1152832508087158,
      "logits/rejected": -1.1346313953399658,
      "logps/chosen": -404.6875,
      "logps/rejected": -555.875,
      "loss": 0.026,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -4.319580078125,
      "rewards/margins": 17.51171875,
      "rewards/rejected": -21.8203125,
      "step": 4550
    },
    {
      "epoch": 2.9346636626971354,
      "grad_norm": 0.8349717835904119,
      "learning_rate": 2.6609787508048936e-07,
      "logits/chosen": -1.2187011241912842,
      "logits/rejected": -1.26025390625,
      "logps/chosen": -419.57501220703125,
      "logps/rejected": -553.0,
      "loss": 0.0051,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -4.408154487609863,
      "rewards/margins": 16.829687118530273,
      "rewards/rejected": -21.245311737060547,
      "step": 4560
    },
    {
      "epoch": 2.94110074026392,
      "grad_norm": 0.0018520052569766796,
      "learning_rate": 2.644880875724404e-07,
      "logits/chosen": -1.1864745616912842,
      "logits/rejected": -1.1776854991912842,
      "logps/chosen": -429.125,
      "logps/rejected": -587.4500122070312,
      "loss": 0.0065,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -5.62286376953125,
      "rewards/margins": 17.575000762939453,
      "rewards/rejected": -23.203125,
      "step": 4570
    },
    {
      "epoch": 2.947537817830705,
      "grad_norm": 0.9827245912908983,
      "learning_rate": 2.628783000643915e-07,
      "logits/chosen": -1.075048804283142,
      "logits/rejected": -1.100341796875,
      "logps/chosen": -450.88751220703125,
      "logps/rejected": -579.8499755859375,
      "loss": 0.0218,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -5.349609375,
      "rewards/margins": 17.246875762939453,
      "rewards/rejected": -22.5859375,
      "step": 4580
    },
    {
      "epoch": 2.9539748953974896,
      "grad_norm": 2.808445023918683,
      "learning_rate": 2.6126851255634253e-07,
      "logits/chosen": -1.074462890625,
      "logits/rejected": -1.122167944908142,
      "logps/chosen": -404.38751220703125,
      "logps/rejected": -577.3499755859375,
      "loss": 0.0465,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -5.451562404632568,
      "rewards/margins": 16.517187118530273,
      "rewards/rejected": -21.970312118530273,
      "step": 4590
    },
    {
      "epoch": 2.960411972964274,
      "grad_norm": 0.31073133311477225,
      "learning_rate": 2.596587250482936e-07,
      "logits/chosen": -1.086157202720642,
      "logits/rejected": -1.089868187904358,
      "logps/chosen": -421.4624938964844,
      "logps/rejected": -573.3499755859375,
      "loss": 0.0195,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -5.573144435882568,
      "rewards/margins": 17.239843368530273,
      "rewards/rejected": -22.795312881469727,
      "step": 4600
    },
    {
      "epoch": 2.9668490505310587,
      "grad_norm": 0.068889903289002,
      "learning_rate": 2.580489375402447e-07,
      "logits/chosen": -1.0750000476837158,
      "logits/rejected": -1.1067383289337158,
      "logps/chosen": -408.54998779296875,
      "logps/rejected": -580.625,
      "loss": 0.0148,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -4.872363090515137,
      "rewards/margins": 17.399999618530273,
      "rewards/rejected": -22.2890625,
      "step": 4610
    },
    {
      "epoch": 2.9732861280978433,
      "grad_norm": 0.6319961977568703,
      "learning_rate": 2.5643915003219575e-07,
      "logits/chosen": -1.137109398841858,
      "logits/rejected": -1.150292992591858,
      "logps/chosen": -435.17498779296875,
      "logps/rejected": -535.0,
      "loss": 0.0127,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -4.769067287445068,
      "rewards/margins": 15.90625,
      "rewards/rejected": -20.671875,
      "step": 4620
    },
    {
      "epoch": 2.9797232056646283,
      "grad_norm": 0.008954633914854768,
      "learning_rate": 2.5482936252414677e-07,
      "logits/chosen": -1.117285132408142,
      "logits/rejected": -1.121679663658142,
      "logps/chosen": -465.875,
      "logps/rejected": -593.4000244140625,
      "loss": 0.0262,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -4.642480373382568,
      "rewards/margins": 16.438282012939453,
      "rewards/rejected": -21.082813262939453,
      "step": 4630
    },
    {
      "epoch": 2.986160283231413,
      "grad_norm": 0.2715041175189798,
      "learning_rate": 2.5321957501609784e-07,
      "logits/chosen": -1.084082007408142,
      "logits/rejected": -1.119775414466858,
      "logps/chosen": -480.92498779296875,
      "logps/rejected": -611.3499755859375,
      "loss": 0.0071,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -4.862011909484863,
      "rewards/margins": 17.3046875,
      "rewards/rejected": -22.178125381469727,
      "step": 4640
    },
    {
      "epoch": 2.9925973607981975,
      "grad_norm": 0.07398779198819364,
      "learning_rate": 2.516097875080489e-07,
      "logits/chosen": -1.11474609375,
      "logits/rejected": -1.156152367591858,
      "logps/chosen": -449.57501220703125,
      "logps/rejected": -600.5999755859375,
      "loss": 0.0117,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -4.337744235992432,
      "rewards/margins": 17.264062881469727,
      "rewards/rejected": -21.604686737060547,
      "step": 4650
    },
    {
      "epoch": 2.9990344383649825,
      "grad_norm": 0.6036221444805209,
      "learning_rate": 2.5e-07,
      "logits/chosen": -1.118505835533142,
      "logits/rejected": -1.121191382408142,
      "logps/chosen": -417.1000061035156,
      "logps/rejected": -584.4000244140625,
      "loss": 0.0179,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -3.984081983566284,
      "rewards/margins": 17.8046875,
      "rewards/rejected": -21.784374237060547,
      "step": 4660
    },
    {
      "epoch": 3.005149662053428,
      "grad_norm": 0.02049975193867326,
      "learning_rate": 2.4839021249195106e-07,
      "logits/chosen": -1.1468955278396606,
      "logits/rejected": -1.1493626832962036,
      "logps/chosen": -438.1052551269531,
      "logps/rejected": -590.6842041015625,
      "loss": 0.0043,
      "rewards/accuracies": 0.9934210777282715,
      "rewards/chosen": -5.279964923858643,
      "rewards/margins": 18.324012756347656,
      "rewards/rejected": -23.601974487304688,
      "step": 4670
    },
    {
      "epoch": 3.0115867396202125,
      "grad_norm": 0.007171380019719538,
      "learning_rate": 2.4678042498390214e-07,
      "logits/chosen": -1.125341773033142,
      "logits/rejected": -1.152099609375,
      "logps/chosen": -459.7749938964844,
      "logps/rejected": -600.5499877929688,
      "loss": 0.0015,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.583300590515137,
      "rewards/margins": 17.621875762939453,
      "rewards/rejected": -23.206249237060547,
      "step": 4680
    },
    {
      "epoch": 3.018023817186997,
      "grad_norm": 0.015194056176862642,
      "learning_rate": 2.4517063747585316e-07,
      "logits/chosen": -1.083105444908142,
      "logits/rejected": -1.109155297279358,
      "logps/chosen": -415.1000061035156,
      "logps/rejected": -594.9500122070312,
      "loss": 0.0173,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -4.552294731140137,
      "rewards/margins": 19.375,
      "rewards/rejected": -23.924999237060547,
      "step": 4690
    },
    {
      "epoch": 3.0244608947537817,
      "grad_norm": 4.468715166239343,
      "learning_rate": 2.4356084996780423e-07,
      "logits/chosen": -1.2006347179412842,
      "logits/rejected": -1.192480444908142,
      "logps/chosen": -440.1000061035156,
      "logps/rejected": -595.4500122070312,
      "loss": 0.0173,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -5.480175971984863,
      "rewards/margins": 18.094532012939453,
      "rewards/rejected": -23.568750381469727,
      "step": 4700
    },
    {
      "epoch": 3.0308979723205667,
      "grad_norm": 0.021122244530159647,
      "learning_rate": 2.419510624597553e-07,
      "logits/chosen": -1.069067358970642,
      "logits/rejected": -1.050115942955017,
      "logps/chosen": -445.5,
      "logps/rejected": -602.0499877929688,
      "loss": 0.0092,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -5.22955322265625,
      "rewards/margins": 17.5,
      "rewards/rejected": -22.737499237060547,
      "step": 4710
    },
    {
      "epoch": 3.0373350498873513,
      "grad_norm": 0.03307507714663866,
      "learning_rate": 2.403412749517064e-07,
      "logits/chosen": -1.173242211341858,
      "logits/rejected": -1.1818358898162842,
      "logps/chosen": -433.1000061035156,
      "logps/rejected": -591.5999755859375,
      "loss": 0.0044,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -4.977221488952637,
      "rewards/margins": 18.759374618530273,
      "rewards/rejected": -23.740625381469727,
      "step": 4720
    },
    {
      "epoch": 3.043772127454136,
      "grad_norm": 0.02934355373683881,
      "learning_rate": 2.3873148744365745e-07,
      "logits/chosen": -1.133935570716858,
      "logits/rejected": -1.1240723133087158,
      "logps/chosen": -412.6499938964844,
      "logps/rejected": -562.75,
      "loss": 0.013,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -5.184765815734863,
      "rewards/margins": 18.248437881469727,
      "rewards/rejected": -23.424999237060547,
      "step": 4730
    },
    {
      "epoch": 3.0502092050209204,
      "grad_norm": 0.019452870159428832,
      "learning_rate": 2.371216999356085e-07,
      "logits/chosen": -1.0964844226837158,
      "logits/rejected": -1.0937011241912842,
      "logps/chosen": -428.2749938964844,
      "logps/rejected": -587.7999877929688,
      "loss": 0.026,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -6.134753227233887,
      "rewards/margins": 17.532032012939453,
      "rewards/rejected": -23.657812118530273,
      "step": 4740
    },
    {
      "epoch": 3.056646282587705,
      "grad_norm": 0.01026781399221231,
      "learning_rate": 2.3551191242755954e-07,
      "logits/chosen": -1.1647460460662842,
      "logits/rejected": -1.160058617591858,
      "logps/chosen": -442.70001220703125,
      "logps/rejected": -598.5499877929688,
      "loss": 0.0043,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -5.059496879577637,
      "rewards/margins": 20.454687118530273,
      "rewards/rejected": -25.529687881469727,
      "step": 4750
    },
    {
      "epoch": 3.06308336015449,
      "grad_norm": 0.28593764965232715,
      "learning_rate": 2.3390212491951062e-07,
      "logits/chosen": -1.155908226966858,
      "logits/rejected": -1.1824219226837158,
      "logps/chosen": -442.8500061035156,
      "logps/rejected": -572.7000122070312,
      "loss": 0.0224,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -5.127831935882568,
      "rewards/margins": 17.984375,
      "rewards/rejected": -23.118749618530273,
      "step": 4760
    },
    {
      "epoch": 3.0695204377212746,
      "grad_norm": 0.003290033796275054,
      "learning_rate": 2.3229233741146166e-07,
      "logits/chosen": -1.203222632408142,
      "logits/rejected": -1.217187523841858,
      "logps/chosen": -454.8500061035156,
      "logps/rejected": -630.7000122070312,
      "loss": 0.0021,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.734375,
      "rewards/margins": 19.578125,
      "rewards/rejected": -25.309375762939453,
      "step": 4770
    },
    {
      "epoch": 3.075957515288059,
      "grad_norm": 0.0017905173138558442,
      "learning_rate": 2.3068254990341274e-07,
      "logits/chosen": -1.1566894054412842,
      "logits/rejected": -1.195703148841858,
      "logps/chosen": -481.125,
      "logps/rejected": -612.9000244140625,
      "loss": 0.0043,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -7.490429878234863,
      "rewards/margins": 18.299999237060547,
      "rewards/rejected": -25.787500381469727,
      "step": 4780
    },
    {
      "epoch": 3.0823945928548437,
      "grad_norm": 1.331154357210038,
      "learning_rate": 2.290727623953638e-07,
      "logits/chosen": -1.2042968273162842,
      "logits/rejected": -1.212011694908142,
      "logps/chosen": -475.0249938964844,
      "logps/rejected": -620.4500122070312,
      "loss": 0.0133,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -7.7841796875,
      "rewards/margins": 19.067188262939453,
      "rewards/rejected": -26.856250762939453,
      "step": 4790
    },
    {
      "epoch": 3.0888316704216288,
      "grad_norm": 0.008913213065396297,
      "learning_rate": 2.2746297488731486e-07,
      "logits/chosen": -1.195410132408142,
      "logits/rejected": -1.2082030773162842,
      "logps/chosen": -421.79998779296875,
      "logps/rejected": -622.4000244140625,
      "loss": 0.0043,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -6.528954982757568,
      "rewards/margins": 19.818750381469727,
      "rewards/rejected": -26.353124618530273,
      "step": 4800
    },
    {
      "epoch": 3.0952687479884133,
      "grad_norm": 0.1029343972347026,
      "learning_rate": 2.2585318737926593e-07,
      "logits/chosen": -1.1962890625,
      "logits/rejected": -1.1948730945587158,
      "logps/chosen": -438.8999938964844,
      "logps/rejected": -587.0499877929688,
      "loss": 0.0029,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.034863471984863,
      "rewards/margins": 19.465625762939453,
      "rewards/rejected": -26.515625,
      "step": 4810
    },
    {
      "epoch": 3.101705825555198,
      "grad_norm": 0.00040497685294049686,
      "learning_rate": 2.2424339987121698e-07,
      "logits/chosen": -1.1691405773162842,
      "logits/rejected": -1.1959960460662842,
      "logps/chosen": -445.5249938964844,
      "logps/rejected": -604.0499877929688,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.252734184265137,
      "rewards/margins": 19.690624237060547,
      "rewards/rejected": -26.931249618530273,
      "step": 4820
    },
    {
      "epoch": 3.1081429031219825,
      "grad_norm": 0.15227806520684364,
      "learning_rate": 2.2263361236316805e-07,
      "logits/chosen": -1.1714355945587158,
      "logits/rejected": -1.162451148033142,
      "logps/chosen": -425.70001220703125,
      "logps/rejected": -615.7000122070312,
      "loss": 0.0054,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -7.599999904632568,
      "rewards/margins": 19.184375762939453,
      "rewards/rejected": -26.774999618530273,
      "step": 4830
    },
    {
      "epoch": 3.1145799806887675,
      "grad_norm": 0.023311318939842125,
      "learning_rate": 2.210238248551191e-07,
      "logits/chosen": -1.174707055091858,
      "logits/rejected": -1.1965820789337158,
      "logps/chosen": -453.04998779296875,
      "logps/rejected": -608.7999877929688,
      "loss": 0.0132,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -7.903124809265137,
      "rewards/margins": 18.704687118530273,
      "rewards/rejected": -26.618749618530273,
      "step": 4840
    },
    {
      "epoch": 3.121017058255552,
      "grad_norm": 10.786296882917435,
      "learning_rate": 2.1941403734707017e-07,
      "logits/chosen": -1.1326172351837158,
      "logits/rejected": -1.1515624523162842,
      "logps/chosen": -422.5249938964844,
      "logps/rejected": -609.1500244140625,
      "loss": 0.0135,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -7.811327934265137,
      "rewards/margins": 18.329687118530273,
      "rewards/rejected": -26.146875381469727,
      "step": 4850
    },
    {
      "epoch": 3.1274541358223367,
      "grad_norm": 0.01329847930022698,
      "learning_rate": 2.1780424983902125e-07,
      "logits/chosen": -1.146520972251892,
      "logits/rejected": -1.1954345703125,
      "logps/chosen": -444.375,
      "logps/rejected": -611.2999877929688,
      "loss": 0.013,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -6.67236328125,
      "rewards/margins": 18.916406631469727,
      "rewards/rejected": -25.596874237060547,
      "step": 4860
    },
    {
      "epoch": 3.1338912133891212,
      "grad_norm": 0.04550344760940479,
      "learning_rate": 2.161944623309723e-07,
      "logits/chosen": -1.194433569908142,
      "logits/rejected": -1.233496069908142,
      "logps/chosen": -418.92498779296875,
      "logps/rejected": -581.0,
      "loss": 0.0054,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -7.391015529632568,
      "rewards/margins": 18.495311737060547,
      "rewards/rejected": -25.868749618530273,
      "step": 4870
    },
    {
      "epoch": 3.140328290955906,
      "grad_norm": 0.11635747842049714,
      "learning_rate": 2.1458467482292337e-07,
      "logits/chosen": -1.123266577720642,
      "logits/rejected": -1.149194359779358,
      "logps/chosen": -495.25,
      "logps/rejected": -631.2000122070312,
      "loss": 0.0056,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -6.9462890625,
      "rewards/margins": 19.1015625,
      "rewards/rejected": -26.037500381469727,
      "step": 4880
    },
    {
      "epoch": 3.146765368522691,
      "grad_norm": 0.0015098347770240576,
      "learning_rate": 2.129748873148744e-07,
      "logits/chosen": -1.099462866783142,
      "logits/rejected": -1.12841796875,
      "logps/chosen": -447.04998779296875,
      "logps/rejected": -605.0499877929688,
      "loss": 0.0045,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -6.196093559265137,
      "rewards/margins": 19.821874618530273,
      "rewards/rejected": -26.018749237060547,
      "step": 4890
    },
    {
      "epoch": 3.1532024460894754,
      "grad_norm": 0.06132780820289997,
      "learning_rate": 2.1136509980682549e-07,
      "logits/chosen": -1.181860327720642,
      "logits/rejected": -1.214599609375,
      "logps/chosen": -458.0249938964844,
      "logps/rejected": -618.4000244140625,
      "loss": 0.0045,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -6.970703125,
      "rewards/margins": 18.651561737060547,
      "rewards/rejected": -25.634374618530273,
      "step": 4900
    },
    {
      "epoch": 3.15963952365626,
      "grad_norm": 0.015670466658027772,
      "learning_rate": 2.0975531229877656e-07,
      "logits/chosen": -1.186669945716858,
      "logits/rejected": -1.179956078529358,
      "logps/chosen": -451.1499938964844,
      "logps/rejected": -623.6500244140625,
      "loss": 0.0173,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -6.311132907867432,
      "rewards/margins": 18.420312881469727,
      "rewards/rejected": -24.734375,
      "step": 4910
    },
    {
      "epoch": 3.1660766012230446,
      "grad_norm": 0.7860096166411907,
      "learning_rate": 2.081455247907276e-07,
      "logits/chosen": -1.139672875404358,
      "logits/rejected": -1.1612060070037842,
      "logps/chosen": -434.07501220703125,
      "logps/rejected": -601.9000244140625,
      "loss": 0.0087,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -5.775292873382568,
      "rewards/margins": 18.990625381469727,
      "rewards/rejected": -24.756250381469727,
      "step": 4920
    },
    {
      "epoch": 3.1725136787898296,
      "grad_norm": 0.009808484495203735,
      "learning_rate": 2.0653573728267868e-07,
      "logits/chosen": -1.0760498046875,
      "logits/rejected": -1.0888671875,
      "logps/chosen": -405.92498779296875,
      "logps/rejected": -592.4500122070312,
      "loss": 0.0256,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -7.089453220367432,
      "rewards/margins": 18.778125762939453,
      "rewards/rejected": -25.860937118530273,
      "step": 4930
    },
    {
      "epoch": 3.178950756356614,
      "grad_norm": 0.0016510487300226008,
      "learning_rate": 2.0492594977462973e-07,
      "logits/chosen": -1.156835913658142,
      "logits/rejected": -1.186132788658142,
      "logps/chosen": -437.20001220703125,
      "logps/rejected": -580.5,
      "loss": 0.0087,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -5.806250095367432,
      "rewards/margins": 18.1875,
      "rewards/rejected": -24.0234375,
      "step": 4940
    },
    {
      "epoch": 3.1853878339233987,
      "grad_norm": 0.04680967562897145,
      "learning_rate": 2.033161622665808e-07,
      "logits/chosen": -1.114355444908142,
      "logits/rejected": -1.157324194908142,
      "logps/chosen": -447.20001220703125,
      "logps/rejected": -595.4000244140625,
      "loss": 0.0044,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -5.834228515625,
      "rewards/margins": 16.8828125,
      "rewards/rejected": -22.717187881469727,
      "step": 4950
    },
    {
      "epoch": 3.1918249114901833,
      "grad_norm": 0.03766884792215408,
      "learning_rate": 2.0170637475853185e-07,
      "logits/chosen": -1.107934594154358,
      "logits/rejected": -1.1571044921875,
      "logps/chosen": -463.6000061035156,
      "logps/rejected": -588.5999755859375,
      "loss": 0.0087,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -5.200463771820068,
      "rewards/margins": 18.962499618530273,
      "rewards/rejected": -24.15625,
      "step": 4960
    },
    {
      "epoch": 3.1982619890569683,
      "grad_norm": 0.032789307952724236,
      "learning_rate": 2.0009658725048292e-07,
      "logits/chosen": -1.143945336341858,
      "logits/rejected": -1.173437476158142,
      "logps/chosen": -460.0,
      "logps/rejected": -598.0499877929688,
      "loss": 0.013,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -5.598474025726318,
      "rewards/margins": 18.3203125,
      "rewards/rejected": -23.903125762939453,
      "step": 4970
    },
    {
      "epoch": 3.204699066623753,
      "grad_norm": 0.014658227990306128,
      "learning_rate": 1.98486799742434e-07,
      "logits/chosen": -1.2394530773162842,
      "logits/rejected": -1.252832055091858,
      "logps/chosen": -451.29998779296875,
      "logps/rejected": -618.7999877929688,
      "loss": 0.0088,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -6.079833984375,
      "rewards/margins": 19.456249237060547,
      "rewards/rejected": -25.534374237060547,
      "step": 4980
    },
    {
      "epoch": 3.2111361441905375,
      "grad_norm": 0.046097116270495335,
      "learning_rate": 1.9687701223438504e-07,
      "logits/chosen": -1.2252929210662842,
      "logits/rejected": -1.2561523914337158,
      "logps/chosen": -442.57501220703125,
      "logps/rejected": -594.75,
      "loss": 0.013,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -6.322241306304932,
      "rewards/margins": 19.120311737060547,
      "rewards/rejected": -25.435937881469727,
      "step": 4990
    },
    {
      "epoch": 3.217573221757322,
      "grad_norm": 0.09069092869768736,
      "learning_rate": 1.9526722472633612e-07,
      "logits/chosen": -1.132568359375,
      "logits/rejected": -1.156494140625,
      "logps/chosen": -425.82501220703125,
      "logps/rejected": -578.7000122070312,
      "loss": 0.0174,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -6.106738090515137,
      "rewards/margins": 18.545312881469727,
      "rewards/rejected": -24.645313262939453,
      "step": 5000
    },
    {
      "epoch": 3.2240102993241067,
      "grad_norm": 0.01590234525948529,
      "learning_rate": 1.9365743721828716e-07,
      "logits/chosen": -1.1720702648162842,
      "logits/rejected": -1.216406226158142,
      "logps/chosen": -410.375,
      "logps/rejected": -574.5999755859375,
      "loss": 0.0216,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -6.863867282867432,
      "rewards/margins": 18.245311737060547,
      "rewards/rejected": -25.118749618530273,
      "step": 5010
    },
    {
      "epoch": 3.2304473768908917,
      "grad_norm": 0.005519486507844155,
      "learning_rate": 1.9204764971023824e-07,
      "logits/chosen": -1.161962866783142,
      "logits/rejected": -1.197119116783142,
      "logps/chosen": -420.7250061035156,
      "logps/rejected": -583.4000244140625,
      "loss": 0.0044,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -6.3515625,
      "rewards/margins": 19.045312881469727,
      "rewards/rejected": -25.3828125,
      "step": 5020
    },
    {
      "epoch": 3.2368844544576763,
      "grad_norm": 0.004607457293818073,
      "learning_rate": 1.904378622021893e-07,
      "logits/chosen": -1.165283203125,
      "logits/rejected": -1.167138695716858,
      "logps/chosen": -445.92498779296875,
      "logps/rejected": -614.1500244140625,
      "loss": 0.0173,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -6.177538871765137,
      "rewards/margins": 19.678125381469727,
      "rewards/rejected": -25.84375,
      "step": 5030
    },
    {
      "epoch": 3.243321532024461,
      "grad_norm": 0.7275701284809951,
      "learning_rate": 1.8882807469414036e-07,
      "logits/chosen": -1.1663086414337158,
      "logits/rejected": -1.1945312023162842,
      "logps/chosen": -407.82501220703125,
      "logps/rejected": -566.5499877929688,
      "loss": 0.0113,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -6.144921779632568,
      "rewards/margins": 18.651561737060547,
      "rewards/rejected": -24.7734375,
      "step": 5040
    },
    {
      "epoch": 3.2497586095912454,
      "grad_norm": 0.00925264942928273,
      "learning_rate": 1.8721828718609143e-07,
      "logits/chosen": -1.0818359851837158,
      "logits/rejected": -1.1124999523162842,
      "logps/chosen": -422.625,
      "logps/rejected": -599.2000122070312,
      "loss": 0.0216,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -5.3525390625,
      "rewards/margins": 18.346874237060547,
      "rewards/rejected": -23.692188262939453,
      "step": 5050
    },
    {
      "epoch": 3.2561956871580304,
      "grad_norm": 0.018748740799270636,
      "learning_rate": 1.8560849967804248e-07,
      "logits/chosen": -1.139746069908142,
      "logits/rejected": -1.134374976158142,
      "logps/chosen": -432.95001220703125,
      "logps/rejected": -592.5,
      "loss": 0.0263,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -6.676465034484863,
      "rewards/margins": 17.557811737060547,
      "rewards/rejected": -24.228124618530273,
      "step": 5060
    },
    {
      "epoch": 3.262632764724815,
      "grad_norm": 0.009384866899820094,
      "learning_rate": 1.8399871216999355e-07,
      "logits/chosen": -1.19677734375,
      "logits/rejected": -1.204199194908142,
      "logps/chosen": -433.67498779296875,
      "logps/rejected": -604.8499755859375,
      "loss": 0.0173,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -5.350488185882568,
      "rewards/margins": 18.135936737060547,
      "rewards/rejected": -23.475000381469727,
      "step": 5070
    },
    {
      "epoch": 3.2690698422915996,
      "grad_norm": 0.02147237650112711,
      "learning_rate": 1.823889246619446e-07,
      "logits/chosen": -1.2500488758087158,
      "logits/rejected": -1.2795898914337158,
      "logps/chosen": -481.875,
      "logps/rejected": -612.6500244140625,
      "loss": 0.0044,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -7.387597560882568,
      "rewards/margins": 17.568750381469727,
      "rewards/rejected": -24.957813262939453,
      "step": 5080
    },
    {
      "epoch": 3.275506919858384,
      "grad_norm": 0.02187166539536812,
      "learning_rate": 1.8077913715389567e-07,
      "logits/chosen": -1.118310570716858,
      "logits/rejected": -1.1315186023712158,
      "logps/chosen": -412.5,
      "logps/rejected": -595.2999877929688,
      "loss": 0.0303,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -5.707812309265137,
      "rewards/margins": 18.108592987060547,
      "rewards/rejected": -23.81640625,
      "step": 5090
    },
    {
      "epoch": 3.281943997425169,
      "grad_norm": 0.13063587935498464,
      "learning_rate": 1.7916934964584674e-07,
      "logits/chosen": -1.181549072265625,
      "logits/rejected": -1.2327148914337158,
      "logps/chosen": -423.9750061035156,
      "logps/rejected": -615.6500244140625,
      "loss": 0.013,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -7.080273628234863,
      "rewards/margins": 18.559375762939453,
      "rewards/rejected": -25.653125762939453,
      "step": 5100
    },
    {
      "epoch": 3.2883810749919538,
      "grad_norm": 0.005783772596752793,
      "learning_rate": 1.775595621377978e-07,
      "logits/chosen": -1.120336890220642,
      "logits/rejected": -1.1363036632537842,
      "logps/chosen": -441.20001220703125,
      "logps/rejected": -580.0499877929688,
      "loss": 0.0087,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -5.895410060882568,
      "rewards/margins": 18.892187118530273,
      "rewards/rejected": -24.774999618530273,
      "step": 5110
    },
    {
      "epoch": 3.2948181525587383,
      "grad_norm": 0.009963166845550958,
      "learning_rate": 1.7594977462974886e-07,
      "logits/chosen": -1.1458008289337158,
      "logits/rejected": -1.1650879383087158,
      "logps/chosen": -396.3500061035156,
      "logps/rejected": -568.3499755859375,
      "loss": 0.013,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -6.274755954742432,
      "rewards/margins": 18.643749237060547,
      "rewards/rejected": -24.907812118530273,
      "step": 5120
    },
    {
      "epoch": 3.301255230125523,
      "grad_norm": 0.06936182148963972,
      "learning_rate": 1.743399871216999e-07,
      "logits/chosen": -1.218896508216858,
      "logits/rejected": -1.230566382408142,
      "logps/chosen": -452.2749938964844,
      "logps/rejected": -606.5999755859375,
      "loss": 0.0101,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -5.982812404632568,
      "rewards/margins": 17.821874618530273,
      "rewards/rejected": -23.8203125,
      "step": 5130
    },
    {
      "epoch": 3.3076923076923075,
      "grad_norm": 0.004261491928143392,
      "learning_rate": 1.7273019961365098e-07,
      "logits/chosen": -1.1884276866912842,
      "logits/rejected": -1.212499976158142,
      "logps/chosen": -437.8999938964844,
      "logps/rejected": -612.7999877929688,
      "loss": 0.0043,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -5.523461818695068,
      "rewards/margins": 20.407812118530273,
      "rewards/rejected": -25.935937881469727,
      "step": 5140
    },
    {
      "epoch": 3.3141293852590925,
      "grad_norm": 0.013833502764908275,
      "learning_rate": 1.7112041210560206e-07,
      "logits/chosen": -1.161230444908142,
      "logits/rejected": -1.194726586341858,
      "logps/chosen": -460.1000061035156,
      "logps/rejected": -585.75,
      "loss": 0.013,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -6.311327934265137,
      "rewards/margins": 17.731250762939453,
      "rewards/rejected": -24.037500381469727,
      "step": 5150
    },
    {
      "epoch": 3.320566462825877,
      "grad_norm": 0.02845853102131737,
      "learning_rate": 1.695106245975531e-07,
      "logits/chosen": -1.135888695716858,
      "logits/rejected": -1.14990234375,
      "logps/chosen": -443.6000061035156,
      "logps/rejected": -636.25,
      "loss": 0.0043,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -5.395410060882568,
      "rewards/margins": 19.323436737060547,
      "rewards/rejected": -24.731250762939453,
      "step": 5160
    },
    {
      "epoch": 3.3270035403926617,
      "grad_norm": 0.0038233653605419843,
      "learning_rate": 1.6790083708950418e-07,
      "logits/chosen": -1.1759765148162842,
      "logits/rejected": -1.20068359375,
      "logps/chosen": -446.75,
      "logps/rejected": -607.7000122070312,
      "loss": 0.013,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -6.467577934265137,
      "rewards/margins": 19.192188262939453,
      "rewards/rejected": -25.649999618530273,
      "step": 5170
    },
    {
      "epoch": 3.3334406179594462,
      "grad_norm": 1.268747881939313,
      "learning_rate": 1.6629104958145523e-07,
      "logits/chosen": -1.114404320716858,
      "logits/rejected": -1.1251952648162842,
      "logps/chosen": -414.70001220703125,
      "logps/rejected": -584.0499877929688,
      "loss": 0.0091,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -6.07421875,
      "rewards/margins": 19.090625762939453,
      "rewards/rejected": -25.174999237060547,
      "step": 5180
    },
    {
      "epoch": 3.3398776955262313,
      "grad_norm": 0.11625811094142115,
      "learning_rate": 1.646812620734063e-07,
      "logits/chosen": -1.169042944908142,
      "logits/rejected": -1.19970703125,
      "logps/chosen": -440.625,
      "logps/rejected": -621.0499877929688,
      "loss": 0.013,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -6.413671970367432,
      "rewards/margins": 18.1328125,
      "rewards/rejected": -24.542186737060547,
      "step": 5190
    },
    {
      "epoch": 3.346314773093016,
      "grad_norm": 0.01457558814733387,
      "learning_rate": 1.6307147456535735e-07,
      "logits/chosen": -1.1357421875,
      "logits/rejected": -1.1613280773162842,
      "logps/chosen": -445.125,
      "logps/rejected": -589.5,
      "loss": 0.0046,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -6.311450004577637,
      "rewards/margins": 18.700000762939453,
      "rewards/rejected": -25.009374618530273,
      "step": 5200
    },
    {
      "epoch": 3.3527518506598004,
      "grad_norm": 0.0318603012676317,
      "learning_rate": 1.6146168705730842e-07,
      "logits/chosen": -1.173583984375,
      "logits/rejected": -1.1763184070587158,
      "logps/chosen": -449.125,
      "logps/rejected": -632.0499877929688,
      "loss": 0.0201,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -6.589745998382568,
      "rewards/margins": 18.8046875,
      "rewards/rejected": -25.396875381469727,
      "step": 5210
    },
    {
      "epoch": 3.359188928226585,
      "grad_norm": 0.037256736610292676,
      "learning_rate": 1.598518995492595e-07,
      "logits/chosen": -1.100341796875,
      "logits/rejected": -1.1125977039337158,
      "logps/chosen": -424.2250061035156,
      "logps/rejected": -582.9000244140625,
      "loss": 0.026,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -6.180517673492432,
      "rewards/margins": 18.310937881469727,
      "rewards/rejected": -24.490625381469727,
      "step": 5220
    },
    {
      "epoch": 3.36562600579337,
      "grad_norm": 0.10211936783870917,
      "learning_rate": 1.5824211204121054e-07,
      "logits/chosen": -1.0896484851837158,
      "logits/rejected": -1.1443359851837158,
      "logps/chosen": -449.375,
      "logps/rejected": -602.0499877929688,
      "loss": 0.0177,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -6.273779392242432,
      "rewards/margins": 17.615625381469727,
      "rewards/rejected": -23.881250381469727,
      "step": 5230
    },
    {
      "epoch": 3.3720630833601546,
      "grad_norm": 0.01454528726990812,
      "learning_rate": 1.566323245331616e-07,
      "logits/chosen": -1.1654052734375,
      "logits/rejected": -1.2077147960662842,
      "logps/chosen": -443.2250061035156,
      "logps/rejected": -612.1500244140625,
      "loss": 0.0173,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -6.409081935882568,
      "rewards/margins": 19.645313262939453,
      "rewards/rejected": -26.056249618530273,
      "step": 5240
    },
    {
      "epoch": 3.378500160926939,
      "grad_norm": 0.9637410547971565,
      "learning_rate": 1.5502253702511266e-07,
      "logits/chosen": -1.1218750476837158,
      "logits/rejected": -1.118749976158142,
      "logps/chosen": -419.0874938964844,
      "logps/rejected": -581.5999755859375,
      "loss": 0.0174,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -6.750048637390137,
      "rewards/margins": 17.667186737060547,
      "rewards/rejected": -24.440624237060547,
      "step": 5250
    },
    {
      "epoch": 3.3849372384937237,
      "grad_norm": 0.03149535057004577,
      "learning_rate": 1.5341274951706373e-07,
      "logits/chosen": -1.111108422279358,
      "logits/rejected": -1.144677758216858,
      "logps/chosen": -461.375,
      "logps/rejected": -606.0999755859375,
      "loss": 0.0216,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -6.674023628234863,
      "rewards/margins": 17.876562118530273,
      "rewards/rejected": -24.568750381469727,
      "step": 5260
    },
    {
      "epoch": 3.3913743160605083,
      "grad_norm": 0.010542321081653615,
      "learning_rate": 1.518029620090148e-07,
      "logits/chosen": -1.197851538658142,
      "logits/rejected": -1.2239258289337158,
      "logps/chosen": -411.45001220703125,
      "logps/rejected": -562.6500244140625,
      "loss": 0.0107,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -5.745898246765137,
      "rewards/margins": 19.893749237060547,
      "rewards/rejected": -25.65625,
      "step": 5270
    },
    {
      "epoch": 3.3978113936272933,
      "grad_norm": 0.009674870957905443,
      "learning_rate": 1.5019317450096585e-07,
      "logits/chosen": -1.1496093273162842,
      "logits/rejected": -1.172216773033142,
      "logps/chosen": -393.3374938964844,
      "logps/rejected": -585.4500122070312,
      "loss": 0.0153,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -5.639721870422363,
      "rewards/margins": 18.8828125,
      "rewards/rejected": -24.532812118530273,
      "step": 5280
    },
    {
      "epoch": 3.404248471194078,
      "grad_norm": 0.03660607691936547,
      "learning_rate": 1.4858338699291693e-07,
      "logits/chosen": -1.12353515625,
      "logits/rejected": -1.0983397960662842,
      "logps/chosen": -398.54998779296875,
      "logps/rejected": -549.8499755859375,
      "loss": 0.0135,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -5.693066596984863,
      "rewards/margins": 18.256250381469727,
      "rewards/rejected": -23.946874618530273,
      "step": 5290
    },
    {
      "epoch": 3.4106855487608625,
      "grad_norm": 0.0018207283503531464,
      "learning_rate": 1.4697359948486797e-07,
      "logits/chosen": -1.1821777820587158,
      "logits/rejected": -1.2270019054412842,
      "logps/chosen": -451.57501220703125,
      "logps/rejected": -590.9500122070312,
      "loss": 0.0045,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -5.689355373382568,
      "rewards/margins": 20.020313262939453,
      "rewards/rejected": -25.712499618530273,
      "step": 5300
    },
    {
      "epoch": 3.417122626327647,
      "grad_norm": 0.013680348716149777,
      "learning_rate": 1.4536381197681905e-07,
      "logits/chosen": -1.1581542491912842,
      "logits/rejected": -1.185937523841858,
      "logps/chosen": -468.75,
      "logps/rejected": -631.25,
      "loss": 0.0002,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -6.91015625,
      "rewards/margins": 17.759374618530273,
      "rewards/rejected": -24.668750762939453,
      "step": 5310
    },
    {
      "epoch": 3.423559703894432,
      "grad_norm": 0.9264470620519801,
      "learning_rate": 1.437540244687701e-07,
      "logits/chosen": -1.13616943359375,
      "logits/rejected": -1.133520483970642,
      "logps/chosen": -420.3999938964844,
      "logps/rejected": -594.5999755859375,
      "loss": 0.013,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -6.451952934265137,
      "rewards/margins": 19.596874237060547,
      "rewards/rejected": -26.049999237060547,
      "step": 5320
    },
    {
      "epoch": 3.4299967814612167,
      "grad_norm": 0.0014284653419422593,
      "learning_rate": 1.4214423696072117e-07,
      "logits/chosen": -1.182226538658142,
      "logits/rejected": -1.187597632408142,
      "logps/chosen": -430.0,
      "logps/rejected": -622.7000122070312,
      "loss": 0.0051,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -5.946875095367432,
      "rewards/margins": 20.178125381469727,
      "rewards/rejected": -26.125,
      "step": 5330
    },
    {
      "epoch": 3.4364338590280012,
      "grad_norm": 0.016144057920363487,
      "learning_rate": 1.4053444945267224e-07,
      "logits/chosen": -1.2000000476837158,
      "logits/rejected": -1.2268555164337158,
      "logps/chosen": -426.4750061035156,
      "logps/rejected": -583.9000244140625,
      "loss": 0.013,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -5.616894721984863,
      "rewards/margins": 18.295312881469727,
      "rewards/rejected": -23.901561737060547,
      "step": 5340
    },
    {
      "epoch": 3.442870936594786,
      "grad_norm": 0.002661825772172003,
      "learning_rate": 1.389246619446233e-07,
      "logits/chosen": -1.1572754383087158,
      "logits/rejected": -1.1572754383087158,
      "logps/chosen": -408.67498779296875,
      "logps/rejected": -589.7999877929688,
      "loss": 0.0177,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -6.3740234375,
      "rewards/margins": 18.325000762939453,
      "rewards/rejected": -24.712499618530273,
      "step": 5350
    },
    {
      "epoch": 3.449308014161571,
      "grad_norm": 0.17931528592870347,
      "learning_rate": 1.3731487443657436e-07,
      "logits/chosen": -1.0957763195037842,
      "logits/rejected": -1.1067383289337158,
      "logps/chosen": -425.45001220703125,
      "logps/rejected": -586.2000122070312,
      "loss": 0.026,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -6.767773628234863,
      "rewards/margins": 17.924999237060547,
      "rewards/rejected": -24.690624237060547,
      "step": 5360
    },
    {
      "epoch": 3.4557450917283554,
      "grad_norm": 0.00752747415982439,
      "learning_rate": 1.357050869285254e-07,
      "logits/chosen": -1.0899658203125,
      "logits/rejected": -1.118408203125,
      "logps/chosen": -446.88751220703125,
      "logps/rejected": -618.4000244140625,
      "loss": 0.013,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -6.139916896820068,
      "rewards/margins": 19.3515625,
      "rewards/rejected": -25.503124237060547,
      "step": 5370
    },
    {
      "epoch": 3.46218216929514,
      "grad_norm": 0.017031868523837528,
      "learning_rate": 1.3409529942047648e-07,
      "logits/chosen": -1.152441382408142,
      "logits/rejected": -1.169335961341858,
      "logps/chosen": -466.92498779296875,
      "logps/rejected": -623.4000244140625,
      "loss": 0.0088,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -6.587695121765137,
      "rewards/margins": 18.764062881469727,
      "rewards/rejected": -25.3515625,
      "step": 5380
    },
    {
      "epoch": 3.4686192468619246,
      "grad_norm": 0.001165475350785279,
      "learning_rate": 1.3248551191242756e-07,
      "logits/chosen": -1.215234398841858,
      "logits/rejected": -1.2233397960662842,
      "logps/chosen": -442.29998779296875,
      "logps/rejected": -611.2000122070312,
      "loss": 0.013,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -6.246972560882568,
      "rewards/margins": 18.779687881469727,
      "rewards/rejected": -25.021875381469727,
      "step": 5390
    },
    {
      "epoch": 3.475056324428709,
      "grad_norm": 0.005232958057430995,
      "learning_rate": 1.308757244043786e-07,
      "logits/chosen": -1.1653320789337158,
      "logits/rejected": -1.1648437976837158,
      "logps/chosen": -410.17498779296875,
      "logps/rejected": -596.4500122070312,
      "loss": 0.0134,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -6.278515815734863,
      "rewards/margins": 17.900781631469727,
      "rewards/rejected": -24.192188262939453,
      "step": 5400
    },
    {
      "epoch": 3.481493401995494,
      "grad_norm": 0.012513547122515394,
      "learning_rate": 1.2926593689632968e-07,
      "logits/chosen": -1.149658203125,
      "logits/rejected": -1.196874976158142,
      "logps/chosen": -399.11248779296875,
      "logps/rejected": -591.7249755859375,
      "loss": 0.0173,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -6.450390815734863,
      "rewards/margins": 19.028125762939453,
      "rewards/rejected": -25.479686737060547,
      "step": 5410
    },
    {
      "epoch": 3.4879304795622788,
      "grad_norm": 0.020245274961717487,
      "learning_rate": 1.2765614938828072e-07,
      "logits/chosen": -1.2703125476837158,
      "logits/rejected": -1.290917992591858,
      "logps/chosen": -479.1000061035156,
      "logps/rejected": -626.1500244140625,
      "loss": 0.0087,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -7.063818454742432,
      "rewards/margins": 20.067188262939453,
      "rewards/rejected": -27.118749618530273,
      "step": 5420
    },
    {
      "epoch": 3.4943675571290633,
      "grad_norm": 0.017114813119376918,
      "learning_rate": 1.260463618802318e-07,
      "logits/chosen": -1.210693359375,
      "logits/rejected": -1.2346680164337158,
      "logps/chosen": -458.2250061035156,
      "logps/rejected": -611.75,
      "loss": 0.0261,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -6.143652439117432,
      "rewards/margins": 19.665624618530273,
      "rewards/rejected": -25.8125,
      "step": 5430
    },
    {
      "epoch": 3.5008046346958483,
      "grad_norm": 0.018696061591920828,
      "learning_rate": 1.2443657437218287e-07,
      "logits/chosen": -1.2316405773162842,
      "logits/rejected": -1.231054663658142,
      "logps/chosen": -460.2749938964844,
      "logps/rejected": -670.75,
      "loss": 0.0044,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -6.941992282867432,
      "rewards/margins": 20.125,
      "rewards/rejected": -27.071874618530273,
      "step": 5440
    },
    {
      "epoch": 3.507241712262633,
      "grad_norm": 0.9238072536774803,
      "learning_rate": 1.2282678686413394e-07,
      "logits/chosen": -1.198486328125,
      "logits/rejected": -1.195947289466858,
      "logps/chosen": -401.70001220703125,
      "logps/rejected": -588.2999877929688,
      "loss": 0.0091,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -6.305371284484863,
      "rewards/margins": 20.212499618530273,
      "rewards/rejected": -26.515625,
      "step": 5450
    },
    {
      "epoch": 3.5136787898294175,
      "grad_norm": 0.059459388376309825,
      "learning_rate": 1.21216999356085e-07,
      "logits/chosen": -1.1687500476837158,
      "logits/rejected": -1.204199194908142,
      "logps/chosen": -428.75,
      "logps/rejected": -604.9500122070312,
      "loss": 0.0087,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -6.602441310882568,
      "rewards/margins": 19.134374618530273,
      "rewards/rejected": -25.737499237060547,
      "step": 5460
    },
    {
      "epoch": 3.520115867396202,
      "grad_norm": 0.27459831092472686,
      "learning_rate": 1.1960721184803606e-07,
      "logits/chosen": -1.142822265625,
      "logits/rejected": -1.189208984375,
      "logps/chosen": -410.29998779296875,
      "logps/rejected": -570.4000244140625,
      "loss": 0.013,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -7.4951171875,
      "rewards/margins": 18.196874618530273,
      "rewards/rejected": -25.700000762939453,
      "step": 5470
    },
    {
      "epoch": 3.5265529449629867,
      "grad_norm": 0.009273735594422142,
      "learning_rate": 1.1799742433998712e-07,
      "logits/chosen": -1.131933569908142,
      "logits/rejected": -1.1398437023162842,
      "logps/chosen": -420.9375,
      "logps/rejected": -592.5999755859375,
      "loss": 0.0216,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -6.311841011047363,
      "rewards/margins": 18.828125,
      "rewards/rejected": -25.142187118530273,
      "step": 5480
    },
    {
      "epoch": 3.5329900225297717,
      "grad_norm": 0.0528647089110024,
      "learning_rate": 1.1638763683193818e-07,
      "logits/chosen": -1.117700219154358,
      "logits/rejected": -1.1470947265625,
      "logps/chosen": -438.29998779296875,
      "logps/rejected": -586.5999755859375,
      "loss": 0.0153,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -6.416796684265137,
      "rewards/margins": 19.087499618530273,
      "rewards/rejected": -25.515625,
      "step": 5490
    },
    {
      "epoch": 3.5394271000965563,
      "grad_norm": 0.055401282213416295,
      "learning_rate": 1.1477784932388924e-07,
      "logits/chosen": -1.223168969154358,
      "logits/rejected": -1.2292969226837158,
      "logps/chosen": -404.4125061035156,
      "logps/rejected": -573.75,
      "loss": 0.0132,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -7.193163871765137,
      "rewards/margins": 18.621875762939453,
      "rewards/rejected": -25.798437118530273,
      "step": 5500
    },
    {
      "epoch": 3.545864177663341,
      "grad_norm": 0.002659691570256985,
      "learning_rate": 1.131680618158403e-07,
      "logits/chosen": -1.1658203601837158,
      "logits/rejected": -1.186425805091858,
      "logps/chosen": -443.2250061035156,
      "logps/rejected": -587.3499755859375,
      "loss": 0.0173,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -5.725976467132568,
      "rewards/margins": 18.939062118530273,
      "rewards/rejected": -24.674999237060547,
      "step": 5510
    },
    {
      "epoch": 3.5523012552301254,
      "grad_norm": 0.014521885488236713,
      "learning_rate": 1.1155827430779136e-07,
      "logits/chosen": -1.2326171398162842,
      "logits/rejected": -1.237036108970642,
      "logps/chosen": -446.38751220703125,
      "logps/rejected": -592.9500122070312,
      "loss": 0.013,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -5.961035251617432,
      "rewards/margins": 18.581249237060547,
      "rewards/rejected": -24.543750762939453,
      "step": 5520
    },
    {
      "epoch": 3.55873833279691,
      "grad_norm": 0.025977628087194685,
      "learning_rate": 1.0994848679974244e-07,
      "logits/chosen": -1.169042944908142,
      "logits/rejected": -1.169335961341858,
      "logps/chosen": -384.70001220703125,
      "logps/rejected": -554.3499755859375,
      "loss": 0.0131,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -5.560546875,
      "rewards/margins": 18.770313262939453,
      "rewards/rejected": -24.334375381469727,
      "step": 5530
    },
    {
      "epoch": 3.565175410363695,
      "grad_norm": 0.0046389029281187245,
      "learning_rate": 1.083386992916935e-07,
      "logits/chosen": -1.123803734779358,
      "logits/rejected": -1.149511694908142,
      "logps/chosen": -401.0249938964844,
      "logps/rejected": -595.0,
      "loss": 0.0163,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -6.404040336608887,
      "rewards/margins": 20.435937881469727,
      "rewards/rejected": -26.846874237060547,
      "step": 5540
    },
    {
      "epoch": 3.5716124879304796,
      "grad_norm": 0.017966434361620324,
      "learning_rate": 1.0672891178364456e-07,
      "logits/chosen": -1.214257836341858,
      "logits/rejected": -1.2414062023162842,
      "logps/chosen": -414.375,
      "logps/rejected": -604.8499755859375,
      "loss": 0.0173,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -6.5302734375,
      "rewards/margins": 18.482812881469727,
      "rewards/rejected": -25.018749237060547,
      "step": 5550
    },
    {
      "epoch": 3.578049565497264,
      "grad_norm": 0.014762859371399277,
      "learning_rate": 1.0511912427559562e-07,
      "logits/chosen": -1.200341820716858,
      "logits/rejected": -1.205175757408142,
      "logps/chosen": -407.8500061035156,
      "logps/rejected": -590.7000122070312,
      "loss": 0.0141,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -5.133984565734863,
      "rewards/margins": 18.453125,
      "rewards/rejected": -23.579687118530273,
      "step": 5560
    },
    {
      "epoch": 3.584486643064049,
      "grad_norm": 0.344430572394362,
      "learning_rate": 1.0350933676754668e-07,
      "logits/chosen": -1.171777367591858,
      "logits/rejected": -1.159082055091858,
      "logps/chosen": -397.9750061035156,
      "logps/rejected": -590.5999755859375,
      "loss": 0.0087,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -5.848437309265137,
      "rewards/margins": 18.840625762939453,
      "rewards/rejected": -24.703125,
      "step": 5570
    },
    {
      "epoch": 3.5909237206308338,
      "grad_norm": 0.009410808435903622,
      "learning_rate": 1.0189954925949774e-07,
      "logits/chosen": -1.302636742591858,
      "logits/rejected": -1.300878882408142,
      "logps/chosen": -390.6499938964844,
      "logps/rejected": -587.9500122070312,
      "loss": 0.0043,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -6.140234470367432,
      "rewards/margins": 18.835155487060547,
      "rewards/rejected": -24.9765625,
      "step": 5580
    },
    {
      "epoch": 3.5973607981976183,
      "grad_norm": 0.47452165527225904,
      "learning_rate": 1.0028976175144881e-07,
      "logits/chosen": -1.177978515625,
      "logits/rejected": -1.1656982898712158,
      "logps/chosen": -415.0,
      "logps/rejected": -600.7999877929688,
      "loss": 0.0133,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -6.214453220367432,
      "rewards/margins": 19.087499618530273,
      "rewards/rejected": -25.3125,
      "step": 5590
    },
    {
      "epoch": 3.603797875764403,
      "grad_norm": 0.05161000381239699,
      "learning_rate": 9.867997424339987e-08,
      "logits/chosen": -1.251953125,
      "logits/rejected": -1.2570312023162842,
      "logps/chosen": -446.625,
      "logps/rejected": -611.25,
      "loss": 0.013,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -5.92431640625,
      "rewards/margins": 18.439062118530273,
      "rewards/rejected": -24.360937118530273,
      "step": 5600
    },
    {
      "epoch": 3.6102349533311875,
      "grad_norm": 0.009289587395709877,
      "learning_rate": 9.707018673535093e-08,
      "logits/chosen": -1.2129395008087158,
      "logits/rejected": -1.230371117591858,
      "logps/chosen": -432.8500061035156,
      "logps/rejected": -588.7999877929688,
      "loss": 0.013,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -5.639208793640137,
      "rewards/margins": 18.514062881469727,
      "rewards/rejected": -24.134374618530273,
      "step": 5610
    },
    {
      "epoch": 3.6166720308979725,
      "grad_norm": 0.011055816630815653,
      "learning_rate": 9.5460399227302e-08,
      "logits/chosen": -1.1920287609100342,
      "logits/rejected": -1.178613305091858,
      "logps/chosen": -444.6499938964844,
      "logps/rejected": -599.0,
      "loss": 0.0216,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -5.476122856140137,
      "rewards/margins": 18.553125381469727,
      "rewards/rejected": -24.029687881469727,
      "step": 5620
    },
    {
      "epoch": 3.623109108464757,
      "grad_norm": 0.561158217771827,
      "learning_rate": 9.385061171925305e-08,
      "logits/chosen": -1.147802710533142,
      "logits/rejected": -1.180761694908142,
      "logps/chosen": -487.7250061035156,
      "logps/rejected": -629.0999755859375,
      "loss": 0.0132,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -6.1337890625,
      "rewards/margins": 18.4140625,
      "rewards/rejected": -24.549999237060547,
      "step": 5630
    },
    {
      "epoch": 3.6295461860315417,
      "grad_norm": 0.09391178158930319,
      "learning_rate": 9.224082421120411e-08,
      "logits/chosen": -1.2050292491912842,
      "logits/rejected": -1.23583984375,
      "logps/chosen": -410.8999938964844,
      "logps/rejected": -601.0999755859375,
      "loss": 0.0303,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -5.978369235992432,
      "rewards/margins": 19.447656631469727,
      "rewards/rejected": -25.4296875,
      "step": 5640
    },
    {
      "epoch": 3.6359832635983262,
      "grad_norm": 0.005353638341178634,
      "learning_rate": 9.063103670315519e-08,
      "logits/chosen": -1.254248023033142,
      "logits/rejected": -1.2638671398162842,
      "logps/chosen": -437.1875,
      "logps/rejected": -608.2000122070312,
      "loss": 0.0087,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -5.216015815734863,
      "rewards/margins": 18.854686737060547,
      "rewards/rejected": -24.067188262939453,
      "step": 5650
    },
    {
      "epoch": 3.642420341165111,
      "grad_norm": 0.0796077309007433,
      "learning_rate": 8.902124919510625e-08,
      "logits/chosen": -1.177978515625,
      "logits/rejected": -1.2056152820587158,
      "logps/chosen": -475.1499938964844,
      "logps/rejected": -607.5,
      "loss": 0.0087,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -6.043749809265137,
      "rewards/margins": 19.282812118530273,
      "rewards/rejected": -25.309375762939453,
      "step": 5660
    },
    {
      "epoch": 3.648857418731896,
      "grad_norm": 0.00354886488066356,
      "learning_rate": 8.741146168705731e-08,
      "logits/chosen": -1.2111327648162842,
      "logits/rejected": -1.214453101158142,
      "logps/chosen": -421.1499938964844,
      "logps/rejected": -611.0999755859375,
      "loss": 0.0043,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -5.758008003234863,
      "rewards/margins": 19.325000762939453,
      "rewards/rejected": -25.079687118530273,
      "step": 5670
    },
    {
      "epoch": 3.6552944962986804,
      "grad_norm": 0.011692323247531729,
      "learning_rate": 8.580167417900837e-08,
      "logits/chosen": -1.193945288658142,
      "logits/rejected": -1.203710913658142,
      "logps/chosen": -440.8999938964844,
      "logps/rejected": -606.0,
      "loss": 0.013,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -6.059960842132568,
      "rewards/margins": 18.110937118530273,
      "rewards/rejected": -24.171875,
      "step": 5680
    },
    {
      "epoch": 3.661731573865465,
      "grad_norm": 0.0032497001850268115,
      "learning_rate": 8.419188667095943e-08,
      "logits/chosen": -1.242285132408142,
      "logits/rejected": -1.253173828125,
      "logps/chosen": -448.1000061035156,
      "logps/rejected": -609.4000244140625,
      "loss": 0.0087,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -6.205370903015137,
      "rewards/margins": 19.162500381469727,
      "rewards/rejected": -25.389062881469727,
      "step": 5690
    },
    {
      "epoch": 3.66816865143225,
      "grad_norm": 0.021910077756192244,
      "learning_rate": 8.258209916291049e-08,
      "logits/chosen": -1.19921875,
      "logits/rejected": -1.2140624523162842,
      "logps/chosen": -414.42498779296875,
      "logps/rejected": -615.6500244140625,
      "loss": 0.0044,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -6.513964653015137,
      "rewards/margins": 19.879688262939453,
      "rewards/rejected": -26.390625,
      "step": 5700
    },
    {
      "epoch": 3.6746057289990346,
      "grad_norm": 0.028081385474183675,
      "learning_rate": 8.097231165486156e-08,
      "logits/chosen": -1.206298828125,
      "logits/rejected": -1.247802734375,
      "logps/chosen": -469.45001220703125,
      "logps/rejected": -612.3499755859375,
      "loss": 0.0044,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -5.853906154632568,
      "rewards/margins": 18.9140625,
      "rewards/rejected": -24.764062881469727,
      "step": 5710
    },
    {
      "epoch": 3.681042806565819,
      "grad_norm": 0.009291367600511937,
      "learning_rate": 7.936252414681262e-08,
      "logits/chosen": -1.1588866710662842,
      "logits/rejected": -1.1977050304412842,
      "logps/chosen": -427.6000061035156,
      "logps/rejected": -570.5499877929688,
      "loss": 0.0174,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -5.973535060882568,
      "rewards/margins": 18.046875,
      "rewards/rejected": -24.015625,
      "step": 5720
    },
    {
      "epoch": 3.6874798841326037,
      "grad_norm": 1.0300650774799196,
      "learning_rate": 7.775273663876368e-08,
      "logits/chosen": -1.214990258216858,
      "logits/rejected": -1.264257788658142,
      "logps/chosen": -436.8999938964844,
      "logps/rejected": -600.75,
      "loss": 0.0086,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -6.286913871765137,
      "rewards/margins": 19.489063262939453,
      "rewards/rejected": -25.778125762939453,
      "step": 5730
    },
    {
      "epoch": 3.6939169616993883,
      "grad_norm": 0.1391319359241606,
      "learning_rate": 7.614294913071474e-08,
      "logits/chosen": -1.2268555164337158,
      "logits/rejected": -1.248144507408142,
      "logps/chosen": -445.8500061035156,
      "logps/rejected": -629.2999877929688,
      "loss": 0.0259,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -6.4736328125,
      "rewards/margins": 19.696874618530273,
      "rewards/rejected": -26.178125381469727,
      "step": 5740
    },
    {
      "epoch": 3.7003540392661733,
      "grad_norm": 0.36740609289586745,
      "learning_rate": 7.45331616226658e-08,
      "logits/chosen": -1.179589867591858,
      "logits/rejected": -1.18310546875,
      "logps/chosen": -397.2250061035156,
      "logps/rejected": -576.75,
      "loss": 0.0217,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -6.327441215515137,
      "rewards/margins": 18.165624618530273,
      "rewards/rejected": -24.484375,
      "step": 5750
    },
    {
      "epoch": 3.706791116832958,
      "grad_norm": 1.4062043903030401,
      "learning_rate": 7.292337411461686e-08,
      "logits/chosen": -1.1822540760040283,
      "logits/rejected": -1.2383301258087158,
      "logps/chosen": -418.6499938964844,
      "logps/rejected": -598.6500244140625,
      "loss": 0.0046,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -5.955420017242432,
      "rewards/margins": 19.715625762939453,
      "rewards/rejected": -25.673437118530273,
      "step": 5760
    },
    {
      "epoch": 3.7132281943997425,
      "grad_norm": 0.010906458911690259,
      "learning_rate": 7.131358660656794e-08,
      "logits/chosen": -1.23486328125,
      "logits/rejected": -1.24169921875,
      "logps/chosen": -412.95001220703125,
      "logps/rejected": -583.5499877929688,
      "loss": 0.013,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -7.052734375,
      "rewards/margins": 18.771875381469727,
      "rewards/rejected": -25.815624237060547,
      "step": 5770
    },
    {
      "epoch": 3.719665271966527,
      "grad_norm": 0.4194311473667511,
      "learning_rate": 6.9703799098519e-08,
      "logits/chosen": -1.142797827720642,
      "logits/rejected": -1.152197241783142,
      "logps/chosen": -451.17498779296875,
      "logps/rejected": -613.25,
      "loss": 0.013,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -6.297070503234863,
      "rewards/margins": 18.793750762939453,
      "rewards/rejected": -25.09375,
      "step": 5780
    },
    {
      "epoch": 3.7261023495333117,
      "grad_norm": 0.14153168186128046,
      "learning_rate": 6.809401159047006e-08,
      "logits/chosen": -1.154870629310608,
      "logits/rejected": -1.183935523033142,
      "logps/chosen": -441.25,
      "logps/rejected": -601.25,
      "loss": 0.0087,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -6.050488471984863,
      "rewards/margins": 19.293750762939453,
      "rewards/rejected": -25.354686737060547,
      "step": 5790
    },
    {
      "epoch": 3.7325394271000967,
      "grad_norm": 0.31575897374121853,
      "learning_rate": 6.648422408242112e-08,
      "logits/chosen": -1.235742211341858,
      "logits/rejected": -1.2470214366912842,
      "logps/chosen": -438.8500061035156,
      "logps/rejected": -564.3499755859375,
      "loss": 0.0047,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -5.980322360992432,
      "rewards/margins": 18.651561737060547,
      "rewards/rejected": -24.625,
      "step": 5800
    },
    {
      "epoch": 3.7389765046668813,
      "grad_norm": 0.06287240602018677,
      "learning_rate": 6.487443657437218e-08,
      "logits/chosen": -1.195947289466858,
      "logits/rejected": -1.205468773841858,
      "logps/chosen": -417.57501220703125,
      "logps/rejected": -602.0999755859375,
      "loss": 0.0196,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -5.490429878234863,
      "rewards/margins": 18.731250762939453,
      "rewards/rejected": -24.228124618530273,
      "step": 5810
    },
    {
      "epoch": 3.745413582233666,
      "grad_norm": 0.0005135924625372928,
      "learning_rate": 6.326464906632324e-08,
      "logits/chosen": -1.2028687000274658,
      "logits/rejected": -1.223486304283142,
      "logps/chosen": -466.2749938964844,
      "logps/rejected": -638.0999755859375,
      "loss": 0.0087,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -5.878710746765137,
      "rewards/margins": 20.1484375,
      "rewards/rejected": -26.024999618530273,
      "step": 5820
    },
    {
      "epoch": 3.751850659800451,
      "grad_norm": 0.03898064290236112,
      "learning_rate": 6.165486155827431e-08,
      "logits/chosen": -1.201318383216858,
      "logits/rejected": -1.215429663658142,
      "logps/chosen": -429.57501220703125,
      "logps/rejected": -586.4500122070312,
      "loss": 0.018,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -5.779492378234863,
      "rewards/margins": 19.001562118530273,
      "rewards/rejected": -24.768749237060547,
      "step": 5830
    },
    {
      "epoch": 3.7582877373672354,
      "grad_norm": 0.013577934214215267,
      "learning_rate": 6.004507405022537e-08,
      "logits/chosen": -1.228417992591858,
      "logits/rejected": -1.2648437023162842,
      "logps/chosen": -484.8999938964844,
      "logps/rejected": -625.7999877929688,
      "loss": 0.005,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -5.214037895202637,
      "rewards/margins": 19.340625762939453,
      "rewards/rejected": -24.549999237060547,
      "step": 5840
    },
    {
      "epoch": 3.76472481493402,
      "grad_norm": 0.016377009732415706,
      "learning_rate": 5.843528654217643e-08,
      "logits/chosen": -1.1907227039337158,
      "logits/rejected": -1.204492211341858,
      "logps/chosen": -413.07501220703125,
      "logps/rejected": -563.0999755859375,
      "loss": 0.009,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -5.839453220367432,
      "rewards/margins": 17.390625,
      "rewards/rejected": -23.234375,
      "step": 5850
    },
    {
      "epoch": 3.7711618925008046,
      "grad_norm": 0.015952358445255128,
      "learning_rate": 5.682549903412749e-08,
      "logits/chosen": -1.08544921875,
      "logits/rejected": -1.106835961341858,
      "logps/chosen": -406.57501220703125,
      "logps/rejected": -596.4000244140625,
      "loss": 0.0217,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -5.909936428070068,
      "rewards/margins": 19.125,
      "rewards/rejected": -25.037500381469727,
      "step": 5860
    },
    {
      "epoch": 3.777598970067589,
      "grad_norm": 0.006201655476189211,
      "learning_rate": 5.521571152607855e-08,
      "logits/chosen": -1.1427733898162842,
      "logits/rejected": -1.167578101158142,
      "logps/chosen": -459.1000061035156,
      "logps/rejected": -633.2999877929688,
      "loss": 0.0217,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -5.592871189117432,
      "rewards/margins": 20.014062881469727,
      "rewards/rejected": -25.606250762939453,
      "step": 5870
    },
    {
      "epoch": 3.784036047634374,
      "grad_norm": 0.009195903984204546,
      "learning_rate": 5.360592401802962e-08,
      "logits/chosen": -1.1323730945587158,
      "logits/rejected": -1.120214819908142,
      "logps/chosen": -447.07501220703125,
      "logps/rejected": -620.5999755859375,
      "loss": 0.0043,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -6.411718845367432,
      "rewards/margins": 19.521875381469727,
      "rewards/rejected": -25.934375762939453,
      "step": 5880
    },
    {
      "epoch": 3.7904731252011588,
      "grad_norm": 6.727471051585618,
      "learning_rate": 5.199613650998068e-08,
      "logits/chosen": -1.192773461341858,
      "logits/rejected": -1.21337890625,
      "logps/chosen": -386.1499938964844,
      "logps/rejected": -546.25,
      "loss": 0.0133,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -5.852783203125,
      "rewards/margins": 18.420312881469727,
      "rewards/rejected": -24.274999618530273,
      "step": 5890
    },
    {
      "epoch": 3.7969102027679433,
      "grad_norm": 0.02099655547602189,
      "learning_rate": 5.038634900193174e-08,
      "logits/chosen": -1.214257836341858,
      "logits/rejected": -1.22705078125,
      "logps/chosen": -445.125,
      "logps/rejected": -612.5,
      "loss": 0.013,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -5.768261909484863,
      "rewards/margins": 17.332813262939453,
      "rewards/rejected": -23.095312118530273,
      "step": 5900
    },
    {
      "epoch": 3.803347280334728,
      "grad_norm": 0.004007814604864058,
      "learning_rate": 4.8776561493882806e-08,
      "logits/chosen": -1.2330811023712158,
      "logits/rejected": -1.25048828125,
      "logps/chosen": -476.54998779296875,
      "logps/rejected": -619.9000244140625,
      "loss": 0.013,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -6.557909965515137,
      "rewards/margins": 18.684375762939453,
      "rewards/rejected": -25.256250381469727,
      "step": 5910
    },
    {
      "epoch": 3.8097843579015125,
      "grad_norm": 0.16369300391096503,
      "learning_rate": 4.7166773985833866e-08,
      "logits/chosen": -1.169335961341858,
      "logits/rejected": -1.1863281726837158,
      "logps/chosen": -452.32501220703125,
      "logps/rejected": -593.3250122070312,
      "loss": 0.0228,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -6.1298828125,
      "rewards/margins": 18.404687881469727,
      "rewards/rejected": -24.515625,
      "step": 5920
    },
    {
      "epoch": 3.8162214354682975,
      "grad_norm": 0.00922647797366643,
      "learning_rate": 4.5556986477784926e-08,
      "logits/chosen": -1.1843993663787842,
      "logits/rejected": -1.206201195716858,
      "logps/chosen": -441.1499938964844,
      "logps/rejected": -621.0999755859375,
      "loss": 0.0173,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -5.243261814117432,
      "rewards/margins": 19.825000762939453,
      "rewards/rejected": -25.057811737060547,
      "step": 5930
    },
    {
      "epoch": 3.822658513035082,
      "grad_norm": 0.013267277094511646,
      "learning_rate": 4.394719896973599e-08,
      "logits/chosen": -1.101464867591858,
      "logits/rejected": -1.147607445716858,
      "logps/chosen": -440.5249938964844,
      "logps/rejected": -587.0999755859375,
      "loss": 0.0303,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -5.225390434265137,
      "rewards/margins": 18.484375,
      "rewards/rejected": -23.71875,
      "step": 5940
    },
    {
      "epoch": 3.8290955906018667,
      "grad_norm": 0.09662645701310577,
      "learning_rate": 4.233741146168705e-08,
      "logits/chosen": -1.1639404296875,
      "logits/rejected": -1.1750977039337158,
      "logps/chosen": -431.9750061035156,
      "logps/rejected": -587.3499755859375,
      "loss": 0.0259,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -5.033447265625,
      "rewards/margins": 19.165624618530273,
      "rewards/rejected": -24.212499618530273,
      "step": 5950
    },
    {
      "epoch": 3.8355326681686517,
      "grad_norm": 0.008851252217354692,
      "learning_rate": 4.072762395363811e-08,
      "logits/chosen": -1.260156273841858,
      "logits/rejected": -1.277929663658142,
      "logps/chosen": -436.2250061035156,
      "logps/rejected": -586.7999877929688,
      "loss": 0.0086,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -6.388281345367432,
      "rewards/margins": 18.201562881469727,
      "rewards/rejected": -24.606250762939453,
      "step": 5960
    },
    {
      "epoch": 3.8419697457354363,
      "grad_norm": 0.0248495029820189,
      "learning_rate": 3.911783644558918e-08,
      "logits/chosen": -1.160009741783142,
      "logits/rejected": -1.194433569908142,
      "logps/chosen": -444.6000061035156,
      "logps/rejected": -621.7000122070312,
      "loss": 0.0173,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -5.681836128234863,
      "rewards/margins": 19.104686737060547,
      "rewards/rejected": -24.806249618530273,
      "step": 5970
    },
    {
      "epoch": 3.848406823302221,
      "grad_norm": 0.03249204827309899,
      "learning_rate": 3.750804893754024e-08,
      "logits/chosen": -1.224755883216858,
      "logits/rejected": -1.24853515625,
      "logps/chosen": -451.82501220703125,
      "logps/rejected": -622.8499755859375,
      "loss": 0.013,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -5.946679592132568,
      "rewards/margins": 19.3359375,
      "rewards/rejected": -25.296875,
      "step": 5980
    },
    {
      "epoch": 3.8548439008690054,
      "grad_norm": 0.0045911375877971785,
      "learning_rate": 3.58982614294913e-08,
      "logits/chosen": -1.176025390625,
      "logits/rejected": -1.178564429283142,
      "logps/chosen": -412.375,
      "logps/rejected": -592.3499755859375,
      "loss": 0.0087,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -5.462890625,
      "rewards/margins": 18.596874237060547,
      "rewards/rejected": -24.068750381469727,
      "step": 5990
    },
    {
      "epoch": 3.86128097843579,
      "grad_norm": 0.022481239633330877,
      "learning_rate": 3.428847392144237e-08,
      "logits/chosen": -1.226953148841858,
      "logits/rejected": -1.242822289466858,
      "logps/chosen": -441.20001220703125,
      "logps/rejected": -612.3499755859375,
      "loss": 0.0043,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -6.344152927398682,
      "rewards/margins": 18.314062118530273,
      "rewards/rejected": -24.653125762939453,
      "step": 6000
    },
    {
      "epoch": 3.867718056002575,
      "grad_norm": 0.007146640596074133,
      "learning_rate": 3.267868641339343e-08,
      "logits/chosen": -1.117163062095642,
      "logits/rejected": -1.1201903820037842,
      "logps/chosen": -428.3125,
      "logps/rejected": -588.6500244140625,
      "loss": 0.0173,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -5.737353324890137,
      "rewards/margins": 18.498437881469727,
      "rewards/rejected": -24.25,
      "step": 6010
    },
    {
      "epoch": 3.8741551335693596,
      "grad_norm": 0.007787262767055397,
      "learning_rate": 3.1068898905344494e-08,
      "logits/chosen": -1.100195288658142,
      "logits/rejected": -1.133642554283142,
      "logps/chosen": -393.17498779296875,
      "logps/rejected": -556.2000122070312,
      "loss": 0.0217,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -6.217382907867432,
      "rewards/margins": 17.943750381469727,
      "rewards/rejected": -24.15625,
      "step": 6020
    },
    {
      "epoch": 3.880592211136144,
      "grad_norm": 0.02099978633163225,
      "learning_rate": 2.9459111397295555e-08,
      "logits/chosen": -1.2048828601837158,
      "logits/rejected": -1.2155272960662842,
      "logps/chosen": -444.3500061035156,
      "logps/rejected": -610.6500244140625,
      "loss": 0.0218,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -6.603125095367432,
      "rewards/margins": 19.350000381469727,
      "rewards/rejected": -25.959375381469727,
      "step": 6030
    },
    {
      "epoch": 3.8870292887029287,
      "grad_norm": 0.09482176849813918,
      "learning_rate": 2.7849323889246618e-08,
      "logits/chosen": -1.2528808116912842,
      "logits/rejected": -1.2414062023162842,
      "logps/chosen": -433.2250061035156,
      "logps/rejected": -581.5999755859375,
      "loss": 0.013,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -5.998827934265137,
      "rewards/margins": 18.329687118530273,
      "rewards/rejected": -24.332813262939453,
      "step": 6040
    },
    {
      "epoch": 3.8934663662697133,
      "grad_norm": 0.8030153239793042,
      "learning_rate": 2.623953638119768e-08,
      "logits/chosen": -1.220117211341858,
      "logits/rejected": -1.2244141101837158,
      "logps/chosen": -459.1499938964844,
      "logps/rejected": -607.8499755859375,
      "loss": 0.0087,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -5.8914794921875,
      "rewards/margins": 18.568750381469727,
      "rewards/rejected": -24.464061737060547,
      "step": 6050
    },
    {
      "epoch": 3.8999034438364983,
      "grad_norm": 1.5912180308467658,
      "learning_rate": 2.4629748873148742e-08,
      "logits/chosen": -1.1505858898162842,
      "logits/rejected": -1.1716797351837158,
      "logps/chosen": -406.6187438964844,
      "logps/rejected": -567.1500244140625,
      "loss": 0.0303,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -5.146173000335693,
      "rewards/margins": 19.529687881469727,
      "rewards/rejected": -24.676563262939453,
      "step": 6060
    },
    {
      "epoch": 3.906340521403283,
      "grad_norm": 0.0016637072817598324,
      "learning_rate": 2.3019961365099805e-08,
      "logits/chosen": -1.2268555164337158,
      "logits/rejected": -1.233789086341858,
      "logps/chosen": -451.6000061035156,
      "logps/rejected": -651.7999877929688,
      "loss": 0.0089,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -5.985547065734863,
      "rewards/margins": 19.717187881469727,
      "rewards/rejected": -25.6953125,
      "step": 6070
    },
    {
      "epoch": 3.9127775989700675,
      "grad_norm": 0.007263086315974875,
      "learning_rate": 2.141017385705087e-08,
      "logits/chosen": -1.098291039466858,
      "logits/rejected": -1.1204102039337158,
      "logps/chosen": -413.7250061035156,
      "logps/rejected": -584.9500122070312,
      "loss": 0.0232,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -6.582226753234863,
      "rewards/margins": 18.423437118530273,
      "rewards/rejected": -25.003124237060547,
      "step": 6080
    },
    {
      "epoch": 3.9192146765368525,
      "grad_norm": 0.005372894252220103,
      "learning_rate": 1.980038634900193e-08,
      "logits/chosen": -1.1232421398162842,
      "logits/rejected": -1.110595703125,
      "logps/chosen": -416.1000061035156,
      "logps/rejected": -576.9500122070312,
      "loss": 0.0216,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -5.376269340515137,
      "rewards/margins": 19.579687118530273,
      "rewards/rejected": -24.953125,
      "step": 6090
    },
    {
      "epoch": 3.925651754103637,
      "grad_norm": 1.223081214457884,
      "learning_rate": 1.8190598840952993e-08,
      "logits/chosen": -1.138671875,
      "logits/rejected": -1.160668969154358,
      "logps/chosen": -426.2749938964844,
      "logps/rejected": -576.2999877929688,
      "loss": 0.0174,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -5.99658203125,
      "rewards/margins": 19.203125,
      "rewards/rejected": -25.189062118530273,
      "step": 6100
    },
    {
      "epoch": 3.9320888316704217,
      "grad_norm": 0.24373284704903744,
      "learning_rate": 1.6580811332904056e-08,
      "logits/chosen": -1.2075684070587158,
      "logits/rejected": -1.2453124523162842,
      "logps/chosen": -456.2749938964844,
      "logps/rejected": -626.9500122070312,
      "loss": 0.0087,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -6.045752048492432,
      "rewards/margins": 19.284374237060547,
      "rewards/rejected": -25.317188262939453,
      "step": 6110
    },
    {
      "epoch": 3.9385259092372062,
      "grad_norm": 0.030940312380477432,
      "learning_rate": 1.497102382485512e-08,
      "logits/chosen": -1.2628905773162842,
      "logits/rejected": -1.289648413658142,
      "logps/chosen": -446.42498779296875,
      "logps/rejected": -595.75,
      "loss": 0.013,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -5.348242282867432,
      "rewards/margins": 19.128124237060547,
      "rewards/rejected": -24.459375381469727,
      "step": 6120
    },
    {
      "epoch": 3.944962986803991,
      "grad_norm": 0.030722284899537575,
      "learning_rate": 1.3361236316806181e-08,
      "logits/chosen": -1.1930663585662842,
      "logits/rejected": -1.2233397960662842,
      "logps/chosen": -457.6000061035156,
      "logps/rejected": -600.6500244140625,
      "loss": 0.0011,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -6.046191215515137,
      "rewards/margins": 18.890625,
      "rewards/rejected": -24.9375,
      "step": 6130
    },
    {
      "epoch": 3.951400064370776,
      "grad_norm": 0.01006862708136734,
      "learning_rate": 1.1751448808757243e-08,
      "logits/chosen": -1.186792016029358,
      "logits/rejected": -1.175756812095642,
      "logps/chosen": -449.125,
      "logps/rejected": -613.1500244140625,
      "loss": 0.0173,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -6.257226467132568,
      "rewards/margins": 18.479686737060547,
      "rewards/rejected": -24.739063262939453,
      "step": 6140
    },
    {
      "epoch": 3.9578371419375604,
      "grad_norm": 0.0269485574039851,
      "learning_rate": 1.0141661300708307e-08,
      "logits/chosen": -1.1842772960662842,
      "logits/rejected": -1.1896483898162842,
      "logps/chosen": -394.6000061035156,
      "logps/rejected": -543.3499755859375,
      "loss": 0.0087,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -6.286718845367432,
      "rewards/margins": 17.590625762939453,
      "rewards/rejected": -23.871875762939453,
      "step": 6150
    },
    {
      "epoch": 3.964274219504345,
      "grad_norm": 0.015970088391158554,
      "learning_rate": 8.531873792659369e-09,
      "logits/chosen": -1.1664307117462158,
      "logits/rejected": -1.1992676258087158,
      "logps/chosen": -432.0,
      "logps/rejected": -593.6500244140625,
      "loss": 0.0087,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -5.511328220367432,
      "rewards/margins": 19.0390625,
      "rewards/rejected": -24.556249618530273,
      "step": 6160
    },
    {
      "epoch": 3.9707112970711296,
      "grad_norm": 0.06187758075741397,
      "learning_rate": 6.922086284610431e-09,
      "logits/chosen": -1.233984351158142,
      "logits/rejected": -1.266992211341858,
      "logps/chosen": -442.20001220703125,
      "logps/rejected": -558.5499877929688,
      "loss": 0.0173,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -5.187597751617432,
      "rewards/margins": 18.693750381469727,
      "rewards/rejected": -23.881250381469727,
      "step": 6170
    },
    {
      "epoch": 3.977148374637914,
      "grad_norm": 0.25424001056387396,
      "learning_rate": 5.312298776561494e-09,
      "logits/chosen": -1.1482422351837158,
      "logits/rejected": -1.1685059070587158,
      "logps/chosen": -441.0249938964844,
      "logps/rejected": -610.4000244140625,
      "loss": 0.013,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -6.648241996765137,
      "rewards/margins": 18.834375381469727,
      "rewards/rejected": -25.481250762939453,
      "step": 6180
    },
    {
      "epoch": 3.983585452204699,
      "grad_norm": 0.006728702542557314,
      "learning_rate": 3.7025112685125563e-09,
      "logits/chosen": -1.1803710460662842,
      "logits/rejected": -1.1916015148162842,
      "logps/chosen": -434.8999938964844,
      "logps/rejected": -569.0499877929688,
      "loss": 0.0087,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -5.6787109375,
      "rewards/margins": 19.334375381469727,
      "rewards/rejected": -25.006250381469727,
      "step": 6190
    },
    {
      "epoch": 3.9900225297714838,
      "grad_norm": 0.0492094714617133,
      "learning_rate": 2.0927237604636186e-09,
      "logits/chosen": -1.182037353515625,
      "logits/rejected": -1.1907927989959717,
      "logps/chosen": -433.1000061035156,
      "logps/rejected": -609.4000244140625,
      "loss": 0.0152,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -6.009033203125,
      "rewards/margins": 18.553125381469727,
      "rewards/rejected": -24.575000762939453,
      "step": 6200
    },
    {
      "epoch": 3.9964596073382683,
      "grad_norm": 0.025755020406968258,
      "learning_rate": 4.829362524146813e-10,
      "logits/chosen": -1.1433594226837158,
      "logits/rejected": -1.1748046875,
      "logps/chosen": -440.6499938964844,
      "logps/rejected": -625.7999877929688,
      "loss": 0.0186,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -6.156201362609863,
      "rewards/margins": 18.5078125,
      "rewards/rejected": -24.65625,
      "step": 6210
    }
  ],
  "logging_steps": 10,
  "max_steps": 6212,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 4,
  "save_steps": 500,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 1,
  "trial_name": null,
  "trial_params": null
}