{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.9998691270776077,
  "eval_steps": 200,
  "global_step": 955,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0010469833791388562,
      "grad_norm": 46.5684700012207,
      "kl": 0.04943779110908508,
      "learning_rate": 0.0,
      "logits/chosen": 442452480.0,
      "logits/rejected": 465586784.0,
      "logps/chosen": -192.3951231060606,
      "logps/rejected": -244.5851814516129,
      "loss": 4.0033,
      "rewards/chosen": -0.002359318913835468,
      "rewards/margins": -0.005345997827551815,
      "rewards/rejected": 0.002986678913716347,
      "step": 1
    },
    {
      "epoch": 0.010469833791388562,
      "grad_norm": 47.87677001953125,
      "kl": 0.0966624915599823,
      "learning_rate": 4.6875e-08,
      "logits/chosen": 497818176.0,
      "logits/rejected": 465912416.0,
      "logps/chosen": -272.4681021973466,
      "logps/rejected": -251.92019581056465,
      "loss": 4.0037,
      "rewards/chosen": -0.0018457298848166394,
      "rewards/margins": -0.002352562184804766,
      "rewards/rejected": 0.0005068322999881265,
      "step": 10
    },
    {
      "epoch": 0.020939667582777124,
      "grad_norm": 52.892616271972656,
      "kl": 0.07839385420084,
      "learning_rate": 9.895833333333332e-08,
      "logits/chosen": 495900416.0,
      "logits/rejected": 479196576.0,
      "logps/chosen": -270.1613023952096,
      "logps/rejected": -259.60822610294116,
      "loss": 3.9998,
      "rewards/chosen": -0.0014621614517565972,
      "rewards/margins": 0.001095909008419559,
      "rewards/rejected": -0.0025580704601761563,
      "step": 20
    },
    {
      "epoch": 0.031409501374165684,
      "grad_norm": 49.37065505981445,
      "kl": 0.12432318925857544,
      "learning_rate": 1.5104166666666664e-07,
      "logits/chosen": 446004160.0,
      "logits/rejected": 416264512.0,
      "logps/chosen": -286.6748310810811,
      "logps/rejected": -239.96737581433226,
      "loss": 3.9988,
      "rewards/chosen": 0.005302351516288322,
      "rewards/margins": 0.001094049639710752,
      "rewards/rejected": 0.00420830187657757,
      "step": 30
    },
    {
      "epoch": 0.04187933516555425,
      "grad_norm": 56.20479965209961,
      "kl": 0.1288590282201767,
      "learning_rate": 2.03125e-07,
      "logits/chosen": 469622016.0,
      "logits/rejected": 471840608.0,
      "logps/chosen": -299.05229591836735,
      "logps/rejected": -275.7191874027994,
      "loss": 4.0024,
      "rewards/chosen": 0.011023993020529275,
      "rewards/margins": -0.0016035611417646294,
      "rewards/rejected": 0.012627554162293905,
      "step": 40
    },
    {
      "epoch": 0.05234916895694281,
      "grad_norm": 53.468544006347656,
      "kl": 0.15856146812438965,
      "learning_rate": 2.552083333333333e-07,
      "logits/chosen": 472065984.0,
      "logits/rejected": 415794880.0,
      "logps/chosen": -301.99,
      "logps/rejected": -266.23648313492066,
      "loss": 3.9892,
      "rewards/chosen": 0.0272962394127479,
      "rewards/margins": 0.012006511944408674,
      "rewards/rejected": 0.015289727468339225,
      "step": 50
    },
    {
      "epoch": 0.06281900274833137,
      "grad_norm": 55.099525451660156,
      "kl": 0.1767362356185913,
      "learning_rate": 3.0729166666666665e-07,
      "logits/chosen": 458564768.0,
      "logits/rejected": 465017408.0,
      "logps/chosen": -291.2200561908517,
      "logps/rejected": -270.51400445046437,
      "loss": 3.9806,
      "rewards/chosen": 0.03807779967972909,
      "rewards/margins": 0.01917116797379836,
      "rewards/rejected": 0.018906631705930727,
      "step": 60
    },
    {
      "epoch": 0.07328883653971993,
      "grad_norm": 59.78743362426758,
      "kl": 0.1566978394985199,
      "learning_rate": 3.59375e-07,
      "logits/chosen": 435970144.0,
      "logits/rejected": 464205728.0,
      "logps/chosen": -276.08103898713824,
      "logps/rejected": -271.9755176671733,
      "loss": 3.974,
      "rewards/chosen": 0.05013132555308449,
      "rewards/margins": 0.027480934417912214,
      "rewards/rejected": 0.02265039113517228,
      "step": 70
    },
    {
      "epoch": 0.0837586703311085,
      "grad_norm": 63.29967498779297,
      "kl": 0.14287321269512177,
      "learning_rate": 4.114583333333333e-07,
      "logits/chosen": 465435904.0,
      "logits/rejected": 445954368.0,
      "logps/chosen": -301.7261437595712,
      "logps/rejected": -253.4555173444976,
      "loss": 3.9459,
      "rewards/chosen": 0.07625812337738083,
      "rewards/margins": 0.05526375141259004,
      "rewards/rejected": 0.020994371964790794,
      "step": 80
    },
    {
      "epoch": 0.09422850412249706,
      "grad_norm": 51.23120880126953,
      "kl": 0.08589013665914536,
      "learning_rate": 4.6354166666666664e-07,
      "logits/chosen": 425856352.0,
      "logits/rejected": 442634752.0,
      "logps/chosen": -250.01865157480316,
      "logps/rejected": -253.97698643410854,
      "loss": 3.9365,
      "rewards/chosen": 0.0922366164800689,
      "rewards/margins": 0.06793148576547496,
      "rewards/rejected": 0.02430513071459393,
      "step": 90
    },
    {
      "epoch": 0.10469833791388562,
      "grad_norm": 53.67849349975586,
      "kl": 0.06646991521120071,
      "learning_rate": 4.999849525959245e-07,
      "logits/chosen": 443815680.0,
      "logits/rejected": 468530592.0,
      "logps/chosen": -292.86275268151815,
      "logps/rejected": -254.13021606083086,
      "loss": 3.8786,
      "rewards/chosen": 0.1379817798979605,
      "rewards/margins": 0.13271600843962053,
      "rewards/rejected": 0.005265771458339974,
      "step": 100
    },
    {
      "epoch": 0.11516817170527417,
      "grad_norm": 55.30476379394531,
      "kl": 0.008740690536797047,
      "learning_rate": 4.997174935782199e-07,
      "logits/chosen": 405325280.0,
      "logits/rejected": 404096320.0,
      "logps/chosen": -281.1175155520995,
      "logps/rejected": -245.54272959183675,
      "loss": 3.8465,
      "rewards/chosen": 0.10175868104071491,
      "rewards/margins": 0.15764128317543147,
      "rewards/rejected": -0.05588260213471657,
      "step": 110
    },
    {
      "epoch": 0.12563800549666274,
      "grad_norm": 53.23088073730469,
      "kl": 0.0,
      "learning_rate": 4.9911605954668e-07,
      "logits/chosen": 413253440.0,
      "logits/rejected": 466272960.0,
      "logps/chosen": -265.9372119815668,
      "logps/rejected": -285.0679401828299,
      "loss": 3.7909,
      "rewards/chosen": 0.09825083297518541,
      "rewards/margins": 0.21899295158017598,
      "rewards/rejected": -0.12074211860499057,
      "step": 120
    },
    {
      "epoch": 0.1361078392880513,
      "grad_norm": 47.935428619384766,
      "kl": 0.0031980276107788086,
      "learning_rate": 4.981814548660135e-07,
      "logits/chosen": 415410240.0,
      "logits/rejected": 464793856.0,
      "logps/chosen": -278.3420138888889,
      "logps/rejected": -253.4058807237814,
      "loss": 3.7401,
      "rewards/chosen": 0.16747628080706495,
      "rewards/margins": 0.27606504747137556,
      "rewards/rejected": -0.10858876666431061,
      "step": 130
    },
    {
      "epoch": 0.14657767307943986,
      "grad_norm": 45.16371154785156,
      "kl": 0.0,
      "learning_rate": 4.969149294871417e-07,
      "logits/chosen": 419959424.0,
      "logits/rejected": 448011200.0,
      "logps/chosen": -259.45691892971246,
      "logps/rejected": -274.75979548929666,
      "loss": 3.6988,
      "rewards/chosen": 0.15218203364850613,
      "rewards/margins": 0.3255889087455651,
      "rewards/rejected": -0.17340687509705896,
      "step": 140
    },
    {
      "epoch": 0.1570475068708284,
      "grad_norm": 45.20751190185547,
      "kl": 0.0,
      "learning_rate": 4.953181772754997e-07,
      "logits/chosen": 419500992.0,
      "logits/rejected": 426865440.0,
      "logps/chosen": -262.6239265267176,
      "logps/rejected": -255.55855,
      "loss": 3.5915,
      "rewards/chosen": 0.18705672635376908,
      "rewards/margins": 0.4586699343615816,
      "rewards/rejected": -0.2716132080078125,
      "step": 150
    },
    {
      "epoch": 0.167517340662217,
      "grad_norm": 43.46672058105469,
      "kl": 0.0,
      "learning_rate": 4.93393333745642e-07,
      "logits/chosen": 402036128.0,
      "logits/rejected": 407421024.0,
      "logps/chosen": -256.07155539772725,
      "logps/rejected": -253.9230515813253,
      "loss": 3.5417,
      "rewards/chosen": 0.35380408051726103,
      "rewards/margins": 0.5338230570275502,
      "rewards/rejected": -0.1800189765102892,
      "step": 160
    },
    {
      "epoch": 0.17798717445360554,
      "grad_norm": 50.28922653198242,
      "kl": 0.0,
      "learning_rate": 4.9114297320518e-07,
      "logits/chosen": 443569248.0,
      "logits/rejected": 446337024.0,
      "logps/chosen": -272.3035854231975,
      "logps/rejected": -268.38152258566976,
      "loss": 3.4368,
      "rewards/chosen": 0.5461393254677703,
      "rewards/margins": 0.6761910660725394,
      "rewards/rejected": -0.13005174060476904,
      "step": 170
    },
    {
      "epoch": 0.18845700824499412,
      "grad_norm": 50.34736633300781,
      "kl": 0.0,
      "learning_rate": 4.885701053118751e-07,
      "logits/chosen": 420526400.0,
      "logits/rejected": 440075456.0,
      "logps/chosen": -267.0804433925811,
      "logps/rejected": -268.7849772906793,
      "loss": 3.3582,
      "rewards/chosen": 0.5984241420739954,
      "rewards/margins": 0.7952823357110659,
      "rewards/rejected": -0.1968581936370705,
      "step": 180
    },
    {
      "epoch": 0.19892684203638267,
      "grad_norm": 46.28329086303711,
      "kl": 0.0,
      "learning_rate": 4.856781710484872e-07,
      "logits/chosen": 400002560.0,
      "logits/rejected": 408255584.0,
      "logps/chosen": -266.1100976874003,
      "logps/rejected": -277.3754067764165,
      "loss": 3.272,
      "rewards/chosen": 0.658042360150643,
      "rewards/margins": 0.9656070181593528,
      "rewards/rejected": -0.3075646580087098,
      "step": 190
    },
    {
      "epoch": 0.20939667582777124,
      "grad_norm": 37.88730239868164,
      "kl": 0.0,
      "learning_rate": 4.824710381207655e-07,
      "logits/chosen": 394069760.0,
      "logits/rejected": 439350080.0,
      "logps/chosen": -280.22834606109325,
      "logps/rejected": -273.49769661854106,
      "loss": 3.2673,
      "rewards/chosen": 0.34384584580203725,
      "rewards/margins": 1.0138983249053806,
      "rewards/rejected": -0.6700524791033434,
      "step": 200
    },
    {
      "epoch": 0.2198665096191598,
      "grad_norm": 50.169498443603516,
      "kl": 0.0,
      "learning_rate": 4.789529957847353e-07,
      "logits/chosen": 409849664.0,
      "logits/rejected": 351628448.0,
      "logps/chosen": -283.19119751908397,
      "logps/rejected": -262.1562,
      "loss": 3.1422,
      "rewards/chosen": 0.7572274943344466,
      "rewards/margins": 1.2206477580063215,
      "rewards/rejected": -0.463420263671875,
      "step": 210
    },
    {
      "epoch": 0.23033634341054834,
      "grad_norm": 54.83884048461914,
      "kl": 0.0,
      "learning_rate": 4.751287491101977e-07,
      "logits/chosen": 400426208.0,
      "logits/rejected": 369194944.0,
      "logps/chosen": -270.1439732142857,
      "logps/rejected": -257.53584414308176,
      "loss": 3.2684,
      "rewards/chosen": 0.4789316402458996,
      "rewards/margins": 1.1337323431866424,
      "rewards/rejected": -0.6548007029407429,
      "step": 220
    },
    {
      "epoch": 0.24080617720193692,
      "grad_norm": 59.397918701171875,
      "kl": 0.0,
      "learning_rate": 4.710034126881159e-07,
      "logits/chosen": 424935232.0,
      "logits/rejected": 324887488.0,
      "logps/chosen": -288.61760826055314,
      "logps/rejected": -282.7670741989882,
      "loss": 3.1502,
      "rewards/chosen": 0.5312555082673762,
      "rewards/margins": 1.3070566024126125,
      "rewards/rejected": -0.7758010941452361,
      "step": 230
    },
    {
      "epoch": 0.25127601099332547,
      "grad_norm": 54.53902053833008,
      "kl": 0.0,
      "learning_rate": 4.665825037903035e-07,
      "logits/chosen": 411758592.0,
      "logits/rejected": 355237504.0,
      "logps/chosen": -266.5797621340524,
      "logps/rejected": -262.10652733755944,
      "loss": 3.1504,
      "rewards/chosen": 0.7458241232003563,
      "rewards/margins": 1.340257552513327,
      "rewards/rejected": -0.5944334293129705,
      "step": 240
    },
    {
      "epoch": 0.261745844784714,
      "grad_norm": 84.02921295166016,
      "kl": 0.0,
      "learning_rate": 4.618719349905619e-07,
      "logits/chosen": 432440224.0,
      "logits/rejected": 374901696.0,
      "logps/chosen": -284.9515267175573,
      "logps/rejected": -268.12985,
      "loss": 3.0524,
      "rewards/chosen": 0.4285235455927958,
      "rewards/margins": 1.5875603619990457,
      "rewards/rejected": -1.15903681640625,
      "step": 250
    },
    {
      "epoch": 0.2722156785761026,
      "grad_norm": 44.28253173828125,
      "kl": 0.0,
      "learning_rate": 4.568780062571374e-07,
      "logits/chosen": 350974560.0,
      "logits/rejected": 379531712.0,
      "logps/chosen": -270.08513621794873,
      "logps/rejected": -280.8126429115854,
      "loss": 3.0586,
      "rewards/chosen": 0.14809322357177734,
      "rewards/margins": 1.5623344560948813,
      "rewards/rejected": -1.414241232523104,
      "step": 260
    },
    {
      "epoch": 0.2826855123674912,
      "grad_norm": 56.65611267089844,
      "kl": 0.0,
      "learning_rate": 4.516073965270717e-07,
      "logits/chosen": 353575936.0,
      "logits/rejected": 341260640.0,
      "logps/chosen": -261.7265013693271,
      "logps/rejected": -284.40742004680186,
      "loss": 3.0462,
      "rewards/chosen": 0.5539226979716843,
      "rewards/margins": 1.646275173808999,
      "rewards/rejected": -1.0923524758373147,
      "step": 270
    },
    {
      "epoch": 0.2931553461588797,
      "grad_norm": 117.23200988769531,
      "kl": 0.0,
      "learning_rate": 4.460671547737158e-07,
      "logits/chosen": 336798048.0,
      "logits/rejected": 346110912.0,
      "logps/chosen": -299.6681069131833,
      "logps/rejected": -268.43320193768994,
      "loss": 3.0478,
      "rewards/chosen": -0.01779557194357133,
      "rewards/margins": 1.6294317994113867,
      "rewards/rejected": -1.6472273713549581,
      "step": 280
    },
    {
      "epoch": 0.3036251799502683,
      "grad_norm": 45.30254364013672,
      "kl": 0.0,
      "learning_rate": 4.40264690579353e-07,
      "logits/chosen": 350954464.0,
      "logits/rejected": 331866784.0,
      "logps/chosen": -285.780487804878,
      "logps/rejected": -266.4627904647436,
      "loss": 2.9457,
      "rewards/chosen": 0.3624885140395746,
      "rewards/margins": 1.9047759982330341,
      "rewards/rejected": -1.5422874841934595,
      "step": 290
    },
    {
      "epoch": 0.3140950137416568,
      "grad_norm": 45.462738037109375,
      "kl": 0.0,
      "learning_rate": 4.3420776422553916e-07,
      "logits/chosen": 356290880.0,
      "logits/rejected": 351119168.0,
      "logps/chosen": -276.1206756498471,
      "logps/rejected": -268.070062899361,
      "loss": 3.0097,
      "rewards/chosen": 0.5417160360820432,
      "rewards/margins": 1.7571349400544374,
      "rewards/rejected": -1.2154189039723942,
      "step": 300
    },
    {
      "epoch": 0.32456484753304543,
      "grad_norm": 47.83991622924805,
      "kl": 0.0,
      "learning_rate": 4.279044763144141e-07,
      "logits/chosen": 334029376.0,
      "logits/rejected": 379953376.0,
      "logps/chosen": -260.419678514377,
      "logps/rejected": -303.30186831039754,
      "loss": 2.9571,
      "rewards/chosen": 0.12563846819697858,
      "rewards/margins": 1.8882139502067263,
      "rewards/rejected": -1.7625754820097477,
      "step": 310
    },
    {
      "epoch": 0.335034681324434,
      "grad_norm": 42.41203308105469,
      "kl": 0.0,
      "learning_rate": 4.213632569348639e-07,
      "logits/chosen": 396048096.0,
      "logits/rejected": 356900800.0,
      "logps/chosen": -283.4178257686676,
      "logps/rejected": -284.24552449748745,
      "loss": 3.0977,
      "rewards/chosen": -0.031586469703328034,
      "rewards/margins": 1.8606463005934684,
      "rewards/rejected": -1.8922327702967965,
      "step": 320
    },
    {
      "epoch": 0.34550451511582253,
      "grad_norm": 38.14891815185547,
      "kl": 0.0,
      "learning_rate": 4.145928543880249e-07,
      "logits/chosen": 355648192.0,
      "logits/rejected": 368038208.0,
      "logps/chosen": -278.34861275671403,
      "logps/rejected": -277.72862731839257,
      "loss": 2.8694,
      "rewards/chosen": 0.5873398622630331,
      "rewards/margins": 2.0932907428295904,
      "rewards/rejected": -1.5059508805665571,
      "step": 330
    },
    {
      "epoch": 0.3559743489072111,
      "grad_norm": 52.64259719848633,
      "kl": 0.0,
      "learning_rate": 4.076023234872057e-07,
      "logits/chosen": 334901248.0,
      "logits/rejected": 373347008.0,
      "logps/chosen": -278.5432905896607,
      "logps/rejected": -275.132209720121,
      "loss": 2.9211,
      "rewards/chosen": 0.4131505716598092,
      "rewards/margins": 1.998353064331897,
      "rewards/rejected": -1.5852024926720878,
      "step": 340
    },
    {
      "epoch": 0.3664441826985997,
      "grad_norm": 47.56734085083008,
      "kl": 0.0,
      "learning_rate": 4.004010134478771e-07,
      "logits/chosen": 377446912.0,
      "logits/rejected": 370444192.0,
      "logps/chosen": -268.446590470679,
      "logps/rejected": -275.46061609968353,
      "loss": 2.8817,
      "rewards/chosen": 0.38401062105908806,
      "rewards/margins": 2.1151071060521653,
      "rewards/rejected": -1.7310964849930774,
      "step": 350
    },
    {
      "epoch": 0.37691401648998824,
      "grad_norm": 44.92960739135742,
      "kl": 0.0,
      "learning_rate": 3.9299855538392534e-07,
      "logits/chosen": 367343424.0,
      "logits/rejected": 372150560.0,
      "logps/chosen": -282.18478154388714,
      "logps/rejected": -279.0036507009346,
      "loss": 2.8888,
      "rewards/chosen": 0.24498392421997453,
      "rewards/margins": 2.0810528114254847,
      "rewards/rejected": -1.8360688872055102,
      "step": 360
    },
    {
      "epoch": 0.3873838502813768,
      "grad_norm": 70.89231872558594,
      "kl": 0.0,
      "learning_rate": 3.8540484942689075e-07,
      "logits/chosen": 341587808.0,
      "logits/rejected": 377246784.0,
      "logps/chosen": -268.64557623407643,
      "logps/rejected": -290.71287864263803,
      "loss": 2.9263,
      "rewards/chosen": 0.4795560897535579,
      "rewards/margins": 2.0588525528598653,
      "rewards/rejected": -1.5792964631063076,
      "step": 370
    },
    {
      "epoch": 0.39785368407276533,
      "grad_norm": 48.053218841552734,
      "kl": 0.0,
      "learning_rate": 3.77630051485419e-07,
      "logits/chosen": 373726304.0,
      "logits/rejected": 332450912.0,
      "logps/chosen": -287.8293694690266,
      "logps/rejected": -280.6202242524917,
      "loss": 2.9653,
      "rewards/chosen": 0.13132064954369468,
      "rewards/margins": 2.172828205736593,
      "rewards/rejected": -2.0415075561928986,
      "step": 380
    },
    {
      "epoch": 0.4083235178641539,
      "grad_norm": 73.88297271728516,
      "kl": 0.0,
      "learning_rate": 3.696845596626342e-07,
      "logits/chosen": 357619776.0,
      "logits/rejected": 357397952.0,
      "logps/chosen": -258.85225694444443,
      "logps/rejected": -277.76314903846156,
      "loss": 2.9633,
      "rewards/chosen": 0.49055417984250993,
      "rewards/margins": 2.095406193063664,
      "rewards/rejected": -1.604852013221154,
      "step": 390
    },
    {
      "epoch": 0.4187933516555425,
      "grad_norm": 59.15121078491211,
      "kl": 0.0,
      "learning_rate": 3.61579000349597e-07,
      "logits/chosen": 388324384.0,
      "logits/rejected": 387673536.0,
      "logps/chosen": -280.22964449541286,
      "logps/rejected": -283.0034944089457,
      "loss": 2.7056,
      "rewards/chosen": 0.5564632590757598,
      "rewards/margins": 2.5239340558009995,
      "rewards/rejected": -1.9674707967252396,
      "step": 400
    },
    {
      "epoch": 0.42926318544693104,
      "grad_norm": 72.18789672851562,
      "kl": 0.0,
      "learning_rate": 3.5332421401344837e-07,
      "logits/chosen": 336218400.0,
      "logits/rejected": 390127072.0,
      "logps/chosen": -281.6167403198653,
      "logps/rejected": -275.0251457725947,
      "loss": 2.935,
      "rewards/chosen": 0.14200486719407618,
      "rewards/margins": 2.1233040867982127,
      "rewards/rejected": -1.9812992196041364,
      "step": 410
    },
    {
      "epoch": 0.4397330192383196,
      "grad_norm": 83.5944595336914,
      "kl": 0.0,
      "learning_rate": 3.4493124069924635e-07,
      "logits/chosen": 369444512.0,
      "logits/rejected": 383078592.0,
      "logps/chosen": -284.5074,
      "logps/rejected": -268.33742843511453,
      "loss": 2.8723,
      "rewards/chosen": 0.2378343994140625,
      "rewards/margins": 2.372110997696505,
      "rewards/rejected": -2.1342765982824425,
      "step": 420
    },
    {
      "epoch": 0.45020285302970814,
      "grad_norm": 52.63675308227539,
      "kl": 0.0,
      "learning_rate": 3.3641130526488335e-07,
      "logits/chosen": 376764608.0,
      "logits/rejected": 398398816.0,
      "logps/chosen": -252.68347723704866,
      "logps/rejected": -299.20040338258167,
      "loss": 3.0279,
      "rewards/chosen": 0.16862698589426756,
      "rewards/margins": 2.0355602834861224,
      "rewards/rejected": -1.8669332975918547,
      "step": 430
    },
    {
      "epoch": 0.4606726868210967,
      "grad_norm": 39.36049270629883,
      "kl": 0.0,
      "learning_rate": 3.2777580236883473e-07,
      "logits/chosen": 365922400.0,
      "logits/rejected": 392508864.0,
      "logps/chosen": -252.91024361022363,
      "logps/rejected": -279.8361525229358,
      "loss": 2.8382,
      "rewards/chosen": 0.5580993262342752,
      "rewards/margins": 2.297070068182861,
      "rewards/rejected": -1.7389707419485856,
      "step": 440
    },
    {
      "epoch": 0.4711425206124853,
      "grad_norm": 57.87444305419922,
      "kl": 0.0,
      "learning_rate": 3.1903628123081196e-07,
      "logits/chosen": 394673920.0,
      "logits/rejected": 370509824.0,
      "logps/chosen": -272.00308132763973,
      "logps/rejected": -274.67553557389937,
      "loss": 2.7844,
      "rewards/chosen": 0.5175159051551582,
      "rewards/margins": 2.5180846069412235,
      "rewards/rejected": -2.000568701786065,
      "step": 450
    },
    {
      "epoch": 0.48161235440387384,
      "grad_norm": 76.35614013671875,
      "kl": 0.0,
      "learning_rate": 3.1020443018570556e-07,
      "logits/chosen": 327263424.0,
      "logits/rejected": 376021536.0,
      "logps/chosen": -267.0524807224026,
      "logps/rejected": -267.99872929216866,
      "loss": 2.9054,
      "rewards/chosen": 0.571973280473189,
      "rewards/margins": 2.1715556308708734,
      "rewards/rejected": -1.5995823503976845,
      "step": 460
    },
    {
      "epoch": 0.4920821881952624,
      "grad_norm": 65.82987976074219,
      "kl": 0.0,
      "learning_rate": 3.0129206105147343e-07,
      "logits/chosen": 370748928.0,
      "logits/rejected": 389477536.0,
      "logps/chosen": -286.8205546492659,
      "logps/rejected": -267.65315779610194,
      "loss": 2.9417,
      "rewards/chosen": 0.08930689825708286,
      "rewards/margins": 2.134248538764954,
      "rewards/rejected": -2.044941640507871,
      "step": 470
    },
    {
      "epoch": 0.5025520219866509,
      "grad_norm": 69.5935287475586,
      "kl": 0.0,
      "learning_rate": 2.923110933318805e-07,
      "logits/chosen": 372539360.0,
      "logits/rejected": 354864128.0,
      "logps/chosen": -271.98661380597014,
      "logps/rejected": -262.8959016393443,
      "loss": 2.9602,
      "rewards/chosen": 0.0067851991795781835,
      "rewards/margins": 2.1713433846944343,
      "rewards/rejected": -2.1645581855148563,
      "step": 480
    },
    {
      "epoch": 0.5130218557780395,
      "grad_norm": 75.75752258300781,
      "kl": 0.0,
      "learning_rate": 2.832735382752194e-07,
      "logits/chosen": 403122240.0,
      "logits/rejected": 371043840.0,
      "logps/chosen": -270.78357101837673,
      "logps/rejected": -284.7364433811802,
      "loss": 2.9662,
      "rewards/chosen": -0.02843327544184593,
      "rewards/margins": 2.2282741669923647,
      "rewards/rejected": -2.2567074424342106,
      "step": 490
    },
    {
      "epoch": 0.523491689569428,
      "grad_norm": 67.58584594726562,
      "kl": 0.0,
      "learning_rate": 2.741914828103307e-07,
      "logits/chosen": 372796864.0,
      "logits/rejected": 383237824.0,
      "logps/chosen": -266.3919005102041,
      "logps/rejected": -270.33850116640747,
      "loss": 2.8509,
      "rewards/chosen": 0.3570691981532697,
      "rewards/margins": 2.358759767566664,
      "rewards/rejected": -2.001690569413394,
      "step": 500
    },
    {
      "epoch": 0.5339615233608166,
      "grad_norm": 93.17213439941406,
      "kl": 0.0,
      "learning_rate": 2.650770733814065e-07,
      "logits/chosen": 373166944.0,
      "logits/rejected": 374013248.0,
      "logps/chosen": -274.2689144736842,
      "logps/rejected": -269.6120549387443,
      "loss": 2.8906,
      "rewards/chosen": 0.45857961554276316,
      "rewards/margins": 2.376228702090428,
      "rewards/rejected": -1.9176490865476645,
      "step": 510
    },
    {
      "epoch": 0.5444313571522053,
      "grad_norm": 41.66130065917969,
      "kl": 0.0,
      "learning_rate": 2.55942499703198e-07,
      "logits/chosen": 398498176.0,
      "logits/rejected": 392743488.0,
      "logps/chosen": -277.3498,
      "logps/rejected": -268.6053673664122,
      "loss": 2.9243,
      "rewards/chosen": 0.3561779296875,
      "rewards/margins": 2.193211826097328,
      "rewards/rejected": -1.8370338964098283,
      "step": 520
    },
    {
      "epoch": 0.5549011909435938,
      "grad_norm": 55.216609954833984,
      "kl": 0.0,
      "learning_rate": 2.467999784583527e-07,
      "logits/chosen": 361071104.0,
      "logits/rejected": 369749184.0,
      "logps/chosen": -261.3549321086262,
      "logps/rejected": -272.8801605504587,
      "loss": 2.8446,
      "rewards/chosen": 0.34026234294659796,
      "rewards/margins": 2.427280370529262,
      "rewards/rejected": -2.087018027582664,
      "step": 530
    },
    {
      "epoch": 0.5653710247349824,
      "grad_norm": 63.85381317138672,
      "kl": 0.0,
      "learning_rate": 2.3766173695868388e-07,
      "logits/chosen": 359954880.0,
      "logits/rejected": 357861440.0,
      "logps/chosen": -279.6579010336907,
      "logps/rejected": -281.34379984051037,
      "loss": 2.9617,
      "rewards/chosen": 0.2960061603446832,
      "rewards/margins": 2.205951756312586,
      "rewards/rejected": -1.9099455959679028,
      "step": 540
    },
    {
      "epoch": 0.5758408585263709,
      "grad_norm": 49.361106872558594,
      "kl": 0.0,
      "learning_rate": 2.285399967922253e-07,
      "logits/chosen": 359154336.0,
      "logits/rejected": 393155392.0,
      "logps/chosen": -259.8390325479233,
      "logps/rejected": -275.05624044342505,
      "loss": 2.7826,
      "rewards/chosen": 0.05572237983679238,
      "rewards/margins": 2.533227845003077,
      "rewards/rejected": -2.4775054651662844,
      "step": 550
    },
    {
      "epoch": 0.5863106923177595,
      "grad_norm": 133.47369384765625,
      "kl": 0.0,
      "learning_rate": 2.194469574779397e-07,
      "logits/chosen": 403979776.0,
      "logits/rejected": 350524512.0,
      "logps/chosen": -281.06496585735965,
      "logps/rejected": -278.4915710547504,
      "loss": 2.9209,
      "rewards/chosen": -0.0682100171565286,
      "rewards/margins": 2.379913441730347,
      "rewards/rejected": -2.448123458886876,
      "step": 560
    },
    {
      "epoch": 0.596780526109148,
      "grad_norm": 52.14252853393555,
      "kl": 0.0,
      "learning_rate": 2.1039478014994441e-07,
      "logits/chosen": 348112192.0,
      "logits/rejected": 400084896.0,
      "logps/chosen": -262.22181181959564,
      "logps/rejected": -282.3291306907378,
      "loss": 2.8607,
      "rewards/chosen": 0.08417754818455057,
      "rewards/margins": 2.4870239519203237,
      "rewards/rejected": -2.402846403735773,
      "step": 570
    },
    {
      "epoch": 0.6072503599005366,
      "grad_norm": 102.21758270263672,
      "kl": 0.0,
      "learning_rate": 2.0139557129307149e-07,
      "logits/chosen": 375497920.0,
      "logits/rejected": 391752224.0,
      "logps/chosen": -287.6094496417197,
      "logps/rejected": -300.8546060199387,
      "loss": 2.8128,
      "rewards/chosen": 0.2170035246830837,
      "rewards/margins": 2.57385417802137,
      "rewards/rejected": -2.3568506533382863,
      "step": 580
    },
    {
      "epoch": 0.6177201936919251,
      "grad_norm": 50.08977127075195,
      "kl": 0.0,
      "learning_rate": 1.9246136655151808e-07,
      "logits/chosen": 391623968.0,
      "logits/rejected": 371523136.0,
      "logps/chosen": -284.78519864341087,
      "logps/rejected": -299.40981791338584,
      "loss": 2.7844,
      "rewards/chosen": -0.06276658523914426,
      "rewards/margins": 2.5907717236929426,
      "rewards/rejected": -2.6535383089320868,
      "step": 590
    },
    {
      "epoch": 0.6281900274833137,
      "grad_norm": 62.64336013793945,
      "kl": 0.0,
      "learning_rate": 1.8360411463223873e-07,
      "logits/chosen": 372225920.0,
      "logits/rejected": 384911008.0,
      "logps/chosen": -274.0559916534181,
      "logps/rejected": -288.66136232718895,
      "loss": 2.8593,
      "rewards/chosen": -0.07598057611947598,
      "rewards/margins": 2.551059179424879,
      "rewards/rejected": -2.627039755544355,
      "step": 600
    },
    {
      "epoch": 0.6386598612747023,
      "grad_norm": 50.6363639831543,
      "kl": 0.0,
      "learning_rate": 1.7483566132460865e-07,
      "logits/chosen": 373666784.0,
      "logits/rejected": 393250304.0,
      "logps/chosen": -285.63343057753167,
      "logps/rejected": -270.8369984567901,
      "loss": 2.8871,
      "rewards/chosen": 0.03572451313839683,
      "rewards/margins": 2.3185132018922547,
      "rewards/rejected": -2.282788688753858,
      "step": 610
    },
    {
      "epoch": 0.6491296950660909,
      "grad_norm": 70.63817596435547,
      "kl": 0.0,
      "learning_rate": 1.66167733657731e-07,
      "logits/chosen": 388653408.0,
      "logits/rejected": 387147584.0,
      "logps/chosen": -286.9971341706539,
      "logps/rejected": -280.6799866003063,
      "loss": 2.9484,
      "rewards/chosen": 0.25652475068063446,
      "rewards/margins": 2.2528504903556916,
      "rewards/rejected": -1.9963257396750573,
      "step": 620
    },
    {
      "epoch": 0.6595995288574794,
      "grad_norm": 42.911075592041016,
      "kl": 0.0,
      "learning_rate": 1.5761192421657456e-07,
      "logits/chosen": 380535040.0,
      "logits/rejected": 384688384.0,
      "logps/chosen": -277.87602336261983,
      "logps/rejected": -287.9544629204893,
      "loss": 2.7116,
      "rewards/chosen": 0.6408637781112719,
      "rewards/margins": 2.699830458515515,
      "rewards/rejected": -2.058966680404243,
      "step": 630
    },
    {
      "epoch": 0.670069362648868,
      "grad_norm": 33.01219177246094,
      "kl": 0.0,
      "learning_rate": 1.491796756379185e-07,
      "logits/chosen": 426926528.0,
      "logits/rejected": 395172800.0,
      "logps/chosen": -296.3998833955224,
      "logps/rejected": -276.0863729508197,
      "loss": 2.8864,
      "rewards/chosen": 0.3844701510756763,
      "rewards/margins": 2.4492122425203484,
      "rewards/rejected": -2.064742091444672,
      "step": 640
    },
    {
      "epoch": 0.6805391964402565,
      "grad_norm": 37.57501220703125,
      "kl": 0.0,
      "learning_rate": 1.4088226530684071e-07,
      "logits/chosen": 419599680.0,
      "logits/rejected": 368290528.0,
      "logps/chosen": -283.03209005376345,
      "logps/rejected": -280.43193561208267,
      "loss": 2.7923,
      "rewards/chosen": 0.5528853690386185,
      "rewards/margins": 2.5762015750079152,
      "rewards/rejected": -2.0233162059692966,
      "step": 650
    },
    {
      "epoch": 0.6910090302316451,
      "grad_norm": 73.64015197753906,
      "kl": 0.0,
      "learning_rate": 1.327307902742142e-07,
      "logits/chosen": 411392192.0,
      "logits/rejected": 383965600.0,
      "logps/chosen": -270.74495192307694,
      "logps/rejected": -289.49122023809525,
      "loss": 2.7692,
      "rewards/chosen": 0.3836714054987981,
      "rewards/margins": 2.714577089712588,
      "rewards/rejected": -2.3309056842137896,
      "step": 660
    },
    {
      "epoch": 0.7014788640230336,
      "grad_norm": 80.35784912109375,
      "kl": 0.0,
      "learning_rate": 1.2473615241538523e-07,
      "logits/chosen": 383171776.0,
      "logits/rejected": 369880512.0,
      "logps/chosen": -261.0426136363636,
      "logps/rejected": -292.5834616174056,
      "loss": 2.893,
      "rewards/chosen": 0.42839020730486216,
      "rewards/margins": 2.40669276047861,
      "rewards/rejected": -1.978302553173748,
      "step": 670
    },
    {
      "epoch": 0.7119486978144222,
      "grad_norm": 42.02663040161133,
      "kl": 0.0,
      "learning_rate": 1.169090438498816e-07,
      "logits/chosen": 406482432.0,
      "logits/rejected": 408727328.0,
      "logps/chosen": -278.5440385367762,
      "logps/rejected": -283.1681942277691,
      "loss": 2.7501,
      "rewards/chosen": 0.6435129467123925,
      "rewards/margins": 2.670884500558921,
      "rewards/rejected": -2.0273715538465287,
      "step": 680
    },
    {
      "epoch": 0.7224185316058107,
      "grad_norm": 44.5306396484375,
      "kl": 0.0,
      "learning_rate": 1.0925993264165045e-07,
      "logits/chosen": 379088128.0,
      "logits/rejected": 389904960.0,
      "logps/chosen": -274.40904552715654,
      "logps/rejected": -286.7942708333333,
      "loss": 2.8273,
      "rewards/chosen": 0.4699281930161741,
      "rewards/margins": 2.5310232645113,
      "rewards/rejected": -2.061095071495126,
      "step": 690
    },
    {
      "epoch": 0.7328883653971994,
      "grad_norm": 51.31837463378906,
      "kl": 0.0,
      "learning_rate": 1.0179904879894998e-07,
      "logits/chosen": 404275392.0,
      "logits/rejected": 392524608.0,
      "logps/chosen": -271.8105407523511,
      "logps/rejected": -288.45069119937693,
      "loss": 2.8177,
      "rewards/chosen": 0.28736071452078027,
      "rewards/margins": 2.677813256929756,
      "rewards/rejected": -2.3904525424089758,
      "step": 700
    },
    {
      "epoch": 0.7433581991885879,
      "grad_norm": 65.66101837158203,
      "kl": 0.0,
      "learning_rate": 9.453637059262117e-08,
      "logits/chosen": 381676384.0,
      "logits/rejected": 391855936.0,
      "logps/chosen": -266.3333492366412,
      "logps/rejected": -267.5372,
      "loss": 2.9462,
      "rewards/chosen": 0.19959532759571805,
      "rewards/margins": 2.291592788533218,
      "rewards/rejected": -2.0919974609375,
      "step": 710
    },
    {
      "epoch": 0.7538280329799765,
      "grad_norm": 87.61113739013672,
      "kl": 0.0,
      "learning_rate": 8.748161121103406e-08,
      "logits/chosen": 391284128.0,
      "logits/rejected": 399996864.0,
      "logps/chosen": -276.3944143700787,
      "logps/rejected": -298.0451792635659,
      "loss": 2.737,
      "rewards/chosen": 0.5421728900098425,
      "rewards/margins": 2.7657016115456954,
      "rewards/rejected": -2.223528721535853,
      "step": 720
    },
    {
      "epoch": 0.764297866771365,
      "grad_norm": 83.69277954101562,
      "kl": 0.0,
      "learning_rate": 8.064420576955965e-08,
      "logits/chosen": 411759936.0,
      "logits/rejected": 421010976.0,
      "logps/chosen": -281.4224365234375,
      "logps/rejected": -293.0370361328125,
      "loss": 2.8698,
      "rewards/chosen": 0.0782856285572052,
      "rewards/margins": 2.5774748146533963,
      "rewards/rejected": -2.4991891860961912,
      "step": 730
    },
    {
      "epoch": 0.7747677005627536,
      "grad_norm": 43.22333526611328,
      "kl": 0.0,
      "learning_rate": 7.403329869193922e-08,
      "logits/chosen": 400044256.0,
      "logits/rejected": 360019264.0,
      "logps/chosen": -269.0556355606759,
      "logps/rejected": -272.2297794117647,
      "loss": 2.5935,
      "rewards/chosen": 0.15169298960133448,
      "rewards/margins": 3.1453841474411597,
      "rewards/rejected": -2.9936911578398253,
      "step": 740
    },
    {
      "epoch": 0.7852375343541421,
      "grad_norm": 55.45348358154297,
      "kl": 0.0,
      "learning_rate": 6.765773148042858e-08,
      "logits/chosen": 402389952.0,
      "logits/rejected": 381027904.0,
      "logps/chosen": -278.2174539170507,
      "logps/rejected": -275.59782392686805,
      "loss": 2.9414,
      "rewards/chosen": 0.10164003621231758,
      "rewards/margins": 2.3010800265243208,
      "rewards/rejected": -2.199439990312003,
      "step": 750
    },
    {
      "epoch": 0.7957073681455307,
      "grad_norm": 88.17606353759766,
      "kl": 0.0,
      "learning_rate": 6.152603089107139e-08,
      "logits/chosen": 405889216.0,
      "logits/rejected": 379471808.0,
      "logps/chosen": -268.80648306697105,
      "logps/rejected": -274.56631219903693,
      "loss": 2.9234,
      "rewards/chosen": 0.23258001365255185,
      "rewards/margins": 2.3599755700580496,
      "rewards/rejected": -2.1273955564054976,
      "step": 760
    },
    {
      "epoch": 0.8061772019369192,
      "grad_norm": 76.30126190185547,
      "kl": 0.0,
      "learning_rate": 5.5646397529920175e-08,
      "logits/chosen": 395753920.0,
      "logits/rejected": 400388992.0,
      "logps/chosen": -295.6583118044515,
      "logps/rejected": -280.01015264976957,
      "loss": 2.6817,
      "rewards/chosen": 0.3095470483049111,
      "rewards/margins": 2.8186074406700223,
      "rewards/rejected": -2.5090603923651114,
      "step": 770
    },
    {
      "epoch": 0.8166470357283078,
      "grad_norm": 82.81681823730469,
      "kl": 0.0,
      "learning_rate": 5.002669488545111e-08,
      "logits/chosen": 367751136.0,
      "logits/rejected": 401200928.0,
      "logps/chosen": -271.7482040229885,
      "logps/rejected": -296.7671153129657,
      "loss": 2.8095,
      "rewards/chosen": 0.1301105543114673,
      "rewards/margins": 2.4850469977028604,
      "rewards/rejected": -2.3549364433913933,
      "step": 780
    },
    {
      "epoch": 0.8271168695196964,
      "grad_norm": 62.69572448730469,
      "kl": 0.0,
      "learning_rate": 4.467443881184646e-08,
      "logits/chosen": 372966144.0,
      "logits/rejected": 386677760.0,
      "logps/chosen": -278.3856132075472,
      "logps/rejected": -270.80277076863354,
      "loss": 2.9023,
      "rewards/chosen": 0.03588581685000246,
      "rewards/margins": 2.3716193608620366,
      "rewards/rejected": -2.3357335440120344,
      "step": 790
    },
    {
      "epoch": 0.837586703311085,
      "grad_norm": 43.35691833496094,
      "kl": 0.0,
      "learning_rate": 3.959678747720488e-08,
      "logits/chosen": 410616064.0,
      "logits/rejected": 374841760.0,
      "logps/chosen": -270.33201388888887,
      "logps/rejected": -281.9245867768595,
      "loss": 2.8421,
      "rewards/chosen": 0.23464515968605323,
      "rewards/margins": 2.7391692267056813,
      "rewards/rejected": -2.504524067019628,
      "step": 800
    },
    {
      "epoch": 0.8480565371024735,
      "grad_norm": 44.70280838012695,
      "kl": 0.0,
      "learning_rate": 3.480053179012654e-08,
      "logits/chosen": 351436864.0,
      "logits/rejected": 401837312.0,
      "logps/chosen": -260.5194256756757,
      "logps/rejected": -283.58179723502303,
      "loss": 3.0422,
      "rewards/chosen": -0.020509217994580775,
      "rewards/margins": 2.058081561815327,
      "rewards/rejected": -2.078590779809908,
      "step": 810
    },
    {
      "epoch": 0.8585263708938621,
      "grad_norm": 96.47203826904297,
      "kl": 0.0,
      "learning_rate": 3.029208631747446e-08,
      "logits/chosen": 404733696.0,
      "logits/rejected": 377852096.0,
      "logps/chosen": -264.6643500766871,
      "logps/rejected": -281.85546875,
      "loss": 2.7119,
      "rewards/chosen": 0.3689906582510544,
      "rewards/margins": 2.8578583511376983,
      "rewards/rejected": -2.488867692886644,
      "step": 820
    },
    {
      "epoch": 0.8689962046852506,
      "grad_norm": 56.39802169799805,
      "kl": 0.0,
      "learning_rate": 2.607748070546037e-08,
      "logits/chosen": 405348512.0,
      "logits/rejected": 421100256.0,
      "logps/chosen": -267.34557373817034,
      "logps/rejected": -290.90634674922603,
      "loss": 2.8455,
      "rewards/chosen": 0.21449558968047613,
      "rewards/margins": 2.5988345354767226,
      "rewards/rejected": -2.3843389457962463,
      "step": 830
    },
    {
      "epoch": 0.8794660384766392,
      "grad_norm": 53.03834533691406,
      "kl": 0.0,
      "learning_rate": 2.2162351615526544e-08,
      "logits/chosen": 399964032.0,
      "logits/rejected": 412186112.0,
      "logps/chosen": -291.91339285714287,
      "logps/rejected": -283.60141826923075,
      "loss": 2.8146,
      "rewards/chosen": 0.25672137548053076,
      "rewards/margins": 2.538224906129569,
      "rewards/rejected": -2.2815035306490383,
      "step": 840
    },
    {
      "epoch": 0.8899358722680277,
      "grad_norm": 44.19123077392578,
      "kl": 0.0,
      "learning_rate": 1.8551935185811717e-08,
      "logits/chosen": 369389504.0,
      "logits/rejected": 379160128.0,
      "logps/chosen": -273.960441468254,
      "logps/rejected": -294.90163461538464,
      "loss": 2.7471,
      "rewards/chosen": 0.26350000775049603,
      "rewards/margins": 2.7092262322096303,
      "rewards/rejected": -2.4457262244591345,
      "step": 850
    },
    {
      "epoch": 0.9004057060594163,
      "grad_norm": 69.50191497802734,
      "kl": 0.0,
      "learning_rate": 1.5251060028279612e-08,
      "logits/chosen": 406441888.0,
      "logits/rejected": 384275776.0,
      "logps/chosen": -261.94189528593506,
      "logps/rejected": -298.96228278041076,
      "loss": 2.88,
      "rewards/chosen": 0.2605755435996667,
      "rewards/margins": 2.450452539921744,
      "rewards/rejected": -2.189876996322077,
      "step": 860
    },
    {
      "epoch": 0.9108755398508048,
      "grad_norm": 49.09358215332031,
      "kl": 0.0,
      "learning_rate": 1.2264140770878839e-08,
      "logits/chosen": 381910016.0,
      "logits/rejected": 408627904.0,
      "logps/chosen": -288.8589045166403,
      "logps/rejected": -287.52084938366716,
      "loss": 2.86,
      "rewards/chosen": 0.2764157931513718,
      "rewards/margins": 2.49960226843787,
      "rewards/rejected": -2.2231864752864983,
      "step": 870
    },
    {
      "epoch": 0.9213453736421934,
      "grad_norm": 94.33980560302734,
      "kl": 0.0,
      "learning_rate": 9.59517215336922e-09,
      "logits/chosen": 337365408.0,
      "logits/rejected": 363262144.0,
      "logps/chosen": -271.76117468701096,
      "logps/rejected": -282.8595700078003,
      "loss": 2.8254,
      "rewards/chosen": 0.10245320494745819,
      "rewards/margins": 2.6755982694219123,
      "rewards/rejected": -2.573145064474454,
      "step": 880
    },
    {
      "epoch": 0.931815207433582,
      "grad_norm": 67.45891571044922,
      "kl": 0.0,
      "learning_rate": 7.247723684711382e-09,
      "logits/chosen": 388323680.0,
      "logits/rejected": 380559360.0,
      "logps/chosen": -261.3919588414634,
      "logps/rejected": -287.53240184294873,
      "loss": 2.7528,
      "rewards/chosen": 0.4676312702458079,
      "rewards/margins": 2.706891625877318,
      "rewards/rejected": -2.2392603556315103,
      "step": 890
    },
    {
      "epoch": 0.9422850412249706,
      "grad_norm": 61.053680419921875,
      "kl": 0.0,
      "learning_rate": 5.224934869164976e-09,
      "logits/chosen": 389374336.0,
      "logits/rejected": 416766144.0,
      "logps/chosen": -281.80756179092384,
      "logps/rejected": -292.0082249245852,
      "loss": 2.9554,
      "rewards/chosen": 0.13213771282177497,
      "rewards/margins": 2.3010161904447957,
      "rewards/rejected": -2.1688784776230206,
      "step": 900
    },
    {
      "epoch": 0.9527548750163591,
      "grad_norm": 57.665740966796875,
      "kl": 0.0,
      "learning_rate": 3.529511007479946e-09,
      "logits/chosen": 392336480.0,
      "logits/rejected": 398449632.0,
      "logps/chosen": -280.89854550691246,
      "logps/rejected": -273.7118441971383,
      "loss": 2.8584,
      "rewards/chosen": 0.3160471879575293,
      "rewards/margins": 2.478805146398507,
      "rewards/rejected": -2.1627579584409777,
      "step": 910
    },
    {
      "epoch": 0.9632247088077477,
      "grad_norm": 62.919960021972656,
      "kl": 0.0,
      "learning_rate": 2.1637195787966857e-09,
      "logits/chosen": 370773184.0,
      "logits/rejected": 438506944.0,
      "logps/chosen": -283.6101524879615,
      "logps/rejected": -277.2926179604262,
      "loss": 2.7438,
      "rewards/chosen": 0.5020495272371589,
      "rewards/margins": 2.5911041864956252,
      "rewards/rejected": -2.0890546592584665,
      "step": 920
    },
    {
      "epoch": 0.9736945425991362,
      "grad_norm": 56.47111129760742,
      "kl": 0.0,
      "learning_rate": 1.1293872080934963e-09,
      "logits/chosen": 367021120.0,
      "logits/rejected": 405562304.0,
      "logps/chosen": -274.2324472402597,
      "logps/rejected": -289.800828313253,
      "loss": 2.7894,
      "rewards/chosen": 0.2850929359336952,
      "rewards/margins": 2.6591384343335447,
      "rewards/rejected": -2.3740454983998496,
      "step": 930
    },
    {
      "epoch": 0.9841643763905248,
      "grad_norm": 55.910579681396484,
      "kl": 0.0,
      "learning_rate": 4.2789722323760546e-10,
      "logits/chosen": 390189216.0,
      "logits/rejected": 378433184.0,
      "logps/chosen": -277.966340755988,
      "logps/rejected": -284.6381229575163,
      "loss": 2.7211,
      "rewards/chosen": 0.5247520857942318,
      "rewards/margins": 2.884224622367455,
      "rewards/rejected": -2.359472536573223,
      "step": 940
    },
    {
      "epoch": 0.9946342101819133,
      "grad_norm": 105.30035400390625,
      "kl": 0.0,
      "learning_rate": 6.018780490690822e-11,
      "logits/chosen": 404358912.0,
      "logits/rejected": 365349088.0,
      "logps/chosen": -274.1581202651515,
      "logps/rejected": -274.12162298387096,
      "loss": 2.7314,
      "rewards/chosen": 0.3820543924967448,
      "rewards/margins": 2.859391448318317,
      "rewards/rejected": -2.4773370558215726,
      "step": 950
    },
    {
      "epoch": 0.9998691270776077,
      "step": 955,
      "total_flos": 0.0,
      "train_loss": 3.070508968892522,
      "train_runtime": 9873.9747,
      "train_samples_per_second": 12.383,
      "train_steps_per_second": 0.097
    }
  ],
  "logging_steps": 10,
  "max_steps": 955,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 500,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": false,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 4,
  "trial_name": null,
  "trial_params": null
}