{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.0,
  "eval_steps": 500,
  "global_step": 196,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.025591810620601407,
      "grad_norm": 10.149239540100098,
      "kl": 0.00701373303309083,
      "learning_rate": 4e-08,
      "logits/chosen": -59719033.2631579,
      "logits/rejected": -50417389.71428572,
      "logps/chosen": -439.0221011513158,
      "logps/rejected": -198.42022414434524,
      "loss": 0.5369,
      "loss/base_kto": 0.49959954619407654,
      "metrics/advantage_var": 0.05083351954817772,
      "regularization/advantage_range": 0.40121540427207947,
      "regularization/tv": 0.040121544152498245,
      "rewards/chosen": 0.0023490378731175474,
      "rewards/margins": 0.0034306578170088,
      "rewards/rejected": -0.0010816199438912527,
      "step": 5
    },
    {
      "epoch": 0.05118362124120281,
      "grad_norm": 8.902235984802246,
      "kl": 0.02517082169651985,
      "learning_rate": 9e-08,
      "logits/chosen": -59301183.16883117,
      "logits/rejected": -50015657.638554215,
      "logps/chosen": -429.9437905844156,
      "logps/rejected": -192.8660815135542,
      "loss": 0.556,
      "loss/base_kto": 0.500069797039032,
      "metrics/advantage_var": 0.07264194637537003,
      "regularization/advantage_range": 0.6553927659988403,
      "regularization/tv": 0.06553927809000015,
      "rewards/chosen": -0.0005267390183040074,
      "rewards/margins": -0.0007132336933313261,
      "rewards/rejected": 0.00018649467502731876,
      "stability/repetition_rate_mean": 0.3566492199897766,
      "stability/response_length_mean": 44.25,
      "stability/response_length_std": 28.13614845275879,
      "stability/response_length_var": 791.6428833007812,
      "stability/token_entropy_mean": 3.3583984375,
      "step": 10
    },
    {
      "epoch": 0.07677543186180422,
      "grad_norm": 10.803398132324219,
      "kl": 0.013909915462136269,
      "learning_rate": 1.4e-07,
      "logits/chosen": -56230992.67878788,
      "logits/rejected": -51926349.62580645,
      "logps/chosen": -404.1289772727273,
      "logps/rejected": -200.66030745967743,
      "loss": 0.5608,
      "loss/base_kto": 0.5003475546836853,
      "metrics/advantage_var": 0.06621196120977402,
      "regularization/advantage_range": 0.580450177192688,
      "regularization/tv": 0.05804501846432686,
      "rewards/chosen": 0.00018446743488311767,
      "rewards/margins": -0.0031913619848989673,
      "rewards/rejected": 0.003375829419782085,
      "stability/repetition_rate_mean": 0.2766926884651184,
      "stability/response_length_mean": 34.0,
      "stability/response_length_std": 32.27117156982422,
      "stability/response_length_var": 1041.4285888671875,
      "stability/token_entropy_mean": 3.552734375,
      "step": 15
    },
    {
      "epoch": 0.10236724248240563,
      "grad_norm": 9.64525032043457,
      "kl": 0.018590977415442467,
      "learning_rate": 1.8999999999999998e-07,
      "logits/chosen": -51663180.8,
      "logits/rejected": -48048432.0,
      "logps/chosen": -343.686181640625,
      "logps/rejected": -225.646044921875,
      "loss": 0.5609,
      "loss/base_kto": 0.5000947117805481,
      "metrics/advantage_var": 0.06219376251101494,
      "regularization/advantage_range": 0.5751846432685852,
      "regularization/tv": 0.05751847103238106,
      "rewards/chosen": 0.0010538341477513314,
      "rewards/margins": -0.0006758462637662887,
      "rewards/rejected": 0.00172968041151762,
      "stability/repetition_rate_mean": 0.20810391008853912,
      "stability/response_length_mean": 37.25,
      "stability/response_length_std": 29.040611267089844,
      "stability/response_length_var": 843.3571166992188,
      "stability/token_entropy_mean": 4.4921875,
      "step": 20
    },
    {
      "epoch": 0.12795905310300704,
      "grad_norm": 9.157170295715332,
      "kl": 0.01860981434583664,
      "learning_rate": 2.4e-07,
      "logits/chosen": -56477703.06206897,
      "logits/rejected": -52019299.474285714,
      "logps/chosen": -378.4747844827586,
      "logps/rejected": -202.77080357142856,
      "loss": 0.5583,
      "loss/base_kto": 0.5000918507575989,
      "metrics/advantage_var": 0.05210736021399498,
      "regularization/advantage_range": 0.5048887133598328,
      "regularization/tv": 0.050488878041505814,
      "rewards/chosen": 0.001669498455935511,
      "rewards/margins": -0.0005333170426890181,
      "rewards/rejected": 0.002202815498624529,
      "stability/repetition_rate_mean": 0.18192441761493683,
      "stability/response_length_mean": 43.375,
      "stability/response_length_std": 28.54538917541504,
      "stability/response_length_var": 814.8392944335938,
      "stability/token_entropy_mean": 4.79296875,
      "step": 25
    },
    {
      "epoch": 0.15355086372360843,
      "grad_norm": 10.262587547302246,
      "kl": 0.009434306994080544,
      "learning_rate": 2.9e-07,
      "logits/chosen": -57731374.389261745,
      "logits/rejected": -51533303.01754386,
      "logps/chosen": -468.63920931208054,
      "logps/rejected": -198.23499177631578,
      "loss": 0.5625,
      "loss/base_kto": 0.5008201003074646,
      "metrics/advantage_var": 0.08361370861530304,
      "regularization/advantage_range": 0.6748486757278442,
      "regularization/tv": 0.06748487055301666,
      "rewards/chosen": -0.005791494510317809,
      "rewards/margins": -0.007079464091185643,
      "rewards/rejected": 0.0012879695808678342,
      "stability/repetition_rate_mean": 0.17027698457241058,
      "stability/response_length_mean": 45.875,
      "stability/response_length_std": 28.246049880981445,
      "stability/response_length_var": 797.8392944335938,
      "stability/token_entropy_mean": 4.546875,
      "step": 30
    },
    {
      "epoch": 0.17914267434420986,
      "grad_norm": 12.831774711608887,
      "kl": 0.015319338999688625,
      "learning_rate": 3.4000000000000003e-07,
      "logits/chosen": -57655516.35443038,
      "logits/rejected": -53911861.72839506,
      "logps/chosen": -385.9513943829114,
      "logps/rejected": -195.11897183641975,
      "loss": 0.5625,
      "loss/base_kto": 0.5003347396850586,
      "metrics/advantage_var": 0.0590394102036953,
      "regularization/advantage_range": 0.563585102558136,
      "regularization/tv": 0.05635851249098778,
      "rewards/chosen": -0.0014883535383622857,
      "rewards/margins": -0.0022382299621126803,
      "rewards/rejected": 0.0007498764237503947,
      "stability/repetition_rate_mean": 0.29817304015159607,
      "stability/response_length_mean": 54.625,
      "stability/response_length_std": 20.248016357421875,
      "stability/response_length_var": 409.9821472167969,
      "stability/token_entropy_mean": 5.41015625,
      "step": 35
    },
    {
      "epoch": 0.20473448496481125,
      "grad_norm": 10.331823348999023,
      "kl": 0.01382284052670002,
      "learning_rate": 3.8999999999999997e-07,
      "logits/chosen": -56653414.4,
      "logits/rejected": -50244768.0,
      "logps/chosen": -421.876953125,
      "logps/rejected": -219.268798828125,
      "loss": 0.5619,
      "loss/base_kto": 0.5002309679985046,
      "metrics/advantage_var": 0.08367353677749634,
      "regularization/advantage_range": 0.7092666029930115,
      "regularization/tv": 0.07092665880918503,
      "rewards/chosen": 0.002568211778998375,
      "rewards/margins": -0.0014746449887752537,
      "rewards/rejected": 0.004042856767773629,
      "stability/repetition_rate_mean": 0.16303454339504242,
      "stability/response_length_mean": 35.0,
      "stability/response_length_std": 31.518701553344727,
      "stability/response_length_var": 993.4285888671875,
      "stability/token_entropy_mean": 4.8271484375,
      "step": 40
    },
    {
      "epoch": 0.23032629558541268,
      "grad_norm": 9.053437232971191,
      "kl": 0.02095024846494198,
      "learning_rate": 4.3999999999999997e-07,
      "logits/chosen": -52719038.7114094,
      "logits/rejected": -50797819.508771926,
      "logps/chosen": -352.31208053691273,
      "logps/rejected": -243.8545321637427,
      "loss": 0.5582,
      "loss/base_kto": 0.5000016093254089,
      "metrics/advantage_var": 0.06015477702021599,
      "regularization/advantage_range": 0.5615599751472473,
      "regularization/tv": 0.05615599825978279,
      "rewards/chosen": 0.0016376149134347904,
      "rewards/margins": -0.00032698764061880057,
      "rewards/rejected": 0.001964602554053591,
      "stability/repetition_rate_mean": 0.0994017943739891,
      "stability/response_length_mean": 29.25,
      "stability/response_length_std": 29.51875877380371,
      "stability/response_length_var": 871.3571166992188,
      "stability/token_entropy_mean": 4.78515625,
      "step": 45
    },
    {
      "epoch": 0.2559181062060141,
      "grad_norm": 9.656952857971191,
      "kl": 0.020553916692733765,
      "learning_rate": 4.9e-07,
      "logits/chosen": -54045991.51677852,
      "logits/rejected": -48306355.649122804,
      "logps/chosen": -375.191956795302,
      "logps/rejected": -180.22372989766083,
      "loss": 0.5563,
      "loss/base_kto": 0.5000076293945312,
      "metrics/advantage_var": 0.05576709657907486,
      "regularization/advantage_range": 0.521994948387146,
      "regularization/tv": 0.05219949409365654,
      "rewards/chosen": 0.0007156767701142587,
      "rewards/margins": -0.0009732905526312598,
      "rewards/rejected": 0.0016889673227455184,
      "stability/repetition_rate_mean": 0.3385762870311737,
      "stability/response_length_mean": 55.375,
      "stability/response_length_std": 16.361431121826172,
      "stability/response_length_var": 267.6964416503906,
      "stability/token_entropy_mean": 4.0732421875,
      "step": 50
    },
    {
      "epoch": 0.28150991682661547,
      "grad_norm": 12.431864738464355,
      "kl": 0.021929491311311722,
      "learning_rate": 5.4e-07,
      "logits/chosen": -56980320.99378882,
      "logits/rejected": -49598167.748427674,
      "logps/chosen": -344.5066236413044,
      "logps/rejected": -230.43226709905662,
      "loss": 0.5603,
      "loss/base_kto": 0.5000991821289062,
      "metrics/advantage_var": 0.06936132162809372,
      "regularization/advantage_range": 0.6200594305992126,
      "regularization/tv": 0.06200593709945679,
      "rewards/chosen": 0.0005392193424035303,
      "rewards/margins": -0.0010378449588293415,
      "rewards/rejected": 0.0015770643012328717,
      "stability/repetition_rate_mean": 0.11199301481246948,
      "stability/response_length_mean": 22.375,
      "stability/response_length_std": 30.236862182617188,
      "stability/response_length_var": 914.2678833007812,
      "stability/token_entropy_mean": 4.3291015625,
      "step": 55
    },
    {
      "epoch": 0.30710172744721687,
      "grad_norm": 10.12047004699707,
      "kl": 0.01495620608329773,
      "learning_rate": 5.9e-07,
      "logits/chosen": -52681796.266666666,
      "logits/rejected": -47855018.11612903,
      "logps/chosen": -359.67945075757575,
      "logps/rejected": -178.17473538306453,
      "loss": 0.5585,
      "loss/base_kto": 0.5001052618026733,
      "metrics/advantage_var": 0.06890353560447693,
      "regularization/advantage_range": 0.5747928619384766,
      "regularization/tv": 0.057479288429021835,
      "rewards/chosen": 0.004134472933682528,
      "rewards/margins": -0.0010833526636498418,
      "rewards/rejected": 0.00521782559733237,
      "stability/repetition_rate_mean": 0.22963818907737732,
      "stability/response_length_mean": 52.625,
      "stability/response_length_std": 21.75800323486328,
      "stability/response_length_var": 473.41070556640625,
      "stability/token_entropy_mean": 5.240234375,
      "step": 60
    },
    {
      "epoch": 0.3326935380678183,
      "grad_norm": 10.53354549407959,
      "kl": 0.015728404745459557,
      "learning_rate": 6.4e-07,
      "logits/chosen": -57591107.716129035,
      "logits/rejected": -47302038.4969697,
      "logps/chosen": -381.9234122983871,
      "logps/rejected": -183.5323271780303,
      "loss": 0.5603,
      "loss/base_kto": 0.5003747344017029,
      "metrics/advantage_var": 0.06578836590051651,
      "regularization/advantage_range": 0.6160892844200134,
      "regularization/tv": 0.06160893663764,
      "rewards/chosen": 0.0023386393823931293,
      "rewards/margins": -0.002662936275422515,
      "rewards/rejected": 0.005001575657815644,
      "stability/repetition_rate_mean": 0.18363864719867706,
      "stability/response_length_mean": 21.125,
      "stability/response_length_std": 28.027729034423828,
      "stability/response_length_var": 785.5535888671875,
      "stability/token_entropy_mean": 5.55859375,
      "step": 65
    },
    {
      "epoch": 0.3582853486884197,
      "grad_norm": 9.509407997131348,
      "kl": 0.023167893290519714,
      "learning_rate": 6.9e-07,
      "logits/chosen": -52795008.825806454,
      "logits/rejected": -49060789.52727272,
      "logps/chosen": -379.1555443548387,
      "logps/rejected": -220.40440340909092,
      "loss": 0.5583,
      "loss/base_kto": 0.500537633895874,
      "metrics/advantage_var": 0.06570049375295639,
      "regularization/advantage_range": 0.6170091032981873,
      "regularization/tv": 0.06170091778039932,
      "rewards/chosen": -0.002706207767609627,
      "rewards/margins": -0.004482276628443922,
      "rewards/rejected": 0.0017760688608342951,
      "stability/repetition_rate_mean": 0.177734375,
      "stability/response_length_mean": 33.875,
      "stability/response_length_std": 32.36151885986328,
      "stability/response_length_var": 1047.267822265625,
      "stability/token_entropy_mean": 5.71875,
      "step": 70
    },
    {
      "epoch": 0.3838771593090211,
      "grad_norm": 9.599165916442871,
      "kl": 0.022589249536395073,
      "learning_rate": 7.4e-07,
      "logits/chosen": -49524611.45945946,
      "logits/rejected": -47927310.88372093,
      "logps/chosen": -314.25604413006755,
      "logps/rejected": -228.89164698401163,
      "loss": 0.56,
      "loss/base_kto": 0.4996826648712158,
      "metrics/advantage_var": 0.06055055186152458,
      "regularization/advantage_range": 0.5569594502449036,
      "regularization/tv": 0.055695947259664536,
      "rewards/chosen": 0.0014766463758172216,
      "rewards/margins": 0.002158018085452313,
      "rewards/rejected": -0.0006813717096350914,
      "stability/repetition_rate_mean": 0.06745792925357819,
      "stability/response_length_mean": 22.75,
      "stability/response_length_std": 31.07479476928711,
      "stability/response_length_var": 965.6428833007812,
      "stability/token_entropy_mean": 2.16015625,
      "step": 75
    },
    {
      "epoch": 0.4094689699296225,
      "grad_norm": 9.964875221252441,
      "kl": 0.03271854668855667,
      "learning_rate": 7.9e-07,
      "logits/chosen": -58666967.31125828,
      "logits/rejected": -53504824.04733728,
      "logps/chosen": -404.0348199503311,
      "logps/rejected": -201.04611224112426,
      "loss": 0.562,
      "loss/base_kto": 0.5000892877578735,
      "metrics/advantage_var": 0.06544683128595352,
      "regularization/advantage_range": 0.5930110216140747,
      "regularization/tv": 0.05930110439658165,
      "rewards/chosen": -0.00047064370272175364,
      "rewards/margins": -0.0014391638079595452,
      "rewards/rejected": 0.0009685201052377915,
      "stability/repetition_rate_mean": 0.18697917461395264,
      "stability/response_length_mean": 42.0,
      "stability/response_length_std": 30.481843948364258,
      "stability/response_length_var": 929.1428833007812,
      "stability/token_entropy_mean": 5.25390625,
      "step": 80
    },
    {
      "epoch": 0.4350607805502239,
      "grad_norm": 9.67324161529541,
      "kl": 0.016080880537629128,
      "learning_rate": 8.399999999999999e-07,
      "logits/chosen": -51517241.80645161,
      "logits/rejected": -47190096.67878788,
      "logps/chosen": -358.3233870967742,
      "logps/rejected": -230.45189393939393,
      "loss": 0.5598,
      "loss/base_kto": 0.49952802062034607,
      "metrics/advantage_var": 0.05629377439618111,
      "regularization/advantage_range": 0.5415768027305603,
      "regularization/tv": 0.05415768548846245,
      "rewards/chosen": 0.0009144691690321892,
      "rewards/margins": 0.003623393166333233,
      "rewards/rejected": -0.002708923997301044,
      "stability/repetition_rate_mean": 0.1272321343421936,
      "stability/response_length_mean": 28.0,
      "stability/response_length_std": 31.181495666503906,
      "stability/response_length_var": 972.2857055664062,
      "stability/token_entropy_mean": 4.953125,
      "step": 85
    },
    {
      "epoch": 0.46065259117082535,
      "grad_norm": 11.245959281921387,
      "kl": 0.02391611598432064,
      "learning_rate": 8.9e-07,
      "logits/chosen": -58483305.025641024,
      "logits/rejected": -50648064.0,
      "logps/chosen": -380.7841796875,
      "logps/rejected": -187.2305878429878,
      "loss": 0.5579,
      "loss/base_kto": 0.4998149871826172,
      "metrics/advantage_var": 0.06445884704589844,
      "regularization/advantage_range": 0.5840460658073425,
      "regularization/tv": 0.05840460583567619,
      "rewards/chosen": 0.0005208480721100783,
      "rewards/margins": 0.0008993004271067479,
      "rewards/rejected": -0.0003784523549966696,
      "stability/repetition_rate_mean": 0.2582859992980957,
      "stability/response_length_mean": 35.125,
      "stability/response_length_std": 31.638301849365234,
      "stability/response_length_var": 1000.9821166992188,
      "stability/token_entropy_mean": 2.94775390625,
      "step": 90
    },
    {
      "epoch": 0.48624440179142675,
      "grad_norm": 12.896798133850098,
      "kl": 0.01629577949643135,
      "learning_rate": 9.399999999999999e-07,
      "logits/chosen": -50465753.11392405,
      "logits/rejected": -49047940.74074074,
      "logps/chosen": -315.00089003164555,
      "logps/rejected": -207.1407455632716,
      "loss": 0.5612,
      "loss/base_kto": 0.5001658797264099,
      "metrics/advantage_var": 0.05674365162849426,
      "regularization/advantage_range": 0.5391353368759155,
      "regularization/tv": 0.05391353368759155,
      "rewards/chosen": -0.0007501794378968734,
      "rewards/margins": -0.0010482627723222226,
      "rewards/rejected": 0.00029808333442534927,
      "stability/repetition_rate_mean": 0.19374233484268188,
      "stability/response_length_mean": 27.0,
      "stability/response_length_std": 31.1264705657959,
      "stability/response_length_var": 968.8571166992188,
      "stability/token_entropy_mean": 3.720703125,
      "step": 95
    },
    {
      "epoch": 0.5118362124120281,
      "grad_norm": 10.303702354431152,
      "kl": 0.020755577832460403,
      "learning_rate": 9.9e-07,
      "logits/chosen": -54792804.324324325,
      "logits/rejected": -51000611.72093023,
      "logps/chosen": -359.86095861486484,
      "logps/rejected": -216.5813726380814,
      "loss": 0.56,
      "loss/base_kto": 0.4998439848423004,
      "metrics/advantage_var": 0.06928826868534088,
      "regularization/advantage_range": 0.6253687143325806,
      "regularization/tv": 0.06253687292337418,
      "rewards/chosen": 0.000921740523866705,
      "rewards/margins": 0.0011683945935676114,
      "rewards/rejected": -0.00024665406970090646,
      "stability/repetition_rate_mean": 0.23201069235801697,
      "stability/response_length_mean": 38.0,
      "stability/response_length_std": 30.10457992553711,
      "stability/response_length_var": 906.2857055664062,
      "stability/token_entropy_mean": 4.138671875,
      "step": 100
    },
    {
      "epoch": 0.5374280230326296,
      "grad_norm": 10.662942886352539,
      "kl": 0.020296735689044,
      "learning_rate": 9.999815554351224e-07,
      "logits/chosen": -59457202.89156626,
      "logits/rejected": -53918879.584415585,
      "logps/chosen": -399.8859186746988,
      "logps/rejected": -210.00055803571428,
      "loss": 0.5607,
      "loss/base_kto": 0.49988850951194763,
      "metrics/advantage_var": 0.07286067306995392,
      "regularization/advantage_range": 0.6427713632583618,
      "regularization/tv": 0.06427713483572006,
      "rewards/chosen": 0.0016550730509930346,
      "rewards/margins": 0.0013284400814278288,
      "rewards/rejected": 0.0003266329695652058,
      "stability/repetition_rate_mean": 0.1858258843421936,
      "stability/response_length_mean": 29.0,
      "stability/response_length_std": 29.89505386352539,
      "stability/response_length_var": 893.7142944335938,
      "stability/token_entropy_mean": 3.388671875,
      "step": 105
    },
    {
      "epoch": 0.5630198336532309,
      "grad_norm": 9.717812538146973,
      "kl": 0.020564759150147438,
      "learning_rate": 9.999066267225444e-07,
      "logits/chosen": -57471142.55421687,
      "logits/rejected": -53720256.83116883,
      "logps/chosen": -378.5656061746988,
      "logps/rejected": -206.05845424107142,
      "loss": 0.5618,
      "loss/base_kto": 0.5004459619522095,
      "metrics/advantage_var": 0.05769955739378929,
      "regularization/advantage_range": 0.5556509494781494,
      "regularization/tv": 0.05556509643793106,
      "rewards/chosen": -0.0002350416258875146,
      "rewards/margins": -0.0032033283584620765,
      "rewards/rejected": 0.002968286732574562,
      "stability/repetition_rate_mean": 0.3163115680217743,
      "stability/response_length_mean": 30.625,
      "stability/response_length_std": 27.7794246673584,
      "stability/response_length_var": 771.6964111328125,
      "stability/token_entropy_mean": 4.0458984375,
      "step": 110
    },
    {
      "epoch": 0.5886116442738324,
      "grad_norm": 9.348725318908691,
      "kl": 0.017028478905558586,
      "learning_rate": 9.997740697079592e-07,
      "logits/chosen": -60715643.1392405,
      "logits/rejected": -53533525.333333336,
      "logps/chosen": -364.5681121439873,
      "logps/rejected": -201.188777970679,
      "loss": 0.5607,
      "loss/base_kto": 0.49982261657714844,
      "metrics/advantage_var": 0.055978983640670776,
      "regularization/advantage_range": 0.5500068664550781,
      "regularization/tv": 0.05500068888068199,
      "rewards/chosen": 0.00244392662108699,
      "rewards/margins": 0.0016650501921002168,
      "rewards/rejected": 0.000778876428986773,
      "stability/repetition_rate_mean": 0.2799479365348816,
      "stability/response_length_mean": 41.375,
      "stability/response_length_std": 31.25442886352539,
      "stability/response_length_var": 976.8392944335938,
      "stability/token_entropy_mean": 4.947265625,
      "step": 115
    },
    {
      "epoch": 0.6142034548944337,
      "grad_norm": 9.511648178100586,
      "kl": 0.017523258924484253,
      "learning_rate": 9.995838996722914e-07,
      "logits/chosen": -57841958.48366013,
      "logits/rejected": -54527546.25149701,
      "logps/chosen": -384.26187193627453,
      "logps/rejected": -212.1956165793413,
      "loss": 0.5592,
      "loss/base_kto": 0.5004550814628601,
      "metrics/advantage_var": 0.06272400915622711,
      "regularization/advantage_range": 0.5939682126045227,
      "regularization/tv": 0.05939681455492973,
      "rewards/chosen": -0.0016958388238171346,
      "rewards/margins": -0.00377208721119384,
      "rewards/rejected": 0.0020762483873767055,
      "stability/repetition_rate_mean": 0.18938209116458893,
      "stability/response_length_mean": 29.25,
      "stability/response_length_std": 29.687660217285156,
      "stability/response_length_var": 881.3571166992188,
      "stability/token_entropy_mean": 4.556640625,
      "step": 120
    },
    {
      "epoch": 0.6397952655150352,
      "grad_norm": 9.419194221496582,
      "kl": 0.01519815530627966,
      "learning_rate": 9.993361385379874e-07,
      "logits/chosen": -60847749.26027397,
      "logits/rejected": -55852126.16091954,
      "logps/chosen": -417.3937285958904,
      "logps/rejected": -216.33616199712642,
      "loss": 0.5598,
      "loss/base_kto": 0.5005448460578918,
      "metrics/advantage_var": 0.0776585265994072,
      "regularization/advantage_range": 0.6543391346931458,
      "regularization/tv": 0.06543391197919846,
      "rewards/chosen": -0.0014389355296957982,
      "rewards/margins": -0.004544129248792366,
      "rewards/rejected": 0.0031051937190965675,
      "stability/repetition_rate_mean": 0.16215278208255768,
      "stability/response_length_mean": 15.25,
      "stability/response_length_std": 24.93276596069336,
      "stability/response_length_var": 621.6428833007812,
      "stability/token_entropy_mean": 3.640625,
      "step": 125
    },
    {
      "epoch": 0.6653870761356366,
      "grad_norm": 9.733044624328613,
      "kl": 0.021911317482590675,
      "learning_rate": 9.99030814866488e-07,
      "logits/chosen": -57533492.8516129,
      "logits/rejected": -54012052.945454545,
      "logps/chosen": -392.6370715725806,
      "logps/rejected": -183.88927556818183,
      "loss": 0.5598,
      "loss/base_kto": 0.5001145601272583,
      "metrics/advantage_var": 0.056465037167072296,
      "regularization/advantage_range": 0.5432730913162231,
      "regularization/tv": 0.054327309131622314,
      "rewards/chosen": 0.0020855096078688097,
      "rewards/margins": -0.0008782148535999737,
      "rewards/rejected": 0.0029637244614687834,
      "stability/repetition_rate_mean": 0.18619318306446075,
      "stability/response_length_mean": 38.0,
      "stability/response_length_std": 29.13515281677246,
      "stability/response_length_var": 848.8571166992188,
      "stability/token_entropy_mean": 3.828125,
      "step": 130
    },
    {
      "epoch": 0.690978886756238,
      "grad_norm": 9.209653854370117,
      "kl": 0.03033452108502388,
      "learning_rate": 9.986679638549358e-07,
      "logits/chosen": -60026842.306748465,
      "logits/rejected": -55231651.05732484,
      "logps/chosen": -401.78712135736197,
      "logps/rejected": -247.8531548566879,
      "loss": 0.5601,
      "loss/base_kto": 0.4998328387737274,
      "metrics/advantage_var": 0.07589313387870789,
      "regularization/advantage_range": 0.6498702168464661,
      "regularization/tv": 0.06498702615499496,
      "rewards/chosen": 0.0029037466809793484,
      "rewards/margins": 0.0011423244553552155,
      "rewards/rejected": 0.001761422225624133,
      "stability/repetition_rate_mean": 0.2230902761220932,
      "stability/response_length_mean": 29.0,
      "stability/response_length_std": 30.20406723022461,
      "stability/response_length_var": 912.2857055664062,
      "stability/token_entropy_mean": 3.412109375,
      "step": 135
    },
    {
      "epoch": 0.7165706973768394,
      "grad_norm": 9.760176658630371,
      "kl": 0.022469190880656242,
      "learning_rate": 9.982476273321175e-07,
      "logits/chosen": -57270967.07878788,
      "logits/rejected": -50321067.76774193,
      "logps/chosen": -367.16176609848486,
      "logps/rejected": -189.99272933467742,
      "loss": 0.5564,
      "loss/base_kto": 0.5001166462898254,
      "metrics/advantage_var": 0.06705542653799057,
      "regularization/advantage_range": 0.6326877474784851,
      "regularization/tv": 0.06326878070831299,
      "rewards/chosen": 0.0027976480397311123,
      "rewards/margins": -0.0013773135425757801,
      "rewards/rejected": 0.004174961582306892,
      "stability/repetition_rate_mean": 0.349609375,
      "stability/response_length_mean": 56.25,
      "stability/response_length_std": 21.920310974121094,
      "stability/response_length_var": 480.5,
      "stability/token_entropy_mean": 4.6923828125,
      "step": 140
    },
    {
      "epoch": 0.7421625079974408,
      "grad_norm": 9.946731567382812,
      "kl": 0.028916647657752037,
      "learning_rate": 9.977698537536417e-07,
      "logits/chosen": -51334526.45783132,
      "logits/rejected": -51897603.32467532,
      "logps/chosen": -330.24228162650604,
      "logps/rejected": -199.45389864042207,
      "loss": 0.5578,
      "loss/base_kto": 0.5000768899917603,
      "metrics/advantage_var": 0.062081992626190186,
      "regularization/advantage_range": 0.5850270390510559,
      "regularization/tv": 0.05850271135568619,
      "rewards/chosen": 0.0029526342110461497,
      "rewards/margins": -0.0004488631437090042,
      "rewards/rejected": 0.003401497354755154,
      "stability/repetition_rate_mean": 0.19453908503055573,
      "stability/response_length_mean": 22.5,
      "stability/response_length_std": 26.47370719909668,
      "stability/response_length_var": 700.8571166992188,
      "stability/token_entropy_mean": 5.62890625,
      "step": 145
    },
    {
      "epoch": 0.7677543186180422,
      "grad_norm": 10.813165664672852,
      "kl": 0.02525208331644535,
      "learning_rate": 9.972346981963546e-07,
      "logits/chosen": -55602320.564705886,
      "logits/rejected": -50432863.57333333,
      "logps/chosen": -397.90363051470587,
      "logps/rejected": -207.71752604166667,
      "loss": 0.5599,
      "loss/base_kto": 0.49969664216041565,
      "metrics/advantage_var": 0.06014819070696831,
      "regularization/advantage_range": 0.5671171545982361,
      "regularization/tv": 0.05671171098947525,
      "rewards/chosen": 0.0030785830581889434,
      "rewards/margins": 0.0022385192677086474,
      "rewards/rejected": 0.0008400637904802959,
      "stability/repetition_rate_mean": 0.2428385466337204,
      "stability/response_length_mean": 35.25,
      "stability/response_length_std": 31.01957893371582,
      "stability/response_length_var": 962.2142944335938,
      "stability/token_entropy_mean": 4.59375,
      "step": 150
    },
    {
      "epoch": 0.7933461292386437,
      "grad_norm": 9.703414916992188,
      "kl": 0.015056160278618336,
      "learning_rate": 9.966422223519885e-07,
      "logits/chosen": -54256543.62352941,
      "logits/rejected": -49605277.013333336,
      "logps/chosen": -361.0272058823529,
      "logps/rejected": -191.29368489583334,
      "loss": 0.5608,
      "loss/base_kto": 0.5006486177444458,
      "metrics/advantage_var": 0.06816024333238602,
      "regularization/advantage_range": 0.6048727631568909,
      "regularization/tv": 0.060487277805805206,
      "rewards/chosen": -0.003791299988241757,
      "rewards/margins": -0.004983574777257209,
      "rewards/rejected": 0.001192274789015452,
      "stability/repetition_rate_mean": 0.31328123807907104,
      "stability/response_length_mean": 61.0,
      "stability/response_length_std": 8.485280990600586,
      "stability/response_length_var": 72.0,
      "stability/token_entropy_mean": 4.5947265625,
      "step": 155
    },
    {
      "epoch": 0.818937939859245,
      "grad_norm": 9.560346603393555,
      "kl": 0.019763818010687828,
      "learning_rate": 9.959924945200523e-07,
      "logits/chosen": -55504676.571428575,
      "logits/rejected": -49278878.315789476,
      "logps/chosen": -416.90545944940476,
      "logps/rejected": -186.884765625,
      "loss": 0.561,
      "loss/base_kto": 0.5002955794334412,
      "metrics/advantage_var": 0.07846514135599136,
      "regularization/advantage_range": 0.6755853891372681,
      "regularization/tv": 0.06755854189395905,
      "rewards/chosen": -0.001014596117394311,
      "rewards/margins": -0.002271914728602072,
      "rewards/rejected": 0.0012573186112077612,
      "stability/repetition_rate_mean": 0.29296875,
      "stability/response_length_mean": 26.25,
      "stability/response_length_std": 31.535694122314453,
      "stability/response_length_var": 994.5,
      "stability/token_entropy_mean": 3.9140625,
      "step": 160
    },
    {
      "epoch": 0.8445297504798465,
      "grad_norm": 10.165249824523926,
      "kl": 0.015461891889572144,
      "learning_rate": 9.952855895999568e-07,
      "logits/chosen": -54690933.70114943,
      "logits/rejected": -50131115.83561644,
      "logps/chosen": -370.9220321479885,
      "logps/rejected": -212.58657962328766,
      "loss": 0.5596,
      "loss/base_kto": 0.5001530051231384,
      "metrics/advantage_var": 0.06636251509189606,
      "regularization/advantage_range": 0.5807952284812927,
      "regularization/tv": 0.05807952210307121,
      "rewards/chosen": -0.000529495378335317,
      "rewards/margins": -0.0010842096791964144,
      "rewards/rejected": 0.0005547143008610973,
      "stability/repetition_rate_mean": 0.20468750596046448,
      "stability/response_length_mean": 12.625,
      "stability/response_length_std": 21.460512161254883,
      "stability/response_length_var": 460.5535583496094,
      "stability/token_entropy_mean": 4.79296875,
      "step": 165
    },
    {
      "epoch": 0.8701215611004478,
      "grad_norm": 10.048286437988281,
      "kl": 0.014135316014289856,
      "learning_rate": 9.9452158908238e-07,
      "logits/chosen": -59401676.49704142,
      "logits/rejected": -52977155.39072847,
      "logps/chosen": -441.3876201923077,
      "logps/rejected": -191.26497826986756,
      "loss": 0.561,
      "loss/base_kto": 0.500499963760376,
      "metrics/advantage_var": 0.07585406303405762,
      "regularization/advantage_range": 0.6783817410469055,
      "regularization/tv": 0.0678381696343422,
      "rewards/chosen": -0.004575217263938407,
      "rewards/margins": -0.0038604529689035562,
      "rewards/rejected": -0.0007147642950348507,
      "stability/repetition_rate_mean": 0.22302038967609406,
      "stability/response_length_mean": 31.625,
      "stability/response_length_std": 33.3163948059082,
      "stability/response_length_var": 1109.982177734375,
      "stability/token_entropy_mean": 3.30322265625,
      "step": 170
    },
    {
      "epoch": 0.8957133717210493,
      "grad_norm": 9.772781372070312,
      "kl": 0.015697255730628967,
      "learning_rate": 9.937005810398745e-07,
      "logits/chosen": -56969629.30120482,
      "logits/rejected": -48460181.61038961,
      "logps/chosen": -356.4288403614458,
      "logps/rejected": -194.79348873782467,
      "loss": 0.5584,
      "loss/base_kto": 0.5000929236412048,
      "metrics/advantage_var": 0.05933871492743492,
      "regularization/advantage_range": 0.5566183924674988,
      "regularization/tv": 0.05566183850169182,
      "rewards/chosen": 0.0008359345865536885,
      "rewards/margins": -0.0005672121634134158,
      "rewards/rejected": 0.0014031467499671043,
      "stability/repetition_rate_mean": 0.2897879481315613,
      "stability/response_length_mean": 32.25,
      "stability/response_length_std": 27.932315826416016,
      "stability/response_length_var": 780.2142944335938,
      "stability/token_entropy_mean": 4.7216796875,
      "step": 175
    },
    {
      "epoch": 0.9213051823416507,
      "grad_norm": 10.050302505493164,
      "kl": 0.022633520886301994,
      "learning_rate": 9.928226601167138e-07,
      "logits/chosen": -52965721.521472394,
      "logits/rejected": -48372876.22929937,
      "logps/chosen": -376.1388036809816,
      "logps/rejected": -211.28087679140128,
      "loss": 0.5602,
      "loss/base_kto": 0.5001126527786255,
      "metrics/advantage_var": 0.0682484433054924,
      "regularization/advantage_range": 0.6192613840103149,
      "regularization/tv": 0.061926137655973434,
      "rewards/chosen": 0.0007036126281586161,
      "rewards/margins": -0.0010293145909649813,
      "rewards/rejected": 0.0017329272191235973,
      "stability/repetition_rate_mean": 0.2578125,
      "stability/response_length_mean": 40.375,
      "stability/response_length_std": 32.61874008178711,
      "stability/response_length_var": 1063.982177734375,
      "stability/token_entropy_mean": 3.068359375,
      "step": 180
    },
    {
      "epoch": 0.946896992962252,
      "grad_norm": 9.547765731811523,
      "kl": 0.029239613562822342,
      "learning_rate": 9.918879275179817e-07,
      "logits/chosen": -54956689.38271605,
      "logits/rejected": -49911691.341772154,
      "logps/chosen": -312.00192901234567,
      "logps/rejected": -197.01856704905063,
      "loss": 0.5556,
      "loss/base_kto": 0.5002581477165222,
      "metrics/advantage_var": 0.05665111541748047,
      "regularization/advantage_range": 0.5235700607299805,
      "regularization/tv": 0.05235700681805611,
      "rewards/chosen": -5.7566451069749434e-05,
      "rewards/margins": -0.0021155726139546185,
      "rewards/rejected": 0.002058006162884869,
      "stability/repetition_rate_mean": 0.17862215638160706,
      "stability/response_length_mean": 22.25,
      "stability/response_length_std": 27.78874397277832,
      "stability/response_length_var": 772.2142944335938,
      "stability/token_entropy_mean": 3.833984375,
      "step": 185
    },
    {
      "epoch": 0.9724888035828535,
      "grad_norm": 9.160686492919922,
      "kl": 0.023898351937532425,
      "learning_rate": 9.90896490997906e-07,
      "logits/chosen": -53441732.21556886,
      "logits/rejected": -50112880.104575165,
      "logps/chosen": -392.382063997006,
      "logps/rejected": -211.1847937091503,
      "loss": 0.5575,
      "loss/base_kto": 0.4998375475406647,
      "metrics/advantage_var": 0.06992342323064804,
      "regularization/advantage_range": 0.6320301294326782,
      "regularization/tv": 0.06320301443338394,
      "rewards/chosen": 0.0023274359231937433,
      "rewards/margins": 0.000948604694794155,
      "rewards/rejected": 0.0013788312283995884,
      "stability/repetition_rate_mean": 0.1551254689693451,
      "stability/response_length_mean": 20.125,
      "stability/response_length_std": 27.986923217773438,
      "stability/response_length_var": 783.2678833007812,
      "stability/token_entropy_mean": 4.71728515625,
      "step": 190
    },
    {
      "epoch": 0.9980806142034548,
      "grad_norm": 10.104104995727539,
      "kl": 0.01778222993016243,
      "learning_rate": 9.898484648474362e-07,
      "logits/chosen": -60544971.1483871,
      "logits/rejected": -54323001.406060606,
      "logps/chosen": -392.7150705645161,
      "logps/rejected": -197.92739109848486,
      "loss": 0.5615,
      "loss/base_kto": 0.49904999136924744,
      "metrics/advantage_var": 0.05891064554452896,
      "regularization/advantage_range": 0.5749723315238953,
      "regularization/tv": 0.05749724060297012,
      "rewards/chosen": 0.006984769913458055,
      "rewards/margins": 0.007790713756548228,
      "rewards/rejected": -0.0008059438430901729,
      "stability/repetition_rate_mean": 0.412109375,
      "stability/response_length_mean": 48.125,
      "stability/response_length_std": 29.39600372314453,
      "stability/response_length_var": 864.125,
      "stability/token_entropy_mean": 4.4453125,
      "step": 195
    },
    {
      "epoch": 1.0,
      "step": 196,
      "total_flos": 3.1861866257511875e+18,
      "train_loss": 0.5588978714480692,
      "train_runtime": 3155.2184,
      "train_samples_per_second": 31.694,
      "train_steps_per_second": 0.062
    }
  ],
  "logging_steps": 5,
  "max_steps": 196,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 200,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 3.1861866257511875e+18,
  "train_batch_size": 8,
  "trial_name": null,
  "trial_params": null
}