{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.0,
  "eval_steps": 500,
  "global_step": 196,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.025591810620601407,
      "grad_norm": 1.2250019311904907,
      "kl": 0.007727213203907013,
      "learning_rate": 4e-08,
      "logits/chosen": -59713280.0,
      "logits/rejected": -50413135.23809524,
      "logps/chosen": -439.02909128289474,
      "logps/rejected": -198.39232235863096,
      "loss": 0.5035,
      "loss/base_kto": 0.5000293254852295,
      "metrics/advantage_var": 0.04804132506251335,
      "regularization/advantage_range": 0.41785937547683716,
      "regularization/tv": 0.004178593400865793,
      "rewards/chosen": 0.0016484160564447705,
      "rewards/margins": -6.007232276120577e-05,
      "rewards/rejected": 0.0017084883792059763,
      "step": 5
    },
    {
      "epoch": 0.05118362124120281,
      "grad_norm": 1.1427640914916992,
      "kl": 0.019967058673501015,
      "learning_rate": 9e-08,
      "logits/chosen": -59292100.15584416,
      "logits/rejected": -50021388.3373494,
      "logps/chosen": -429.93242694805195,
      "logps/rejected": -192.85899849397592,
      "loss": 0.506,
      "loss/base_kto": 0.5000237822532654,
      "metrics/advantage_var": 0.07722852379083633,
      "regularization/advantage_range": 0.6631855368614197,
      "regularization/tv": 0.006631854921579361,
      "rewards/chosen": 0.0006063263621423152,
      "rewards/margins": -0.00028858899526931065,
      "rewards/rejected": 0.0008949153574116258,
      "stability/repetition_rate_mean": 0.3512222170829773,
      "stability/response_length_mean": 46.125,
      "stability/response_length_std": 27.231481552124023,
      "stability/response_length_var": 741.5535888671875,
      "stability/token_entropy_mean": 3.4091796875,
      "step": 10
    },
    {
      "epoch": 0.07677543186180422,
      "grad_norm": 1.1430689096450806,
      "kl": 0.01295405812561512,
      "learning_rate": 1.4e-07,
      "logits/chosen": -56228826.763636366,
      "logits/rejected": -51931185.548387095,
      "logps/chosen": -404.1435606060606,
      "logps/rejected": -200.67100554435484,
      "loss": 0.5058,
      "loss/base_kto": 0.5004787445068359,
      "metrics/advantage_var": 0.06713377684354782,
      "regularization/advantage_range": 0.5917664766311646,
      "regularization/tv": 0.005917664151638746,
      "rewards/chosen": -0.0012753700668161566,
      "rewards/margins": -0.003580832927108161,
      "rewards/rejected": 0.002305462860292004,
      "stability/repetition_rate_mean": 0.2476128488779068,
      "stability/response_length_mean": 27.125,
      "stability/response_length_std": 30.79163932800293,
      "stability/response_length_var": 948.125,
      "stability/token_entropy_mean": 3.7978515625,
      "step": 15
    },
    {
      "epoch": 0.10236724248240563,
      "grad_norm": 1.2928932905197144,
      "kl": 0.021185090765357018,
      "learning_rate": 1.8999999999999998e-07,
      "logits/chosen": -51667904.0,
      "logits/rejected": -48060899.2,
      "logps/chosen": -343.6844482421875,
      "logps/rejected": -225.60615234375,
      "loss": 0.5065,
      "loss/base_kto": 0.5005955100059509,
      "metrics/advantage_var": 0.05965769290924072,
      "regularization/advantage_range": 0.5542503595352173,
      "regularization/tv": 0.0055425032041966915,
      "rewards/chosen": 0.0012270104140043258,
      "rewards/margins": -0.004491991177201272,
      "rewards/rejected": 0.005719001591205597,
      "stability/repetition_rate_mean": 0.2155257910490036,
      "stability/response_length_mean": 44.0,
      "stability/response_length_std": 28.0611572265625,
      "stability/response_length_var": 787.4285888671875,
      "stability/token_entropy_mean": 4.4296875,
      "step": 20
    },
    {
      "epoch": 0.12795905310300704,
      "grad_norm": 1.1084049940109253,
      "kl": 0.01974540390074253,
      "learning_rate": 2.4e-07,
      "logits/chosen": -56471661.46206897,
      "logits/rejected": -52027626.057142854,
      "logps/chosen": -378.47896012931034,
      "logps/rejected": -202.78183035714287,
      "loss": 0.5061,
      "loss/base_kto": 0.4999264180660248,
      "metrics/advantage_var": 0.056819237768650055,
      "regularization/advantage_range": 0.5499088168144226,
      "regularization/tv": 0.005499088205397129,
      "rewards/chosen": 0.0012486030315530712,
      "rewards/margins": 0.00014683637712976623,
      "rewards/rejected": 0.001101766654423305,
      "stability/repetition_rate_mean": 0.17801816761493683,
      "stability/response_length_mean": 43.375,
      "stability/response_length_std": 28.54538917541504,
      "stability/response_length_var": 814.8392944335938,
      "stability/token_entropy_mean": 4.806640625,
      "step": 25
    },
    {
      "epoch": 0.15355086372360843,
      "grad_norm": 1.1647928953170776,
      "kl": 0.012587564997375011,
      "learning_rate": 2.9e-07,
      "logits/chosen": -57726900.40268456,
      "logits/rejected": -51525524.21052632,
      "logps/chosen": -468.54829068791946,
      "logps/rejected": -198.2200292397661,
      "loss": 0.5063,
      "loss/base_kto": 0.49992284178733826,
      "metrics/advantage_var": 0.07996314764022827,
      "regularization/advantage_range": 0.6584316492080688,
      "regularization/tv": 0.0065843164920806885,
      "rewards/chosen": 0.003298740258952915,
      "rewards/margins": 0.0005137763221193048,
      "rewards/rejected": 0.0027849639368336104,
      "stability/repetition_rate_mean": 0.1486235111951828,
      "stability/response_length_mean": 43.0,
      "stability/response_length_std": 29.602123260498047,
      "stability/response_length_var": 876.2857055664062,
      "stability/token_entropy_mean": 4.56640625,
      "step": 30
    },
    {
      "epoch": 0.17914267434420986,
      "grad_norm": 1.2508803606033325,
      "kl": 0.018261268734931946,
      "learning_rate": 3.4000000000000003e-07,
      "logits/chosen": -57639534.17721519,
      "logits/rejected": -53902835.358024694,
      "logps/chosen": -385.93008306962025,
      "logps/rejected": -195.11032744984567,
      "loss": 0.5061,
      "loss/base_kto": 0.5001189708709717,
      "metrics/advantage_var": 0.07392420619726181,
      "regularization/advantage_range": 0.6383035778999329,
      "regularization/tv": 0.006383036263287067,
      "rewards/chosen": 0.0006445885836323605,
      "rewards/margins": -0.0009711290555254411,
      "rewards/rejected": 0.0016157176391578016,
      "stability/repetition_rate_mean": 0.3546627163887024,
      "stability/response_length_mean": 53.375,
      "stability/response_length_std": 21.14533042907715,
      "stability/response_length_var": 447.125,
      "stability/token_entropy_mean": 5.37890625,
      "step": 35
    },
    {
      "epoch": 0.20473448496481125,
      "grad_norm": 1.1856293678283691,
      "kl": 0.024992099031805992,
      "learning_rate": 3.8999999999999997e-07,
      "logits/chosen": -56638668.8,
      "logits/rejected": -50249897.6,
      "logps/chosen": -421.91201171875,
      "logps/rejected": -219.274609375,
      "loss": 0.5064,
      "loss/base_kto": 0.5005558729171753,
      "metrics/advantage_var": 0.06863617151975632,
      "regularization/advantage_range": 0.6221620440483093,
      "regularization/tv": 0.006221619900316,
      "rewards/chosen": -0.0009344659745693206,
      "rewards/margins": -0.004399620369076729,
      "rewards/rejected": 0.0034651543945074083,
      "stability/repetition_rate_mean": 0.15522204339504242,
      "stability/response_length_mean": 35.0,
      "stability/response_length_std": 31.518701553344727,
      "stability/response_length_var": 993.4285888671875,
      "stability/token_entropy_mean": 4.7978515625,
      "step": 40
    },
    {
      "epoch": 0.23032629558541268,
      "grad_norm": 1.183511734008789,
      "kl": 0.02745547890663147,
      "learning_rate": 4.3999999999999997e-07,
      "logits/chosen": -52711602.684563756,
      "logits/rejected": -50789016.701754384,
      "logps/chosen": -352.2836881291946,
      "logps/rejected": -243.8523163377193,
      "loss": 0.5057,
      "loss/base_kto": 0.49970006942749023,
      "metrics/advantage_var": 0.06934358179569244,
      "regularization/advantage_range": 0.6039530634880066,
      "regularization/tv": 0.006039530038833618,
      "rewards/chosen": 0.00447692487063824,
      "rewards/margins": 0.0022907486723704423,
      "rewards/rejected": 0.0021861761982677973,
      "stability/repetition_rate_mean": 0.1033080443739891,
      "stability/response_length_mean": 28.875,
      "stability/response_length_std": 29.87324333190918,
      "stability/response_length_var": 892.4107055664062,
      "stability/token_entropy_mean": 4.744140625,
      "step": 45
    },
    {
      "epoch": 0.2559181062060141,
      "grad_norm": 1.0992074012756348,
      "kl": 0.03547530993819237,
      "learning_rate": 4.9e-07,
      "logits/chosen": -54046757.798657715,
      "logits/rejected": -48303999.25146199,
      "logps/chosen": -375.16906984060404,
      "logps/rejected": -180.20630025584796,
      "loss": 0.5059,
      "loss/base_kto": 0.49996376037597656,
      "metrics/advantage_var": 0.05242161080241203,
      "regularization/advantage_range": 0.5211772322654724,
      "regularization/tv": 0.005211771931499243,
      "rewards/chosen": 0.003006344873633161,
      "rewards/margins": -0.00042433604826718147,
      "rewards/rejected": 0.0034306809219003423,
      "stability/repetition_rate_mean": 0.4206075370311737,
      "stability/response_length_mean": 55.375,
      "stability/response_length_std": 16.361431121826172,
      "stability/response_length_var": 267.6964416503906,
      "stability/token_entropy_mean": 3.8271484375,
      "step": 50
    },
    {
      "epoch": 0.28150991682661547,
      "grad_norm": 1.2209643125534058,
      "kl": 0.04145348072052002,
      "learning_rate": 5.4e-07,
      "logits/chosen": -56948996.770186335,
      "logits/rejected": -49573849.358490564,
      "logps/chosen": -344.536636257764,
      "logps/rejected": -230.4120233883648,
      "loss": 0.5062,
      "loss/base_kto": 0.5007476210594177,
      "metrics/advantage_var": 0.06931664049625397,
      "regularization/advantage_range": 0.6143749952316284,
      "regularization/tv": 0.006143749691545963,
      "rewards/chosen": -0.0024640397255465112,
      "rewards/margins": -0.006064461667051799,
      "rewards/rejected": 0.003600421941505288,
      "stability/repetition_rate_mean": 0.1015625,
      "stability/response_length_mean": 24.5,
      "stability/response_length_std": 32.710853576660156,
      "stability/response_length_var": 1070.0,
      "stability/token_entropy_mean": 4.419921875,
      "step": 55
    },
    {
      "epoch": 0.30710172744721687,
      "grad_norm": 1.0752596855163574,
      "kl": 0.03181527182459831,
      "learning_rate": 5.9e-07,
      "logits/chosen": -52678984.92121212,
      "logits/rejected": -47834769.341935486,
      "logps/chosen": -359.6562026515152,
      "logps/rejected": -178.18324092741935,
      "loss": 0.5058,
      "loss/base_kto": 0.4996957778930664,
      "metrics/advantage_var": 0.058318037539720535,
      "regularization/advantage_range": 0.5441433191299438,
      "regularization/tv": 0.005441432818770409,
      "rewards/chosen": 0.006462704051624645,
      "rewards/margins": 0.0020943700043686677,
      "rewards/rejected": 0.0043683340472559775,
      "stability/repetition_rate_mean": 0.21373416483402252,
      "stability/response_length_mean": 52.625,
      "stability/response_length_std": 21.75800323486328,
      "stability/response_length_var": 473.41070556640625,
      "stability/token_entropy_mean": 5.291015625,
      "step": 60
    },
    {
      "epoch": 0.3326935380678183,
      "grad_norm": 1.1203948259353638,
      "kl": 0.03296849876642227,
      "learning_rate": 6.4e-07,
      "logits/chosen": -57569332.8516129,
      "logits/rejected": -47285291.442424245,
      "logps/chosen": -381.85897177419355,
      "logps/rejected": -183.5492305871212,
      "loss": 0.5057,
      "loss/base_kto": 0.4993171691894531,
      "metrics/advantage_var": 0.059292834252119064,
      "regularization/advantage_range": 0.5489024519920349,
      "regularization/tv": 0.0054890247993171215,
      "rewards/chosen": 0.008781992235491353,
      "rewards/margins": 0.005469901925307915,
      "rewards/rejected": 0.0033120903101834386,
      "stability/repetition_rate_mean": 0.1857219785451889,
      "stability/response_length_mean": 20.625,
      "stability/response_length_std": 27.176342010498047,
      "stability/response_length_var": 738.5535888671875,
      "stability/token_entropy_mean": 5.544921875,
      "step": 65
    },
    {
      "epoch": 0.3582853486884197,
      "grad_norm": 1.0599192380905151,
      "kl": 0.03457256406545639,
      "learning_rate": 6.9e-07,
      "logits/chosen": -52787147.1483871,
      "logits/rejected": -49043046.4,
      "logps/chosen": -379.08941532258063,
      "logps/rejected": -220.3997159090909,
      "loss": 0.5056,
      "loss/base_kto": 0.49978336691856384,
      "metrics/advantage_var": 0.061428315937519073,
      "regularization/advantage_range": 0.5580489039421082,
      "regularization/tv": 0.00558048952370882,
      "rewards/chosen": 0.0039039734871156753,
      "rewards/margins": 0.00166090367942728,
      "rewards/rejected": 0.002243069807688395,
      "stability/repetition_rate_mean": 0.166015625,
      "stability/response_length_mean": 33.875,
      "stability/response_length_std": 32.36151885986328,
      "stability/response_length_var": 1047.267822265625,
      "stability/token_entropy_mean": 5.80078125,
      "step": 70
    },
    {
      "epoch": 0.3838771593090211,
      "grad_norm": 1.1175593137741089,
      "kl": 0.03155888244509697,
      "learning_rate": 7.4e-07,
      "logits/chosen": -49525379.45945946,
      "logits/rejected": -47913144.55813953,
      "logps/chosen": -314.1897434543919,
      "logps/rejected": -228.8531976744186,
      "loss": 0.5059,
      "loss/base_kto": 0.4994099736213684,
      "metrics/advantage_var": 0.0698389783501625,
      "regularization/advantage_range": 0.6349931359291077,
      "regularization/tv": 0.006349931005388498,
      "rewards/chosen": 0.00810670611020681,
      "rewards/margins": 0.004941528282549155,
      "rewards/rejected": 0.0031651778276576554,
      "stability/repetition_rate_mean": 0.08308292925357819,
      "stability/response_length_mean": 22.75,
      "stability/response_length_std": 31.07479476928711,
      "stability/response_length_var": 965.6428833007812,
      "stability/token_entropy_mean": 2.1484375,
      "step": 75
    },
    {
      "epoch": 0.4094689699296225,
      "grad_norm": 1.1559727191925049,
      "kl": 0.049778860062360764,
      "learning_rate": 7.9e-07,
      "logits/chosen": -58659575.52317881,
      "logits/rejected": -53490039.66863906,
      "logps/chosen": -403.96926738410593,
      "logps/rejected": -201.02965514053255,
      "loss": 0.5058,
      "loss/base_kto": 0.49956369400024414,
      "metrics/advantage_var": 0.052713096141815186,
      "regularization/advantage_range": 0.5138265490531921,
      "regularization/tv": 0.005138265434652567,
      "rewards/chosen": 0.006083738330184229,
      "rewards/margins": 0.0034679851436592566,
      "rewards/rejected": 0.0026157531865249723,
      "stability/repetition_rate_mean": 0.19283854961395264,
      "stability/response_length_mean": 42.0,
      "stability/response_length_std": 30.481843948364258,
      "stability/response_length_var": 929.1428833007812,
      "stability/token_entropy_mean": 5.26171875,
      "step": 80
    },
    {
      "epoch": 0.4350607805502239,
      "grad_norm": 1.0702959299087524,
      "kl": 0.03699667379260063,
      "learning_rate": 8.399999999999999e-07,
      "logits/chosen": -51509360.30967742,
      "logits/rejected": -47167947.24848485,
      "logps/chosen": -358.27303427419355,
      "logps/rejected": -230.4253787878788,
      "loss": 0.5055,
      "loss/base_kto": 0.49921008944511414,
      "metrics/advantage_var": 0.056530844420194626,
      "regularization/advantage_range": 0.5624624490737915,
      "regularization/tv": 0.0056246244348585606,
      "rewards/chosen": 0.005946203970140026,
      "rewards/margins": 0.006002022184701026,
      "rewards/rejected": -5.5818214561000016e-05,
      "stability/repetition_rate_mean": 0.1396019458770752,
      "stability/response_length_mean": 26.0,
      "stability/response_length_std": 28.97782325744629,
      "stability/response_length_var": 839.7142944335938,
      "stability/token_entropy_mean": 4.982421875,
      "step": 85
    },
    {
      "epoch": 0.46065259117082535,
      "grad_norm": 1.1454795598983765,
      "kl": 0.04819156974554062,
      "learning_rate": 8.9e-07,
      "logits/chosen": -58464859.897435896,
      "logits/rejected": -50616251.317073174,
      "logps/chosen": -380.70262419871796,
      "logps/rejected": -187.1805330602134,
      "loss": 0.5055,
      "loss/base_kto": 0.4994789659976959,
      "metrics/advantage_var": 0.07163967937231064,
      "regularization/advantage_range": 0.6174459457397461,
      "regularization/tv": 0.006174459587782621,
      "rewards/chosen": 0.00867634858840551,
      "rewards/margins": 0.004048479691828094,
      "rewards/rejected": 0.004627868896577417,
      "stability/repetition_rate_mean": 0.20654930174350739,
      "stability/response_length_mean": 17.375,
      "stability/response_length_std": 20.542203903198242,
      "stability/response_length_var": 421.9821472167969,
      "stability/token_entropy_mean": 4.6875,
      "step": 90
    },
    {
      "epoch": 0.48624440179142675,
      "grad_norm": 1.1122380495071411,
      "kl": 0.04422915354371071,
      "learning_rate": 9.399999999999999e-07,
      "logits/chosen": -50433743.39240506,
      "logits/rejected": -49027005.62962963,
      "logps/chosen": -314.96761273734177,
      "logps/rejected": -207.14773823302468,
      "loss": 0.5057,
      "loss/base_kto": 0.4996590316295624,
      "metrics/advantage_var": 0.06781669706106186,
      "regularization/advantage_range": 0.585126519203186,
      "regularization/tv": 0.005851265043020248,
      "rewards/chosen": 0.0025773750075811074,
      "rewards/margins": 0.002979051076652371,
      "rewards/rejected": -0.0004016760690712634,
      "stability/repetition_rate_mean": 0.19569545984268188,
      "stability/response_length_mean": 27.0,
      "stability/response_length_std": 31.1264705657959,
      "stability/response_length_var": 968.8571166992188,
      "stability/token_entropy_mean": 3.669921875,
      "step": 95
    },
    {
      "epoch": 0.5118362124120281,
      "grad_norm": 1.089025616645813,
      "kl": 0.04536796733736992,
      "learning_rate": 9.9e-07,
      "logits/chosen": -54773109.62162162,
      "logits/rejected": -50978804.093023255,
      "logps/chosen": -359.8231102195946,
      "logps/rejected": -216.5610465116279,
      "loss": 0.5055,
      "loss/base_kto": 0.49959760904312134,
      "metrics/advantage_var": 0.06900867074728012,
      "regularization/advantage_range": 0.6156648993492126,
      "regularization/tv": 0.006156648974865675,
      "rewards/chosen": 0.004704180198746758,
      "rewards/margins": 0.002917736122474814,
      "rewards/rejected": 0.0017864440762719444,
      "stability/repetition_rate_mean": 0.23591694235801697,
      "stability/response_length_mean": 38.0,
      "stability/response_length_std": 30.10457992553711,
      "stability/response_length_var": 906.2857055664062,
      "stability/token_entropy_mean": 4.140625,
      "step": 100
    },
    {
      "epoch": 0.5374280230326296,
      "grad_norm": 1.1396074295043945,
      "kl": 0.038910795003175735,
      "learning_rate": 9.999815554351224e-07,
      "logits/chosen": -59440202.024096385,
      "logits/rejected": -53880659.116883114,
      "logps/chosen": -399.8339608433735,
      "logps/rejected": -209.9449320211039,
      "loss": 0.5058,
      "loss/base_kto": 0.4999099671840668,
      "metrics/advantage_var": 0.10025227069854736,
      "regularization/advantage_range": 0.8001343011856079,
      "regularization/tv": 0.008001342415809631,
      "rewards/chosen": 0.0068486849945711805,
      "rewards/margins": 0.0009596462336304307,
      "rewards/rejected": 0.00588903876094075,
      "stability/repetition_rate_mean": 0.1702008843421936,
      "stability/response_length_mean": 28.0,
      "stability/response_length_std": 28.665559768676758,
      "stability/response_length_var": 821.7142944335938,
      "stability/token_entropy_mean": 3.41015625,
      "step": 105
    },
    {
      "epoch": 0.5630198336532309,
      "grad_norm": 1.0489435195922852,
      "kl": 0.04320025071501732,
      "learning_rate": 9.999066267225444e-07,
      "logits/chosen": -57453222.55421687,
      "logits/rejected": -53678379.22077922,
      "logps/chosen": -378.54155685240966,
      "logps/rejected": -206.06934862012986,
      "loss": 0.5056,
      "loss/base_kto": 0.49997711181640625,
      "metrics/advantage_var": 0.07415717095136642,
      "regularization/advantage_range": 0.637352466583252,
      "regularization/tv": 0.0063735246658325195,
      "rewards/chosen": 0.002172079431005271,
      "rewards/margins": 0.00029375342215903744,
      "rewards/rejected": 0.0018783260088462335,
      "stability/repetition_rate_mean": 0.30853456258773804,
      "stability/response_length_mean": 28.875,
      "stability/response_length_std": 29.911476135253906,
      "stability/response_length_var": 894.6964111328125,
      "stability/token_entropy_mean": 4.4052734375,
      "step": 110
    },
    {
      "epoch": 0.5886116442738324,
      "grad_norm": 1.0511547327041626,
      "kl": 0.04815962538123131,
      "learning_rate": 9.997740697079592e-07,
      "logits/chosen": -60699414.6835443,
      "logits/rejected": -53509745.777777776,
      "logps/chosen": -364.49673655063293,
      "logps/rejected": -201.1546947337963,
      "loss": 0.5057,
      "loss/base_kto": 0.49935370683670044,
      "metrics/advantage_var": 0.077999047935009,
      "regularization/advantage_range": 0.6492432951927185,
      "regularization/tv": 0.006492432206869125,
      "rewards/chosen": 0.00958028322533716,
      "rewards/margins": 0.005393363415477684,
      "rewards/rejected": 0.004186919809859476,
      "stability/repetition_rate_mean": 0.2701823115348816,
      "stability/response_length_mean": 41.375,
      "stability/response_length_std": 31.25442886352539,
      "stability/response_length_var": 976.8392944335938,
      "stability/token_entropy_mean": 4.880859375,
      "step": 115
    },
    {
      "epoch": 0.6142034548944337,
      "grad_norm": 1.1143871545791626,
      "kl": 0.036807831376791,
      "learning_rate": 9.995838996722914e-07,
      "logits/chosen": -57830406.69281046,
      "logits/rejected": -54497776.670658685,
      "logps/chosen": -384.1935253267974,
      "logps/rejected": -212.19879771706587,
      "loss": 0.5057,
      "loss/base_kto": 0.49962493777275085,
      "metrics/advantage_var": 0.07512163370847702,
      "regularization/advantage_range": 0.6422988772392273,
      "regularization/tv": 0.006422989070415497,
      "rewards/chosen": 0.005139833182291268,
      "rewards/margins": 0.0033822947565830225,
      "rewards/rejected": 0.0017575384257082455,
      "stability/repetition_rate_mean": 0.17961646616458893,
      "stability/response_length_mean": 29.375,
      "stability/response_length_std": 29.548688888549805,
      "stability/response_length_var": 873.125,
      "stability/token_entropy_mean": 5.30859375,
      "step": 120
    },
    {
      "epoch": 0.6397952655150352,
      "grad_norm": 1.1321357488632202,
      "kl": 0.03840151056647301,
      "learning_rate": 9.993361385379874e-07,
      "logits/chosen": -60829906.4109589,
      "logits/rejected": -55816921.74712644,
      "logps/chosen": -417.32681399828766,
      "logps/rejected": -216.3238595545977,
      "loss": 0.5058,
      "loss/base_kto": 0.49982985854148865,
      "metrics/advantage_var": 0.08392415195703506,
      "regularization/advantage_range": 0.7117211222648621,
      "regularization/tv": 0.007117211353033781,
      "rewards/chosen": 0.005247564348455978,
      "rewards/margins": 0.00091411963802044,
      "rewards/rejected": 0.004333444710435538,
      "stability/repetition_rate_mean": 0.08598090708255768,
      "stability/response_length_mean": 15.25,
      "stability/response_length_std": 24.93276596069336,
      "stability/response_length_var": 621.6428833007812,
      "stability/token_entropy_mean": 3.7578125,
      "step": 125
    },
    {
      "epoch": 0.6653870761356366,
      "grad_norm": 1.0781404972076416,
      "kl": 0.037142276763916016,
      "learning_rate": 9.99030814866488e-07,
      "logits/chosen": -57524488.258064516,
      "logits/rejected": -53994719.41818182,
      "logps/chosen": -392.6314516129032,
      "logps/rejected": -183.91096117424243,
      "loss": 0.5057,
      "loss/base_kto": 0.4997527301311493,
      "metrics/advantage_var": 0.06489270180463791,
      "regularization/advantage_range": 0.6006088852882385,
      "regularization/tv": 0.006006088573485613,
      "rewards/chosen": 0.0026500949936528358,
      "rewards/margins": 0.0018541186727503293,
      "rewards/rejected": 0.0007959763209025065,
      "stability/repetition_rate_mean": 0.18814630806446075,
      "stability/response_length_mean": 38.0,
      "stability/response_length_std": 29.13515281677246,
      "stability/response_length_var": 848.8571166992188,
      "stability/token_entropy_mean": 3.921875,
      "step": 130
    },
    {
      "epoch": 0.690978886756238,
      "grad_norm": 1.0503453016281128,
      "kl": 0.04310822859406471,
      "learning_rate": 9.986679638549358e-07,
      "logits/chosen": -60006789.496932514,
      "logits/rejected": -55216936.76433121,
      "logps/chosen": -401.7418999233129,
      "logps/rejected": -247.84676054936307,
      "loss": 0.5058,
      "loss/base_kto": 0.4993572235107422,
      "metrics/advantage_var": 0.09088700264692307,
      "regularization/advantage_range": 0.7087545394897461,
      "regularization/tv": 0.00708754500374198,
      "rewards/chosen": 0.007426799440676449,
      "rewards/margins": 0.005029037647782827,
      "rewards/rejected": 0.0023977617928936223,
      "stability/repetition_rate_mean": 0.2250434011220932,
      "stability/response_length_mean": 29.0,
      "stability/response_length_std": 30.20406723022461,
      "stability/response_length_var": 912.2857055664062,
      "stability/token_entropy_mean": 3.4453125,
      "step": 135
    },
    {
      "epoch": 0.7165706973768394,
      "grad_norm": 1.0176606178283691,
      "kl": 0.03307735547423363,
      "learning_rate": 9.982476273321175e-07,
      "logits/chosen": -57250648.43636364,
      "logits/rejected": -50280144.103225805,
      "logps/chosen": -367.0908380681818,
      "logps/rejected": -190.04115423387097,
      "loss": 0.5052,
      "loss/base_kto": 0.4986254870891571,
      "metrics/advantage_var": 0.08752826601266861,
      "regularization/advantage_range": 0.7223478555679321,
      "regularization/tv": 0.007223478052765131,
      "rewards/chosen": 0.009888766028664328,
      "rewards/margins": 0.010556670874794206,
      "rewards/rejected": -0.0006679048461298789,
      "stability/repetition_rate_mean": 0.328125,
      "stability/response_length_mean": 56.25,
      "stability/response_length_std": 21.920310974121094,
      "stability/response_length_var": 480.5,
      "stability/token_entropy_mean": 4.919921875,
      "step": 140
    },
    {
      "epoch": 0.7421625079974408,
      "grad_norm": 1.0782142877578735,
      "kl": 0.04704681783914566,
      "learning_rate": 9.977698537536417e-07,
      "logits/chosen": -51311600.578313254,
      "logits/rejected": -51882898.28571428,
      "logps/chosen": -330.21498493975906,
      "logps/rejected": -199.51471185064935,
      "loss": 0.5056,
      "loss/base_kto": 0.49895191192626953,
      "metrics/advantage_var": 0.06373737007379532,
      "regularization/advantage_range": 0.6130368113517761,
      "regularization/tv": 0.006130368448793888,
      "rewards/chosen": 0.005680616361549102,
      "rewards/margins": 0.008360414097630312,
      "rewards/rejected": -0.00267979773608121,
      "stability/repetition_rate_mean": 0.25034263730049133,
      "stability/response_length_mean": 28.75,
      "stability/response_length_std": 29.86517333984375,
      "stability/response_length_var": 891.9285888671875,
      "stability/token_entropy_mean": 5.29296875,
      "step": 145
    },
    {
      "epoch": 0.7677543186180422,
      "grad_norm": 1.090654969215393,
      "kl": 0.04447220265865326,
      "learning_rate": 9.972346981963546e-07,
      "logits/chosen": -55583340.42352941,
      "logits/rejected": -50427801.6,
      "logps/chosen": -397.8378216911765,
      "logps/rejected": -207.74522135416666,
      "loss": 0.5051,
      "loss/base_kto": 0.4984525740146637,
      "metrics/advantage_var": 0.06993703544139862,
      "regularization/advantage_range": 0.6044992804527283,
      "regularization/tv": 0.006044992245733738,
      "rewards/chosen": 0.009661375775056727,
      "rewards/margins": 0.01159093508533403,
      "rewards/rejected": -0.0019295593102773032,
      "stability/repetition_rate_mean": 0.2467447966337204,
      "stability/response_length_mean": 35.25,
      "stability/response_length_std": 31.01957893371582,
      "stability/response_length_var": 962.2142944335938,
      "stability/token_entropy_mean": 4.5859375,
      "step": 150
    },
    {
      "epoch": 0.7933461292386437,
      "grad_norm": 1.003922700881958,
      "kl": 0.03585250303149223,
      "learning_rate": 9.966422223519885e-07,
      "logits/chosen": -54226715.105882354,
      "logits/rejected": -49573730.986666664,
      "logps/chosen": -360.94910386029414,
      "logps/rejected": -191.28990885416667,
      "loss": 0.5056,
      "loss/base_kto": 0.499664306640625,
      "metrics/advantage_var": 0.07342761009931564,
      "regularization/advantage_range": 0.6386498808860779,
      "regularization/tv": 0.006386497523635626,
      "rewards/chosen": 0.0040183291715734145,
      "rewards/margins": 0.0024476325395060517,
      "rewards/rejected": 0.0015706966320673625,
      "stability/repetition_rate_mean": 0.3152253031730652,
      "stability/response_length_mean": 61.375,
      "stability/response_length_std": 7.424621105194092,
      "stability/response_length_var": 55.125,
      "stability/token_entropy_mean": 4.56640625,
      "step": 155
    },
    {
      "epoch": 0.818937939859245,
      "grad_norm": 0.9930980801582336,
      "kl": 0.04733538627624512,
      "learning_rate": 9.959924945200523e-07,
      "logits/chosen": -55488585.14285714,
      "logits/rejected": -49255141.05263158,
      "logps/chosen": -416.8314267113095,
      "logps/rejected": -186.89907997532896,
      "loss": 0.5062,
      "loss/base_kto": 0.4992094933986664,
      "metrics/advantage_var": 0.09384613484144211,
      "regularization/advantage_range": 0.7354105114936829,
      "regularization/tv": 0.007354104425758123,
      "rewards/chosen": 0.006389315639223371,
      "rewards/margins": 0.006562433261564351,
      "rewards/rejected": -0.00017311762234098032,
      "stability/repetition_rate_mean": 0.205078125,
      "stability/response_length_mean": 26.25,
      "stability/response_length_std": 31.535694122314453,
      "stability/response_length_var": 994.5,
      "stability/token_entropy_mean": 3.8994140625,
      "step": 160
    },
    {
      "epoch": 0.8445297504798465,
      "grad_norm": 1.0706591606140137,
      "kl": 0.039895690977573395,
      "learning_rate": 9.952855895999568e-07,
      "logits/chosen": -54670759.72413793,
      "logits/rejected": -50123060.60273973,
      "logps/chosen": -370.83548850574715,
      "logps/rejected": -212.59262628424656,
      "loss": 0.5055,
      "loss/base_kto": 0.4989406168460846,
      "metrics/advantage_var": 0.06013846397399902,
      "regularization/advantage_range": 0.5651509165763855,
      "regularization/tv": 0.005651508923619986,
      "rewards/chosen": 0.008127493419866452,
      "rewards/margins": 0.008178862493935108,
      "rewards/rejected": -5.136907406865734e-05,
      "stability/repetition_rate_mean": 0.20468750596046448,
      "stability/response_length_mean": 12.625,
      "stability/response_length_std": 21.460512161254883,
      "stability/response_length_var": 460.5535583496094,
      "stability/token_entropy_mean": 4.80078125,
      "step": 165
    },
    {
      "epoch": 0.8701215611004478,
      "grad_norm": 1.0094552040100098,
      "kl": 0.06539209932088852,
      "learning_rate": 9.9452158908238e-07,
      "logits/chosen": -59408686.95857988,
      "logits/rejected": -52972838.993377484,
      "logps/chosen": -441.2704326923077,
      "logps/rejected": -191.2410492549669,
      "loss": 0.5052,
      "loss/base_kto": 0.499387264251709,
      "metrics/advantage_var": 0.07106225937604904,
      "regularization/advantage_range": 0.6181744337081909,
      "regularization/tv": 0.006181744392961264,
      "rewards/chosen": 0.007143582112690401,
      "rewards/margins": 0.005464405191603806,
      "rewards/rejected": 0.0016791769210865956,
      "stability/repetition_rate_mean": 0.2809279263019562,
      "stability/response_length_mean": 28.25,
      "stability/response_length_std": 30.844078063964844,
      "stability/response_length_var": 951.3571166992188,
      "stability/token_entropy_mean": 3.12353515625,
      "step": 170
    },
    {
      "epoch": 0.8957133717210493,
      "grad_norm": 1.0981041193008423,
      "kl": 0.06210724264383316,
      "learning_rate": 9.937005810398745e-07,
      "logits/chosen": -56948748.3373494,
      "logits/rejected": -48448538.597402595,
      "logps/chosen": -356.33819653614455,
      "logps/rejected": -194.78391335227272,
      "loss": 0.5054,
      "loss/base_kto": 0.4990459382534027,
      "metrics/advantage_var": 0.08322405070066452,
      "regularization/advantage_range": 0.7160508632659912,
      "regularization/tv": 0.007160508539527655,
      "rewards/chosen": 0.009901675115148705,
      "rewards/margins": 0.007540365315867672,
      "rewards/rejected": 0.0023613097992810335,
      "stability/repetition_rate_mean": 0.2936941981315613,
      "stability/response_length_mean": 32.25,
      "stability/response_length_std": 27.932315826416016,
      "stability/response_length_var": 780.2142944335938,
      "stability/token_entropy_mean": 4.6943359375,
      "step": 175
    },
    {
      "epoch": 0.9213051823416507,
      "grad_norm": 1.1260960102081299,
      "kl": 0.06741955131292343,
      "learning_rate": 9.928226601167138e-07,
      "logits/chosen": -52939769.71779141,
      "logits/rejected": -48362903.643312104,
      "logps/chosen": -376.04083588957053,
      "logps/rejected": -211.24893013535032,
      "loss": 0.5055,
      "loss/base_kto": 0.49932271242141724,
      "metrics/advantage_var": 0.07661912590265274,
      "regularization/advantage_range": 0.6519152522087097,
      "regularization/tv": 0.006519151385873556,
      "rewards/chosen": 0.010499875238336669,
      "rewards/margins": 0.005572689548009614,
      "rewards/rejected": 0.004927185690327055,
      "stability/repetition_rate_mean": 0.16796875,
      "stability/response_length_mean": 40.375,
      "stability/response_length_std": 32.61874008178711,
      "stability/response_length_var": 1063.982177734375,
      "stability/token_entropy_mean": 3.35546875,
      "step": 180
    },
    {
      "epoch": 0.946896992962252,
      "grad_norm": 1.127066731452942,
      "kl": 0.0625443160533905,
      "learning_rate": 9.918879275179817e-07,
      "logits/chosen": -54950197.72839506,
      "logits/rejected": -49892630.6835443,
      "logps/chosen": -311.92269483024694,
      "logps/rejected": -197.00294204905063,
      "loss": 0.5051,
      "loss/base_kto": 0.4994753897190094,
      "metrics/advantage_var": 0.06317470967769623,
      "regularization/advantage_range": 0.5881640315055847,
      "regularization/tv": 0.0058816405944526196,
      "rewards/chosen": 0.007868100095678258,
      "rewards/margins": 0.004246503771385339,
      "rewards/rejected": 0.0036215963242929193,
      "stability/repetition_rate_mean": 0.20791903138160706,
      "stability/response_length_mean": 26.25,
      "stability/response_length_std": 31.45404815673828,
      "stability/response_length_var": 989.3571166992188,
      "stability/token_entropy_mean": 3.822265625,
      "step": 185
    },
    {
      "epoch": 0.9724888035828535,
      "grad_norm": 1.0826705694198608,
      "kl": 0.06414353102445602,
      "learning_rate": 9.90896490997906e-07,
      "logits/chosen": -53417732.598802395,
      "logits/rejected": -50093725.28104575,
      "logps/chosen": -392.3040793413174,
      "logps/rejected": -211.1696027369281,
      "loss": 0.5055,
      "loss/base_kto": 0.4990668296813965,
      "metrics/advantage_var": 0.0792350098490715,
      "regularization/advantage_range": 0.6623881459236145,
      "regularization/tv": 0.006623882334679365,
      "rewards/chosen": 0.010125834070993755,
      "rewards/margins": 0.007227105145575524,
      "rewards/rejected": 0.0028987289254182305,
      "stability/repetition_rate_mean": 0.1258285939693451,
      "stability/response_length_mean": 12.375,
      "stability/response_length_std": 22.057958602905273,
      "stability/response_length_var": 486.5535583496094,
      "stability/token_entropy_mean": 5.375,
      "step": 190
    },
    {
      "epoch": 0.9980806142034548,
      "grad_norm": 1.1859703063964844,
      "kl": 0.06095733866095543,
      "learning_rate": 9.898484648474362e-07,
      "logits/chosen": -60520414.96774194,
      "logits/rejected": -54298518.4969697,
      "logps/chosen": -392.6244455645161,
      "logps/rejected": -197.9101799242424,
      "loss": 0.5054,
      "loss/base_kto": 0.4981659948825836,
      "metrics/advantage_var": 0.0902789831161499,
      "regularization/advantage_range": 0.683096706867218,
      "regularization/tv": 0.0068309674970805645,
      "rewards/chosen": 0.016048112992317447,
      "rewards/margins": 0.01513210601179714,
      "rewards/rejected": 0.0009160069805203062,
      "stability/repetition_rate_mean": 0.412109375,
      "stability/response_length_mean": 48.125,
      "stability/response_length_std": 29.39600372314453,
      "stability/response_length_var": 864.125,
      "stability/token_entropy_mean": 4.4453125,
      "step": 195
    },
    {
      "epoch": 1.0,
      "step": 196,
      "total_flos": 3.1861866257511875e+18,
      "train_loss": 0.5056634673050472,
      "train_runtime": 3151.7479,
      "train_samples_per_second": 31.728,
      "train_steps_per_second": 0.062
    }
  ],
  "logging_steps": 5,
  "max_steps": 196,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 200,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 3.1861866257511875e+18,
  "train_batch_size": 8,
  "trial_name": null,
  "trial_params": null
}