{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.0,
  "eval_steps": 500,
  "global_step": 196,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.025591810620601407,
      "grad_norm": 0.764057993888855,
      "kl": 0.007318343501538038,
      "learning_rate": 4e-08,
      "logits/chosen": -59712633.2631579,
      "logits/rejected": -50420315.428571425,
      "logps/chosen": -438.9966591282895,
      "logps/rejected": -198.41592261904762,
      "loss": 0.5016,
      "loss/base_kto": 0.499319463968277,
      "metrics/advantage_var": 0.04550802707672119,
      "regularization/advantage_range": 0.4100130498409271,
      "regularization/tv": 0.002050065202638507,
      "rewards/chosen": 0.004893930334793894,
      "rewards/margins": 0.005545394334073802,
      "rewards/rejected": -0.0006514639992799078,
      "step": 5
    },
    {
      "epoch": 0.05118362124120281,
      "grad_norm": 0.8658815026283264,
      "kl": 0.02441932074725628,
      "learning_rate": 9e-08,
      "logits/chosen": -59297033.97402597,
      "logits/rejected": -50017699.469879515,
      "logps/chosen": -429.97412743506493,
      "logps/rejected": -192.85963384789156,
      "loss": 0.503,
      "loss/base_kto": 0.5005311965942383,
      "metrics/advantage_var": 0.06798487901687622,
      "regularization/advantage_range": 0.6393987536430359,
      "regularization/tv": 0.0031969938427209854,
      "rewards/chosen": -0.003558432901060426,
      "rewards/margins": -0.004388217616810557,
      "rewards/rejected": 0.0008297847157501312,
      "stability/repetition_rate_mean": 0.3707534670829773,
      "stability/response_length_mean": 46.125,
      "stability/response_length_std": 27.231481552124023,
      "stability/response_length_var": 741.5535888671875,
      "stability/token_entropy_mean": 3.25927734375,
      "step": 10
    },
    {
      "epoch": 0.07677543186180422,
      "grad_norm": 0.7427634000778198,
      "kl": 0.01599900797009468,
      "learning_rate": 1.4e-07,
      "logits/chosen": -56240252.121212125,
      "logits/rejected": -51923195.04516129,
      "logps/chosen": -404.1088068181818,
      "logps/rejected": -200.65425907258066,
      "loss": 0.5028,
      "loss/base_kto": 0.50014728307724,
      "metrics/advantage_var": 0.06211945042014122,
      "regularization/advantage_range": 0.574948787689209,
      "regularization/tv": 0.002874743891879916,
      "rewards/chosen": 0.0022001774022073458,
      "rewards/margins": -0.0017795889666586207,
      "rewards/rejected": 0.0039797663688659664,
      "stability/repetition_rate_mean": 0.2651909589767456,
      "stability/response_length_mean": 27.125,
      "stability/response_length_std": 30.79163932800293,
      "stability/response_length_var": 948.125,
      "stability/token_entropy_mean": 3.8212890625,
      "step": 15
    },
    {
      "epoch": 0.10236724248240563,
      "grad_norm": 0.8416574001312256,
      "kl": 0.017363430932164192,
      "learning_rate": 1.8999999999999998e-07,
      "logits/chosen": -51677280.0,
      "logits/rejected": -48070777.6,
      "logps/chosen": -343.6998291015625,
      "logps/rejected": -225.650537109375,
      "loss": 0.5035,
      "loss/base_kto": 0.500237226486206,
      "metrics/advantage_var": 0.07353738695383072,
      "regularization/advantage_range": 0.6486077308654785,
      "regularization/tv": 0.003243038197979331,
      "rewards/chosen": -0.0003127170260995626,
      "rewards/margins": -0.0015936737414449452,
      "rewards/rejected": 0.0012809567153453827,
      "stability/repetition_rate_mean": 0.2040860652923584,
      "stability/response_length_mean": 42.125,
      "stability/response_length_std": 27.01553726196289,
      "stability/response_length_var": 729.8392944335938,
      "stability/token_entropy_mean": 4.455078125,
      "step": 20
    },
    {
      "epoch": 0.12795905310300704,
      "grad_norm": 0.7867552638053894,
      "kl": 0.01633894257247448,
      "learning_rate": 2.4e-07,
      "logits/chosen": -56472922.04137931,
      "logits/rejected": -52007742.90285714,
      "logps/chosen": -378.4763469827586,
      "logps/rejected": -202.78308035714286,
      "loss": 0.5031,
      "loss/base_kto": 0.499965101480484,
      "metrics/advantage_var": 0.05515791103243828,
      "regularization/advantage_range": 0.5312573313713074,
      "regularization/tv": 0.0026562868151813745,
      "rewards/chosen": 0.0015089573531315245,
      "rewards/margins": 0.0005325866949382087,
      "rewards/rejected": 0.0009763706581933158,
      "stability/repetition_rate_mean": 0.17997129261493683,
      "stability/response_length_mean": 43.375,
      "stability/response_length_std": 28.54538917541504,
      "stability/response_length_var": 814.8392944335938,
      "stability/token_entropy_mean": 4.78125,
      "step": 25
    },
    {
      "epoch": 0.15355086372360843,
      "grad_norm": 0.7133483290672302,
      "kl": 0.019256237894296646,
      "learning_rate": 2.9e-07,
      "logits/chosen": -57728900.29530201,
      "logits/rejected": -51520781.473684214,
      "logps/chosen": -468.5885591442953,
      "logps/rejected": -198.25054824561403,
      "loss": 0.5032,
      "loss/base_kto": 0.5000054240226746,
      "metrics/advantage_var": 0.06904762983322144,
      "regularization/advantage_range": 0.5958181619644165,
      "regularization/tv": 0.0029790906701236963,
      "rewards/chosen": -0.0007292804301985158,
      "rewards/margins": -0.00046352471233261096,
      "rewards/rejected": -0.00026575571786590486,
      "stability/repetition_rate_mean": 0.1564360111951828,
      "stability/response_length_mean": 43.0,
      "stability/response_length_std": 29.602123260498047,
      "stability/response_length_var": 876.2857055664062,
      "stability/token_entropy_mean": 4.646484375,
      "step": 30
    },
    {
      "epoch": 0.17914267434420986,
      "grad_norm": 0.811491847038269,
      "kl": 0.01968337967991829,
      "learning_rate": 3.4000000000000003e-07,
      "logits/chosen": -57647920.60759494,
      "logits/rejected": -53901805.03703704,
      "logps/chosen": -385.90234375,
      "logps/rejected": -195.12280574845678,
      "loss": 0.503,
      "loss/base_kto": 0.49970123171806335,
      "metrics/advantage_var": 0.06940528005361557,
      "regularization/advantage_range": 0.6135316491127014,
      "regularization/tv": 0.0030676580499857664,
      "rewards/chosen": 0.0034183078928838803,
      "rewards/margins": 0.003051768577942757,
      "rewards/rejected": 0.00036653931494112367,
      "stability/repetition_rate_mean": 0.3264508843421936,
      "stability/response_length_mean": 56.875,
      "stability/response_length_std": 20.152544021606445,
      "stability/response_length_var": 406.125,
      "stability/token_entropy_mean": 5.412109375,
      "step": 35
    },
    {
      "epoch": 0.20473448496481125,
      "grad_norm": 0.7757749557495117,
      "kl": 0.02234978787600994,
      "learning_rate": 3.8999999999999997e-07,
      "logits/chosen": -56634176.0,
      "logits/rejected": -50232592.0,
      "logps/chosen": -421.89375,
      "logps/rejected": -219.286474609375,
      "loss": 0.503,
      "loss/base_kto": 0.5002043843269348,
      "metrics/advantage_var": 0.07796385139226913,
      "regularization/advantage_range": 0.6802285313606262,
      "regularization/tv": 0.0034011423122137785,
      "rewards/chosen": 0.0008893679827451706,
      "rewards/margins": -0.001387944445014,
      "rewards/rejected": 0.0022773124277591706,
      "stability/repetition_rate_mean": 0.16303454339504242,
      "stability/response_length_mean": 35.0,
      "stability/response_length_std": 31.518701553344727,
      "stability/response_length_var": 993.4285888671875,
      "stability/token_entropy_mean": 4.8037109375,
      "step": 40
    },
    {
      "epoch": 0.23032629558541268,
      "grad_norm": 0.8078201413154602,
      "kl": 0.0388161800801754,
      "learning_rate": 4.3999999999999997e-07,
      "logits/chosen": -52696600.053691275,
      "logits/rejected": -50773459.0877193,
      "logps/chosen": -352.2889576342282,
      "logps/rejected": -243.84699378654972,
      "loss": 0.503,
      "loss/base_kto": 0.499786376953125,
      "metrics/advantage_var": 0.07753210514783859,
      "regularization/advantage_range": 0.659938633441925,
      "regularization/tv": 0.0032996931113302708,
      "rewards/chosen": 0.003950149420923835,
      "rewards/margins": 0.0012341703662664956,
      "rewards/rejected": 0.0027159790546573393,
      "stability/repetition_rate_mean": 0.0935424193739891,
      "stability/response_length_mean": 28.875,
      "stability/response_length_std": 29.87324333190918,
      "stability/response_length_var": 892.4107055664062,
      "stability/token_entropy_mean": 4.767578125,
      "step": 45
    },
    {
      "epoch": 0.2559181062060141,
      "grad_norm": 0.7449148297309875,
      "kl": 0.045445796102285385,
      "learning_rate": 4.9e-07,
      "logits/chosen": -54023140.51006711,
      "logits/rejected": -48295337.169590645,
      "logps/chosen": -375.1488307466443,
      "logps/rejected": -180.20160590277777,
      "loss": 0.5027,
      "loss/base_kto": 0.4997333586215973,
      "metrics/advantage_var": 0.05920093134045601,
      "regularization/advantage_range": 0.5474037528038025,
      "regularization/tv": 0.0027370189782232046,
      "rewards/chosen": 0.005029791153517345,
      "rewards/margins": 0.0011288783834635485,
      "rewards/rejected": 0.0039009127700537966,
      "stability/repetition_rate_mean": 0.3366231620311737,
      "stability/response_length_mean": 55.375,
      "stability/response_length_std": 16.361431121826172,
      "stability/response_length_var": 267.6964416503906,
      "stability/token_entropy_mean": 4.048828125,
      "step": 50
    },
    {
      "epoch": 0.28150991682661547,
      "grad_norm": 0.8423920273780823,
      "kl": 0.046494752168655396,
      "learning_rate": 5.4e-07,
      "logits/chosen": -56944411.0310559,
      "logits/rejected": -49572194.213836476,
      "logps/chosen": -344.4786733307453,
      "logps/rejected": -230.38880699685535,
      "loss": 0.5029,
      "loss/base_kto": 0.5002700686454773,
      "metrics/advantage_var": 0.06208331137895584,
      "regularization/advantage_range": 0.5707374811172485,
      "regularization/tv": 0.002853687386959791,
      "rewards/chosen": 0.003334273832925358,
      "rewards/margins": -0.002587133482790696,
      "rewards/rejected": 0.005921407315716054,
      "stability/repetition_rate_mean": 0.123046875,
      "stability/response_length_mean": 24.5,
      "stability/response_length_std": 32.710853576660156,
      "stability/response_length_var": 1070.0,
      "stability/token_entropy_mean": 4.369140625,
      "step": 55
    },
    {
      "epoch": 0.30710172744721687,
      "grad_norm": 0.7453679442405701,
      "kl": 0.03794847056269646,
      "learning_rate": 5.9e-07,
      "logits/chosen": -52685048.24242424,
      "logits/rejected": -47840071.019354835,
      "logps/chosen": -359.64857954545454,
      "logps/rejected": -178.18482862903227,
      "loss": 0.5031,
      "loss/base_kto": 0.4995456337928772,
      "metrics/advantage_var": 0.07329487800598145,
      "regularization/advantage_range": 0.6358262300491333,
      "regularization/tv": 0.003179130842909217,
      "rewards/chosen": 0.00722200075785319,
      "rewards/margins": 0.003012931475075342,
      "rewards/rejected": 0.004209069282777848,
      "stability/repetition_rate_mean": 0.20424756407737732,
      "stability/response_length_mean": 52.625,
      "stability/response_length_std": 21.75800323486328,
      "stability/response_length_var": 473.41070556640625,
      "stability/token_entropy_mean": 5.306640625,
      "step": 60
    },
    {
      "epoch": 0.3326935380678183,
      "grad_norm": 0.7001720666885376,
      "kl": 0.050350647419691086,
      "learning_rate": 6.4e-07,
      "logits/chosen": -57567350.91612903,
      "logits/rejected": -47286489.21212121,
      "logps/chosen": -381.86129032258066,
      "logps/rejected": -183.51529356060607,
      "loss": 0.5026,
      "loss/base_kto": 0.49978089332580566,
      "metrics/advantage_var": 0.059636857360601425,
      "regularization/advantage_range": 0.5542006492614746,
      "regularization/tv": 0.0027710029389709234,
      "rewards/chosen": 0.008551975988572644,
      "rewards/margins": 0.0018449087879408377,
      "rewards/rejected": 0.006707067200631806,
      "stability/repetition_rate_mean": 0.18097305297851562,
      "stability/response_length_mean": 17.875,
      "stability/response_length_std": 23.485177993774414,
      "stability/response_length_var": 551.5535888671875,
      "stability/token_entropy_mean": 5.6640625,
      "step": 65
    },
    {
      "epoch": 0.3582853486884197,
      "grad_norm": 0.6989223957061768,
      "kl": 0.060984399169683456,
      "learning_rate": 6.9e-07,
      "logits/chosen": -52774723.716129035,
      "logits/rejected": -49045562.95757576,
      "logps/chosen": -379.0774193548387,
      "logps/rejected": -220.37597064393938,
      "loss": 0.5025,
      "loss/base_kto": 0.49995157122612,
      "metrics/advantage_var": 0.06866636127233505,
      "regularization/advantage_range": 0.628393828868866,
      "regularization/tv": 0.00314196920953691,
      "rewards/chosen": 0.005101763817571825,
      "rewards/margins": 0.0004828384195255048,
      "rewards/rejected": 0.00461892539804632,
      "stability/repetition_rate_mean": 0.19140625,
      "stability/response_length_mean": 33.875,
      "stability/response_length_std": 32.36151885986328,
      "stability/response_length_var": 1047.267822265625,
      "stability/token_entropy_mean": 5.82421875,
      "step": 70
    },
    {
      "epoch": 0.3838771593090211,
      "grad_norm": 0.7225419878959656,
      "kl": 0.054728973656892776,
      "learning_rate": 7.4e-07,
      "logits/chosen": -49520840.64864865,
      "logits/rejected": -47904303.62790698,
      "logps/chosen": -314.1862331081081,
      "logps/rejected": -228.84690679505815,
      "loss": 0.5028,
      "loss/base_kto": 0.49936389923095703,
      "metrics/advantage_var": 0.07274273782968521,
      "regularization/advantage_range": 0.6523584127426147,
      "regularization/tv": 0.0032617920078337193,
      "rewards/chosen": 0.008458393651085932,
      "rewards/margins": 0.004663693397244442,
      "rewards/rejected": 0.003794700253841489,
      "stability/repetition_rate_mean": 0.08308292925357819,
      "stability/response_length_mean": 22.75,
      "stability/response_length_std": 31.07479476928711,
      "stability/response_length_var": 965.6428833007812,
      "stability/token_entropy_mean": 2.15234375,
      "step": 75
    },
    {
      "epoch": 0.4094689699296225,
      "grad_norm": 0.6908777952194214,
      "kl": 0.07101626694202423,
      "learning_rate": 7.9e-07,
      "logits/chosen": -58633704.26490066,
      "logits/rejected": -53482677.77514793,
      "logps/chosen": -403.94557119205297,
      "logps/rejected": -201.00848280325442,
      "loss": 0.5027,
      "loss/base_kto": 0.49961891770362854,
      "metrics/advantage_var": 0.07172282040119171,
      "regularization/advantage_range": 0.6103575825691223,
      "regularization/tv": 0.0030517878476530313,
      "rewards/chosen": 0.008451506791525329,
      "rewards/margins": 0.003719542008675052,
      "rewards/rejected": 0.004731964782850277,
      "stability/repetition_rate_mean": 0.20260417461395264,
      "stability/response_length_mean": 42.0,
      "stability/response_length_std": 30.481843948364258,
      "stability/response_length_var": 929.1428833007812,
      "stability/token_entropy_mean": 5.19140625,
      "step": 80
    },
    {
      "epoch": 0.4350607805502239,
      "grad_norm": 0.6996161341667175,
      "kl": 0.05506693944334984,
      "learning_rate": 8.399999999999999e-07,
      "logits/chosen": -51499308.59354839,
      "logits/rejected": -47171608.82424243,
      "logps/chosen": -358.26514616935486,
      "logps/rejected": -230.41628787878787,
      "loss": 0.5025,
      "loss/base_kto": 0.4992435574531555,
      "metrics/advantage_var": 0.06620263308286667,
      "regularization/advantage_range": 0.586564302444458,
      "regularization/tv": 0.002932821400463581,
      "rewards/chosen": 0.00673757137790803,
      "rewards/margins": 0.005884982836328061,
      "rewards/rejected": 0.000852588541579969,
      "stability/repetition_rate_mean": 0.115234375,
      "stability/response_length_mean": 32.5,
      "stability/response_length_std": 33.67915725708008,
      "stability/response_length_var": 1134.2857666015625,
      "stability/token_entropy_mean": 4.892578125,
      "step": 85
    },
    {
      "epoch": 0.46065259117082535,
      "grad_norm": 0.7025343775749207,
      "kl": 0.08351220190525055,
      "learning_rate": 8.9e-07,
      "logits/chosen": -58432590.76923077,
      "logits/rejected": -50594678.63414634,
      "logps/chosen": -380.6888020833333,
      "logps/rejected": -187.20376810213415,
      "loss": 0.5023,
      "loss/base_kto": 0.4989584982395172,
      "metrics/advantage_var": 0.0777924656867981,
      "regularization/advantage_range": 0.6482706069946289,
      "regularization/tv": 0.0032413527369499207,
      "rewards/chosen": 0.010058989891639123,
      "rewards/margins": 0.0077552105847040215,
      "rewards/rejected": 0.002303779306935101,
      "stability/repetition_rate_mean": 0.2608901560306549,
      "stability/response_length_mean": 28.625,
      "stability/response_length_std": 30.042530059814453,
      "stability/response_length_var": 902.5535888671875,
      "stability/token_entropy_mean": 4.2890625,
      "step": 90
    },
    {
      "epoch": 0.48624440179142675,
      "grad_norm": 0.7479750514030457,
      "kl": 0.06547798216342926,
      "learning_rate": 9.399999999999999e-07,
      "logits/chosen": -50417741.7721519,
      "logits/rejected": -49019410.96296296,
      "logps/chosen": -314.919551028481,
      "logps/rejected": -207.13025655864197,
      "loss": 0.5026,
      "loss/base_kto": 0.4992534816265106,
      "metrics/advantage_var": 0.07280212640762329,
      "regularization/advantage_range": 0.623410701751709,
      "regularization/tv": 0.0031170533038675785,
      "rewards/chosen": 0.0073801558228987684,
      "rewards/margins": 0.006031447308084745,
      "rewards/rejected": 0.0013487085148140236,
      "stability/repetition_rate_mean": 0.19374233484268188,
      "stability/response_length_mean": 27.0,
      "stability/response_length_std": 31.1264705657959,
      "stability/response_length_var": 968.8571166992188,
      "stability/token_entropy_mean": 3.75390625,
      "step": 95
    },
    {
      "epoch": 0.5118362124120281,
      "grad_norm": 0.7327731251716614,
      "kl": 0.06663308292627335,
      "learning_rate": 9.9e-07,
      "logits/chosen": -54740666.81081081,
      "logits/rejected": -50959592.18604651,
      "logps/chosen": -359.78283361486484,
      "logps/rejected": -216.5646121002907,
      "loss": 0.5025,
      "loss/base_kto": 0.49907180666923523,
      "metrics/advantage_var": 0.07108690589666367,
      "regularization/advantage_range": 0.6231733560562134,
      "regularization/tv": 0.0031158665660768747,
      "rewards/chosen": 0.008731751828580289,
      "rewards/margins": 0.00730434040194184,
      "rewards/rejected": 0.001427411426638448,
      "stability/repetition_rate_mean": 0.23591694235801697,
      "stability/response_length_mean": 38.0,
      "stability/response_length_std": 30.10457992553711,
      "stability/response_length_var": 906.2857055664062,
      "stability/token_entropy_mean": 4.123046875,
      "step": 100
    },
    {
      "epoch": 0.5374280230326296,
      "grad_norm": 0.7179022431373596,
      "kl": 0.06615767627954483,
      "learning_rate": 9.999815554351224e-07,
      "logits/chosen": -59415317.590361446,
      "logits/rejected": -53838402.49350649,
      "logps/chosen": -399.7782379518072,
      "logps/rejected": -209.96453936688312,
      "loss": 0.5024,
      "loss/base_kto": 0.49892622232437134,
      "metrics/advantage_var": 0.08854730427265167,
      "regularization/advantage_range": 0.7208102345466614,
      "regularization/tv": 0.003604050725698471,
      "rewards/chosen": 0.012420005108936724,
      "rewards/margins": 0.008493082747364655,
      "rewards/rejected": 0.003926922361572068,
      "stability/repetition_rate_mean": 0.1858258843421936,
      "stability/response_length_mean": 29.0,
      "stability/response_length_std": 29.89505386352539,
      "stability/response_length_var": 893.7142944335938,
      "stability/token_entropy_mean": 3.375,
      "step": 105
    },
    {
      "epoch": 0.5630198336532309,
      "grad_norm": 0.7244800925254822,
      "kl": 0.06936220079660416,
      "learning_rate": 9.999066267225444e-07,
      "logits/chosen": -57426684.915662654,
      "logits/rejected": -53662198.02597403,
      "logps/chosen": -378.4852692018072,
      "logps/rejected": -206.07779525162337,
      "loss": 0.5022,
      "loss/base_kto": 0.49917060136795044,
      "metrics/advantage_var": 0.07539087533950806,
      "regularization/advantage_range": 0.661239743232727,
      "regularization/tv": 0.0033061986323446035,
      "rewards/chosen": 0.007796992738562894,
      "rewards/margins": 0.006763640752123697,
      "rewards/rejected": 0.001033351986439197,
      "stability/repetition_rate_mean": 0.3186553120613098,
      "stability/response_length_mean": 31.125,
      "stability/response_length_std": 28.412460327148438,
      "stability/response_length_var": 807.2678833007812,
      "stability/token_entropy_mean": 4.0693359375,
      "step": 110
    },
    {
      "epoch": 0.5886116442738324,
      "grad_norm": 0.7003039717674255,
      "kl": 0.07836370915174484,
      "learning_rate": 9.997740697079592e-07,
      "logits/chosen": -60664832.0,
      "logits/rejected": -53468248.493827164,
      "logps/chosen": -364.47947982594934,
      "logps/rejected": -201.1459297839506,
      "loss": 0.5022,
      "loss/base_kto": 0.49925246834754944,
      "metrics/advantage_var": 0.06936754286289215,
      "regularization/advantage_range": 0.6166824698448181,
      "regularization/tv": 0.003083412069827318,
      "rewards/chosen": 0.011307053173644633,
      "rewards/margins": 0.006244304921035003,
      "rewards/rejected": 0.00506274825260963,
      "stability/repetition_rate_mean": 0.2916666865348816,
      "stability/response_length_mean": 41.375,
      "stability/response_length_std": 31.25442886352539,
      "stability/response_length_var": 976.8392944335938,
      "stability/token_entropy_mean": 4.95703125,
      "step": 115
    },
    {
      "epoch": 0.6142034548944337,
      "grad_norm": 0.7075148820877075,
      "kl": 0.057178642600774765,
      "learning_rate": 9.995838996722914e-07,
      "logits/chosen": -57793315.1372549,
      "logits/rejected": -54451776.38323353,
      "logps/chosen": -384.1260212418301,
      "logps/rejected": -212.2142122005988,
      "loss": 0.5023,
      "loss/base_kto": 0.4985807538032532,
      "metrics/advantage_var": 0.09316372126340866,
      "regularization/advantage_range": 0.7302612662315369,
      "regularization/tv": 0.0036513060331344604,
      "rewards/chosen": 0.011887521525613622,
      "rewards/margins": 0.011674204398685646,
      "rewards/rejected": 0.0002133171269279754,
      "stability/repetition_rate_mean": 0.16154514253139496,
      "stability/response_length_mean": 27.625,
      "stability/response_length_std": 30.486238479614258,
      "stability/response_length_var": 929.4107055664062,
      "stability/token_entropy_mean": 4.67578125,
      "step": 120
    },
    {
      "epoch": 0.6397952655150352,
      "grad_norm": 0.7155240774154663,
      "kl": 0.07378504425287247,
      "learning_rate": 9.993361385379874e-07,
      "logits/chosen": -60778180.38356164,
      "logits/rejected": -55767816.827586204,
      "logps/chosen": -417.27097602739724,
      "logps/rejected": -216.31503681752875,
      "loss": 0.5025,
      "loss/base_kto": 0.4993002116680145,
      "metrics/advantage_var": 0.09819125384092331,
      "regularization/advantage_range": 0.771293580532074,
      "regularization/tv": 0.003856467781588435,
      "rewards/chosen": 0.010833015180613897,
      "rewards/margins": 0.005618527838985219,
      "rewards/rejected": 0.005214487341628678,
      "stability/repetition_rate_mean": 0.16215278208255768,
      "stability/response_length_mean": 15.25,
      "stability/response_length_std": 24.93276596069336,
      "stability/response_length_var": 621.6428833007812,
      "stability/token_entropy_mean": 3.63671875,
      "step": 125
    },
    {
      "epoch": 0.6653870761356366,
      "grad_norm": 0.6972976326942444,
      "kl": 0.07307125627994537,
      "learning_rate": 9.99030814866488e-07,
      "logits/chosen": -57474946.477419354,
      "logits/rejected": -53936748.60606061,
      "logps/chosen": -392.5390625,
      "logps/rejected": -183.91380208333334,
      "loss": 0.5024,
      "loss/base_kto": 0.49857017397880554,
      "metrics/advantage_var": 0.1106613278388977,
      "regularization/advantage_range": 0.8060771822929382,
      "regularization/tv": 0.00403038552030921,
      "rewards/chosen": 0.01188800488748858,
      "rewards/margins": 0.011378020909000352,
      "rewards/rejected": 0.0005099839784882286,
      "stability/repetition_rate_mean": 0.20767755806446075,
      "stability/response_length_mean": 38.0,
      "stability/response_length_std": 29.13515281677246,
      "stability/response_length_var": 848.8571166992188,
      "stability/token_entropy_mean": 4.00390625,
      "step": 130
    },
    {
      "epoch": 0.690978886756238,
      "grad_norm": 0.6732994318008423,
      "kl": 0.08451961725950241,
      "learning_rate": 9.986679638549358e-07,
      "logits/chosen": -59954722.55214724,
      "logits/rejected": -55130353.32484076,
      "logps/chosen": -401.636071702454,
      "logps/rejected": -247.8638535031847,
      "loss": 0.5022,
      "loss/base_kto": 0.49775609374046326,
      "metrics/advantage_var": 0.12270352989435196,
      "regularization/advantage_range": 0.8722677230834961,
      "regularization/tv": 0.004361338447779417,
      "rewards/chosen": 0.018009210656757004,
      "rewards/margins": 0.017320391667471007,
      "rewards/rejected": 0.0006888189892859975,
      "stability/repetition_rate_mean": 0.22056502103805542,
      "stability/response_length_mean": 29.75,
      "stability/response_length_std": 30.221799850463867,
      "stability/response_length_var": 913.3571166992188,
      "stability/token_entropy_mean": 3.40234375,
      "step": 135
    },
    {
      "epoch": 0.7165706973768394,
      "grad_norm": 0.6431248784065247,
      "kl": 0.07150302082300186,
      "learning_rate": 9.982476273321175e-07,
      "logits/chosen": -57209874.61818182,
      "logits/rejected": -50235841.23870968,
      "logps/chosen": -367.0889204545455,
      "logps/rejected": -190.05186491935484,
      "loss": 0.502,
      "loss/base_kto": 0.4985456168651581,
      "metrics/advantage_var": 0.096021369099617,
      "regularization/advantage_range": 0.753349781036377,
      "regularization/tv": 0.0037667490541934967,
      "rewards/chosen": 0.0100834203488899,
      "rewards/margins": 0.01182123529829471,
      "rewards/rejected": -0.0017378149494048087,
      "stability/repetition_rate_mean": 0.2682291567325592,
      "stability/response_length_mean": 52.75,
      "stability/response_length_std": 22.72663688659668,
      "stability/response_length_var": 516.5,
      "stability/token_entropy_mean": 4.888671875,
      "step": 140
    },
    {
      "epoch": 0.7421625079974408,
      "grad_norm": 0.640091598033905,
      "kl": 0.08613911271095276,
      "learning_rate": 9.977698537536417e-07,
      "logits/chosen": -51269098.409638554,
      "logits/rejected": -51827958.02597403,
      "logps/chosen": -330.1547439759036,
      "logps/rejected": -199.49535815746754,
      "loss": 0.5021,
      "loss/base_kto": 0.49847277998924255,
      "metrics/advantage_var": 0.0968799814581871,
      "regularization/advantage_range": 0.7232274413108826,
      "regularization/tv": 0.003616137197241187,
      "rewards/chosen": 0.01170891307922731,
      "rewards/margins": 0.012454076898459855,
      "rewards/rejected": -0.0007451638192325443,
      "stability/repetition_rate_mean": 0.23393641412258148,
      "stability/response_length_mean": 25.875,
      "stability/response_length_std": 26.744491577148438,
      "stability/response_length_var": 715.2678833007812,
      "stability/token_entropy_mean": 5.328125,
      "step": 145
    },
    {
      "epoch": 0.7677543186180422,
      "grad_norm": 0.7514075636863708,
      "kl": 0.09408855438232422,
      "learning_rate": 9.972346981963546e-07,
      "logits/chosen": -55563324.23529412,
      "logits/rejected": -50373949.44,
      "logps/chosen": -397.7233455882353,
      "logps/rejected": -207.72268229166667,
      "loss": 0.5024,
      "loss/base_kto": 0.4973035752773285,
      "metrics/advantage_var": 0.10378842800855637,
      "regularization/advantage_range": 0.7775968909263611,
      "regularization/tv": 0.0038879842031747103,
      "rewards/chosen": 0.02110713229459875,
      "rewards/margins": 0.020782324630840152,
      "rewards/rejected": 0.00032480766375859576,
      "stability/repetition_rate_mean": 0.23066024482250214,
      "stability/response_length_mean": 29.375,
      "stability/response_length_std": 29.193626403808594,
      "stability/response_length_var": 852.2678833007812,
      "stability/token_entropy_mean": 4.578125,
      "step": 150
    },
    {
      "epoch": 0.7933461292386437,
      "grad_norm": 0.6408149600028992,
      "kl": 0.08004111051559448,
      "learning_rate": 9.966422223519885e-07,
      "logits/chosen": -54190971.48235294,
      "logits/rejected": -49517864.96,
      "logps/chosen": -360.90234375,
      "logps/rejected": -191.31059895833334,
      "loss": 0.5022,
      "loss/base_kto": 0.49881210923194885,
      "metrics/advantage_var": 0.08827362209558487,
      "regularization/advantage_range": 0.7308141589164734,
      "regularization/tv": 0.0036540706641972065,
      "rewards/chosen": 0.008696890578550451,
      "rewards/margins": 0.00919545874583955,
      "rewards/rejected": -0.0004985681672890981,
      "stability/repetition_rate_mean": 0.3074128031730652,
      "stability/response_length_mean": 61.375,
      "stability/response_length_std": 7.424621105194092,
      "stability/response_length_var": 55.125,
      "stability/token_entropy_mean": 4.58984375,
      "step": 155
    },
    {
      "epoch": 0.818937939859245,
      "grad_norm": 0.7288516759872437,
      "kl": 0.089725062251091,
      "learning_rate": 9.959924945200523e-07,
      "logits/chosen": -55450593.52380952,
      "logits/rejected": -49211429.05263158,
      "logps/chosen": -416.73158482142856,
      "logps/rejected": -186.90829307154604,
      "loss": 0.5023,
      "loss/base_kto": 0.4977860152721405,
      "metrics/advantage_var": 0.14824461936950684,
      "regularization/advantage_range": 0.9315454363822937,
      "regularization/tv": 0.00465772720053792,
      "rewards/chosen": 0.016371272859119233,
      "rewards/margins": 0.01746604460429279,
      "rewards/rejected": -0.0010947717451735546,
      "stability/repetition_rate_mean": 0.291015625,
      "stability/response_length_mean": 26.25,
      "stability/response_length_std": 31.535694122314453,
      "stability/response_length_var": 994.5,
      "stability/token_entropy_mean": 3.8037109375,
      "step": 160
    },
    {
      "epoch": 0.8445297504798465,
      "grad_norm": 0.7331513166427612,
      "kl": 0.08966539055109024,
      "learning_rate": 9.952855895999568e-07,
      "logits/chosen": -54638721.47126437,
      "logits/rejected": -50055630.90410959,
      "logps/chosen": -370.7618085488506,
      "logps/rejected": -212.5884792380137,
      "loss": 0.5021,
      "loss/base_kto": 0.4981077313423157,
      "metrics/advantage_var": 0.09710618853569031,
      "regularization/advantage_range": 0.714181125164032,
      "regularization/tv": 0.003570905653759837,
      "rewards/chosen": 0.015497415915302846,
      "rewards/margins": 0.015133659842046785,
      "rewards/rejected": 0.0003637560732560615,
      "stability/repetition_rate_mean": 0.20468750596046448,
      "stability/response_length_mean": 12.625,
      "stability/response_length_std": 21.460512161254883,
      "stability/response_length_var": 460.5535583496094,
      "stability/token_entropy_mean": 4.81640625,
      "step": 165
    },
    {
      "epoch": 0.8701215611004478,
      "grad_norm": 0.6250264644622803,
      "kl": 0.1153709888458252,
      "learning_rate": 9.9452158908238e-07,
      "logits/chosen": -59361898.035502955,
      "logits/rejected": -52922496.84768212,
      "logps/chosen": -441.13951553254435,
      "logps/rejected": -191.25437189569536,
      "loss": 0.5019,
      "loss/base_kto": 0.4975743293762207,
      "metrics/advantage_var": 0.12109291553497314,
      "regularization/advantage_range": 0.871537983417511,
      "regularization/tv": 0.004357689525932074,
      "rewards/chosen": 0.020231455740844003,
      "rewards/margins": 0.01988455187277508,
      "rewards/rejected": 0.00034690386806892243,
      "stability/repetition_rate_mean": 0.25176572799682617,
      "stability/response_length_mean": 35.5,
      "stability/response_length_std": 30.06184196472168,
      "stability/response_length_var": 903.7142944335938,
      "stability/token_entropy_mean": 3.73876953125,
      "step": 170
    },
    {
      "epoch": 0.8957133717210493,
      "grad_norm": 0.6219585537910461,
      "kl": 0.12609218060970306,
      "learning_rate": 9.937005810398745e-07,
      "logits/chosen": -56909719.13253012,
      "logits/rejected": -48392531.116883114,
      "logps/chosen": -356.2319041792169,
      "logps/rejected": -194.7689351663961,
      "loss": 0.5017,
      "loss/base_kto": 0.49790093302726746,
      "metrics/advantage_var": 0.14611347019672394,
      "regularization/advantage_range": 0.942708432674408,
      "regularization/tv": 0.0047135422937572,
      "rewards/chosen": 0.020526822791042097,
      "rewards/margins": 0.01666777347701384,
      "rewards/rejected": 0.0038590493140282568,
      "stability/repetition_rate_mean": 0.2936941981315613,
      "stability/response_length_mean": 32.25,
      "stability/response_length_std": 27.932315826416016,
      "stability/response_length_var": 780.2142944335938,
      "stability/token_entropy_mean": 4.69921875,
      "step": 175
    },
    {
      "epoch": 0.9213051823416507,
      "grad_norm": 0.6889165639877319,
      "kl": 0.13772912323474884,
      "learning_rate": 9.928226601167138e-07,
      "logits/chosen": -52906668.7607362,
      "logits/rejected": -48310288.30573248,
      "logps/chosen": -375.950512845092,
      "logps/rejected": -211.2246218152866,
      "loss": 0.5019,
      "loss/base_kto": 0.49842366576194763,
      "metrics/advantage_var": 0.13614535331726074,
      "regularization/advantage_range": 0.8898941874504089,
      "regularization/tv": 0.004449470899999142,
      "rewards/chosen": 0.019533021318400564,
      "rewards/margins": 0.012176497610227141,
      "rewards/rejected": 0.007356523708173424,
      "stability/repetition_rate_mean": 0.16015625,
      "stability/response_length_mean": 40.375,
      "stability/response_length_std": 32.61874008178711,
      "stability/response_length_var": 1063.982177734375,
      "stability/token_entropy_mean": 3.36328125,
      "step": 180
    },
    {
      "epoch": 0.946896992962252,
      "grad_norm": 0.6404728889465332,
      "kl": 0.13586579263210297,
      "learning_rate": 9.918879275179817e-07,
      "logits/chosen": -54910508.24691358,
      "logits/rejected": -49845928.50632911,
      "logps/chosen": -311.828800154321,
      "logps/rejected": -196.98788568037975,
      "loss": 0.5022,
      "loss/base_kto": 0.49845775961875916,
      "metrics/advantage_var": 0.10770604759454727,
      "regularization/advantage_range": 0.8012159466743469,
      "regularization/tv": 0.004006079863756895,
      "rewards/chosen": 0.01726006872860002,
      "rewards/margins": 0.012132867753906238,
      "rewards/rejected": 0.005127200974693782,
      "stability/repetition_rate_mean": 0.18998579680919647,
      "stability/response_length_mean": 22.25,
      "stability/response_length_std": 27.78874397277832,
      "stability/response_length_var": 772.2142944335938,
      "stability/token_entropy_mean": 3.83203125,
      "step": 185
    },
    {
      "epoch": 0.9724888035828535,
      "grad_norm": 0.624788224697113,
      "kl": 0.12165651470422745,
      "learning_rate": 9.90896490997906e-07,
      "logits/chosen": -53395456.0,
      "logits/rejected": -50061977.93464052,
      "logps/chosen": -392.174120508982,
      "logps/rejected": -211.14671415441177,
      "loss": 0.5019,
      "loss/base_kto": 0.497698038816452,
      "metrics/advantage_var": 0.12404324114322662,
      "regularization/advantage_range": 0.8396191000938416,
      "regularization/tv": 0.004198095295578241,
      "rewards/chosen": 0.023119575249220797,
      "rewards/margins": 0.017932860871163053,
      "rewards/rejected": 0.005186714378057742,
      "stability/repetition_rate_mean": 0.1047247052192688,
      "stability/response_length_mean": 11.375,
      "stability/response_length_std": 21.738296508789062,
      "stability/response_length_var": 472.5535583496094,
      "stability/token_entropy_mean": 5.3671875,
      "step": 190
    },
    {
      "epoch": 0.9980806142034548,
      "grad_norm": 0.6496215462684631,
      "kl": 0.11633441597223282,
      "learning_rate": 9.898484648474362e-07,
      "logits/chosen": -60478417.75483871,
      "logits/rejected": -54269747.2,
      "logps/chosen": -392.54798387096776,
      "logps/rejected": -197.8989109848485,
      "loss": 0.5019,
      "loss/base_kto": 0.4972642958164215,
      "metrics/advantage_var": 0.12972894310951233,
      "regularization/advantage_range": 0.8943235278129578,
      "regularization/tv": 0.0044716172851622105,
      "rewards/chosen": 0.023692352541031376,
      "rewards/margins": 0.021650432876128254,
      "rewards/rejected": 0.0020419196649031205,
      "stability/repetition_rate_mean": 0.41796875,
      "stability/response_length_mean": 48.125,
      "stability/response_length_std": 29.39600372314453,
      "stability/response_length_var": 864.125,
      "stability/token_entropy_mean": 4.4609375,
      "step": 195
    },
    {
      "epoch": 1.0,
      "step": 196,
      "total_flos": 3.1861866257511875e+18,
      "train_loss": 0.5024726828750299,
      "train_runtime": 3154.4483,
      "train_samples_per_second": 31.701,
      "train_steps_per_second": 0.062
    }
  ],
  "logging_steps": 5,
  "max_steps": 196,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 200,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 3.1861866257511875e+18,
  "train_batch_size": 8,
  "trial_name": null,
  "trial_params": null
}