{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.0,
  "eval_steps": 500,
  "global_step": 196,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.025591810620601407,
      "grad_norm": 0.5678192377090454,
      "kl": 0.009494942612946033,
      "learning_rate": 4e-08,
      "logits/chosen": -59717214.315789476,
      "logits/rejected": -50418761.14285714,
      "logps/chosen": -439.0025699013158,
      "logps/rejected": -198.39857700892858,
      "loss": 0.5319,
      "loss/base_kto": 0.49962836503982544,
      "metrics/advantage_var": 0.04574162885546684,
      "regularization/lipschitz_norm": 36.127017974853516,
      "regularization/w1": 0.03612701967358589,
      "rewards/chosen": 0.004304667836741397,
      "rewards/margins": 0.0032234100442855877,
      "rewards/rejected": 0.0010812577924558095,
      "step": 5
    },
    {
      "epoch": 0.05118362124120281,
      "grad_norm": 0.6058618426322937,
      "kl": 0.017178865149617195,
      "learning_rate": 9e-08,
      "logits/chosen": -59305279.16883117,
      "logits/rejected": -50020561.73493976,
      "logps/chosen": -429.8796672077922,
      "logps/rejected": -192.88354198042168,
      "loss": 0.5474,
      "loss/base_kto": 0.4991108477115631,
      "metrics/advantage_var": 0.07220327109098434,
      "regularization/lipschitz_norm": 51.88991165161133,
      "regularization/w1": 0.051889922469854355,
      "rewards/chosen": 0.00588285071509225,
      "rewards/margins": 0.007441361728100276,
      "rewards/rejected": -0.0015585110130080257,
      "stability/repetition_rate_mean": 0.3605554699897766,
      "stability/response_length_mean": 44.25,
      "stability/response_length_std": 28.13614845275879,
      "stability/response_length_var": 791.6428833007812,
      "stability/token_entropy_mean": 3.3544921875,
      "step": 10
    },
    {
      "epoch": 0.07677543186180422,
      "grad_norm": 0.6088977456092834,
      "kl": 0.013535099104046822,
      "learning_rate": 1.4e-07,
      "logits/chosen": -56235889.26060606,
      "logits/rejected": -51918445.006451614,
      "logps/chosen": -404.12386363636364,
      "logps/rejected": -200.66649445564516,
      "loss": 0.5528,
      "loss/base_kto": 0.5002279281616211,
      "metrics/advantage_var": 0.057587385177612305,
      "regularization/lipschitz_norm": 47.16032409667969,
      "regularization/w1": 0.047160327434539795,
      "rewards/chosen": 0.0006950111100167939,
      "rewards/margins": -0.0020621884311981795,
      "rewards/rejected": 0.0027571995412149736,
      "stability/repetition_rate_mean": 0.2651909589767456,
      "stability/response_length_mean": 27.125,
      "stability/response_length_std": 30.79163932800293,
      "stability/response_length_var": 948.125,
      "stability/token_entropy_mean": 3.8203125,
      "step": 15
    },
    {
      "epoch": 0.10236724248240563,
      "grad_norm": 0.6186808347702026,
      "kl": 0.014954889193177223,
      "learning_rate": 1.8999999999999998e-07,
      "logits/chosen": -51655552.0,
      "logits/rejected": -48047257.6,
      "logps/chosen": -343.684423828125,
      "logps/rejected": -225.644677734375,
      "loss": 0.5531,
      "loss/base_kto": 0.500113308429718,
      "metrics/advantage_var": 0.0725272074341774,
      "regularization/lipschitz_norm": 51.32513427734375,
      "regularization/w1": 0.05132513865828514,
      "rewards/chosen": 0.0012291071005165577,
      "rewards/margins": -0.0006385699845850468,
      "rewards/rejected": 0.0018676770851016045,
      "stability/repetition_rate_mean": 0.2174789160490036,
      "stability/response_length_mean": 44.0,
      "stability/response_length_std": 28.0611572265625,
      "stability/response_length_var": 787.4285888671875,
      "stability/token_entropy_mean": 4.462890625,
      "step": 20
    },
    {
      "epoch": 0.12795905310300704,
      "grad_norm": 0.5771578550338745,
      "kl": 0.019569864496588707,
      "learning_rate": 2.4e-07,
      "logits/chosen": -56482780.68965517,
      "logits/rejected": -52020662.85714286,
      "logps/chosen": -378.46988146551723,
      "logps/rejected": -202.78185267857143,
      "loss": 0.5539,
      "loss/base_kto": 0.49986138939857483,
      "metrics/advantage_var": 0.05885173752903938,
      "regularization/lipschitz_norm": 46.89044189453125,
      "regularization/w1": 0.04689044505357742,
      "rewards/chosen": 0.002159434351427802,
      "rewards/margins": 0.0010595623583629214,
      "rewards/rejected": 0.0010998719930648804,
      "stability/repetition_rate_mean": 0.28109976649284363,
      "stability/response_length_mean": 50.25,
      "stability/response_length_std": 25.549671173095703,
      "stability/response_length_var": 652.7857055664062,
      "stability/token_entropy_mean": 4.5078125,
      "step": 25
    },
    {
      "epoch": 0.15355086372360843,
      "grad_norm": 0.603465735912323,
      "kl": 0.018669696524739265,
      "learning_rate": 2.9e-07,
      "logits/chosen": -57732102.87248322,
      "logits/rejected": -51518679.578947365,
      "logps/chosen": -468.6152999161074,
      "logps/rejected": -198.2326845760234,
      "loss": 0.5527,
      "loss/base_kto": 0.5005181431770325,
      "metrics/advantage_var": 0.07389438152313232,
      "regularization/lipschitz_norm": 52.765899658203125,
      "regularization/w1": 0.05276590213179588,
      "rewards/chosen": -0.003406525858296644,
      "rewards/margins": -0.004925517465271106,
      "rewards/rejected": 0.0015189916069744624,
      "stability/repetition_rate_mean": 0.158203125,
      "stability/response_length_mean": 48.375,
      "stability/response_length_std": 28.93311309814453,
      "stability/response_length_var": 837.125,
      "stability/token_entropy_mean": 4.595703125,
      "step": 30
    },
    {
      "epoch": 0.17914267434420986,
      "grad_norm": 0.6781691908836365,
      "kl": 0.03436283394694328,
      "learning_rate": 3.4000000000000003e-07,
      "logits/chosen": -57640635.94936709,
      "logits/rejected": -53904560.98765432,
      "logps/chosen": -385.9036293512658,
      "logps/rejected": -195.09850019290124,
      "loss": 0.5528,
      "loss/base_kto": 0.4999621510505676,
      "metrics/advantage_var": 0.05386241897940636,
      "regularization/lipschitz_norm": 46.16049575805664,
      "regularization/w1": 0.0461605004966259,
      "rewards/chosen": 0.003288782095607323,
      "rewards/margins": 0.0004902812913537566,
      "rewards/rejected": 0.0027985008042535663,
      "stability/repetition_rate_mean": 0.3537946343421936,
      "stability/response_length_mean": 56.875,
      "stability/response_length_std": 20.152544021606445,
      "stability/response_length_var": 406.125,
      "stability/token_entropy_mean": 5.416015625,
      "step": 35
    },
    {
      "epoch": 0.20473448496481125,
      "grad_norm": 0.5843126177787781,
      "kl": 0.027318641543388367,
      "learning_rate": 3.8999999999999997e-07,
      "logits/chosen": -56632812.8,
      "logits/rejected": -50224707.2,
      "logps/chosen": -421.8916015625,
      "logps/rejected": -219.2817138671875,
      "loss": 0.5509,
      "loss/base_kto": 0.5002434849739075,
      "metrics/advantage_var": 0.07726090401411057,
      "regularization/lipschitz_norm": 52.84709548950195,
      "regularization/w1": 0.05284709855914116,
      "rewards/chosen": 0.0011051620356738567,
      "rewards/margins": -0.0016500151716172696,
      "rewards/rejected": 0.0027551772072911263,
      "stability/repetition_rate_mean": 0.13671875,
      "stability/response_length_mean": 33.125,
      "stability/response_length_std": 33.01704025268555,
      "stability/response_length_var": 1090.125,
      "stability/token_entropy_mean": 4.8935546875,
      "step": 40
    },
    {
      "epoch": 0.23032629558541268,
      "grad_norm": 0.6399968862533569,
      "kl": 0.04743483290076256,
      "learning_rate": 4.3999999999999997e-07,
      "logits/chosen": -52704039.51677852,
      "logits/rejected": -50776093.94152047,
      "logps/chosen": -352.3003355704698,
      "logps/rejected": -243.84438961988303,
      "loss": 0.5532,
      "loss/base_kto": 0.4999566078186035,
      "metrics/advantage_var": 0.07218363136053085,
      "regularization/lipschitz_norm": 56.39221954345703,
      "regularization/w1": 0.05639222264289856,
      "rewards/chosen": 0.002814074090663219,
      "rewards/margins": -0.00016493607860582982,
      "rewards/rejected": 0.0029790101692690487,
      "stability/repetition_rate_mean": 0.0994017943739891,
      "stability/response_length_mean": 28.875,
      "stability/response_length_std": 29.87324333190918,
      "stability/response_length_var": 892.4107055664062,
      "stability/token_entropy_mean": 4.767578125,
      "step": 45
    },
    {
      "epoch": 0.2559181062060141,
      "grad_norm": 0.5783849954605103,
      "kl": 0.051512785255908966,
      "learning_rate": 4.9e-07,
      "logits/chosen": -54039232.4295302,
      "logits/rejected": -48303834.57309941,
      "logps/chosen": -375.13357277684565,
      "logps/rejected": -180.2006807383041,
      "loss": 0.5494,
      "loss/base_kto": 0.4995388090610504,
      "metrics/advantage_var": 0.04794999957084656,
      "regularization/lipschitz_norm": 41.682228088378906,
      "regularization/w1": 0.04168223217129707,
      "rewards/chosen": 0.006553848717836726,
      "rewards/margins": 0.0025606038216785853,
      "rewards/rejected": 0.003993244896158141,
      "stability/repetition_rate_mean": 0.3249044120311737,
      "stability/response_length_mean": 55.375,
      "stability/response_length_std": 16.361431121826172,
      "stability/response_length_var": 267.6964416503906,
      "stability/token_entropy_mean": 4.068359375,
      "step": 50
    },
    {
      "epoch": 0.28150991682661547,
      "grad_norm": 0.6621458530426025,
      "kl": 0.05490356683731079,
      "learning_rate": 5.4e-07,
      "logits/chosen": -56948316.22360248,
      "logits/rejected": -49577713.50943396,
      "logps/chosen": -344.4683618012422,
      "logps/rejected": -230.42428262578616,
      "loss": 0.5531,
      "loss/base_kto": 0.4997487962245941,
      "metrics/advantage_var": 0.05243971571326256,
      "regularization/lipschitz_norm": 45.507625579833984,
      "regularization/w1": 0.04550762102007866,
      "rewards/chosen": 0.0043661816519980105,
      "rewards/margins": 0.0019911858475652076,
      "rewards/rejected": 0.002374995804432803,
      "stability/repetition_rate_mean": 0.1015625,
      "stability/response_length_mean": 24.5,
      "stability/response_length_std": 32.710853576660156,
      "stability/response_length_var": 1070.0,
      "stability/token_entropy_mean": 4.6484375,
      "step": 55
    },
    {
      "epoch": 0.30710172744721687,
      "grad_norm": 0.5836600661277771,
      "kl": 0.05875139310956001,
      "learning_rate": 5.9e-07,
      "logits/chosen": -52669911.660606064,
      "logits/rejected": -47820423.43225806,
      "logps/chosen": -359.61126893939394,
      "logps/rejected": -178.15209173387098,
      "loss": 0.5496,
      "loss/base_kto": 0.4995574951171875,
      "metrics/advantage_var": 0.05988660454750061,
      "regularization/lipschitz_norm": 46.30636215209961,
      "regularization/w1": 0.046306367963552475,
      "rewards/chosen": 0.010953712463378906,
      "rewards/margins": 0.0034705385085075126,
      "rewards/rejected": 0.007483173954871393,
      "stability/repetition_rate_mean": 0.20815381407737732,
      "stability/response_length_mean": 52.625,
      "stability/response_length_std": 21.75800323486328,
      "stability/response_length_var": 473.41070556640625,
      "stability/token_entropy_mean": 5.314453125,
      "step": 60
    },
    {
      "epoch": 0.3326935380678183,
      "grad_norm": 0.5745318531990051,
      "kl": 0.08697053045034409,
      "learning_rate": 6.4e-07,
      "logits/chosen": -57561999.69032258,
      "logits/rejected": -47265897.5030303,
      "logps/chosen": -381.825,
      "logps/rejected": -183.50355113636363,
      "loss": 0.5544,
      "loss/base_kto": 0.4995245635509491,
      "metrics/advantage_var": 0.0773346945643425,
      "regularization/lipschitz_norm": 53.16814041137695,
      "regularization/w1": 0.053168147802352905,
      "rewards/chosen": 0.012180716760696904,
      "rewards/margins": 0.00430064315553518,
      "rewards/rejected": 0.007880073605161724,
      "stability/repetition_rate_mean": 0.20286151766777039,
      "stability/response_length_mean": 17.875,
      "stability/response_length_std": 23.485177993774414,
      "stability/response_length_var": 551.5535888671875,
      "stability/token_entropy_mean": 5.6875,
      "step": 65
    },
    {
      "epoch": 0.3582853486884197,
      "grad_norm": 0.5525624752044678,
      "kl": 0.09267856180667877,
      "learning_rate": 6.9e-07,
      "logits/chosen": -52762746.21935484,
      "logits/rejected": -49031695.515151516,
      "logps/chosen": -379.0071572580645,
      "logps/rejected": -220.3510890151515,
      "loss": 0.5558,
      "loss/base_kto": 0.4993644654750824,
      "metrics/advantage_var": 0.07783074676990509,
      "regularization/lipschitz_norm": 53.91382598876953,
      "regularization/w1": 0.05391382798552513,
      "rewards/chosen": 0.012132589278682585,
      "rewards/margins": 0.005023109807996106,
      "rewards/rejected": 0.007109479470686479,
      "stability/repetition_rate_mean": 0.119140625,
      "stability/response_length_mean": 33.875,
      "stability/response_length_std": 32.36151885986328,
      "stability/response_length_var": 1047.267822265625,
      "stability/token_entropy_mean": 5.3349609375,
      "step": 70
    },
    {
      "epoch": 0.3838771593090211,
      "grad_norm": 0.6093087792396545,
      "kl": 0.10173000395298004,
      "learning_rate": 7.4e-07,
      "logits/chosen": -49496755.89189189,
      "logits/rejected": -47891048.18604651,
      "logps/chosen": -314.1371674408784,
      "logps/rejected": -228.80718568313952,
      "loss": 0.5574,
      "loss/base_kto": 0.4992505609989166,
      "metrics/advantage_var": 0.07528817653656006,
      "regularization/lipschitz_norm": 53.78229904174805,
      "regularization/w1": 0.05378229543566704,
      "rewards/chosen": 0.013364916717683946,
      "rewards/margins": 0.0056003187528426805,
      "rewards/rejected": 0.007764597964841266,
      "stability/repetition_rate_mean": 0.06745792925357819,
      "stability/response_length_mean": 22.75,
      "stability/response_length_std": 31.07479476928711,
      "stability/response_length_var": 965.6428833007812,
      "stability/token_entropy_mean": 2.16015625,
      "step": 75
    },
    {
      "epoch": 0.4094689699296225,
      "grad_norm": 0.6032807230949402,
      "kl": 0.13186340034008026,
      "learning_rate": 7.9e-07,
      "logits/chosen": -58616967.62913907,
      "logits/rejected": -53431344.47337278,
      "logps/chosen": -403.86936051324506,
      "logps/rejected": -200.9628328402367,
      "loss": 0.5561,
      "loss/base_kto": 0.4992029666900635,
      "metrics/advantage_var": 0.0667102262377739,
      "regularization/lipschitz_norm": 47.66132354736328,
      "regularization/w1": 0.047661323100328445,
      "rewards/chosen": 0.01607713478290482,
      "rewards/margins": 0.006778749535228491,
      "rewards/rejected": 0.00929838524767633,
      "stability/repetition_rate_mean": 0.17916667461395264,
      "stability/response_length_mean": 42.0,
      "stability/response_length_std": 30.481843948364258,
      "stability/response_length_var": 929.1428833007812,
      "stability/token_entropy_mean": 5.25390625,
      "step": 80
    },
    {
      "epoch": 0.4350607805502239,
      "grad_norm": 0.6124517321586609,
      "kl": 0.1272238940000534,
      "learning_rate": 8.399999999999999e-07,
      "logits/chosen": -51464935.22580645,
      "logits/rejected": -47132777.5030303,
      "logps/chosen": -358.1264616935484,
      "logps/rejected": -230.3289772727273,
      "loss": 0.5594,
      "loss/base_kto": 0.4986273944377899,
      "metrics/advantage_var": 0.0937027856707573,
      "regularization/lipschitz_norm": 55.11994552612305,
      "regularization/w1": 0.055119942873716354,
      "rewards/chosen": 0.020604954996416645,
      "rewards/margins": 0.011022517827948516,
      "rewards/rejected": 0.00958243716846813,
      "stability/repetition_rate_mean": 0.11328125,
      "stability/response_length_mean": 32.5,
      "stability/response_length_std": 33.67915725708008,
      "stability/response_length_var": 1134.2857666015625,
      "stability/token_entropy_mean": 4.9453125,
      "step": 85
    },
    {
      "epoch": 0.46065259117082535,
      "grad_norm": 0.6401332020759583,
      "kl": 0.1703931838274002,
      "learning_rate": 8.9e-07,
      "logits/chosen": -58414500.102564104,
      "logits/rejected": -50554680.19512195,
      "logps/chosen": -380.5234124599359,
      "logps/rejected": -187.12747713414635,
      "loss": 0.5657,
      "loss/base_kto": 0.4978005886077881,
      "metrics/advantage_var": 0.12542474269866943,
      "regularization/lipschitz_norm": 66.9811782836914,
      "regularization/w1": 0.06698118895292282,
      "rewards/chosen": 0.026595650575099848,
      "rewards/margins": 0.016664286864556843,
      "rewards/rejected": 0.009931363710543005,
      "stability/repetition_rate_mean": 0.3812144994735718,
      "stability/response_length_mean": 30.625,
      "stability/response_length_std": 28.779644012451172,
      "stability/response_length_var": 828.2678833007812,
      "stability/token_entropy_mean": 3.736328125,
      "step": 90
    },
    {
      "epoch": 0.48624440179142675,
      "grad_norm": 0.6237964034080505,
      "kl": 0.16741442680358887,
      "learning_rate": 9.399999999999999e-07,
      "logits/chosen": -50386140.35443038,
      "logits/rejected": -48979835.25925926,
      "logps/chosen": -314.7296034414557,
      "logps/rejected": -207.09116994598764,
      "loss": 0.5682,
      "loss/base_kto": 0.49736830592155457,
      "metrics/advantage_var": 0.1331707090139389,
      "regularization/lipschitz_norm": 66.74359130859375,
      "regularization/w1": 0.06674359738826752,
      "rewards/chosen": 0.026378269436993177,
      "rewards/margins": 0.021123482745445267,
      "rewards/rejected": 0.005254786691547912,
      "stability/repetition_rate_mean": 0.20546108484268188,
      "stability/response_length_mean": 27.0,
      "stability/response_length_std": 31.1264705657959,
      "stability/response_length_var": 968.8571166992188,
      "stability/token_entropy_mean": 3.689453125,
      "step": 95
    },
    {
      "epoch": 0.5118362124120281,
      "grad_norm": 0.6090478301048279,
      "kl": 0.20850256085395813,
      "learning_rate": 9.9e-07,
      "logits/chosen": -54716139.24324324,
      "logits/rejected": -50905552.37209302,
      "logps/chosen": -359.56579919763516,
      "logps/rejected": -216.39162427325581,
      "loss": 0.5714,
      "loss/base_kto": 0.4984969198703766,
      "metrics/advantage_var": 0.13034458458423615,
      "regularization/lipschitz_norm": 67.18830871582031,
      "regularization/w1": 0.0671883150935173,
      "rewards/chosen": 0.030437920544598554,
      "rewards/margins": 0.011710877204226068,
      "rewards/rejected": 0.018727043340372486,
      "stability/repetition_rate_mean": 0.24010416865348816,
      "stability/response_length_mean": 40.75,
      "stability/response_length_std": 31.093177795410156,
      "stability/response_length_var": 966.7857055664062,
      "stability/token_entropy_mean": 4.140625,
      "step": 100
    },
    {
      "epoch": 0.5374280230326296,
      "grad_norm": 0.6770948171615601,
      "kl": 0.2234809845685959,
      "learning_rate": 9.999815554351224e-07,
      "logits/chosen": -59376942.26506024,
      "logits/rejected": -53793147.01298701,
      "logps/chosen": -399.5205666415663,
      "logps/rejected": -209.8428622159091,
      "loss": 0.5793,
      "loss/base_kto": 0.4972439706325531,
      "metrics/advantage_var": 0.22079260647296906,
      "regularization/lipschitz_norm": 90.85485076904297,
      "regularization/w1": 0.09085484594106674,
      "rewards/chosen": 0.03818985353033227,
      "rewards/margins": 0.022094057341581444,
      "rewards/rejected": 0.016095796188750823,
      "stability/repetition_rate_mean": 0.2014508843421936,
      "stability/response_length_mean": 29.0,
      "stability/response_length_std": 29.89505386352539,
      "stability/response_length_var": 893.7142944335938,
      "stability/token_entropy_mean": 3.40625,
      "step": 105
    },
    {
      "epoch": 0.5630198336532309,
      "grad_norm": 0.6547015309333801,
      "kl": 0.26629599928855896,
      "learning_rate": 9.999066267225444e-07,
      "logits/chosen": -57391949.10843374,
      "logits/rejected": -53613863.896103896,
      "logps/chosen": -378.19820689006025,
      "logps/rejected": -205.89108157467533,
      "loss": 0.5834,
      "loss/base_kto": 0.4977465569972992,
      "metrics/advantage_var": 0.21738283336162567,
      "regularization/lipschitz_norm": 89.9118881225586,
      "regularization/w1": 0.08991190046072006,
      "rewards/chosen": 0.036506069711892,
      "rewards/margins": 0.01679964862767379,
      "rewards/rejected": 0.01970642108421821,
      "stability/repetition_rate_mean": 0.31048768758773804,
      "stability/response_length_mean": 28.875,
      "stability/response_length_std": 29.911476135253906,
      "stability/response_length_var": 894.6964111328125,
      "stability/token_entropy_mean": 4.3857421875,
      "step": 110
    },
    {
      "epoch": 0.5886116442738324,
      "grad_norm": 0.6483462452888489,
      "kl": 0.32535991072654724,
      "learning_rate": 9.997740697079592e-07,
      "logits/chosen": -60606263.088607594,
      "logits/rejected": -53428704.39506173,
      "logps/chosen": -364.06833465189874,
      "logps/rejected": -200.9859905478395,
      "loss": 0.5942,
      "loss/base_kto": 0.49606046080589294,
      "metrics/advantage_var": 0.26823315024375916,
      "regularization/lipschitz_norm": 103.65421295166016,
      "regularization/w1": 0.10365422070026398,
      "rewards/chosen": 0.052421219741241844,
      "rewards/margins": 0.031364458764897865,
      "rewards/rejected": 0.02105676097634398,
      "stability/repetition_rate_mean": 0.3151041865348816,
      "stability/response_length_mean": 49.0,
      "stability/response_length_std": 27.795169830322266,
      "stability/response_length_var": 772.5714111328125,
      "stability/token_entropy_mean": 4.830078125,
      "step": 115
    },
    {
      "epoch": 0.6142034548944337,
      "grad_norm": 0.6184374690055847,
      "kl": 0.3553595244884491,
      "learning_rate": 9.995838996722914e-07,
      "logits/chosen": -57748794.5620915,
      "logits/rejected": -54355410.01197605,
      "logps/chosen": -383.7023845996732,
      "logps/rejected": -211.95403723802394,
      "loss": 0.6004,
      "loss/base_kto": 0.4964432418346405,
      "metrics/advantage_var": 0.3870581090450287,
      "regularization/lipschitz_norm": 109.87580871582031,
      "regularization/w1": 0.10987582057714462,
      "rewards/chosen": 0.05425496818193423,
      "rewards/margins": 0.028023887709821817,
      "rewards/rejected": 0.026231080472112416,
      "stability/repetition_rate_mean": 0.1723366528749466,
      "stability/response_length_mean": 27.875,
      "stability/response_length_std": 30.31943130493164,
      "stability/response_length_var": 919.2678833007812,
      "stability/token_entropy_mean": 4.64453125,
      "step": 120
    },
    {
      "epoch": 0.6397952655150352,
      "grad_norm": 0.6122660636901855,
      "kl": 0.39908552169799805,
      "learning_rate": 9.993361385379874e-07,
      "logits/chosen": -60685662.68493151,
      "logits/rejected": -55654294.06896552,
      "logps/chosen": -416.8324058219178,
      "logps/rejected": -216.1103627873563,
      "loss": 0.6157,
      "loss/base_kto": 0.4964618682861328,
      "metrics/advantage_var": 0.5276581645011902,
      "regularization/lipschitz_norm": 126.17388153076172,
      "regularization/w1": 0.12617388367652893,
      "rewards/chosen": 0.05468781353676156,
      "rewards/margins": 0.029002990971923985,
      "rewards/rejected": 0.025684822564837576,
      "stability/repetition_rate_mean": 0.15937499701976776,
      "stability/response_length_mean": 15.875,
      "stability/response_length_std": 24.584766387939453,
      "stability/response_length_var": 604.4107055664062,
      "stability/token_entropy_mean": 3.49609375,
      "step": 125
    },
    {
      "epoch": 0.6653870761356366,
      "grad_norm": 0.5939815044403076,
      "kl": 0.44654884934425354,
      "learning_rate": 9.99030814866488e-07,
      "logits/chosen": -57391097.393548384,
      "logits/rejected": -53830773.915151514,
      "logps/chosen": -391.98465221774194,
      "logps/rejected": -183.64661458333333,
      "loss": 0.6243,
      "loss/base_kto": 0.49498701095581055,
      "metrics/advantage_var": 0.7337404489517212,
      "regularization/lipschitz_norm": 144.0272216796875,
      "regularization/w1": 0.1440272331237793,
      "rewards/chosen": 0.06732863149335308,
      "rewards/margins": 0.04009907965785947,
      "rewards/rejected": 0.02722955183549361,
      "stability/repetition_rate_mean": 0.22330255806446075,
      "stability/response_length_mean": 38.0,
      "stability/response_length_std": 29.13515281677246,
      "stability/response_length_var": 848.8571166992188,
      "stability/token_entropy_mean": 3.91796875,
      "step": 130
    },
    {
      "epoch": 0.690978886756238,
      "grad_norm": 0.5939186215400696,
      "kl": 0.4990025460720062,
      "learning_rate": 9.986679638549358e-07,
      "logits/chosen": -59841391.50920245,
      "logits/rejected": -55011230.1656051,
      "logps/chosen": -401.0247795245399,
      "logps/rejected": -247.52191978503186,
      "loss": 0.6335,
      "loss/base_kto": 0.494366854429245,
      "metrics/advantage_var": 0.9336435198783875,
      "regularization/lipschitz_norm": 167.570068359375,
      "regularization/w1": 0.167570099234581,
      "rewards/chosen": 0.07913663197148797,
      "rewards/margins": 0.044253570179806793,
      "rewards/rejected": 0.03488306179168118,
      "stability/repetition_rate_mean": 0.2270585298538208,
      "stability/response_length_mean": 29.875,
      "stability/response_length_std": 29.502723693847656,
      "stability/response_length_var": 870.4107055664062,
      "stability/token_entropy_mean": 3.9013671875,
      "step": 135
    },
    {
      "epoch": 0.7165706973768394,
      "grad_norm": 0.7301244139671326,
      "kl": 0.4819028973579407,
      "learning_rate": 9.982476273321175e-07,
      "logits/chosen": -57072553.11515152,
      "logits/rejected": -50065289.08387097,
      "logps/chosen": -366.36526988636365,
      "logps/rejected": -189.91263860887096,
      "loss": 0.6499,
      "loss/base_kto": 0.4916805922985077,
      "metrics/advantage_var": 1.7661739587783813,
      "regularization/lipschitz_norm": 183.31504821777344,
      "regularization/w1": 0.18331506848335266,
      "rewards/chosen": 0.08244560126102332,
      "rewards/margins": 0.0702617435278198,
      "rewards/rejected": 0.012183857733203518,
      "stability/repetition_rate_mean": 0.228515625,
      "stability/response_length_mean": 56.25,
      "stability/response_length_std": 21.920310974121094,
      "stability/response_length_var": 480.5,
      "stability/token_entropy_mean": 5.134765625,
      "step": 140
    },
    {
      "epoch": 0.7421625079974408,
      "grad_norm": 0.6743220090866089,
      "kl": 0.5741755366325378,
      "learning_rate": 9.977698537536417e-07,
      "logits/chosen": -51148250.98795181,
      "logits/rejected": -51665840.20779221,
      "logps/chosen": -329.439406061747,
      "logps/rejected": -199.1564275568182,
      "loss": 0.6604,
      "loss/base_kto": 0.49365338683128357,
      "metrics/advantage_var": 0.856822669506073,
      "regularization/lipschitz_norm": 163.44825744628906,
      "regularization/w1": 0.1634482592344284,
      "rewards/chosen": 0.08324084224471126,
      "rewards/margins": 0.050092330581051434,
      "rewards/rejected": 0.03314851166365983,
      "stability/repetition_rate_mean": 0.21016408503055573,
      "stability/response_length_mean": 22.625,
      "stability/response_length_std": 26.359804153442383,
      "stability/response_length_var": 694.8392944335938,
      "stability/token_entropy_mean": 5.6640625,
      "step": 145
    },
    {
      "epoch": 0.7677543186180422,
      "grad_norm": 0.6430969834327698,
      "kl": 0.6443031430244446,
      "learning_rate": 9.972346981963546e-07,
      "logits/chosen": -55426325.08235294,
      "logits/rejected": -50193100.8,
      "logps/chosen": -396.8122242647059,
      "logps/rejected": -207.33588541666666,
      "loss": 0.6752,
      "loss/base_kto": 0.4903334677219391,
      "metrics/advantage_var": 1.2321922779083252,
      "regularization/lipschitz_norm": 185.36038208007812,
      "regularization/w1": 0.1853603571653366,
      "rewards/chosen": 0.11221859875847312,
      "rewards/margins": 0.07321333511202943,
      "rewards/rejected": 0.03900526364644368,
      "stability/repetition_rate_mean": 0.23066024482250214,
      "stability/response_length_mean": 29.375,
      "stability/response_length_std": 29.193626403808594,
      "stability/response_length_var": 852.2678833007812,
      "stability/token_entropy_mean": 4.58984375,
      "step": 150
    },
    {
      "epoch": 0.7933461292386437,
      "grad_norm": 0.6289855241775513,
      "kl": 0.6507260799407959,
      "learning_rate": 9.966422223519885e-07,
      "logits/chosen": -54025167.8117647,
      "logits/rejected": -49363421.86666667,
      "logps/chosen": -360.06833639705883,
      "logps/rejected": -191.04104166666667,
      "loss": 0.7031,
      "loss/base_kto": 0.49213314056396484,
      "metrics/advantage_var": 2.246683120727539,
      "regularization/lipschitz_norm": 223.3385467529297,
      "regularization/w1": 0.2233385592699051,
      "rewards/chosen": 0.09209441016702091,
      "rewards/margins": 0.06563791106728947,
      "rewards/rejected": 0.026456499099731447,
      "stability/repetition_rate_mean": 0.3115246295928955,
      "stability/response_length_mean": 58.125,
      "stability/response_length_std": 10.960155487060547,
      "stability/response_length_var": 120.125,
      "stability/token_entropy_mean": 4.5673828125,
      "step": 155
    },
    {
      "epoch": 0.818937939859245,
      "grad_norm": 0.619614839553833,
      "kl": 0.7176083326339722,
      "learning_rate": 9.959924945200523e-07,
      "logits/chosen": -55248286.47619048,
      "logits/rejected": -48961236.21052632,
      "logps/chosen": -415.52762276785717,
      "logps/rejected": -186.48067434210526,
      "loss": 0.7229,
      "loss/base_kto": 0.487854927778244,
      "metrics/advantage_var": 2.0578365325927734,
      "regularization/lipschitz_norm": 231.88279724121094,
      "regularization/w1": 0.23188279569149017,
      "rewards/chosen": 0.13677164486476354,
      "rewards/margins": 0.09510390426879539,
      "rewards/rejected": 0.041667740595968145,
      "stability/repetition_rate_mean": 0.30851560831069946,
      "stability/response_length_mean": 32.5,
      "stability/response_length_std": 30.528675079345703,
      "stability/response_length_var": 932.0,
      "stability/token_entropy_mean": 4.484375,
      "step": 160
    },
    {
      "epoch": 0.8445297504798465,
      "grad_norm": 0.7249348163604736,
      "kl": 0.770777702331543,
      "learning_rate": 9.952855895999568e-07,
      "logits/chosen": -54404449.10344828,
      "logits/rejected": -49778060.2739726,
      "logps/chosen": -369.70171066810343,
      "logps/rejected": -212.1182041952055,
      "loss": 0.7436,
      "loss/base_kto": 0.49073153734207153,
      "metrics/advantage_var": 1.810006022453308,
      "regularization/lipschitz_norm": 221.78726196289062,
      "regularization/w1": 0.22178728878498077,
      "rewards/chosen": 0.12150281051109577,
      "rewards/margins": 0.07411109539603195,
      "rewards/rejected": 0.047391715115063814,
      "stability/repetition_rate_mean": 0.2411106526851654,
      "stability/response_length_mean": 22.5,
      "stability/response_length_std": 28.33977508544922,
      "stability/response_length_var": 803.1428833007812,
      "stability/token_entropy_mean": 4.87109375,
      "step": 165
    },
    {
      "epoch": 0.8701215611004478,
      "grad_norm": 0.621860682964325,
      "kl": 0.9223539233207703,
      "learning_rate": 9.9452158908238e-07,
      "logits/chosen": -59132721.988165684,
      "logits/rejected": -52588998.357615896,
      "logps/chosen": -439.69253883136093,
      "logps/rejected": -190.58294960678808,
      "loss": 0.7621,
      "loss/base_kto": 0.48786044120788574,
      "metrics/advantage_var": 2.32456111907959,
      "regularization/lipschitz_norm": 273.0086975097656,
      "regularization/w1": 0.27300870418548584,
      "rewards/chosen": 0.16493017433663093,
      "rewards/margins": 0.09744154819757748,
      "rewards/rejected": 0.06748862613905345,
      "stability/repetition_rate_mean": 0.3512404263019562,
      "stability/response_length_mean": 44.125,
      "stability/response_length_std": 28.43256378173828,
      "stability/response_length_var": 808.4107055664062,
      "stability/token_entropy_mean": 3.59228515625,
      "step": 170
    },
    {
      "epoch": 0.8957133717210493,
      "grad_norm": 0.7261084318161011,
      "kl": 0.9431293606758118,
      "learning_rate": 9.937005810398745e-07,
      "logits/chosen": -56663225.06024096,
      "logits/rejected": -48029942.02597403,
      "logps/chosen": -354.8965549698795,
      "logps/rejected": -194.3030768060065,
      "loss": 0.7733,
      "loss/base_kto": 0.4871121346950531,
      "metrics/advantage_var": 4.2855119705200195,
      "regularization/lipschitz_norm": 322.96240234375,
      "regularization/w1": 0.3229624629020691,
      "rewards/chosen": 0.15406293754118033,
      "rewards/margins": 0.1036179753988305,
      "rewards/rejected": 0.050444962142349836,
      "stability/repetition_rate_mean": 0.19270984828472137,
      "stability/response_length_mean": 26.875,
      "stability/response_length_std": 25.570281982421875,
      "stability/response_length_var": 653.8392944335938,
      "stability/token_entropy_mean": 4.849609375,
      "step": 175
    },
    {
      "epoch": 0.9213051823416507,
      "grad_norm": 0.6424126625061035,
      "kl": 1.0352585315704346,
      "learning_rate": 9.928226601167138e-07,
      "logits/chosen": -52662171.48466258,
      "logits/rejected": -47984167.133757964,
      "logps/chosen": -374.4056269171779,
      "logps/rejected": -210.5522243232484,
      "loss": 0.8001,
      "loss/base_kto": 0.4874431788921356,
      "metrics/advantage_var": 4.169354438781738,
      "regularization/lipschitz_norm": 315.4919128417969,
      "regularization/w1": 0.3154919147491455,
      "rewards/chosen": 0.1740191524014151,
      "rewards/margins": 0.0994211801875728,
      "rewards/rejected": 0.0745979722138423,
      "stability/repetition_rate_mean": 0.2008928656578064,
      "stability/response_length_mean": 35.25,
      "stability/response_length_std": 31.129911422729492,
      "stability/response_length_var": 969.0714111328125,
      "stability/token_entropy_mean": 3.46875,
      "step": 180
    },
    {
      "epoch": 0.946896992962252,
      "grad_norm": 0.6501648426055908,
      "kl": 1.078233003616333,
      "learning_rate": 9.918879275179817e-07,
      "logits/chosen": -54614294.12345679,
      "logits/rejected": -49507992.30379747,
      "logps/chosen": -310.3076774691358,
      "logps/rejected": -196.53772745253164,
      "loss": 0.8358,
      "loss/base_kto": 0.48679420351982117,
      "metrics/advantage_var": 7.409839630126953,
      "regularization/lipschitz_norm": 392.8832702636719,
      "regularization/w1": 0.3928832709789276,
      "rewards/chosen": 0.16936941500063296,
      "rewards/margins": 0.11922688233813265,
      "rewards/rejected": 0.05014253266250031,
      "stability/repetition_rate_mean": 0.19140625,
      "stability/response_length_mean": 32.875,
      "stability/response_length_std": 33.30996322631836,
      "stability/response_length_var": 1109.5535888671875,
      "stability/token_entropy_mean": 3.705078125,
      "step": 185
    },
    {
      "epoch": 0.9724888035828535,
      "grad_norm": 0.6980246901512146,
      "kl": 1.122229814529419,
      "learning_rate": 9.90896490997906e-07,
      "logits/chosen": -53080229.556886226,
      "logits/rejected": -49681528.47058824,
      "logps/chosen": -390.28709767964074,
      "logps/rejected": -210.45874183006535,
      "loss": 0.8981,
      "loss/base_kto": 0.483151912689209,
      "metrics/advantage_var": 7.226253032684326,
      "regularization/lipschitz_norm": 438.9554748535156,
      "regularization/w1": 0.4389554560184479,
      "rewards/chosen": 0.21182136764069517,
      "rewards/margins": 0.13783560089950975,
      "rewards/rejected": 0.07398576674118541,
      "stability/repetition_rate_mean": 0.19138340651988983,
      "stability/response_length_mean": 18.625,
      "stability/response_length_std": 26.52189064025879,
      "stability/response_length_var": 703.4107055664062,
      "stability/token_entropy_mean": 4.580078125,
      "step": 190
    },
    {
      "epoch": 0.9980806142034548,
      "grad_norm": 0.6413615942001343,
      "kl": 1.1505008935928345,
      "learning_rate": 9.898484648474362e-07,
      "logits/chosen": -60036446.14193548,
      "logits/rejected": -53804826.375757575,
      "logps/chosen": -390.84732862903223,
      "logps/rejected": -197.08839962121212,
      "loss": 0.8839,
      "loss/base_kto": 0.4860563278198242,
      "metrics/advantage_var": 4.479440212249756,
      "regularization/lipschitz_norm": 372.052001953125,
      "regularization/w1": 0.37205204367637634,
      "rewards/chosen": 0.19376148100822202,
      "rewards/margins": 0.11066871057624107,
      "rewards/rejected": 0.08309277043198095,
      "stability/repetition_rate_mean": 0.3934151828289032,
      "stability/response_length_mean": 49.0,
      "stability/response_length_std": 27.82085609436035,
      "stability/response_length_var": 774.0,
      "stability/token_entropy_mean": 5.1875,
      "step": 195
    },
    {
      "epoch": 1.0,
      "step": 196,
      "total_flos": 3.1861866257511875e+18,
      "train_loss": 0.6270372308030421,
      "train_runtime": 3152.8646,
      "train_samples_per_second": 31.717,
      "train_steps_per_second": 0.062
    }
  ],
  "logging_steps": 5,
  "max_steps": 196,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 200,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 3.1861866257511875e+18,
  "train_batch_size": 8,
  "trial_name": null,
  "trial_params": null
}