{ "best_global_step": null, "best_metric": null, "best_model_checkpoint": null, "epoch": 1.0, "eval_steps": 100, "global_step": 681, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "KL/chosen_KL_mean": 0.05419158935546875, "KL/mean": 0.023573115468025208, "KL/rejected_KL_mean": -0.007045745849609375, "KL/std": 0.33097243309020996, "epoch": 0.0014684287812041115, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.06123492121696472, "fcm_dpo/q_t": 0.4984707534313202, "grad_norm": 38.81883239746094, "learning_rate": 0.0, "logits/chosen": 0.2677389979362488, "logits/rejected": 0.6191319227218628, "logps/chosen": -85.37664031982422, "logps/ref_chosen": -85.43083190917969, "logps/ref_rejected": -79.90458679199219, "logps/rejected": -79.91163635253906, "loss": 1.3807, "margin_dpo/margin_mean": 0.061235010623931885, "margin_dpo/margin_std": 0.4424324035644531, "step": 1 }, { "KL/chosen_KL_mean": -0.044902801513671875, "KL/mean": 0.017303764820098877, "KL/rejected_KL_mean": 0.07951736450195312, "KL/std": 0.2933448553085327, "epoch": 0.002936857562408223, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.12442219257354736, "fcm_dpo/q_t": 0.5031089186668396, "grad_norm": 29.756135940551758, "learning_rate": 7.246376811594203e-09, "logits/chosen": 0.47771453857421875, "logits/rejected": 0.5908545851707458, "logps/chosen": -82.11383056640625, "logps/ref_chosen": -82.06893157958984, "logps/ref_rejected": -81.65457916259766, "logps/rejected": -81.57505798339844, "loss": 1.3992, "margin_dpo/margin_mean": -0.12442246079444885, "margin_dpo/margin_std": 0.41358208656311035, "step": 2 }, { "KL/chosen_KL_mean": 0.0287322998046875, "KL/mean": 0.014838777482509613, "KL/rejected_KL_mean": 0.000946044921875, "KL/std": 0.2968166172504425, "epoch": 0.004405286343612335, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.027784809470176697, "fcm_dpo/q_t": 0.4993055462837219, "grad_norm": 26.00088119506836, "learning_rate": 1.4492753623188406e-08, "logits/chosen": 0.427190363407135, "logits/rejected": 0.6503462195396423, "logps/chosen": -93.78225708007812, "logps/ref_chosen": -93.81098937988281, "logps/ref_rejected": -74.22950744628906, "logps/rejected": -74.22856140136719, "loss": 1.3841, "margin_dpo/margin_mean": 0.027785077691078186, "margin_dpo/margin_std": 0.46310335397720337, "step": 3 }, { "KL/chosen_KL_mean": -0.006504058837890625, "KL/mean": -0.0196978822350502, "KL/rejected_KL_mean": -0.032894134521484375, "KL/std": 0.340604305267334, "epoch": 0.005873715124816446, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.02638496458530426, "fcm_dpo/q_t": 0.4993400573730469, "grad_norm": 32.68545913696289, "learning_rate": 2.1739130434782606e-08, "logits/chosen": 0.6464122533798218, "logits/rejected": 0.7782456874847412, "logps/chosen": -87.2989730834961, "logps/ref_chosen": -87.29246520996094, "logps/ref_rejected": -93.82425689697266, "logps/rejected": -93.85714721679688, "loss": 1.3842, "margin_dpo/margin_mean": 0.026384904980659485, "margin_dpo/margin_std": 0.4554966688156128, "step": 4 }, { "KL/chosen_KL_mean": -0.019741058349609375, "KL/mean": -0.04135115444660187, "KL/rejected_KL_mean": -0.06296539306640625, "KL/std": 0.30629420280456543, "epoch": 0.007342143906020558, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.043223947286605835, "fcm_dpo/q_t": 0.4989202618598938, "grad_norm": 35.385406494140625, "learning_rate": 2.898550724637681e-08, "logits/chosen": 0.4457821547985077, "logits/rejected": 0.656906247138977, "logps/chosen": -89.35649871826172, "logps/ref_chosen": -89.33676147460938, "logps/ref_rejected": -88.74783325195312, "logps/rejected": -88.81079864501953, "loss": 1.3824, "margin_dpo/margin_mean": 0.043224036693573, "margin_dpo/margin_std": 0.40450048446655273, "step": 5 }, { "KL/chosen_KL_mean": -0.01629638671875, "KL/mean": -0.016572266817092896, "KL/rejected_KL_mean": -0.016849517822265625, "KL/std": 0.3018956780433655, "epoch": 0.00881057268722467, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.0005494058132171631, "fcm_dpo/q_t": 0.499985009431839, "grad_norm": 35.97159194946289, "learning_rate": 3.6231884057971014e-08, "logits/chosen": 0.4856405258178711, "logits/rejected": 0.8005138635635376, "logps/chosen": -97.3377685546875, "logps/ref_chosen": -97.32147216796875, "logps/ref_rejected": -97.88345336914062, "logps/rejected": -97.90029907226562, "loss": 1.3868, "margin_dpo/margin_mean": 0.0005492866039276123, "margin_dpo/margin_std": 0.4187016189098358, "step": 6 }, { "KL/chosen_KL_mean": -0.01277923583984375, "KL/mean": -0.02249749004840851, "KL/rejected_KL_mean": -0.032222747802734375, "KL/std": 0.30508697032928467, "epoch": 0.010279001468428781, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.019439488649368286, "fcm_dpo/q_t": 0.49951377511024475, "grad_norm": 37.99931716918945, "learning_rate": 4.347826086956521e-08, "logits/chosen": 0.49927210807800293, "logits/rejected": 0.5469467639923096, "logps/chosen": -86.66131591796875, "logps/ref_chosen": -86.64852905273438, "logps/ref_rejected": -109.61618041992188, "logps/rejected": -109.64840698242188, "loss": 1.3847, "margin_dpo/margin_mean": 0.019439280033111572, "margin_dpo/margin_std": 0.3890848755836487, "step": 7 }, { "KL/chosen_KL_mean": -0.046047210693359375, "KL/mean": 0.021343939006328583, "KL/rejected_KL_mean": 0.08873748779296875, "KL/std": 0.34344005584716797, "epoch": 0.011747430249632892, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.1347811073064804, "fcm_dpo/q_t": 0.5033661723136902, "grad_norm": 32.892112731933594, "learning_rate": 5.0724637681159424e-08, "logits/chosen": 0.41771817207336426, "logits/rejected": 0.7306615710258484, "logps/chosen": -89.9697265625, "logps/ref_chosen": -89.9236831665039, "logps/ref_rejected": -86.22804260253906, "logps/rejected": -86.13929748535156, "loss": 1.4004, "margin_dpo/margin_mean": -0.13478055596351624, "margin_dpo/margin_std": 0.4848729074001312, "step": 8 }, { "KL/chosen_KL_mean": 0.03583526611328125, "KL/mean": 0.028834082186222076, "KL/rejected_KL_mean": 0.021839141845703125, "KL/std": 0.353157639503479, "epoch": 0.013215859030837005, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.013986274600028992, "fcm_dpo/q_t": 0.49964839220046997, "grad_norm": 36.429988861083984, "learning_rate": 5.797101449275362e-08, "logits/chosen": 0.6385010480880737, "logits/rejected": 0.8165180087089539, "logps/chosen": -103.82129669189453, "logps/ref_chosen": -103.85713195800781, "logps/ref_rejected": -104.31932067871094, "logps/rejected": -104.2974853515625, "loss": 1.3855, "margin_dpo/margin_mean": 0.01398603618144989, "margin_dpo/margin_std": 0.47145622968673706, "step": 9 }, { "KL/chosen_KL_mean": 0.01584625244140625, "KL/mean": 0.06282958388328552, "KL/rejected_KL_mean": 0.10980987548828125, "KL/std": 0.2959418296813965, "epoch": 0.014684287812041116, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.09396779537200928, "fcm_dpo/q_t": 0.5023472309112549, "grad_norm": 33.73389434814453, "learning_rate": 6.521739130434782e-08, "logits/chosen": 0.7877804040908813, "logits/rejected": 1.013244390487671, "logps/chosen": -76.18852233886719, "logps/ref_chosen": -76.20436096191406, "logps/ref_rejected": -87.152099609375, "logps/rejected": -87.04229736328125, "loss": 1.3962, "margin_dpo/margin_mean": -0.09396710991859436, "margin_dpo/margin_std": 0.44119375944137573, "step": 10 }, { "KL/chosen_KL_mean": 0.08171844482421875, "KL/mean": 0.03994722664356232, "KL/rejected_KL_mean": -0.001819610595703125, "KL/std": 0.2809763550758362, "epoch": 0.016152716593245228, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.08353599905967712, "fcm_dpo/q_t": 0.4979124665260315, "grad_norm": 37.10492706298828, "learning_rate": 7.246376811594203e-08, "logits/chosen": 0.24867631494998932, "logits/rejected": 0.4622611701488495, "logps/chosen": -82.28477478027344, "logps/ref_chosen": -82.36650085449219, "logps/ref_rejected": -94.26461791992188, "logps/rejected": -94.26644134521484, "loss": 1.3783, "margin_dpo/margin_mean": 0.0835367739200592, "margin_dpo/margin_std": 0.3704025447368622, "step": 11 }, { "KL/chosen_KL_mean": -0.013729095458984375, "KL/mean": -0.0020117759704589844, "KL/rejected_KL_mean": 0.00970458984375, "KL/std": 0.3588986396789551, "epoch": 0.01762114537444934, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.023437470197677612, "fcm_dpo/q_t": 0.5005854368209839, "grad_norm": 40.423004150390625, "learning_rate": 7.971014492753623e-08, "logits/chosen": 0.31419628858566284, "logits/rejected": 0.43920350074768066, "logps/chosen": -99.11923217773438, "logps/ref_chosen": -99.10549926757812, "logps/ref_rejected": -110.27140808105469, "logps/rejected": -110.26170349121094, "loss": 1.3894, "margin_dpo/margin_mean": -0.02343699336051941, "margin_dpo/margin_std": 0.5481129884719849, "step": 12 }, { "KL/chosen_KL_mean": -0.013057708740234375, "KL/mean": 0.00017010420560836792, "KL/rejected_KL_mean": 0.013397216796875, "KL/std": 0.31984561681747437, "epoch": 0.01908957415565345, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.02644781768321991, "fcm_dpo/q_t": 0.5006613731384277, "grad_norm": 41.72941589355469, "learning_rate": 8.695652173913042e-08, "logits/chosen": 0.5206282138824463, "logits/rejected": 0.7794650793075562, "logps/chosen": -90.57278442382812, "logps/ref_chosen": -90.55972290039062, "logps/ref_rejected": -93.69110107421875, "logps/rejected": -93.67770385742188, "loss": 1.3896, "margin_dpo/margin_mean": -0.026447921991348267, "margin_dpo/margin_std": 0.5109332799911499, "step": 13 }, { "KL/chosen_KL_mean": 0.032375335693359375, "KL/mean": 0.05151733011007309, "KL/rejected_KL_mean": 0.07065963745117188, "KL/std": 0.31700772047042847, "epoch": 0.020558002936857563, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.038283541798591614, "fcm_dpo/q_t": 0.5009565949440002, "grad_norm": 35.847442626953125, "learning_rate": 9.420289855072464e-08, "logits/chosen": 0.9608452320098877, "logits/rejected": 1.051961898803711, "logps/chosen": -99.7947998046875, "logps/ref_chosen": -99.82717895507812, "logps/ref_rejected": -108.94200134277344, "logps/rejected": -108.871337890625, "loss": 1.3906, "margin_dpo/margin_mean": -0.03828379511833191, "margin_dpo/margin_std": 0.43059903383255005, "step": 14 }, { "KL/chosen_KL_mean": -0.022243499755859375, "KL/mean": -0.02837357670068741, "KL/rejected_KL_mean": -0.034503936767578125, "KL/std": 0.31871163845062256, "epoch": 0.022026431718061675, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.012257859110832214, "fcm_dpo/q_t": 0.49969547986984253, "grad_norm": 31.83334732055664, "learning_rate": 1.0144927536231885e-07, "logits/chosen": 0.47015249729156494, "logits/rejected": 0.6424407958984375, "logps/chosen": -78.93221282958984, "logps/ref_chosen": -78.90997314453125, "logps/ref_rejected": -90.06234741210938, "logps/rejected": -90.09684753417969, "loss": 1.3857, "margin_dpo/margin_mean": 0.012257441878318787, "margin_dpo/margin_std": 0.48806464672088623, "step": 15 }, { "KL/chosen_KL_mean": 0.004421234130859375, "KL/mean": -0.016099005937576294, "KL/rejected_KL_mean": -0.036624908447265625, "KL/std": 0.31061846017837524, "epoch": 0.023494860499265784, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.04104405641555786, "fcm_dpo/q_t": 0.49897444248199463, "grad_norm": 34.45712661743164, "learning_rate": 1.0869565217391303e-07, "logits/chosen": 0.45026156306266785, "logits/rejected": 0.695430338382721, "logps/chosen": -97.41885375976562, "logps/ref_chosen": -97.42327880859375, "logps/ref_rejected": -90.59945678710938, "logps/rejected": -90.6360855102539, "loss": 1.3826, "margin_dpo/margin_mean": 0.04104390740394592, "margin_dpo/margin_std": 0.3858179450035095, "step": 16 }, { "KL/chosen_KL_mean": -0.028636932373046875, "KL/mean": -0.012308835983276367, "KL/rejected_KL_mean": 0.004016876220703125, "KL/std": 0.29504311084747314, "epoch": 0.024963289280469897, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.032649219036102295, "fcm_dpo/q_t": 0.500815749168396, "grad_norm": 35.82335662841797, "learning_rate": 1.1594202898550725e-07, "logits/chosen": 0.540476381778717, "logits/rejected": 0.7764678001403809, "logps/chosen": -104.3929443359375, "logps/ref_chosen": -104.36430358886719, "logps/ref_rejected": -90.46772766113281, "logps/rejected": -90.46370697021484, "loss": 1.39, "margin_dpo/margin_mean": -0.03264951705932617, "margin_dpo/margin_std": 0.40924182534217834, "step": 17 }, { "KL/chosen_KL_mean": -0.016056060791015625, "KL/mean": -0.005333468317985535, "KL/rejected_KL_mean": 0.005397796630859375, "KL/std": 0.32694047689437866, "epoch": 0.02643171806167401, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.021458178758621216, "fcm_dpo/q_t": 0.5005350708961487, "grad_norm": 41.65411376953125, "learning_rate": 1.2318840579710146e-07, "logits/chosen": 0.29909655451774597, "logits/rejected": 0.6327582001686096, "logps/chosen": -87.10801696777344, "logps/ref_chosen": -87.09195709228516, "logps/ref_rejected": -81.85072326660156, "logps/rejected": -81.84532165527344, "loss": 1.389, "margin_dpo/margin_mean": -0.02145804464817047, "margin_dpo/margin_std": 0.46991294622421265, "step": 18 }, { "KL/chosen_KL_mean": -0.030239105224609375, "KL/mean": -0.011079050600528717, "KL/rejected_KL_mean": 0.00807952880859375, "KL/std": 0.33012986183166504, "epoch": 0.027900146842878122, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.03832681477069855, "fcm_dpo/q_t": 0.5009571313858032, "grad_norm": 31.52723503112793, "learning_rate": 1.3043478260869563e-07, "logits/chosen": 0.6189084649085999, "logits/rejected": 0.897288978099823, "logps/chosen": -105.90377807617188, "logps/ref_chosen": -105.87354278564453, "logps/ref_rejected": -96.93023681640625, "logps/rejected": -96.92214965820312, "loss": 1.3907, "margin_dpo/margin_mean": -0.038326412439346313, "margin_dpo/margin_std": 0.4727324843406677, "step": 19 }, { "KL/chosen_KL_mean": 0.045955657958984375, "KL/mean": 0.016004234552383423, "KL/rejected_KL_mean": -0.01395416259765625, "KL/std": 0.3282163143157959, "epoch": 0.02936857562408223, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.05990403890609741, "fcm_dpo/q_t": 0.4985044598579407, "grad_norm": 32.48124694824219, "learning_rate": 1.3768115942028986e-07, "logits/chosen": 0.32923829555511475, "logits/rejected": 0.6138110160827637, "logps/chosen": -90.71216583251953, "logps/ref_chosen": -90.75811767578125, "logps/ref_rejected": -85.91232299804688, "logps/rejected": -85.92626953125, "loss": 1.3808, "margin_dpo/margin_mean": 0.05990391969680786, "margin_dpo/margin_std": 0.42108580470085144, "step": 20 }, { "KL/chosen_KL_mean": 0.056896209716796875, "KL/mean": 0.009724318981170654, "KL/rejected_KL_mean": -0.03745269775390625, "KL/std": 0.27067482471466064, "epoch": 0.030837004405286344, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.09435027837753296, "fcm_dpo/q_t": 0.49764198064804077, "grad_norm": 31.47742462158203, "learning_rate": 1.4492753623188405e-07, "logits/chosen": 0.7178741693496704, "logits/rejected": 0.9084056615829468, "logps/chosen": -80.27656555175781, "logps/ref_chosen": -80.33345794677734, "logps/ref_rejected": -83.93373107910156, "logps/rejected": -83.97119140625, "loss": 1.3773, "margin_dpo/margin_mean": 0.09435057640075684, "margin_dpo/margin_std": 0.3899233043193817, "step": 21 }, { "KL/chosen_KL_mean": 0.016254425048828125, "KL/mean": 0.0019180327653884888, "KL/rejected_KL_mean": -0.012416839599609375, "KL/std": 0.3457253873348236, "epoch": 0.032305433186490456, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.028666764497756958, "fcm_dpo/q_t": 0.49928390979766846, "grad_norm": 42.878578186035156, "learning_rate": 1.5217391304347825e-07, "logits/chosen": 0.7784150838851929, "logits/rejected": 0.9052686095237732, "logps/chosen": -95.37904357910156, "logps/ref_chosen": -95.39530181884766, "logps/ref_rejected": -103.47351837158203, "logps/rejected": -103.48593139648438, "loss": 1.384, "margin_dpo/margin_mean": 0.02866724133491516, "margin_dpo/margin_std": 0.46230798959732056, "step": 22 }, { "KL/chosen_KL_mean": 0.036510467529296875, "KL/mean": 0.029525823891162872, "KL/rejected_KL_mean": 0.022548675537109375, "KL/std": 0.30836766958236694, "epoch": 0.033773861967694566, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.013959124684333801, "fcm_dpo/q_t": 0.4996510446071625, "grad_norm": 31.371408462524414, "learning_rate": 1.5942028985507245e-07, "logits/chosen": 0.7268781065940857, "logits/rejected": 0.9055923223495483, "logps/chosen": -90.60099792480469, "logps/ref_chosen": -90.63751220703125, "logps/ref_rejected": -86.59425354003906, "logps/rejected": -86.57170867919922, "loss": 1.3854, "margin_dpo/margin_mean": 0.01395919919013977, "margin_dpo/margin_std": 0.4396651089191437, "step": 23 }, { "KL/chosen_KL_mean": 0.030887603759765625, "KL/mean": 0.00558733195066452, "KL/rejected_KL_mean": -0.01971435546875, "KL/std": 0.30953043699264526, "epoch": 0.03524229074889868, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.05059997737407684, "fcm_dpo/q_t": 0.4987359046936035, "grad_norm": 44.12531280517578, "learning_rate": 1.6666666666666665e-07, "logits/chosen": 0.8018275499343872, "logits/rejected": 0.8294210433959961, "logps/chosen": -69.88640594482422, "logps/ref_chosen": -69.91728973388672, "logps/ref_rejected": -106.60990142822266, "logps/rejected": -106.6296157836914, "loss": 1.3817, "margin_dpo/margin_mean": 0.05060113966464996, "margin_dpo/margin_std": 0.4125809371471405, "step": 24 }, { "KL/chosen_KL_mean": -0.04108428955078125, "KL/mean": -0.04277415573596954, "KL/rejected_KL_mean": -0.044467926025390625, "KL/std": 0.3419887125492096, "epoch": 0.03671071953010279, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.0033724606037139893, "fcm_dpo/q_t": 0.49991559982299805, "grad_norm": 36.8920783996582, "learning_rate": 1.7391304347826085e-07, "logits/chosen": 0.5042506456375122, "logits/rejected": 0.6724146008491516, "logps/chosen": -80.86656951904297, "logps/ref_chosen": -80.82548522949219, "logps/ref_rejected": -95.95710754394531, "logps/rejected": -96.00157165527344, "loss": 1.3866, "margin_dpo/margin_mean": 0.0033729970455169678, "margin_dpo/margin_std": 0.4977954924106598, "step": 25 }, { "KL/chosen_KL_mean": -0.022045135498046875, "KL/mean": -0.0279066264629364, "KL/rejected_KL_mean": -0.0337677001953125, "KL/std": 0.3201514184474945, "epoch": 0.0381791483113069, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.011723220348358154, "fcm_dpo/q_t": 0.4997059106826782, "grad_norm": 41.05452346801758, "learning_rate": 1.8115942028985507e-07, "logits/chosen": 0.38126444816589355, "logits/rejected": 0.5917342901229858, "logps/chosen": -88.9232177734375, "logps/ref_chosen": -88.90116882324219, "logps/ref_rejected": -109.82819366455078, "logps/rejected": -109.8619613647461, "loss": 1.3856, "margin_dpo/margin_mean": 0.011723697185516357, "margin_dpo/margin_std": 0.43292662501335144, "step": 26 }, { "KL/chosen_KL_mean": -0.000385284423828125, "KL/mean": -0.05045069009065628, "KL/rejected_KL_mean": -0.10052108764648438, "KL/std": 0.3165472745895386, "epoch": 0.039647577092511016, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.10013644397258759, "fcm_dpo/q_t": 0.4974977672100067, "grad_norm": 44.33415985107422, "learning_rate": 1.8840579710144927e-07, "logits/chosen": 0.019319038838148117, "logits/rejected": 0.20480017364025116, "logps/chosen": -77.59639739990234, "logps/ref_chosen": -77.59600830078125, "logps/ref_rejected": -103.93850708007812, "logps/rejected": -104.03903198242188, "loss": 1.3768, "margin_dpo/margin_mean": 0.10013610124588013, "margin_dpo/margin_std": 0.4272763133049011, "step": 27 }, { "KL/chosen_KL_mean": 0.050746917724609375, "KL/mean": -0.0067250728607177734, "KL/rejected_KL_mean": -0.0641937255859375, "KL/std": 0.327208936214447, "epoch": 0.041116005873715125, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.11494022607803345, "fcm_dpo/q_t": 0.497127503156662, "grad_norm": 36.51122283935547, "learning_rate": 1.9565217391304347e-07, "logits/chosen": 0.4654392600059509, "logits/rejected": 0.671930193901062, "logps/chosen": -102.17782592773438, "logps/ref_chosen": -102.22856903076172, "logps/ref_rejected": -96.9594955444336, "logps/rejected": -97.02368927001953, "loss": 1.3754, "margin_dpo/margin_mean": 0.11494016647338867, "margin_dpo/margin_std": 0.47538506984710693, "step": 28 }, { "KL/chosen_KL_mean": -0.035511016845703125, "KL/mean": -0.07765653729438782, "KL/rejected_KL_mean": -0.1197967529296875, "KL/std": 0.3336794376373291, "epoch": 0.042584434654919234, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.08429092168807983, "fcm_dpo/q_t": 0.4978937804698944, "grad_norm": 42.448341369628906, "learning_rate": 2.028985507246377e-07, "logits/chosen": 0.30568623542785645, "logits/rejected": 0.5627601146697998, "logps/chosen": -88.68256378173828, "logps/ref_chosen": -88.64704895019531, "logps/ref_rejected": -102.96011352539062, "logps/rejected": -103.07991027832031, "loss": 1.3784, "margin_dpo/margin_mean": 0.08429181575775146, "margin_dpo/margin_std": 0.4542495012283325, "step": 29 }, { "KL/chosen_KL_mean": -0.03495025634765625, "KL/mean": -0.07996026426553726, "KL/rejected_KL_mean": -0.12497711181640625, "KL/std": 0.3174794912338257, "epoch": 0.04405286343612335, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.09002189338207245, "fcm_dpo/q_t": 0.4977508783340454, "grad_norm": 39.160457611083984, "learning_rate": 2.1014492753623187e-07, "logits/chosen": 0.6724940538406372, "logits/rejected": 0.7918111085891724, "logps/chosen": -88.42333984375, "logps/ref_chosen": -88.38838195800781, "logps/ref_rejected": -102.31889343261719, "logps/rejected": -102.4438705444336, "loss": 1.3777, "margin_dpo/margin_mean": 0.09002181887626648, "margin_dpo/margin_std": 0.40099483728408813, "step": 30 }, { "KL/chosen_KL_mean": -0.0133514404296875, "KL/mean": -0.06873812526464462, "KL/rejected_KL_mean": -0.12412261962890625, "KL/std": 0.36953017115592957, "epoch": 0.04552129221732746, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.11076883971691132, "fcm_dpo/q_t": 0.4972333312034607, "grad_norm": 31.31879425048828, "learning_rate": 2.1739130434782607e-07, "logits/chosen": 0.6688675880432129, "logits/rejected": 0.9099244475364685, "logps/chosen": -101.13900756835938, "logps/ref_chosen": -101.12565612792969, "logps/ref_rejected": -79.8584213256836, "logps/rejected": -79.9825439453125, "loss": 1.3757, "margin_dpo/margin_mean": 0.11076858639717102, "margin_dpo/margin_std": 0.43215447664260864, "step": 31 }, { "KL/chosen_KL_mean": -0.046009063720703125, "KL/mean": -0.10444005578756332, "KL/rejected_KL_mean": -0.16286849975585938, "KL/std": 0.3813129663467407, "epoch": 0.04698972099853157, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.11686711013317108, "fcm_dpo/q_t": 0.49708083271980286, "grad_norm": 35.11892318725586, "learning_rate": 2.2463768115942027e-07, "logits/chosen": 0.44755664467811584, "logits/rejected": 0.5893191695213318, "logps/chosen": -96.66304016113281, "logps/ref_chosen": -96.61702728271484, "logps/ref_rejected": -96.224365234375, "logps/rejected": -96.38723754882812, "loss": 1.3754, "margin_dpo/margin_mean": 0.11686784029006958, "margin_dpo/margin_std": 0.5499824285507202, "step": 32 }, { "KL/chosen_KL_mean": -0.0162353515625, "KL/mean": -0.07665270566940308, "KL/rejected_KL_mean": -0.1370697021484375, "KL/std": 0.3481675386428833, "epoch": 0.048458149779735685, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.12083667516708374, "fcm_dpo/q_t": 0.49698129296302795, "grad_norm": 33.354393005371094, "learning_rate": 2.318840579710145e-07, "logits/chosen": 0.5735636353492737, "logits/rejected": 0.7545783519744873, "logps/chosen": -81.53724670410156, "logps/ref_chosen": -81.52101135253906, "logps/ref_rejected": -93.80595397949219, "logps/rejected": -93.94302368164062, "loss": 1.3749, "margin_dpo/margin_mean": 0.12083682417869568, "margin_dpo/margin_std": 0.4927264153957367, "step": 33 }, { "KL/chosen_KL_mean": 0.009197235107421875, "KL/mean": -0.08236097544431686, "KL/rejected_KL_mean": -0.173919677734375, "KL/std": 0.3521924614906311, "epoch": 0.049926578560939794, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.18311984837055206, "fcm_dpo/q_t": 0.4954279661178589, "grad_norm": 40.758296966552734, "learning_rate": 2.391304347826087e-07, "logits/chosen": 0.5477845072746277, "logits/rejected": 0.7202606201171875, "logps/chosen": -77.19284057617188, "logps/ref_chosen": -77.20204162597656, "logps/ref_rejected": -106.71876525878906, "logps/rejected": -106.89268493652344, "loss": 1.3687, "margin_dpo/margin_mean": 0.1831202507019043, "margin_dpo/margin_std": 0.49747803807258606, "step": 34 }, { "KL/chosen_KL_mean": 0.01877593994140625, "KL/mean": -0.08079488575458527, "KL/rejected_KL_mean": -0.18036651611328125, "KL/std": 0.3320920169353485, "epoch": 0.0513950073421439, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.1991446316242218, "fcm_dpo/q_t": 0.4950249195098877, "grad_norm": 42.67302703857422, "learning_rate": 2.463768115942029e-07, "logits/chosen": 0.21599772572517395, "logits/rejected": 0.37262865900993347, "logps/chosen": -77.55157470703125, "logps/ref_chosen": -77.57035064697266, "logps/ref_rejected": -112.18855285644531, "logps/rejected": -112.3689193725586, "loss": 1.367, "margin_dpo/margin_mean": 0.19914525747299194, "margin_dpo/margin_std": 0.45872658491134644, "step": 35 }, { "KL/chosen_KL_mean": -0.02880096435546875, "KL/mean": -0.10909676551818848, "KL/rejected_KL_mean": -0.18939590454101562, "KL/std": 0.39818480610847473, "epoch": 0.05286343612334802, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.1605931520462036, "fcm_dpo/q_t": 0.4959890842437744, "grad_norm": 32.48418045043945, "learning_rate": 2.536231884057971e-07, "logits/chosen": 0.40590041875839233, "logits/rejected": 0.5949779152870178, "logps/chosen": -83.31704711914062, "logps/ref_chosen": -83.28823852539062, "logps/ref_rejected": -91.75741577148438, "logps/rejected": -91.94680786132812, "loss": 1.3712, "margin_dpo/margin_mean": 0.16059304773807526, "margin_dpo/margin_std": 0.5829658508300781, "step": 36 }, { "KL/chosen_KL_mean": -0.037395477294921875, "KL/mean": -0.17052027583122253, "KL/rejected_KL_mean": -0.3036460876464844, "KL/std": 0.46341902017593384, "epoch": 0.05433186490455213, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.26624488830566406, "fcm_dpo/q_t": 0.49335789680480957, "grad_norm": 35.963470458984375, "learning_rate": 2.6086956521739126e-07, "logits/chosen": 0.6156303882598877, "logits/rejected": 0.7719565629959106, "logps/chosen": -94.80848693847656, "logps/ref_chosen": -94.77108764648438, "logps/ref_rejected": -85.71726989746094, "logps/rejected": -86.02091217041016, "loss": 1.3608, "margin_dpo/margin_mean": 0.26624536514282227, "margin_dpo/margin_std": 0.6278531551361084, "step": 37 }, { "KL/chosen_KL_mean": 0.06585311889648438, "KL/mean": -0.07702397555112839, "KL/rejected_KL_mean": -0.21990203857421875, "KL/std": 0.4237167239189148, "epoch": 0.055800293685756244, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.28575462102890015, "fcm_dpo/q_t": 0.4928654730319977, "grad_norm": 36.88115692138672, "learning_rate": 2.681159420289855e-07, "logits/chosen": 0.4460694193840027, "logits/rejected": 0.5844659805297852, "logps/chosen": -75.91265106201172, "logps/ref_chosen": -75.97850036621094, "logps/ref_rejected": -104.10401916503906, "logps/rejected": -104.32392120361328, "loss": 1.3587, "margin_dpo/margin_mean": 0.28575465083122253, "margin_dpo/margin_std": 0.5612478256225586, "step": 38 }, { "KL/chosen_KL_mean": -0.020092010498046875, "KL/mean": -0.14587973058223724, "KL/rejected_KL_mean": -0.27166748046875, "KL/std": 0.4051820933818817, "epoch": 0.05726872246696035, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.25157061219215393, "fcm_dpo/q_t": 0.49371784925460815, "grad_norm": 36.32796859741211, "learning_rate": 2.753623188405797e-07, "logits/chosen": 0.3755003809928894, "logits/rejected": 0.5563080310821533, "logps/chosen": -81.20587158203125, "logps/ref_chosen": -81.18577575683594, "logps/ref_rejected": -84.1959228515625, "logps/rejected": -84.46759033203125, "loss": 1.3621, "margin_dpo/margin_mean": 0.25157058238983154, "margin_dpo/margin_std": 0.5538491010665894, "step": 39 }, { "KL/chosen_KL_mean": -0.034717559814453125, "KL/mean": -0.09988433122634888, "KL/rejected_KL_mean": -0.1650543212890625, "KL/std": 0.34656572341918945, "epoch": 0.05873715124816446, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.13033652305603027, "fcm_dpo/q_t": 0.49674469232559204, "grad_norm": 29.362964630126953, "learning_rate": 2.8260869565217386e-07, "logits/chosen": 0.7400258183479309, "logits/rejected": 0.9457363486289978, "logps/chosen": -83.36729431152344, "logps/ref_chosen": -83.33257293701172, "logps/ref_rejected": -80.25591278076172, "logps/rejected": -80.42096710205078, "loss": 1.3738, "margin_dpo/margin_mean": 0.1303362250328064, "margin_dpo/margin_std": 0.4470806121826172, "step": 40 }, { "KL/chosen_KL_mean": 0.00399017333984375, "KL/mean": -0.14051032066345215, "KL/rejected_KL_mean": -0.2850151062011719, "KL/std": 0.4360182285308838, "epoch": 0.06020558002936858, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.2890109419822693, "fcm_dpo/q_t": 0.4927857220172882, "grad_norm": 33.39806365966797, "learning_rate": 2.898550724637681e-07, "logits/chosen": 0.5981994867324829, "logits/rejected": 0.7830361127853394, "logps/chosen": -93.14466857910156, "logps/ref_chosen": -93.14866638183594, "logps/ref_rejected": -102.07920837402344, "logps/rejected": -102.36422729492188, "loss": 1.3585, "margin_dpo/margin_mean": 0.2890111207962036, "margin_dpo/margin_std": 0.572950005531311, "step": 41 }, { "KL/chosen_KL_mean": 0.08095169067382812, "KL/mean": -0.13394027948379517, "KL/rejected_KL_mean": -0.3488349914550781, "KL/std": 0.526868462562561, "epoch": 0.06167400881057269, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.4297866225242615, "fcm_dpo/q_t": 0.4892716407775879, "grad_norm": 40.88289260864258, "learning_rate": 2.971014492753623e-07, "logits/chosen": 0.37874874472618103, "logits/rejected": 0.5197323560714722, "logps/chosen": -90.62066650390625, "logps/ref_chosen": -90.70162200927734, "logps/ref_rejected": -113.8646469116211, "logps/rejected": -114.21348571777344, "loss": 1.3448, "margin_dpo/margin_mean": 0.42978644371032715, "margin_dpo/margin_std": 0.6271623373031616, "step": 42 }, { "KL/chosen_KL_mean": 0.0426025390625, "KL/mean": -0.1768113523721695, "KL/rejected_KL_mean": -0.3962287902832031, "KL/std": 0.523507833480835, "epoch": 0.0631424375917768, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.43883296847343445, "fcm_dpo/q_t": 0.48906266689300537, "grad_norm": 37.787845611572266, "learning_rate": 3.043478260869565e-07, "logits/chosen": 0.593334436416626, "logits/rejected": 0.7848832607269287, "logps/chosen": -89.60142517089844, "logps/ref_chosen": -89.64402770996094, "logps/ref_rejected": -103.95185852050781, "logps/rejected": -104.34808349609375, "loss": 1.3444, "margin_dpo/margin_mean": 0.438833087682724, "margin_dpo/margin_std": 0.7530083656311035, "step": 43 }, { "KL/chosen_KL_mean": -0.08082199096679688, "KL/mean": -0.2548077404499054, "KL/rejected_KL_mean": -0.42879486083984375, "KL/std": 0.46162915229797363, "epoch": 0.06461086637298091, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.34797805547714233, "fcm_dpo/q_t": 0.4913100004196167, "grad_norm": 34.453399658203125, "learning_rate": 3.115942028985507e-07, "logits/chosen": 0.8032479286193848, "logits/rejected": 0.9257111549377441, "logps/chosen": -81.95913696289062, "logps/ref_chosen": -81.8783187866211, "logps/ref_rejected": -113.421630859375, "logps/rejected": -113.85042572021484, "loss": 1.3525, "margin_dpo/margin_mean": 0.34797826409339905, "margin_dpo/margin_std": 0.5304656624794006, "step": 44 }, { "KL/chosen_KL_mean": -0.14659500122070312, "KL/mean": -0.23287206888198853, "KL/rejected_KL_mean": -0.3191490173339844, "KL/std": 0.45169106125831604, "epoch": 0.06607929515418502, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.17255784571170807, "fcm_dpo/q_t": 0.49568986892700195, "grad_norm": 24.54340934753418, "learning_rate": 3.188405797101449e-07, "logits/chosen": 0.6045481562614441, "logits/rejected": 0.7149960994720459, "logps/chosen": -77.49119567871094, "logps/ref_chosen": -77.34459686279297, "logps/ref_rejected": -84.24774169921875, "logps/rejected": -84.56689453125, "loss": 1.3698, "margin_dpo/margin_mean": 0.1725585013628006, "margin_dpo/margin_std": 0.5388465523719788, "step": 45 }, { "KL/chosen_KL_mean": 0.00597381591796875, "KL/mean": -0.21935032308101654, "KL/rejected_KL_mean": -0.4446754455566406, "KL/std": 0.5215157270431519, "epoch": 0.06754772393538913, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.4506525695323944, "fcm_dpo/q_t": 0.48874691128730774, "grad_norm": 33.07174301147461, "learning_rate": 3.260869565217391e-07, "logits/chosen": 0.8180408477783203, "logits/rejected": 1.0767744779586792, "logps/chosen": -90.33485412597656, "logps/ref_chosen": -90.3408203125, "logps/ref_rejected": -100.676513671875, "logps/rejected": -101.1211929321289, "loss": 1.3425, "margin_dpo/margin_mean": 0.4506520628929138, "margin_dpo/margin_std": 0.5685445070266724, "step": 46 }, { "KL/chosen_KL_mean": -0.03304290771484375, "KL/mean": -0.2129550576210022, "KL/rejected_KL_mean": -0.3928718566894531, "KL/std": 0.5051518082618713, "epoch": 0.06901615271659324, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.35982322692871094, "fcm_dpo/q_t": 0.4910237193107605, "grad_norm": 34.059898376464844, "learning_rate": 3.333333333333333e-07, "logits/chosen": 0.3814537525177002, "logits/rejected": 0.559528112411499, "logps/chosen": -104.44434356689453, "logps/ref_chosen": -104.41130065917969, "logps/ref_rejected": -101.25489807128906, "logps/rejected": -101.64776611328125, "loss": 1.3522, "margin_dpo/margin_mean": 0.3598228096961975, "margin_dpo/margin_std": 0.7777154445648193, "step": 47 }, { "KL/chosen_KL_mean": 0.06990432739257812, "KL/mean": -0.23917576670646667, "KL/rejected_KL_mean": -0.5482559204101562, "KL/std": 0.727091908454895, "epoch": 0.07048458149779736, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.6181590557098389, "fcm_dpo/q_t": 0.48459839820861816, "grad_norm": 38.380374908447266, "learning_rate": 3.4057971014492755e-07, "logits/chosen": 0.38576287031173706, "logits/rejected": 0.5973563194274902, "logps/chosen": -91.03036499023438, "logps/ref_chosen": -91.10027313232422, "logps/ref_rejected": -95.08058166503906, "logps/rejected": -95.62882995605469, "loss": 1.3278, "margin_dpo/margin_mean": 0.6181597113609314, "margin_dpo/margin_std": 0.9664177894592285, "step": 48 }, { "KL/chosen_KL_mean": 0.11545562744140625, "KL/mean": -0.2904796898365021, "KL/rejected_KL_mean": -0.6964149475097656, "KL/std": 0.8246832489967346, "epoch": 0.07195301027900147, "fcm_dpo/beta": 0.10157443583011627, "fcm_dpo/delta": 0.1550159603357315, "fcm_dpo/margin": 0.8118634223937988, "fcm_dpo/q_t": 0.4797772169113159, "grad_norm": 40.171722412109375, "learning_rate": 3.478260869565217e-07, "logits/chosen": -0.09916692972183228, "logits/rejected": 0.3478316366672516, "logps/chosen": -92.88822937011719, "logps/ref_chosen": -93.00367736816406, "logps/ref_rejected": -91.74899291992188, "logps/rejected": -92.4454116821289, "loss": 1.3077, "margin_dpo/margin_mean": 0.8118633031845093, "margin_dpo/margin_std": 0.9465253353118896, "step": 49 }, { "KL/chosen_KL_mean": -0.12697982788085938, "KL/mean": -0.44208717346191406, "KL/rejected_KL_mean": -0.7571830749511719, "KL/std": 0.7458336353302002, "epoch": 0.07342143906020558, "fcm_dpo/beta": 0.10314887762069702, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.630203127861023, "fcm_dpo/q_t": 0.4838068187236786, "grad_norm": 34.289432525634766, "learning_rate": 3.5507246376811595e-07, "logits/chosen": 0.4334755539894104, "logits/rejected": 0.6004021167755127, "logps/chosen": -94.75379180908203, "logps/ref_chosen": -94.62681579589844, "logps/ref_rejected": -103.57435607910156, "logps/rejected": -104.33154296875, "loss": 1.3247, "margin_dpo/margin_mean": 0.630203127861023, "margin_dpo/margin_std": 0.9408136010169983, "step": 50 }, { "KL/chosen_KL_mean": -0.141815185546875, "KL/mean": -0.4506117105484009, "KL/rejected_KL_mean": -0.7594108581542969, "KL/std": 0.8958399295806885, "epoch": 0.07488986784140969, "fcm_dpo/beta": 0.10314887762069702, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.6175858974456787, "fcm_dpo/q_t": 0.48417747020721436, "grad_norm": 30.859676361083984, "learning_rate": 3.6231884057971015e-07, "logits/chosen": 0.5725871324539185, "logits/rejected": 0.7114852666854858, "logps/chosen": -87.64909362792969, "logps/ref_chosen": -87.50727844238281, "logps/ref_rejected": -83.47235107421875, "logps/rejected": -84.23176574707031, "loss": 1.3271, "margin_dpo/margin_mean": 0.6175858974456787, "margin_dpo/margin_std": 1.1345471143722534, "step": 51 }, { "KL/chosen_KL_mean": -0.008541107177734375, "KL/mean": -0.4418838918209076, "KL/rejected_KL_mean": -0.8752326965332031, "KL/std": 0.7967055439949036, "epoch": 0.0763582966226138, "fcm_dpo/beta": 0.10803128033876419, "fcm_dpo/delta": 0.30932843685150146, "fcm_dpo/margin": 0.8666830658912659, "fcm_dpo/q_t": 0.4774218797683716, "grad_norm": 34.28953552246094, "learning_rate": 3.695652173913043e-07, "logits/chosen": 0.5145243406295776, "logits/rejected": 0.7332535982131958, "logps/chosen": -90.63880920410156, "logps/ref_chosen": -90.63026428222656, "logps/ref_rejected": -87.0390625, "logps/rejected": -87.91429138183594, "loss": 1.2975, "margin_dpo/margin_mean": 0.8666827082633972, "margin_dpo/margin_std": 0.9383734464645386, "step": 52 }, { "KL/chosen_KL_mean": -0.046649932861328125, "KL/mean": -0.5514045357704163, "KL/rejected_KL_mean": -1.0561561584472656, "KL/std": 1.0689395666122437, "epoch": 0.07782672540381791, "fcm_dpo/beta": 0.11109478771686554, "fcm_dpo/delta": 0.1226954460144043, "fcm_dpo/margin": 1.00951087474823, "fcm_dpo/q_t": 0.4726340174674988, "grad_norm": 43.298675537109375, "learning_rate": 3.7681159420289855e-07, "logits/chosen": 0.2125859409570694, "logits/rejected": 0.32951170206069946, "logps/chosen": -81.62973022460938, "logps/ref_chosen": -81.58307647705078, "logps/ref_rejected": -95.66152954101562, "logps/rejected": -96.71768188476562, "loss": 1.2832, "margin_dpo/margin_mean": 1.0095107555389404, "margin_dpo/margin_std": 1.3466317653656006, "step": 53 }, { "KL/chosen_KL_mean": -0.046512603759765625, "KL/mean": -0.6142176985740662, "KL/rejected_KL_mean": -1.1819190979003906, "KL/std": 1.314652681350708, "epoch": 0.07929515418502203, "fcm_dpo/beta": 0.1171455830335617, "fcm_dpo/delta": 0.27051880955696106, "fcm_dpo/margin": 1.1354037523269653, "fcm_dpo/q_t": 0.46792617440223694, "grad_norm": 45.60523986816406, "learning_rate": 3.8405797101449274e-07, "logits/chosen": 0.32095086574554443, "logits/rejected": 0.4970032870769501, "logps/chosen": -88.95668029785156, "logps/ref_chosen": -88.91017150878906, "logps/ref_rejected": -99.1175537109375, "logps/rejected": -100.29946899414062, "loss": 1.2633, "margin_dpo/margin_mean": 1.1354036331176758, "margin_dpo/margin_std": 1.2828892469406128, "step": 54 }, { "KL/chosen_KL_mean": 0.10482025146484375, "KL/mean": -0.4709744155406952, "KL/rejected_KL_mean": -1.0467643737792969, "KL/std": 1.2181047201156616, "epoch": 0.08076358296622614, "fcm_dpo/beta": 0.12369932234287262, "fcm_dpo/delta": 0.2606315016746521, "fcm_dpo/margin": 1.1515867710113525, "fcm_dpo/q_t": 0.46573761105537415, "grad_norm": 42.0218505859375, "learning_rate": 3.9130434782608694e-07, "logits/chosen": 0.09679587185382843, "logits/rejected": 0.40507882833480835, "logps/chosen": -92.35110473632812, "logps/ref_chosen": -92.45591735839844, "logps/ref_rejected": -92.97093200683594, "logps/rejected": -94.0177001953125, "loss": 1.2585, "margin_dpo/margin_mean": 1.1515865325927734, "margin_dpo/margin_std": 1.571197748184204, "step": 55 }, { "KL/chosen_KL_mean": -0.08965682983398438, "KL/mean": -0.5943702459335327, "KL/rejected_KL_mean": -1.0990829467773438, "KL/std": 1.1371006965637207, "epoch": 0.08223201174743025, "fcm_dpo/beta": 0.13020451366901398, "fcm_dpo/delta": 0.2722901403903961, "fcm_dpo/margin": 1.0094265937805176, "fcm_dpo/q_t": 0.4686237573623657, "grad_norm": 41.74551773071289, "learning_rate": 3.9855072463768114e-07, "logits/chosen": 0.19714362919330597, "logits/rejected": 0.5365463495254517, "logps/chosen": -87.32630920410156, "logps/ref_chosen": -87.23665618896484, "logps/ref_rejected": -100.64553833007812, "logps/rejected": -101.74462890625, "loss": 1.2701, "margin_dpo/margin_mean": 1.0094258785247803, "margin_dpo/margin_std": 1.5956398248672485, "step": 56 }, { "KL/chosen_KL_mean": -0.07779693603515625, "KL/mean": -0.6693606376647949, "KL/rejected_KL_mean": -1.2609329223632812, "KL/std": 1.2961477041244507, "epoch": 0.08370044052863436, "fcm_dpo/beta": 0.13702607154846191, "fcm_dpo/delta": 0.2416207194328308, "fcm_dpo/margin": 1.1831443309783936, "fcm_dpo/q_t": 0.4611359238624573, "grad_norm": 43.754173278808594, "learning_rate": 4.057971014492754e-07, "logits/chosen": 0.3084503412246704, "logits/rejected": 0.5258380770683289, "logps/chosen": -98.2285385131836, "logps/ref_chosen": -98.15074157714844, "logps/ref_rejected": -101.05284118652344, "logps/rejected": -102.31376647949219, "loss": 1.2427, "margin_dpo/margin_mean": 1.183143973350525, "margin_dpo/margin_std": 1.6165554523468018, "step": 57 }, { "KL/chosen_KL_mean": 0.24322509765625, "KL/mean": -0.518633246421814, "KL/rejected_KL_mean": -1.280487060546875, "KL/std": 1.427086591720581, "epoch": 0.08516886930983847, "fcm_dpo/beta": 0.14289551973342896, "fcm_dpo/delta": 0.18566016852855682, "fcm_dpo/margin": 1.5237128734588623, "fcm_dpo/q_t": 0.44805383682250977, "grad_norm": 49.98211669921875, "learning_rate": 4.1304347826086954e-07, "logits/chosen": 0.3037877678871155, "logits/rejected": 0.6340248584747314, "logps/chosen": -99.3377456665039, "logps/ref_chosen": -99.58097076416016, "logps/ref_rejected": -91.22227478027344, "logps/rejected": -92.50276184082031, "loss": 1.1988, "margin_dpo/margin_mean": 1.5237126350402832, "margin_dpo/margin_std": 1.879228115081787, "step": 58 }, { "KL/chosen_KL_mean": 0.0708160400390625, "KL/mean": -0.565733015537262, "KL/rejected_KL_mean": -1.2022781372070312, "KL/std": 1.2017815113067627, "epoch": 0.08663729809104258, "fcm_dpo/beta": 0.14867879450321198, "fcm_dpo/delta": 0.21456587314605713, "fcm_dpo/margin": 1.2730885744094849, "fcm_dpo/q_t": 0.4544857144355774, "grad_norm": 47.67547607421875, "learning_rate": 4.2028985507246374e-07, "logits/chosen": 0.022571925073862076, "logits/rejected": 0.2062058448791504, "logps/chosen": -89.73150634765625, "logps/ref_chosen": -89.80232238769531, "logps/ref_rejected": -94.27667236328125, "logps/rejected": -95.47895812988281, "loss": 1.2171, "margin_dpo/margin_mean": 1.2730882167816162, "margin_dpo/margin_std": 1.4308891296386719, "step": 59 }, { "KL/chosen_KL_mean": 0.017635345458984375, "KL/mean": -0.5863430500030518, "KL/rejected_KL_mean": -1.1903266906738281, "KL/std": 1.2484078407287598, "epoch": 0.0881057268722467, "fcm_dpo/beta": 0.15513595938682556, "fcm_dpo/delta": 0.21661323308944702, "fcm_dpo/margin": 1.2079617977142334, "fcm_dpo/q_t": 0.45489397644996643, "grad_norm": 41.734779357910156, "learning_rate": 4.2753623188405794e-07, "logits/chosen": 0.4172214865684509, "logits/rejected": 0.6895511746406555, "logps/chosen": -95.13807678222656, "logps/ref_chosen": -95.15571594238281, "logps/ref_rejected": -91.3724365234375, "logps/rejected": -92.56275939941406, "loss": 1.2187, "margin_dpo/margin_mean": 1.2079614400863647, "margin_dpo/margin_std": 1.3682575225830078, "step": 60 }, { "KL/chosen_KL_mean": -0.17267608642578125, "KL/mean": -0.7395094037055969, "KL/rejected_KL_mean": -1.3063430786132812, "KL/std": 1.3454945087432861, "epoch": 0.08957415565345081, "fcm_dpo/beta": 0.16227757930755615, "fcm_dpo/delta": 0.2197197675704956, "fcm_dpo/margin": 1.1336653232574463, "fcm_dpo/q_t": 0.4565168619155884, "grad_norm": 42.114280700683594, "learning_rate": 4.3478260869565214e-07, "logits/chosen": 0.3746405243873596, "logits/rejected": 0.5182892084121704, "logps/chosen": -85.90498352050781, "logps/ref_chosen": -85.73230743408203, "logps/ref_rejected": -97.96575927734375, "logps/rejected": -99.27210235595703, "loss": 1.2303, "margin_dpo/margin_mean": 1.133665919303894, "margin_dpo/margin_std": 1.7271978855133057, "step": 61 }, { "KL/chosen_KL_mean": -0.1939849853515625, "KL/mean": -0.7869770526885986, "KL/rejected_KL_mean": -1.3799667358398438, "KL/std": 1.448983907699585, "epoch": 0.09104258443465492, "fcm_dpo/beta": 0.16903537511825562, "fcm_dpo/delta": 0.20344430208206177, "fcm_dpo/margin": 1.1859891414642334, "fcm_dpo/q_t": 0.452701210975647, "grad_norm": 42.73322677612305, "learning_rate": 4.420289855072464e-07, "logits/chosen": 0.3216652572154999, "logits/rejected": 0.524534285068512, "logps/chosen": -81.8293685913086, "logps/ref_chosen": -81.63538360595703, "logps/ref_rejected": -84.03831481933594, "logps/rejected": -85.41827392578125, "loss": 1.2246, "margin_dpo/margin_mean": 1.185987949371338, "margin_dpo/margin_std": 1.9363813400268555, "step": 62 }, { "KL/chosen_KL_mean": 0.004486083984375, "KL/mean": -0.7590829133987427, "KL/rejected_KL_mean": -1.5226516723632812, "KL/std": 1.3426729440689087, "epoch": 0.09251101321585903, "fcm_dpo/beta": 0.1738523244857788, "fcm_dpo/delta": 0.13837175071239471, "fcm_dpo/margin": 1.5271415710449219, "fcm_dpo/q_t": 0.43629300594329834, "grad_norm": 54.37081527709961, "learning_rate": 4.4927536231884053e-07, "logits/chosen": 0.41804039478302, "logits/rejected": 0.6320738196372986, "logps/chosen": -103.61956787109375, "logps/ref_chosen": -103.62405395507812, "logps/ref_rejected": -103.40303039550781, "logps/rejected": -104.9256820678711, "loss": 1.1566, "margin_dpo/margin_mean": 1.5271413326263428, "margin_dpo/margin_std": 1.5446617603302002, "step": 63 }, { "KL/chosen_KL_mean": -0.10556793212890625, "KL/mean": -0.9591192603111267, "KL/rejected_KL_mean": -1.8126792907714844, "KL/std": 1.7861369848251343, "epoch": 0.09397944199706314, "fcm_dpo/beta": 0.1773890256881714, "fcm_dpo/delta": 0.10029230266809464, "fcm_dpo/margin": 1.7071115970611572, "fcm_dpo/q_t": 0.4290154278278351, "grad_norm": 52.18312454223633, "learning_rate": 4.5652173913043473e-07, "logits/chosen": 0.4455505609512329, "logits/rejected": 0.6594642400741577, "logps/chosen": -87.1071548461914, "logps/ref_chosen": -87.0015869140625, "logps/ref_rejected": -100.5854721069336, "logps/rejected": -102.39814758300781, "loss": 1.1388, "margin_dpo/margin_mean": 1.7071114778518677, "margin_dpo/margin_std": 1.985020399093628, "step": 64 }, { "KL/chosen_KL_mean": -0.0731964111328125, "KL/mean": -0.9616843461990356, "KL/rejected_KL_mean": -1.8501777648925781, "KL/std": 1.6791359186172485, "epoch": 0.09544787077826726, "fcm_dpo/beta": 0.1812608242034912, "fcm_dpo/delta": 0.0804394856095314, "fcm_dpo/margin": 1.7769804000854492, "fcm_dpo/q_t": 0.4243488013744354, "grad_norm": 58.16350555419922, "learning_rate": 4.63768115942029e-07, "logits/chosen": -0.10358546674251556, "logits/rejected": 0.04005581885576248, "logps/chosen": -91.29511260986328, "logps/ref_chosen": -91.22191619873047, "logps/ref_rejected": -115.33553314208984, "logps/rejected": -117.18571472167969, "loss": 1.1242, "margin_dpo/margin_mean": 1.7769802808761597, "margin_dpo/margin_std": 2.103424072265625, "step": 65 }, { "KL/chosen_KL_mean": -0.3371429443359375, "KL/mean": -0.9625747203826904, "KL/rejected_KL_mean": -1.5880088806152344, "KL/std": 1.520745873451233, "epoch": 0.09691629955947137, "fcm_dpo/beta": 0.18575233221054077, "fcm_dpo/delta": 0.17199671268463135, "fcm_dpo/margin": 1.250862717628479, "fcm_dpo/q_t": 0.4455907940864563, "grad_norm": 41.983543395996094, "learning_rate": 4.7101449275362313e-07, "logits/chosen": 0.02338741533458233, "logits/rejected": 0.2185419499874115, "logps/chosen": -84.12136840820312, "logps/ref_chosen": -83.78422546386719, "logps/ref_rejected": -82.7520980834961, "logps/rejected": -84.34010314941406, "loss": 1.1977, "margin_dpo/margin_mean": 1.2508630752563477, "margin_dpo/margin_std": 1.8010478019714355, "step": 66 }, { "KL/chosen_KL_mean": -0.8406600952148438, "KL/mean": -1.393054723739624, "KL/rejected_KL_mean": -1.9454421997070312, "KL/std": 1.989898920059204, "epoch": 0.09838472834067548, "fcm_dpo/beta": 0.19371409714221954, "fcm_dpo/delta": 0.19003048539161682, "fcm_dpo/margin": 1.1047859191894531, "fcm_dpo/q_t": 0.4527943730354309, "grad_norm": 44.70664978027344, "learning_rate": 4.782608695652174e-07, "logits/chosen": 0.41904449462890625, "logits/rejected": 0.7008070945739746, "logps/chosen": -88.51362609863281, "logps/ref_chosen": -87.67295837402344, "logps/ref_rejected": -79.08674621582031, "logps/rejected": -81.03219604492188, "loss": 1.247, "margin_dpo/margin_mean": 1.1047863960266113, "margin_dpo/margin_std": 2.6726999282836914, "step": 67 }, { "KL/chosen_KL_mean": -0.22499465942382812, "KL/mean": -1.239583134651184, "KL/rejected_KL_mean": -2.254161834716797, "KL/std": 2.1746928691864014, "epoch": 0.09985315712187959, "fcm_dpo/beta": 0.19569340348243713, "fcm_dpo/delta": 0.0030235094018280506, "fcm_dpo/margin": 2.0291647911071777, "fcm_dpo/q_t": 0.41154956817626953, "grad_norm": 44.33134841918945, "learning_rate": 4.855072463768116e-07, "logits/chosen": -0.00035918503999710083, "logits/rejected": 0.2960222363471985, "logps/chosen": -97.30384063720703, "logps/ref_chosen": -97.07884216308594, "logps/ref_rejected": -84.11872863769531, "logps/rejected": -86.37289428710938, "loss": 1.0921, "margin_dpo/margin_mean": 2.0291647911071777, "margin_dpo/margin_std": 2.771623134613037, "step": 68 }, { "KL/chosen_KL_mean": -0.28879547119140625, "KL/mean": -1.3724240064620972, "KL/rejected_KL_mean": -2.4560470581054688, "KL/std": 2.380420207977295, "epoch": 0.1013215859030837, "fcm_dpo/beta": 0.19525523483753204, "fcm_dpo/delta": -0.024201039224863052, "fcm_dpo/margin": 2.1672563552856445, "fcm_dpo/q_t": 0.4055878818035126, "grad_norm": 46.5745964050293, "learning_rate": 4.927536231884058e-07, "logits/chosen": 0.2287139594554901, "logits/rejected": 0.33843833208084106, "logps/chosen": -86.00850677490234, "logps/ref_chosen": -85.71971130371094, "logps/ref_rejected": -109.48020935058594, "logps/rejected": -111.93624877929688, "loss": 1.0795, "margin_dpo/margin_mean": 2.1672568321228027, "margin_dpo/margin_std": 2.958550453186035, "step": 69 }, { "KL/chosen_KL_mean": -0.5231552124023438, "KL/mean": -1.785667896270752, "KL/rejected_KL_mean": -3.0481796264648438, "KL/std": 2.7011332511901855, "epoch": 0.1027900146842878, "fcm_dpo/beta": 0.19221171736717224, "fcm_dpo/delta": -0.08966369926929474, "fcm_dpo/margin": 2.525022268295288, "fcm_dpo/q_t": 0.3948443531990051, "grad_norm": 42.88922882080078, "learning_rate": 5e-07, "logits/chosen": 0.22474244236946106, "logits/rejected": 0.3976325988769531, "logps/chosen": -95.5331039428711, "logps/ref_chosen": -95.00994873046875, "logps/ref_rejected": -96.21272277832031, "logps/rejected": -99.26090240478516, "loss": 1.0486, "margin_dpo/margin_mean": 2.525021553039551, "margin_dpo/margin_std": 3.3418660163879395, "step": 70 }, { "KL/chosen_KL_mean": 0.030101776123046875, "KL/mean": -1.723403811454773, "KL/rejected_KL_mean": -3.4769134521484375, "KL/std": 3.3195619583129883, "epoch": 0.10425844346549193, "fcm_dpo/beta": 0.1830272674560547, "fcm_dpo/delta": -0.2604806125164032, "fcm_dpo/margin": 3.5070133209228516, "fcm_dpo/q_t": 0.36002588272094727, "grad_norm": 48.0555419921875, "learning_rate": 4.999967061337492e-07, "logits/chosen": -0.36086946725845337, "logits/rejected": -0.016953080892562866, "logps/chosen": -90.94725036621094, "logps/ref_chosen": -90.97735595703125, "logps/ref_rejected": -102.59103393554688, "logps/rejected": -106.06794738769531, "loss": 0.9386, "margin_dpo/margin_mean": 3.5070126056671143, "margin_dpo/margin_std": 3.792950391769409, "step": 71 }, { "KL/chosen_KL_mean": -0.5298118591308594, "KL/mean": -1.9587078094482422, "KL/rejected_KL_mean": -3.3876075744628906, "KL/std": 2.862316608428955, "epoch": 0.10572687224669604, "fcm_dpo/beta": 0.17804387211799622, "fcm_dpo/delta": -0.11504629254341125, "fcm_dpo/margin": 2.8578004837036133, "fcm_dpo/q_t": 0.3855014443397522, "grad_norm": 41.55047607421875, "learning_rate": 4.999868246217933e-07, "logits/chosen": -0.43137094378471375, "logits/rejected": -0.13316676020622253, "logps/chosen": -98.42134094238281, "logps/ref_chosen": -97.89152526855469, "logps/ref_rejected": -100.19171142578125, "logps/rejected": -103.5793228149414, "loss": 1.0182, "margin_dpo/margin_mean": 2.8578004837036133, "margin_dpo/margin_std": 3.3293988704681396, "step": 72 }, { "KL/chosen_KL_mean": -0.721466064453125, "KL/mean": -2.238034725189209, "KL/rejected_KL_mean": -3.7546043395996094, "KL/std": 3.688286781311035, "epoch": 0.10719530102790015, "fcm_dpo/beta": 0.17389589548110962, "fcm_dpo/delta": -0.13456253707408905, "fcm_dpo/margin": 3.0331432819366455, "fcm_dpo/q_t": 0.3930155634880066, "grad_norm": 39.34903335571289, "learning_rate": 4.999703557245192e-07, "logits/chosen": -0.4581258296966553, "logits/rejected": -0.16796040534973145, "logps/chosen": -96.49048614501953, "logps/ref_chosen": -95.7690200805664, "logps/ref_rejected": -95.93243408203125, "logps/rejected": -99.68704223632812, "loss": 1.0607, "margin_dpo/margin_mean": 3.0331430435180664, "margin_dpo/margin_std": 4.717763900756836, "step": 73 }, { "KL/chosen_KL_mean": -0.8892173767089844, "KL/mean": -2.579256772994995, "KL/rejected_KL_mean": -4.269294738769531, "KL/std": 3.6131372451782227, "epoch": 0.10866372980910426, "fcm_dpo/beta": 0.16779246926307678, "fcm_dpo/delta": -0.17777913808822632, "fcm_dpo/margin": 3.3800764083862305, "fcm_dpo/q_t": 0.3775174617767334, "grad_norm": 39.307743072509766, "learning_rate": 4.999472998758977e-07, "logits/chosen": -0.15416769683361053, "logits/rejected": -0.01179670263081789, "logps/chosen": -79.69761657714844, "logps/ref_chosen": -78.80839538574219, "logps/ref_rejected": -101.64677429199219, "logps/rejected": -105.91606140136719, "loss": 1.0049, "margin_dpo/margin_mean": 3.3800764083862305, "margin_dpo/margin_std": 4.235011577606201, "step": 74 }, { "KL/chosen_KL_mean": -0.985992431640625, "KL/mean": -3.125307083129883, "KL/rejected_KL_mean": -5.264621734619141, "KL/std": 4.328281402587891, "epoch": 0.11013215859030837, "fcm_dpo/beta": 0.16025742888450623, "fcm_dpo/delta": -0.3056366443634033, "fcm_dpo/margin": 4.278628826141357, "fcm_dpo/q_t": 0.3579943776130676, "grad_norm": 39.74001693725586, "learning_rate": 4.999176576834721e-07, "logits/chosen": -0.24804621934890747, "logits/rejected": -0.047132182866334915, "logps/chosen": -79.2678451538086, "logps/ref_chosen": -78.28185272216797, "logps/ref_rejected": -115.40311431884766, "logps/rejected": -120.66773986816406, "loss": 0.9458, "margin_dpo/margin_mean": 4.278629302978516, "margin_dpo/margin_std": 5.1031293869018555, "step": 75 }, { "KL/chosen_KL_mean": -1.3166313171386719, "KL/mean": -2.8327572345733643, "KL/rejected_KL_mean": -4.348884582519531, "KL/std": 3.701216220855713, "epoch": 0.11160058737151249, "fcm_dpo/beta": 0.1550556868314743, "fcm_dpo/delta": -0.07364660501480103, "fcm_dpo/margin": 3.032254219055176, "fcm_dpo/q_t": 0.3989346921443939, "grad_norm": 32.30008316040039, "learning_rate": 4.998814299283415e-07, "logits/chosen": -0.6593241691589355, "logits/rejected": -0.2927325963973999, "logps/chosen": -89.19377136230469, "logps/ref_chosen": -87.87713623046875, "logps/ref_rejected": -85.71968078613281, "logps/rejected": -90.06856536865234, "loss": 1.0743, "margin_dpo/margin_mean": 3.032254695892334, "margin_dpo/margin_std": 4.514759540557861, "step": 76 }, { "KL/chosen_KL_mean": -1.4705429077148438, "KL/mean": -3.095768451690674, "KL/rejected_KL_mean": -4.720996856689453, "KL/std": 4.19633150100708, "epoch": 0.1130690161527166, "fcm_dpo/beta": 0.15173056721687317, "fcm_dpo/delta": -0.09861107915639877, "fcm_dpo/margin": 3.2504522800445557, "fcm_dpo/q_t": 0.3927234411239624, "grad_norm": 37.87125778198242, "learning_rate": 4.998386175651409e-07, "logits/chosen": 0.0034573785960674286, "logits/rejected": 0.38357114791870117, "logps/chosen": -101.17089080810547, "logps/ref_chosen": -99.70034790039062, "logps/ref_rejected": -98.20576477050781, "logps/rejected": -102.9267578125, "loss": 1.0726, "margin_dpo/margin_mean": 3.2504520416259766, "margin_dpo/margin_std": 4.8470377922058105, "step": 77 }, { "KL/chosen_KL_mean": -0.6389923095703125, "KL/mean": -2.3557515144348145, "KL/rejected_KL_mean": -4.072505950927734, "KL/std": 3.4795303344726562, "epoch": 0.1145374449339207, "fcm_dpo/beta": 0.14879390597343445, "fcm_dpo/delta": -0.11692792177200317, "fcm_dpo/margin": 3.4335122108459473, "fcm_dpo/q_t": 0.3858852982521057, "grad_norm": 31.79231834411621, "learning_rate": 4.997892217220159e-07, "logits/chosen": 0.07325912266969681, "logits/rejected": 0.30155330896377563, "logps/chosen": -90.93569946289062, "logps/ref_chosen": -90.29670715332031, "logps/ref_rejected": -91.13772583007812, "logps/rejected": -95.21023559570312, "loss": 1.0191, "margin_dpo/margin_mean": 3.433511972427368, "margin_dpo/margin_std": 4.102811813354492, "step": 78 }, { "KL/chosen_KL_mean": -0.7737541198730469, "KL/mean": -3.0345816612243652, "KL/rejected_KL_mean": -5.295417785644531, "KL/std": 5.100748062133789, "epoch": 0.11600587371512482, "fcm_dpo/beta": 0.1426713764667511, "fcm_dpo/delta": -0.2615537643432617, "fcm_dpo/margin": 4.521660804748535, "fcm_dpo/q_t": 0.36319810152053833, "grad_norm": 32.46731185913086, "learning_rate": 4.997332437005931e-07, "logits/chosen": -0.26485198736190796, "logits/rejected": 0.13713252544403076, "logps/chosen": -87.15208435058594, "logps/ref_chosen": -86.37832641601562, "logps/ref_rejected": -94.10777282714844, "logps/rejected": -99.40318298339844, "loss": 0.9707, "margin_dpo/margin_mean": 4.521660327911377, "margin_dpo/margin_std": 5.635766983032227, "step": 79 }, { "KL/chosen_KL_mean": -1.3233757019042969, "KL/mean": -3.144094944000244, "KL/rejected_KL_mean": -4.964817047119141, "KL/std": 4.357922554016113, "epoch": 0.11747430249632893, "fcm_dpo/beta": 0.13804489374160767, "fcm_dpo/delta": -0.10826519876718521, "fcm_dpo/margin": 3.64143705368042, "fcm_dpo/q_t": 0.3902570605278015, "grad_norm": 31.544845581054688, "learning_rate": 4.996706849759452e-07, "logits/chosen": -0.6535516977310181, "logits/rejected": -0.30604004859924316, "logps/chosen": -95.29369354248047, "logps/ref_chosen": -93.97032165527344, "logps/ref_rejected": -92.57441711425781, "logps/rejected": -97.53923034667969, "loss": 1.0359, "margin_dpo/margin_mean": 3.6414365768432617, "margin_dpo/margin_std": 4.712320327758789, "step": 80 }, { "KL/chosen_KL_mean": -0.8677406311035156, "KL/mean": -3.436732769012451, "KL/rejected_KL_mean": -6.005718231201172, "KL/std": 5.486260414123535, "epoch": 0.11894273127753303, "fcm_dpo/beta": 0.13054189085960388, "fcm_dpo/delta": -0.293806791305542, "fcm_dpo/margin": 5.137979030609131, "fcm_dpo/q_t": 0.3613430857658386, "grad_norm": 31.406797409057617, "learning_rate": 4.996015471965529e-07, "logits/chosen": -0.6966358423233032, "logits/rejected": -0.28589504957199097, "logps/chosen": -100.69786071777344, "logps/ref_chosen": -99.83012390136719, "logps/ref_rejected": -133.67247009277344, "logps/rejected": -139.67819213867188, "loss": 0.9624, "margin_dpo/margin_mean": 5.137979030609131, "margin_dpo/margin_std": 6.480086326599121, "step": 81 }, { "KL/chosen_KL_mean": -1.7725448608398438, "KL/mean": -3.5209622383117676, "KL/rejected_KL_mean": -5.269378662109375, "KL/std": 4.265264987945557, "epoch": 0.12041116005873716, "fcm_dpo/beta": 0.12946073710918427, "fcm_dpo/delta": -0.05587388575077057, "fcm_dpo/margin": 3.4968299865722656, "fcm_dpo/q_t": 0.3991600275039673, "grad_norm": 31.16779136657715, "learning_rate": 4.995258321842611e-07, "logits/chosen": -0.2508911192417145, "logits/rejected": -0.041120532900094986, "logps/chosen": -84.81852722167969, "logps/ref_chosen": -83.04598236083984, "logps/ref_rejected": -94.52595520019531, "logps/rejected": -99.79533386230469, "loss": 1.0709, "margin_dpo/margin_mean": 3.4968299865722656, "margin_dpo/margin_std": 4.684732913970947, "step": 82 }, { "KL/chosen_KL_mean": -1.99755859375, "KL/mean": -4.024735450744629, "KL/rejected_KL_mean": -6.051914215087891, "KL/std": 4.803101062774658, "epoch": 0.12187958883994127, "fcm_dpo/beta": 0.12589313089847565, "fcm_dpo/delta": -0.1164156049489975, "fcm_dpo/margin": 4.054357528686523, "fcm_dpo/q_t": 0.3907790780067444, "grad_norm": 32.27962875366211, "learning_rate": 4.994435419342304e-07, "logits/chosen": -0.35282906889915466, "logits/rejected": 0.0684627816081047, "logps/chosen": -94.17377471923828, "logps/ref_chosen": -92.17621612548828, "logps/ref_rejected": -107.74464416503906, "logps/rejected": -113.79655456542969, "loss": 1.0613, "margin_dpo/margin_mean": 4.054357528686523, "margin_dpo/margin_std": 5.970000267028809, "step": 83 }, { "KL/chosen_KL_mean": -2.5833091735839844, "KL/mean": -3.8056838512420654, "KL/rejected_KL_mean": -5.028064727783203, "KL/std": 4.186389923095703, "epoch": 0.12334801762114538, "fcm_dpo/beta": 0.1258600503206253, "fcm_dpo/delta": 0.09445232897996902, "fcm_dpo/margin": 2.444753646850586, "fcm_dpo/q_t": 0.43287602066993713, "grad_norm": 35.470428466796875, "learning_rate": 4.993546786148857e-07, "logits/chosen": -0.16349703073501587, "logits/rejected": 0.23118868470191956, "logps/chosen": -104.10980224609375, "logps/ref_chosen": -101.5264892578125, "logps/ref_rejected": -92.42608642578125, "logps/rejected": -97.45414733886719, "loss": 1.1835, "margin_dpo/margin_mean": 2.444753885269165, "margin_dpo/margin_std": 4.468890190124512, "step": 84 }, { "KL/chosen_KL_mean": -2.2268829345703125, "KL/mean": -3.8887100219726562, "KL/rejected_KL_mean": -5.550537109375, "KL/std": 4.63828706741333, "epoch": 0.12481644640234948, "fcm_dpo/beta": 0.1276790201663971, "fcm_dpo/delta": -0.02574189379811287, "fcm_dpo/margin": 3.323657512664795, "fcm_dpo/q_t": 0.40794801712036133, "grad_norm": 33.49395751953125, "learning_rate": 4.992592445678582e-07, "logits/chosen": -0.1548769623041153, "logits/rejected": 0.17547939717769623, "logps/chosen": -98.35426330566406, "logps/ref_chosen": -96.12738037109375, "logps/ref_rejected": -85.05519104003906, "logps/rejected": -90.60572814941406, "loss": 1.1254, "margin_dpo/margin_mean": 3.323657989501953, "margin_dpo/margin_std": 5.6712565422058105, "step": 85 }, { "KL/chosen_KL_mean": -1.7762565612792969, "KL/mean": -3.8771860599517822, "KL/rejected_KL_mean": -5.978122711181641, "KL/std": 5.417524337768555, "epoch": 0.1262848751835536, "fcm_dpo/beta": 0.12431152909994125, "fcm_dpo/delta": -0.12935477495193481, "fcm_dpo/margin": 4.201868534088135, "fcm_dpo/q_t": 0.3937848210334778, "grad_norm": 26.12570571899414, "learning_rate": 4.991572423079235e-07, "logits/chosen": -0.6697604656219482, "logits/rejected": -0.4364662766456604, "logps/chosen": -83.48051452636719, "logps/ref_chosen": -81.70426177978516, "logps/ref_rejected": -93.65550231933594, "logps/rejected": -99.63362121582031, "loss": 1.0658, "margin_dpo/margin_mean": 4.201868534088135, "margin_dpo/margin_std": 6.64492654800415, "step": 86 }, { "KL/chosen_KL_mean": -2.0185470581054688, "KL/mean": -4.270969390869141, "KL/rejected_KL_mean": -6.523387908935547, "KL/std": 5.914112091064453, "epoch": 0.1277533039647577, "fcm_dpo/beta": 0.12055070698261261, "fcm_dpo/delta": -0.15196214616298676, "fcm_dpo/margin": 4.50484561920166, "fcm_dpo/q_t": 0.3831990957260132, "grad_norm": 30.394954681396484, "learning_rate": 4.990486745229364e-07, "logits/chosen": -0.6615973711013794, "logits/rejected": -0.2949981689453125, "logps/chosen": -94.70451354980469, "logps/ref_chosen": -92.68596649169922, "logps/ref_rejected": -102.91818237304688, "logps/rejected": -109.44156646728516, "loss": 1.069, "margin_dpo/margin_mean": 4.504846096038818, "margin_dpo/margin_std": 7.093955039978027, "step": 87 }, { "KL/chosen_KL_mean": -2.471027374267578, "KL/mean": -4.286229133605957, "KL/rejected_KL_mean": -6.101421356201172, "KL/std": 5.662147521972656, "epoch": 0.12922173274596183, "fcm_dpo/beta": 0.11859438568353653, "fcm_dpo/delta": -0.03308258578181267, "fcm_dpo/margin": 3.6303815841674805, "fcm_dpo/q_t": 0.4060899317264557, "grad_norm": 28.10683250427246, "learning_rate": 4.989335440737586e-07, "logits/chosen": -0.2797431945800781, "logits/rejected": -0.09978815168142319, "logps/chosen": -103.2340087890625, "logps/ref_chosen": -100.76298522949219, "logps/ref_rejected": -113.15037536621094, "logps/rejected": -119.25179290771484, "loss": 1.1098, "margin_dpo/margin_mean": 3.6303815841674805, "margin_dpo/margin_std": 5.808981895446777, "step": 88 }, { "KL/chosen_KL_mean": -2.0373306274414062, "KL/mean": -3.961242914199829, "KL/rejected_KL_mean": -5.885154724121094, "KL/std": 4.8695068359375, "epoch": 0.13069016152716592, "fcm_dpo/beta": 0.11782899498939514, "fcm_dpo/delta": -0.056305523961782455, "fcm_dpo/margin": 3.847822666168213, "fcm_dpo/q_t": 0.4058462977409363, "grad_norm": 26.600008010864258, "learning_rate": 4.988118539941847e-07, "logits/chosen": -0.7034850120544434, "logits/rejected": -0.3789067268371582, "logps/chosen": -91.72840881347656, "logps/ref_chosen": -89.69108581542969, "logps/ref_rejected": -88.56832885742188, "logps/rejected": -94.4534912109375, "loss": 1.0975, "margin_dpo/margin_mean": 3.847822666168213, "margin_dpo/margin_std": 6.464529991149902, "step": 89 }, { "KL/chosen_KL_mean": -1.8579521179199219, "KL/mean": -4.348388671875, "KL/rejected_KL_mean": -6.8388214111328125, "KL/std": 5.588225364685059, "epoch": 0.13215859030837004, "fcm_dpo/beta": 0.11510182917118073, "fcm_dpo/delta": -0.1834583282470703, "fcm_dpo/margin": 4.980863571166992, "fcm_dpo/q_t": 0.3773382902145386, "grad_norm": 28.754634857177734, "learning_rate": 4.986836074908615e-07, "logits/chosen": -0.47218501567840576, "logits/rejected": -0.34603163599967957, "logps/chosen": -83.24050903320312, "logps/ref_chosen": -81.38255310058594, "logps/ref_rejected": -117.77714538574219, "logps/rejected": -124.615966796875, "loss": 1.014, "margin_dpo/margin_mean": 4.980863571166992, "margin_dpo/margin_std": 6.593569278717041, "step": 90 }, { "KL/chosen_KL_mean": -2.0075416564941406, "KL/mean": -4.232414722442627, "KL/rejected_KL_mean": -6.457286834716797, "KL/std": 5.631711483001709, "epoch": 0.13362701908957417, "fcm_dpo/beta": 0.11195258796215057, "fcm_dpo/delta": -0.10325415432453156, "fcm_dpo/margin": 4.449746131896973, "fcm_dpo/q_t": 0.39433497190475464, "grad_norm": 27.465124130249023, "learning_rate": 4.985488079432037e-07, "logits/chosen": -0.2441919445991516, "logits/rejected": 0.16743159294128418, "logps/chosen": -99.22942352294922, "logps/ref_chosen": -97.22187805175781, "logps/ref_rejected": -92.97674560546875, "logps/rejected": -99.43403625488281, "loss": 1.0585, "margin_dpo/margin_mean": 4.449746131896973, "margin_dpo/margin_std": 6.4690046310424805, "step": 91 }, { "KL/chosen_KL_mean": -2.3746337890625, "KL/mean": -4.26043701171875, "KL/rejected_KL_mean": -6.146236419677734, "KL/std": 5.5553483963012695, "epoch": 0.13509544787077826, "fcm_dpo/beta": 0.11114701628684998, "fcm_dpo/delta": -0.020065290853381157, "fcm_dpo/margin": 3.7716033458709717, "fcm_dpo/q_t": 0.4088805913925171, "grad_norm": 25.592466354370117, "learning_rate": 4.984074589033043e-07, "logits/chosen": -0.759026050567627, "logits/rejected": -0.401542603969574, "logps/chosen": -86.9049072265625, "logps/ref_chosen": -84.5302734375, "logps/ref_rejected": -84.5013198852539, "logps/rejected": -90.64755249023438, "loss": 1.1054, "margin_dpo/margin_mean": 3.7716031074523926, "margin_dpo/margin_std": 5.932674884796143, "step": 92 }, { "KL/chosen_KL_mean": -2.4927330017089844, "KL/mean": -4.345421314239502, "KL/rejected_KL_mean": -6.198112487792969, "KL/std": 5.266005992889404, "epoch": 0.13656387665198239, "fcm_dpo/beta": 0.1108069196343422, "fcm_dpo/delta": -0.011053171940147877, "fcm_dpo/margin": 3.705378532409668, "fcm_dpo/q_t": 0.41096413135528564, "grad_norm": 28.39630889892578, "learning_rate": 4.982595640958425e-07, "logits/chosen": -0.7877769470214844, "logits/rejected": -0.22196027636528015, "logps/chosen": -92.7431640625, "logps/ref_chosen": -90.25043487548828, "logps/ref_rejected": -84.0942153930664, "logps/rejected": -90.29232788085938, "loss": 1.133, "margin_dpo/margin_mean": 3.705378532409668, "margin_dpo/margin_std": 6.444093704223633, "step": 93 }, { "KL/chosen_KL_mean": -2.511890411376953, "KL/mean": -5.2273759841918945, "KL/rejected_KL_mean": -7.942863464355469, "KL/std": 6.0470476150512695, "epoch": 0.13803230543318648, "fcm_dpo/beta": 0.10653319954872131, "fcm_dpo/delta": -0.192873015999794, "fcm_dpo/margin": 5.430976390838623, "fcm_dpo/q_t": 0.3727499842643738, "grad_norm": 27.296619415283203, "learning_rate": 4.98105127417984e-07, "logits/chosen": -0.5182806253433228, "logits/rejected": -0.22217103838920593, "logps/chosen": -94.96611022949219, "logps/ref_chosen": -92.4542236328125, "logps/ref_rejected": -105.2472915649414, "logps/rejected": -113.19015502929688, "loss": 1.0121, "margin_dpo/margin_mean": 5.430975437164307, "margin_dpo/margin_std": 6.9383649826049805, "step": 94 }, { "KL/chosen_KL_mean": -2.1095314025878906, "KL/mean": -4.291229248046875, "KL/rejected_KL_mean": -6.472919464111328, "KL/std": 5.184084415435791, "epoch": 0.1395007342143906, "fcm_dpo/beta": 0.10587987303733826, "fcm_dpo/delta": -0.06512865424156189, "fcm_dpo/margin": 4.363385200500488, "fcm_dpo/q_t": 0.40065836906433105, "grad_norm": 23.576265335083008, "learning_rate": 4.979441529392784e-07, "logits/chosen": -0.8746038675308228, "logits/rejected": -0.5554424524307251, "logps/chosen": -80.98323822021484, "logps/ref_chosen": -78.87370300292969, "logps/ref_rejected": -83.59121704101562, "logps/rejected": -90.06413269042969, "loss": 1.0769, "margin_dpo/margin_mean": 4.363385200500488, "margin_dpo/margin_std": 6.325905799865723, "step": 95 }, { "KL/chosen_KL_mean": -1.6287765502929688, "KL/mean": -4.376608848571777, "KL/rejected_KL_mean": -7.124446868896484, "KL/std": 6.129695892333984, "epoch": 0.14096916299559473, "fcm_dpo/beta": 0.10212016850709915, "fcm_dpo/delta": -0.17143580317497253, "fcm_dpo/margin": 5.4956769943237305, "fcm_dpo/q_t": 0.37709975242614746, "grad_norm": 23.83893585205078, "learning_rate": 4.977766449015534e-07, "logits/chosen": -0.699251651763916, "logits/rejected": -0.27587854862213135, "logps/chosen": -108.22093963623047, "logps/ref_chosen": -106.5921630859375, "logps/ref_rejected": -101.76802825927734, "logps/rejected": -108.89247131347656, "loss": 1.0043, "margin_dpo/margin_mean": 5.4956769943237305, "margin_dpo/margin_std": 6.660904884338379, "step": 96 }, { "KL/chosen_KL_mean": -2.734821319580078, "KL/mean": -5.1897172927856445, "KL/rejected_KL_mean": -7.644611358642578, "KL/std": 5.290375709533691, "epoch": 0.14243759177679882, "fcm_dpo/beta": 0.10074971616268158, "fcm_dpo/delta": -0.09995999187231064, "fcm_dpo/margin": 4.909787178039551, "fcm_dpo/q_t": 0.38822782039642334, "grad_norm": 25.695350646972656, "learning_rate": 4.976026077188012e-07, "logits/chosen": -0.445474237203598, "logits/rejected": -0.051459938287734985, "logps/chosen": -91.41470336914062, "logps/ref_chosen": -88.67988586425781, "logps/ref_rejected": -84.81229400634766, "logps/rejected": -92.4569091796875, "loss": 1.0278, "margin_dpo/margin_mean": 4.909787178039551, "margin_dpo/margin_std": 5.577547073364258, "step": 97 }, { "KL/chosen_KL_mean": -1.9879875183105469, "KL/mean": -4.727114677429199, "KL/rejected_KL_mean": -7.466239929199219, "KL/std": 5.264442443847656, "epoch": 0.14390602055800295, "fcm_dpo/beta": 0.09693628549575806, "fcm_dpo/delta": -0.1396963745355606, "fcm_dpo/margin": 5.478252410888672, "fcm_dpo/q_t": 0.3804682195186615, "grad_norm": 25.31107521057129, "learning_rate": 4.974220459770639e-07, "logits/chosen": -0.5954234004020691, "logits/rejected": -0.33950358629226685, "logps/chosen": -94.23048400878906, "logps/ref_chosen": -92.24249267578125, "logps/ref_rejected": -101.51948547363281, "logps/rejected": -108.98572540283203, "loss": 1.0001, "margin_dpo/margin_mean": 5.478251934051514, "margin_dpo/margin_std": 5.961429595947266, "step": 98 }, { "KL/chosen_KL_mean": -1.5430984497070312, "KL/mean": -4.509799003601074, "KL/rejected_KL_mean": -7.476505279541016, "KL/std": 6.127899646759033, "epoch": 0.14537444933920704, "fcm_dpo/beta": 0.09429366886615753, "fcm_dpo/delta": -0.16905245184898376, "fcm_dpo/margin": 5.933409690856934, "fcm_dpo/q_t": 0.3814810514450073, "grad_norm": 24.039554595947266, "learning_rate": 4.972349644343108e-07, "logits/chosen": -0.7258409261703491, "logits/rejected": -0.513698160648346, "logps/chosen": -73.72775268554688, "logps/ref_chosen": -72.18464660644531, "logps/ref_rejected": -91.88131713867188, "logps/rejected": -99.35782623291016, "loss": 1.0086, "margin_dpo/margin_mean": 5.933409214019775, "margin_dpo/margin_std": 7.743303298950195, "step": 99 }, { "KL/chosen_KL_mean": -1.6868171691894531, "KL/mean": -3.9807302951812744, "KL/rejected_KL_mean": -6.274646759033203, "KL/std": 4.8207855224609375, "epoch": 0.14684287812041116, "fcm_dpo/beta": 0.0928003340959549, "fcm_dpo/delta": -0.02729947492480278, "fcm_dpo/margin": 4.587823867797852, "fcm_dpo/q_t": 0.4014517068862915, "grad_norm": 20.777606964111328, "learning_rate": 4.970413680203148e-07, "logits/chosen": -0.5201370120048523, "logits/rejected": -0.040442489087581635, "logps/chosen": -91.20064544677734, "logps/ref_chosen": -89.51382446289062, "logps/ref_rejected": -81.21713256835938, "logps/rejected": -87.49177551269531, "loss": 1.0557, "margin_dpo/margin_mean": 4.587823867797852, "margin_dpo/margin_std": 5.030085563659668, "step": 100 }, { "KL/chosen_KL_mean": -2.9650650024414062, "KL/mean": -4.947567939758301, "KL/rejected_KL_mean": -6.9300689697265625, "KL/std": 6.018884658813477, "epoch": 0.14831130690161526, "fcm_dpo/beta": 0.0931055098772049, "fcm_dpo/delta": 0.031832434237003326, "fcm_dpo/margin": 3.96500301361084, "fcm_dpo/q_t": 0.41612300276756287, "grad_norm": 24.822505950927734, "learning_rate": 4.968412618365215e-07, "logits/chosen": -0.6434340476989746, "logits/rejected": -0.37895065546035767, "logps/chosen": -97.36825561523438, "logps/ref_chosen": -94.4031982421875, "logps/ref_rejected": -89.91001892089844, "logps/rejected": -96.840087890625, "loss": 1.1362, "margin_dpo/margin_mean": 3.9650025367736816, "margin_dpo/margin_std": 6.469620704650879, "step": 101 }, { "KL/chosen_KL_mean": -3.4672584533691406, "KL/mean": -5.168554306030273, "KL/rejected_KL_mean": -6.869850158691406, "KL/std": 5.730302810668945, "epoch": 0.14977973568281938, "fcm_dpo/beta": 0.09549254179000854, "fcm_dpo/delta": 0.07547120749950409, "fcm_dpo/margin": 3.402599573135376, "fcm_dpo/q_t": 0.4264376759529114, "grad_norm": 30.050506591796875, "learning_rate": 4.966346511559149e-07, "logits/chosen": -0.7771315574645996, "logits/rejected": -0.424927294254303, "logps/chosen": -95.0766830444336, "logps/ref_chosen": -91.60942077636719, "logps/ref_rejected": -75.7689208984375, "logps/rejected": -82.6387710571289, "loss": 1.1776, "margin_dpo/margin_mean": 3.4026002883911133, "margin_dpo/margin_std": 6.387094497680664, "step": 102 }, { "KL/chosen_KL_mean": -2.6219825744628906, "KL/mean": -5.435605049133301, "KL/rejected_KL_mean": -8.24923324584961, "KL/std": 6.036181449890137, "epoch": 0.1512481644640235, "fcm_dpo/beta": 0.09307645261287689, "fcm_dpo/delta": -0.13084331154823303, "fcm_dpo/margin": 5.627254009246826, "fcm_dpo/q_t": 0.38143083453178406, "grad_norm": 24.261157989501953, "learning_rate": 4.964215414228785e-07, "logits/chosen": -0.5672892332077026, "logits/rejected": -0.17940247058868408, "logps/chosen": -100.02436828613281, "logps/ref_chosen": -97.40238952636719, "logps/ref_rejected": -98.981689453125, "logps/rejected": -107.23091888427734, "loss": 1.021, "margin_dpo/margin_mean": 5.627254486083984, "margin_dpo/margin_std": 6.770306587219238, "step": 103 }, { "KL/chosen_KL_mean": -1.9034423828125, "KL/mean": -5.001184940338135, "KL/rejected_KL_mean": -8.098930358886719, "KL/std": 6.745368957519531, "epoch": 0.1527165932452276, "fcm_dpo/beta": 0.09018650650978088, "fcm_dpo/delta": -0.16839677095413208, "fcm_dpo/margin": 6.195486545562744, "fcm_dpo/q_t": 0.3795998692512512, "grad_norm": 20.265514373779297, "learning_rate": 4.96201938253052e-07, "logits/chosen": -0.7911609411239624, "logits/rejected": -0.4488197863101959, "logps/chosen": -96.75980377197266, "logps/ref_chosen": -94.85636138916016, "logps/ref_rejected": -97.02669525146484, "logps/rejected": -105.12562561035156, "loss": 1.0036, "margin_dpo/margin_mean": 6.195486068725586, "margin_dpo/margin_std": 7.8650665283203125, "step": 104 }, { "KL/chosen_KL_mean": -2.6060142517089844, "KL/mean": -5.590325832366943, "KL/rejected_KL_mean": -8.574634552001953, "KL/std": 6.004928112030029, "epoch": 0.15418502202643172, "fcm_dpo/beta": 0.08819557726383209, "fcm_dpo/delta": -0.13326339423656464, "fcm_dpo/margin": 5.968611717224121, "fcm_dpo/q_t": 0.38267964124679565, "grad_norm": 23.567642211914062, "learning_rate": 4.959758474331832e-07, "logits/chosen": -0.8084490299224854, "logits/rejected": -0.34298020601272583, "logps/chosen": -99.3639144897461, "logps/ref_chosen": -96.75790405273438, "logps/ref_rejected": -101.5774917602539, "logps/rejected": -110.15213012695312, "loss": 1.0257, "margin_dpo/margin_mean": 5.9686126708984375, "margin_dpo/margin_std": 7.493904113769531, "step": 105 }, { "KL/chosen_KL_mean": -2.5444602966308594, "KL/mean": -5.530681610107422, "KL/rejected_KL_mean": -8.516895294189453, "KL/std": 6.404468536376953, "epoch": 0.15565345080763582, "fcm_dpo/beta": 0.08598188310861588, "fcm_dpo/delta": -0.11951804161071777, "fcm_dpo/margin": 5.972438812255859, "fcm_dpo/q_t": 0.3876997232437134, "grad_norm": 20.605487823486328, "learning_rate": 4.957432749209755e-07, "logits/chosen": -0.5573980212211609, "logits/rejected": -0.12907004356384277, "logps/chosen": -94.98236846923828, "logps/ref_chosen": -92.43791198730469, "logps/ref_rejected": -91.00970458984375, "logps/rejected": -99.52659606933594, "loss": 1.0305, "margin_dpo/margin_mean": 5.972437858581543, "margin_dpo/margin_std": 7.6408185958862305, "step": 106 }, { "KL/chosen_KL_mean": -1.7960433959960938, "KL/mean": -5.146856307983398, "KL/rejected_KL_mean": -8.497676849365234, "KL/std": 7.944108009338379, "epoch": 0.15712187958883994, "fcm_dpo/beta": 0.0830974131822586, "fcm_dpo/delta": -0.16603294014930725, "fcm_dpo/margin": 6.701631546020508, "fcm_dpo/q_t": 0.3860039710998535, "grad_norm": 21.146089553833008, "learning_rate": 4.955042268449307e-07, "logits/chosen": -0.7076393365859985, "logits/rejected": -0.2110217660665512, "logps/chosen": -106.92105102539062, "logps/ref_chosen": -105.12500762939453, "logps/ref_rejected": -101.1824951171875, "logps/rejected": -109.68017578125, "loss": 1.0275, "margin_dpo/margin_mean": 6.701630592346191, "margin_dpo/margin_std": 9.880438804626465, "step": 107 }, { "KL/chosen_KL_mean": -2.772655487060547, "KL/mean": -5.908755302429199, "KL/rejected_KL_mean": -9.044857025146484, "KL/std": 7.226125717163086, "epoch": 0.15859030837004406, "fcm_dpo/beta": 0.08060231804847717, "fcm_dpo/delta": -0.11210188269615173, "fcm_dpo/margin": 6.272194862365723, "fcm_dpo/q_t": 0.38858166337013245, "grad_norm": 20.896970748901367, "learning_rate": 4.952587095041881e-07, "logits/chosen": -0.920397162437439, "logits/rejected": -0.4233282506465912, "logps/chosen": -91.98201751708984, "logps/ref_chosen": -89.20936584472656, "logps/ref_rejected": -101.83784484863281, "logps/rejected": -110.88270568847656, "loss": 1.0349, "margin_dpo/margin_mean": 6.272195816040039, "margin_dpo/margin_std": 7.891620635986328, "step": 108 }, { "KL/chosen_KL_mean": -2.20849609375, "KL/mean": -6.104813575744629, "KL/rejected_KL_mean": -10.00113296508789, "KL/std": 7.493965148925781, "epoch": 0.16005873715124816, "fcm_dpo/beta": 0.07772471755743027, "fcm_dpo/delta": -0.21963942050933838, "fcm_dpo/margin": 7.79263162612915, "fcm_dpo/q_t": 0.36738187074661255, "grad_norm": 20.502532958984375, "learning_rate": 4.95006729368358e-07, "logits/chosen": -0.5712006092071533, "logits/rejected": -0.35399770736694336, "logps/chosen": -102.39558410644531, "logps/ref_chosen": -100.18708801269531, "logps/ref_rejected": -103.53739929199219, "logps/rejected": -113.53852844238281, "loss": 0.9756, "margin_dpo/margin_mean": 7.792632579803467, "margin_dpo/margin_std": 9.155248641967773, "step": 109 }, { "KL/chosen_KL_mean": -2.8511276245117188, "KL/mean": -5.234687805175781, "KL/rejected_KL_mean": -7.618255615234375, "KL/std": 6.038027286529541, "epoch": 0.16152716593245228, "fcm_dpo/beta": 0.07707220315933228, "fcm_dpo/delta": 0.03354433551430702, "fcm_dpo/margin": 4.767132759094238, "fcm_dpo/q_t": 0.4162892699241638, "grad_norm": 20.441862106323242, "learning_rate": 4.947482930773511e-07, "logits/chosen": -0.35412779450416565, "logits/rejected": 0.023226439952850342, "logps/chosen": -109.74684143066406, "logps/ref_chosen": -106.89572143554688, "logps/ref_rejected": -85.81120300292969, "logps/rejected": -93.42945861816406, "loss": 1.1192, "margin_dpo/margin_mean": 4.76713228225708, "margin_dpo/margin_std": 6.876535415649414, "step": 110 }, { "KL/chosen_KL_mean": -2.4928359985351562, "KL/mean": -7.023989677429199, "KL/rejected_KL_mean": -11.55514144897461, "KL/std": 8.625679016113281, "epoch": 0.16299559471365638, "fcm_dpo/beta": 0.07386095821857452, "fcm_dpo/delta": -0.2910497486591339, "fcm_dpo/margin": 9.062307357788086, "fcm_dpo/q_t": 0.35250282287597656, "grad_norm": 19.32626724243164, "learning_rate": 4.944834074412042e-07, "logits/chosen": -0.8727514147758484, "logits/rejected": -0.5925191044807434, "logps/chosen": -95.82220458984375, "logps/ref_chosen": -93.3293685913086, "logps/ref_rejected": -104.177490234375, "logps/rejected": -115.73262786865234, "loss": 0.9297, "margin_dpo/margin_mean": 9.062307357788086, "margin_dpo/margin_std": 9.354782104492188, "step": 111 }, { "KL/chosen_KL_mean": -2.8206138610839844, "KL/mean": -5.230242729187012, "KL/rejected_KL_mean": -7.639869689941406, "KL/std": 4.961908340454102, "epoch": 0.1644640234948605, "fcm_dpo/beta": 0.07362090051174164, "fcm_dpo/delta": 0.04689092934131622, "fcm_dpo/margin": 4.8192548751831055, "fcm_dpo/q_t": 0.4180588126182556, "grad_norm": 17.997583389282227, "learning_rate": 4.942120794399002e-07, "logits/chosen": -1.011833906173706, "logits/rejected": -0.5217633247375488, "logps/chosen": -85.56765747070312, "logps/ref_chosen": -82.74703979492188, "logps/ref_rejected": -71.5584487915039, "logps/rejected": -79.19831848144531, "loss": 1.1177, "margin_dpo/margin_mean": 4.819255828857422, "margin_dpo/margin_std": 6.565848350524902, "step": 112 }, { "KL/chosen_KL_mean": -3.8011207580566406, "KL/mean": -6.33805513381958, "KL/rejected_KL_mean": -8.874992370605469, "KL/std": 6.594950199127197, "epoch": 0.16593245227606462, "fcm_dpo/beta": 0.0742044523358345, "fcm_dpo/delta": 0.02437446266412735, "fcm_dpo/margin": 5.0738725662231445, "fcm_dpo/q_t": 0.4133363962173462, "grad_norm": 21.608713150024414, "learning_rate": 4.939343162231841e-07, "logits/chosen": -0.7599122524261475, "logits/rejected": -0.23744072020053864, "logps/chosen": -101.70061492919922, "logps/ref_chosen": -97.89949035644531, "logps/ref_rejected": -84.41960144042969, "logps/rejected": -93.29458618164062, "loss": 1.1044, "margin_dpo/margin_mean": 5.073872089385986, "margin_dpo/margin_std": 6.467991828918457, "step": 113 }, { "KL/chosen_KL_mean": -3.5090103149414062, "KL/mean": -7.049571990966797, "KL/rejected_KL_mean": -10.590129852294922, "KL/std": 7.784989833831787, "epoch": 0.16740088105726872, "fcm_dpo/beta": 0.07264983654022217, "fcm_dpo/delta": -0.12131000310182571, "fcm_dpo/margin": 7.081119537353516, "fcm_dpo/q_t": 0.3894423842430115, "grad_norm": 21.042259216308594, "learning_rate": 4.936501251103751e-07, "logits/chosen": -0.8794021010398865, "logits/rejected": -0.302768349647522, "logps/chosen": -100.0041275024414, "logps/ref_chosen": -96.4951171875, "logps/ref_rejected": -92.01536560058594, "logps/rejected": -102.6054916381836, "loss": 1.0516, "margin_dpo/margin_mean": 7.081119537353516, "margin_dpo/margin_std": 9.983999252319336, "step": 114 }, { "KL/chosen_KL_mean": -3.4762954711914062, "KL/mean": -7.0945024490356445, "KL/rejected_KL_mean": -10.712711334228516, "KL/std": 8.220789909362793, "epoch": 0.16886930983847284, "fcm_dpo/beta": 0.07118728756904602, "fcm_dpo/delta": -0.12134180963039398, "fcm_dpo/margin": 7.23641300201416, "fcm_dpo/q_t": 0.38561272621154785, "grad_norm": 18.499309539794922, "learning_rate": 4.933595135901732e-07, "logits/chosen": -0.8688855171203613, "logits/rejected": -0.4737377166748047, "logps/chosen": -106.65641784667969, "logps/ref_chosen": -103.18013000488281, "logps/ref_rejected": -105.71595764160156, "logps/rejected": -116.42866516113281, "loss": 1.0254, "margin_dpo/margin_mean": 7.23641300201416, "margin_dpo/margin_std": 8.860952377319336, "step": 115 }, { "KL/chosen_KL_mean": -3.1250762939453125, "KL/mean": -6.184640884399414, "KL/rejected_KL_mean": -9.244197845458984, "KL/std": 6.424797058105469, "epoch": 0.17033773861967694, "fcm_dpo/beta": 0.07058180868625641, "fcm_dpo/delta": -0.03343556821346283, "fcm_dpo/margin": 6.119123935699463, "fcm_dpo/q_t": 0.40130478143692017, "grad_norm": 17.80445671081543, "learning_rate": 4.930624893204624e-07, "logits/chosen": -0.8932570815086365, "logits/rejected": -0.4679751396179199, "logps/chosen": -84.283447265625, "logps/ref_chosen": -81.15837097167969, "logps/ref_rejected": -86.30586242675781, "logps/rejected": -95.55005645751953, "loss": 1.0643, "margin_dpo/margin_mean": 6.119124412536621, "margin_dpo/margin_std": 7.4036335945129395, "step": 116 }, { "KL/chosen_KL_mean": -3.9713096618652344, "KL/mean": -6.5189900398254395, "KL/rejected_KL_mean": -9.066669464111328, "KL/std": 6.564092636108398, "epoch": 0.17180616740088106, "fcm_dpo/beta": 0.07026030123233795, "fcm_dpo/delta": 0.04305105283856392, "fcm_dpo/margin": 5.0953569412231445, "fcm_dpo/q_t": 0.4182448387145996, "grad_norm": 23.900850296020508, "learning_rate": 4.927590601281083e-07, "logits/chosen": -0.5702371001243591, "logits/rejected": -0.16222698986530304, "logps/chosen": -101.05226135253906, "logps/ref_chosen": -97.0809555053711, "logps/ref_rejected": -75.1399154663086, "logps/rejected": -84.20658874511719, "loss": 1.1334, "margin_dpo/margin_mean": 5.0953569412231445, "margin_dpo/margin_std": 7.812357425689697, "step": 117 }, { "KL/chosen_KL_mean": -2.755229949951172, "KL/mean": -6.253632545471191, "KL/rejected_KL_mean": -9.752025604248047, "KL/std": 7.184199333190918, "epoch": 0.17327459618208516, "fcm_dpo/beta": 0.0697828084230423, "fcm_dpo/delta": -0.0929437130689621, "fcm_dpo/margin": 6.996795654296875, "fcm_dpo/q_t": 0.3871319890022278, "grad_norm": 17.51601219177246, "learning_rate": 4.924492340087524e-07, "logits/chosen": -1.0840113162994385, "logits/rejected": -0.7586953639984131, "logps/chosen": -89.29146575927734, "logps/ref_chosen": -86.53623962402344, "logps/ref_rejected": -82.93563842773438, "logps/rejected": -92.68766784667969, "loss": 1.018, "margin_dpo/margin_mean": 6.996794700622559, "margin_dpo/margin_std": 7.442378044128418, "step": 118 }, { "KL/chosen_KL_mean": -4.215473175048828, "KL/mean": -7.875664710998535, "KL/rejected_KL_mean": -11.53585433959961, "KL/std": 8.516252517700195, "epoch": 0.17474302496328928, "fcm_dpo/beta": 0.06828521192073822, "fcm_dpo/delta": -0.10553038865327835, "fcm_dpo/margin": 7.320374488830566, "fcm_dpo/q_t": 0.38795772194862366, "grad_norm": 19.437278747558594, "learning_rate": 4.92133019126601e-07, "logits/chosen": -1.1288388967514038, "logits/rejected": -0.8196889758110046, "logps/chosen": -108.82455444335938, "logps/ref_chosen": -104.60908508300781, "logps/ref_rejected": -109.15780639648438, "logps/rejected": -120.69366455078125, "loss": 1.0633, "margin_dpo/margin_mean": 7.320374488830566, "margin_dpo/margin_std": 10.596355438232422, "step": 119 }, { "KL/chosen_KL_mean": -3.299694061279297, "KL/mean": -8.188924789428711, "KL/rejected_KL_mean": -13.078155517578125, "KL/std": 10.441339492797852, "epoch": 0.1762114537444934, "fcm_dpo/beta": 0.06549836695194244, "fcm_dpo/delta": -0.25712794065475464, "fcm_dpo/margin": 9.778461456298828, "fcm_dpo/q_t": 0.3653952479362488, "grad_norm": 21.754716873168945, "learning_rate": 4.918104238142103e-07, "logits/chosen": -1.1223708391189575, "logits/rejected": -0.5759496688842773, "logps/chosen": -117.98712158203125, "logps/ref_chosen": -114.68742370605469, "logps/ref_rejected": -111.18586730957031, "logps/rejected": -124.26402282714844, "loss": 0.9987, "margin_dpo/margin_mean": 9.778462409973145, "margin_dpo/margin_std": 13.166465759277344, "step": 120 }, { "KL/chosen_KL_mean": -2.9996414184570312, "KL/mean": -7.3607025146484375, "KL/rejected_KL_mean": -11.72177505493164, "KL/std": 8.121614456176758, "epoch": 0.1776798825256975, "fcm_dpo/beta": 0.06294788420200348, "fcm_dpo/delta": -0.15851938724517822, "fcm_dpo/margin": 8.722132682800293, "fcm_dpo/q_t": 0.3754175305366516, "grad_norm": 19.538516998291016, "learning_rate": 4.91481456572267e-07, "logits/chosen": -0.9023872017860413, "logits/rejected": -0.5733996629714966, "logps/chosen": -95.11630249023438, "logps/ref_chosen": -92.11666107177734, "logps/ref_rejected": -114.972900390625, "logps/rejected": -126.69467163085938, "loss": 0.9828, "margin_dpo/margin_mean": 8.722132682800293, "margin_dpo/margin_std": 9.071338653564453, "step": 121 }, { "KL/chosen_KL_mean": -3.319255828857422, "KL/mean": -8.319995880126953, "KL/rejected_KL_mean": -13.320728302001953, "KL/std": 8.855789184570312, "epoch": 0.17914831130690162, "fcm_dpo/beta": 0.06085660308599472, "fcm_dpo/delta": -0.22164666652679443, "fcm_dpo/margin": 10.001470565795898, "fcm_dpo/q_t": 0.36208459734916687, "grad_norm": 20.05810546875, "learning_rate": 4.911461260693638e-07, "logits/chosen": -0.6284879446029663, "logits/rejected": -0.5239280462265015, "logps/chosen": -88.19682312011719, "logps/ref_chosen": -84.87757110595703, "logps/ref_rejected": -111.0829849243164, "logps/rejected": -124.40371704101562, "loss": 0.9549, "margin_dpo/margin_mean": 10.001470565795898, "margin_dpo/margin_std": 10.196475982666016, "step": 122 }, { "KL/chosen_KL_mean": -3.8244400024414062, "KL/mean": -7.453975200653076, "KL/rejected_KL_mean": -11.083511352539062, "KL/std": 8.134807586669922, "epoch": 0.18061674008810572, "fcm_dpo/beta": 0.05927310511469841, "fcm_dpo/delta": -0.03266262263059616, "fcm_dpo/margin": 7.259067535400391, "fcm_dpo/q_t": 0.4017595946788788, "grad_norm": 16.85076332092285, "learning_rate": 4.908044411417711e-07, "logits/chosen": -1.1842890977859497, "logits/rejected": -0.8801992535591125, "logps/chosen": -102.30319213867188, "logps/ref_chosen": -98.47874450683594, "logps/ref_rejected": -93.85417175292969, "logps/rejected": -104.93768310546875, "loss": 1.0727, "margin_dpo/margin_mean": 7.259067058563232, "margin_dpo/margin_std": 9.186704635620117, "step": 123 }, { "KL/chosen_KL_mean": -3.568645477294922, "KL/mean": -8.02293872833252, "KL/rejected_KL_mean": -12.477226257324219, "KL/std": 10.25880241394043, "epoch": 0.18208516886930984, "fcm_dpo/beta": 0.05845661461353302, "fcm_dpo/delta": -0.1273563802242279, "fcm_dpo/margin": 8.908575057983398, "fcm_dpo/q_t": 0.39150458574295044, "grad_norm": 16.979604721069336, "learning_rate": 4.904564107932048e-07, "logits/chosen": -1.0423192977905273, "logits/rejected": -0.7813848257064819, "logps/chosen": -106.33154296875, "logps/ref_chosen": -102.76290130615234, "logps/ref_rejected": -123.005615234375, "logps/rejected": -135.48284912109375, "loss": 1.0562, "margin_dpo/margin_mean": 8.908575057983398, "margin_dpo/margin_std": 13.310579299926758, "step": 124 }, { "KL/chosen_KL_mean": -5.067901611328125, "KL/mean": -8.381734848022461, "KL/rejected_KL_mean": -11.695571899414062, "KL/std": 8.317784309387207, "epoch": 0.18355359765051396, "fcm_dpo/beta": 0.058141790330410004, "fcm_dpo/delta": 0.015242844820022583, "fcm_dpo/margin": 6.627668380737305, "fcm_dpo/q_t": 0.41335079073905945, "grad_norm": 17.033164978027344, "learning_rate": 4.90102044194588e-07, "logits/chosen": -0.7098629474639893, "logits/rejected": -0.4930803179740906, "logps/chosen": -95.49417877197266, "logps/ref_chosen": -90.42627716064453, "logps/ref_rejected": -89.24901580810547, "logps/rejected": -100.94458770751953, "loss": 1.1178, "margin_dpo/margin_mean": 6.627668857574463, "margin_dpo/margin_std": 10.111738204956055, "step": 125 }, { "KL/chosen_KL_mean": -4.006725311279297, "KL/mean": -8.802370071411133, "KL/rejected_KL_mean": -13.598018646240234, "KL/std": 8.956310272216797, "epoch": 0.18502202643171806, "fcm_dpo/beta": 0.05693250894546509, "fcm_dpo/delta": -0.15443840622901917, "fcm_dpo/margin": 9.591299057006836, "fcm_dpo/q_t": 0.3733636140823364, "grad_norm": 16.95452880859375, "learning_rate": 4.897413506838102e-07, "logits/chosen": -0.819510817527771, "logits/rejected": -0.5395961403846741, "logps/chosen": -98.42961120605469, "logps/ref_chosen": -94.42288208007812, "logps/ref_rejected": -103.88156127929688, "logps/rejected": -117.47957611083984, "loss": 0.9756, "margin_dpo/margin_mean": 9.591299057006836, "margin_dpo/margin_std": 9.193231582641602, "step": 126 }, { "KL/chosen_KL_mean": -4.071666717529297, "KL/mean": -7.683848857879639, "KL/rejected_KL_mean": -11.296028137207031, "KL/std": 8.331953048706055, "epoch": 0.18649045521292218, "fcm_dpo/beta": 0.05655498057603836, "fcm_dpo/delta": -0.00909978337585926, "fcm_dpo/margin": 7.224360466003418, "fcm_dpo/q_t": 0.40840333700180054, "grad_norm": 16.871807098388672, "learning_rate": 4.89374339765481e-07, "logits/chosen": -0.7480254173278809, "logits/rejected": -0.4045577943325043, "logps/chosen": -88.82272338867188, "logps/ref_chosen": -84.75105285644531, "logps/ref_rejected": -83.54523468017578, "logps/rejected": -94.84126281738281, "loss": 1.098, "margin_dpo/margin_mean": 7.224360466003418, "margin_dpo/margin_std": 10.39965534210205, "step": 127 }, { "KL/chosen_KL_mean": -4.096443176269531, "KL/mean": -7.83958625793457, "KL/rejected_KL_mean": -11.582723617553711, "KL/std": 10.307234764099121, "epoch": 0.18795888399412627, "fcm_dpo/beta": 0.05641629546880722, "fcm_dpo/delta": -0.023651301860809326, "fcm_dpo/margin": 7.4862871170043945, "fcm_dpo/q_t": 0.40452003479003906, "grad_norm": 18.850421905517578, "learning_rate": 4.890010211106795e-07, "logits/chosen": -0.5200753808021545, "logits/rejected": -0.039977431297302246, "logps/chosen": -94.7160873413086, "logps/ref_chosen": -90.61964416503906, "logps/ref_rejected": -83.10347747802734, "logps/rejected": -94.68620300292969, "loss": 1.1135, "margin_dpo/margin_mean": 7.486286163330078, "margin_dpo/margin_std": 12.478761672973633, "step": 128 }, { "KL/chosen_KL_mean": -4.534244537353516, "KL/mean": -8.429698944091797, "KL/rejected_KL_mean": -12.325157165527344, "KL/std": 8.425018310546875, "epoch": 0.1894273127753304, "fcm_dpo/beta": 0.05594721436500549, "fcm_dpo/delta": -0.037587955594062805, "fcm_dpo/margin": 7.790910720825195, "fcm_dpo/q_t": 0.4046151638031006, "grad_norm": 15.356521606445312, "learning_rate": 4.88621404556699e-07, "logits/chosen": -0.8842312693595886, "logits/rejected": -0.5879188776016235, "logps/chosen": -103.45295715332031, "logps/ref_chosen": -98.91870880126953, "logps/ref_rejected": -103.28419494628906, "logps/rejected": -115.6093521118164, "loss": 1.0891, "margin_dpo/margin_mean": 7.7909111976623535, "margin_dpo/margin_std": 11.217774391174316, "step": 129 }, { "KL/chosen_KL_mean": -3.622638702392578, "KL/mean": -8.684480667114258, "KL/rejected_KL_mean": -13.746326446533203, "KL/std": 10.050458908081055, "epoch": 0.19089574155653452, "fcm_dpo/beta": 0.054750699549913406, "fcm_dpo/delta": -0.16326026618480682, "fcm_dpo/margin": 10.12369441986084, "fcm_dpo/q_t": 0.3784770965576172, "grad_norm": 14.988670349121094, "learning_rate": 4.882355001067891e-07, "logits/chosen": -1.274229884147644, "logits/rejected": -1.0331367254257202, "logps/chosen": -76.80632781982422, "logps/ref_chosen": -73.18368530273438, "logps/ref_rejected": -88.02154541015625, "logps/rejected": -101.76786804199219, "loss": 1.005, "margin_dpo/margin_mean": 10.123693466186523, "margin_dpo/margin_std": 11.845293045043945, "step": 130 }, { "KL/chosen_KL_mean": -3.076427459716797, "KL/mean": -7.559879302978516, "KL/rejected_KL_mean": -12.0433349609375, "KL/std": 9.85338020324707, "epoch": 0.19236417033773862, "fcm_dpo/beta": 0.0535312220454216, "fcm_dpo/delta": -0.08426669985055923, "fcm_dpo/margin": 8.966897964477539, "fcm_dpo/q_t": 0.3919743299484253, "grad_norm": 17.167770385742188, "learning_rate": 4.878433179298909e-07, "logits/chosen": -1.0256226062774658, "logits/rejected": -0.8109716176986694, "logps/chosen": -77.03286743164062, "logps/ref_chosen": -73.9564437866211, "logps/ref_rejected": -94.45582580566406, "logps/rejected": -106.49916076660156, "loss": 1.0475, "margin_dpo/margin_mean": 8.966897964477539, "margin_dpo/margin_std": 11.166572570800781, "step": 131 }, { "KL/chosen_KL_mean": -4.623268127441406, "KL/mean": -9.239850997924805, "KL/rejected_KL_mean": -13.856426239013672, "KL/std": 10.357924461364746, "epoch": 0.19383259911894274, "fcm_dpo/beta": 0.052110929042100906, "fcm_dpo/delta": -0.08587256819009781, "fcm_dpo/margin": 9.233155250549316, "fcm_dpo/q_t": 0.39031627774238586, "grad_norm": 16.96406364440918, "learning_rate": 4.874448683603694e-07, "logits/chosen": -1.225928544998169, "logits/rejected": -0.8144073486328125, "logps/chosen": -97.06275177001953, "logps/ref_chosen": -92.43948364257812, "logps/ref_rejected": -94.13252258300781, "logps/rejected": -107.98894500732422, "loss": 1.0451, "margin_dpo/margin_mean": 9.233156204223633, "margin_dpo/margin_std": 11.620232582092285, "step": 132 }, { "KL/chosen_KL_mean": -3.7442398071289062, "KL/mean": -8.595797538757324, "KL/rejected_KL_mean": -13.44735336303711, "KL/std": 10.539609909057617, "epoch": 0.19530102790014683, "fcm_dpo/beta": 0.05134374648332596, "fcm_dpo/delta": -0.10325406491756439, "fcm_dpo/margin": 9.703120231628418, "fcm_dpo/q_t": 0.38738375902175903, "grad_norm": 15.742362976074219, "learning_rate": 4.870401618977415e-07, "logits/chosen": -1.539555549621582, "logits/rejected": -1.1550912857055664, "logps/chosen": -96.86090087890625, "logps/ref_chosen": -93.11666107177734, "logps/ref_rejected": -102.09490966796875, "logps/rejected": -115.54226684570312, "loss": 1.0194, "margin_dpo/margin_mean": 9.703121185302734, "margin_dpo/margin_std": 10.955461502075195, "step": 133 }, { "KL/chosen_KL_mean": -4.014629364013672, "KL/mean": -9.231218338012695, "KL/rejected_KL_mean": -14.447795867919922, "KL/std": 10.043886184692383, "epoch": 0.19676945668135096, "fcm_dpo/beta": 0.05035899206995964, "fcm_dpo/delta": -0.13238482177257538, "fcm_dpo/margin": 10.433167457580566, "fcm_dpo/q_t": 0.3798108398914337, "grad_norm": 14.930169105529785, "learning_rate": 4.866292092063986e-07, "logits/chosen": -1.7105650901794434, "logits/rejected": -1.3886322975158691, "logps/chosen": -87.56475830078125, "logps/ref_chosen": -83.55012512207031, "logps/ref_rejected": -93.8815689086914, "logps/rejected": -108.32936096191406, "loss": 1.0004, "margin_dpo/margin_mean": 10.433167457580566, "margin_dpo/margin_std": 10.795623779296875, "step": 134 }, { "KL/chosen_KL_mean": -3.9065189361572266, "KL/mean": -8.95309066772461, "KL/rejected_KL_mean": -13.999664306640625, "KL/std": 11.034330368041992, "epoch": 0.19823788546255505, "fcm_dpo/beta": 0.04906691610813141, "fcm_dpo/delta": -0.10007939487695694, "fcm_dpo/margin": 10.093135833740234, "fcm_dpo/q_t": 0.39028918743133545, "grad_norm": 19.812477111816406, "learning_rate": 4.862120211153265e-07, "logits/chosen": -1.3110158443450928, "logits/rejected": -1.177363395690918, "logps/chosen": -81.48914337158203, "logps/ref_chosen": -77.58262634277344, "logps/ref_rejected": -121.05648040771484, "logps/rejected": -135.05615234375, "loss": 1.0331, "margin_dpo/margin_mean": 10.09313678741455, "margin_dpo/margin_std": 12.252727508544922, "step": 135 }, { "KL/chosen_KL_mean": -4.367889404296875, "KL/mean": -9.437177658081055, "KL/rejected_KL_mean": -14.506477355957031, "KL/std": 11.110933303833008, "epoch": 0.19970631424375918, "fcm_dpo/beta": 0.048004090785980225, "fcm_dpo/delta": -0.09121304750442505, "fcm_dpo/margin": 10.138586044311523, "fcm_dpo/q_t": 0.3883233070373535, "grad_norm": 14.456236839294434, "learning_rate": 4.857886086178193e-07, "logits/chosen": -1.469016194343567, "logits/rejected": -1.0410784482955933, "logps/chosen": -98.54061889648438, "logps/ref_chosen": -94.1727294921875, "logps/ref_rejected": -103.26428985595703, "logps/rejected": -117.77076721191406, "loss": 1.0189, "margin_dpo/margin_mean": 10.138586044311523, "margin_dpo/margin_std": 10.917642593383789, "step": 136 }, { "KL/chosen_KL_mean": -3.4312705993652344, "KL/mean": -11.352218627929688, "KL/rejected_KL_mean": -19.273162841796875, "KL/std": 14.324772834777832, "epoch": 0.2011747430249633, "fcm_dpo/beta": 0.04513707011938095, "fcm_dpo/delta": -0.3427148163318634, "fcm_dpo/margin": 15.84189510345459, "fcm_dpo/q_t": 0.3430503010749817, "grad_norm": 17.338064193725586, "learning_rate": 4.853589828711902e-07, "logits/chosen": -1.3237829208374023, "logits/rejected": -1.1018874645233154, "logps/chosen": -81.55001068115234, "logps/ref_chosen": -78.11874389648438, "logps/ref_rejected": -117.92237091064453, "logps/rejected": -137.19552612304688, "loss": 0.8944, "margin_dpo/margin_mean": 15.841897964477539, "margin_dpo/margin_std": 15.57695198059082, "step": 137 }, { "KL/chosen_KL_mean": -6.108768463134766, "KL/mean": -10.983161926269531, "KL/rejected_KL_mean": -15.857555389404297, "KL/std": 10.617330551147461, "epoch": 0.2026431718061674, "fcm_dpo/beta": 0.04433932900428772, "fcm_dpo/delta": -0.03385394066572189, "fcm_dpo/margin": 9.74878978729248, "fcm_dpo/q_t": 0.40105772018432617, "grad_norm": 13.538736343383789, "learning_rate": 4.849231551964771e-07, "logits/chosen": -1.0424013137817383, "logits/rejected": -0.7919163107872009, "logps/chosen": -98.93871307373047, "logps/ref_chosen": -92.82994079589844, "logps/ref_rejected": -98.64840698242188, "logps/rejected": -114.5059585571289, "loss": 1.0625, "margin_dpo/margin_mean": 9.74878978729248, "margin_dpo/margin_std": 11.774738311767578, "step": 138 }, { "KL/chosen_KL_mean": -4.222953796386719, "KL/mean": -9.955503463745117, "KL/rejected_KL_mean": -15.688056945800781, "KL/std": 11.721033096313477, "epoch": 0.20411160058737152, "fcm_dpo/beta": 0.04356386139988899, "fcm_dpo/delta": -0.10503016412258148, "fcm_dpo/margin": 11.465106964111328, "fcm_dpo/q_t": 0.3840472400188446, "grad_norm": 13.765968322753906, "learning_rate": 4.844811370781446e-07, "logits/chosen": -1.2012462615966797, "logits/rejected": -0.8696339726448059, "logps/chosen": -83.22596740722656, "logps/ref_chosen": -79.00301361083984, "logps/ref_rejected": -85.82733154296875, "logps/rejected": -101.51538848876953, "loss": 1.028, "margin_dpo/margin_mean": 11.465106964111328, "margin_dpo/margin_std": 13.759885787963867, "step": 139 }, { "KL/chosen_KL_mean": -4.602313995361328, "KL/mean": -9.842817306518555, "KL/rejected_KL_mean": -15.083324432373047, "KL/std": 11.091232299804688, "epoch": 0.2055800293685756, "fcm_dpo/beta": 0.04309472069144249, "fcm_dpo/delta": -0.05410316586494446, "fcm_dpo/margin": 10.481010437011719, "fcm_dpo/q_t": 0.3955029249191284, "grad_norm": 13.048848152160645, "learning_rate": 4.840329401637809e-07, "logits/chosen": -1.837430715560913, "logits/rejected": -1.4507932662963867, "logps/chosen": -92.95475769042969, "logps/ref_chosen": -88.35244750976562, "logps/ref_rejected": -90.57476806640625, "logps/rejected": -105.65809631347656, "loss": 1.043, "margin_dpo/margin_mean": 10.481010437011719, "margin_dpo/margin_std": 11.646371841430664, "step": 140 }, { "KL/chosen_KL_mean": -6.091522216796875, "KL/mean": -12.133913040161133, "KL/rejected_KL_mean": -18.17630386352539, "KL/std": 14.393857955932617, "epoch": 0.20704845814977973, "fcm_dpo/beta": 0.042237911373376846, "fcm_dpo/delta": -0.11633803695440292, "fcm_dpo/margin": 12.084778785705566, "fcm_dpo/q_t": 0.38337308168411255, "grad_norm": 15.751829147338867, "learning_rate": 4.83578576263792e-07, "logits/chosen": -1.3877817392349243, "logits/rejected": -1.0123834609985352, "logps/chosen": -113.31168365478516, "logps/ref_chosen": -107.22016143798828, "logps/ref_rejected": -104.71223449707031, "logps/rejected": -122.88853454589844, "loss": 1.0231, "margin_dpo/margin_mean": 12.084778785705566, "margin_dpo/margin_std": 14.433948516845703, "step": 141 }, { "KL/chosen_KL_mean": -6.500453948974609, "KL/mean": -12.393810272216797, "KL/rejected_KL_mean": -18.28716278076172, "KL/std": 13.482120513916016, "epoch": 0.20851688693098386, "fcm_dpo/beta": 0.04131639003753662, "fcm_dpo/delta": -0.09171706438064575, "fcm_dpo/margin": 11.786710739135742, "fcm_dpo/q_t": 0.3909595012664795, "grad_norm": 13.148305892944336, "learning_rate": 4.83118057351089e-07, "logits/chosen": -1.2163926362991333, "logits/rejected": -0.959374189376831, "logps/chosen": -95.22940063476562, "logps/ref_chosen": -88.72894287109375, "logps/ref_rejected": -99.73038482666016, "logps/rejected": -118.01754760742188, "loss": 1.0298, "margin_dpo/margin_mean": 11.786711692810059, "margin_dpo/margin_std": 14.061651229858398, "step": 142 }, { "KL/chosen_KL_mean": -4.677314758300781, "KL/mean": -9.565406799316406, "KL/rejected_KL_mean": -14.4534912109375, "KL/std": 11.330511093139648, "epoch": 0.20998531571218795, "fcm_dpo/beta": 0.04103311523795128, "fcm_dpo/delta": -0.0015337169170379639, "fcm_dpo/margin": 9.776169776916504, "fcm_dpo/q_t": 0.4075244069099426, "grad_norm": 13.02778434753418, "learning_rate": 4.826513955607734e-07, "logits/chosen": -1.4969639778137207, "logits/rejected": -1.03965163230896, "logps/chosen": -90.18189239501953, "logps/ref_chosen": -85.50457763671875, "logps/ref_rejected": -85.8056640625, "logps/rejected": -100.2591552734375, "loss": 1.0867, "margin_dpo/margin_mean": 9.776169776916504, "margin_dpo/margin_std": 12.41860580444336, "step": 143 }, { "KL/chosen_KL_mean": -5.759288787841797, "KL/mean": -10.669565200805664, "KL/rejected_KL_mean": -15.579845428466797, "KL/std": 11.9352388381958, "epoch": 0.21145374449339208, "fcm_dpo/beta": 0.04139728099107742, "fcm_dpo/delta": -0.007507693022489548, "fcm_dpo/margin": 9.82055950164795, "fcm_dpo/q_t": 0.40822726488113403, "grad_norm": 16.787689208984375, "learning_rate": 4.821786031898176e-07, "logits/chosen": -1.3392337560653687, "logits/rejected": -0.793165922164917, "logps/chosen": -110.44638061523438, "logps/ref_chosen": -104.68708801269531, "logps/ref_rejected": -87.71908569335938, "logps/rejected": -103.2989273071289, "loss": 1.1083, "margin_dpo/margin_mean": 9.82055950164795, "margin_dpo/margin_std": 14.784820556640625, "step": 144 }, { "KL/chosen_KL_mean": -4.976615905761719, "KL/mean": -10.697338104248047, "KL/rejected_KL_mean": -16.418060302734375, "KL/std": 11.404111862182617, "epoch": 0.21292217327459617, "fcm_dpo/beta": 0.040709398686885834, "fcm_dpo/delta": -0.06899532675743103, "fcm_dpo/margin": 11.441442489624023, "fcm_dpo/q_t": 0.3924832344055176, "grad_norm": 13.521733283996582, "learning_rate": 4.816996926967401e-07, "logits/chosen": -1.0364314317703247, "logits/rejected": -0.44955042004585266, "logps/chosen": -93.49650573730469, "logps/ref_chosen": -88.51988983154297, "logps/ref_rejected": -83.76513671875, "logps/rejected": -100.18319702148438, "loss": 1.0368, "margin_dpo/margin_mean": 11.44144344329834, "margin_dpo/margin_std": 12.873907089233398, "step": 145 }, { "KL/chosen_KL_mean": -5.420673370361328, "KL/mean": -11.78217887878418, "KL/rejected_KL_mean": -18.143680572509766, "KL/std": 12.539497375488281, "epoch": 0.2143906020558003, "fcm_dpo/beta": 0.03969257324934006, "fcm_dpo/delta": -0.11174594610929489, "fcm_dpo/margin": 12.723002433776855, "fcm_dpo/q_t": 0.38346797227859497, "grad_norm": 13.268388748168945, "learning_rate": 4.812146767012779e-07, "logits/chosen": -1.5605173110961914, "logits/rejected": -0.9217730760574341, "logps/chosen": -108.3265380859375, "logps/ref_chosen": -102.90586853027344, "logps/ref_rejected": -88.9493637084961, "logps/rejected": -107.09304809570312, "loss": 1.0128, "margin_dpo/margin_mean": 12.723002433776855, "margin_dpo/margin_std": 13.708772659301758, "step": 146 }, { "KL/chosen_KL_mean": -5.205101013183594, "KL/mean": -11.823930740356445, "KL/rejected_KL_mean": -18.442760467529297, "KL/std": 12.090557098388672, "epoch": 0.21585903083700442, "fcm_dpo/beta": 0.03897321596741676, "fcm_dpo/delta": -0.12218107283115387, "fcm_dpo/margin": 13.237667083740234, "fcm_dpo/q_t": 0.3804128170013428, "grad_norm": 12.336647033691406, "learning_rate": 4.807235679840536e-07, "logits/chosen": -1.5315604209899902, "logits/rejected": -0.9582536220550537, "logps/chosen": -94.46728515625, "logps/ref_chosen": -89.26219177246094, "logps/ref_rejected": -76.50011444091797, "logps/rejected": -94.94287109375, "loss": 0.993, "margin_dpo/margin_mean": 13.237667083740234, "margin_dpo/margin_std": 12.947975158691406, "step": 147 }, { "KL/chosen_KL_mean": -3.6500091552734375, "KL/mean": -9.690452575683594, "KL/rejected_KL_mean": -15.730903625488281, "KL/std": 12.451555252075195, "epoch": 0.2173274596182085, "fcm_dpo/beta": 0.03861015662550926, "fcm_dpo/delta": -0.07036314159631729, "fcm_dpo/margin": 12.080889701843262, "fcm_dpo/q_t": 0.39576423168182373, "grad_norm": 13.279841423034668, "learning_rate": 4.802263794862384e-07, "logits/chosen": -1.4245572090148926, "logits/rejected": -1.0224802494049072, "logps/chosen": -95.41288757324219, "logps/ref_chosen": -91.76287841796875, "logps/ref_rejected": -110.38552856445312, "logps/rejected": -126.1164321899414, "loss": 1.0494, "margin_dpo/margin_mean": 12.080888748168945, "margin_dpo/margin_std": 14.16989803314209, "step": 148 }, { "KL/chosen_KL_mean": -6.8024749755859375, "KL/mean": -12.577699661254883, "KL/rejected_KL_mean": -18.352920532226562, "KL/std": 13.2476167678833, "epoch": 0.21879588839941264, "fcm_dpo/beta": 0.03781084716320038, "fcm_dpo/delta": -0.03885117545723915, "fcm_dpo/margin": 11.550439834594727, "fcm_dpo/q_t": 0.40449076890945435, "grad_norm": 12.889254570007324, "learning_rate": 4.797231243092118e-07, "logits/chosen": -1.3868000507354736, "logits/rejected": -1.064367413520813, "logps/chosen": -100.28649139404297, "logps/ref_chosen": -93.48401641845703, "logps/ref_rejected": -105.6436996459961, "logps/rejected": -123.99662017822266, "loss": 1.0827, "margin_dpo/margin_mean": 11.550440788269043, "margin_dpo/margin_std": 16.422531127929688, "step": 149 }, { "KL/chosen_KL_mean": -5.386837005615234, "KL/mean": -11.571063995361328, "KL/rejected_KL_mean": -17.755287170410156, "KL/std": 14.125550270080566, "epoch": 0.22026431718061673, "fcm_dpo/beta": 0.03741835057735443, "fcm_dpo/delta": -0.06613145768642426, "fcm_dpo/margin": 12.368448257446289, "fcm_dpo/q_t": 0.3956707715988159, "grad_norm": 11.549813270568848, "learning_rate": 4.792138157142157e-07, "logits/chosen": -1.116275429725647, "logits/rejected": -0.8437342047691345, "logps/chosen": -81.16552734375, "logps/ref_chosen": -75.7786865234375, "logps/ref_rejected": -90.02676391601562, "logps/rejected": -107.78205108642578, "loss": 1.0505, "margin_dpo/margin_mean": 12.368448257446289, "margin_dpo/margin_std": 15.55107307434082, "step": 150 }, { "KL/chosen_KL_mean": -7.2343902587890625, "KL/mean": -13.206653594970703, "KL/rejected_KL_mean": -19.178909301757812, "KL/std": 13.935263633728027, "epoch": 0.22173274596182085, "fcm_dpo/beta": 0.03712046146392822, "fcm_dpo/delta": -0.04541153833270073, "fcm_dpo/margin": 11.944517135620117, "fcm_dpo/q_t": 0.39984050393104553, "grad_norm": 14.090399742126465, "learning_rate": 4.786984671220053e-07, "logits/chosen": -2.013538360595703, "logits/rejected": -1.4218746423721313, "logps/chosen": -109.96861267089844, "logps/ref_chosen": -102.73422241210938, "logps/ref_rejected": -106.03781127929688, "logps/rejected": -125.21672058105469, "loss": 1.0668, "margin_dpo/margin_mean": 11.944517135620117, "margin_dpo/margin_std": 15.739160537719727, "step": 151 }, { "KL/chosen_KL_mean": -4.539939880371094, "KL/mean": -11.727033615112305, "KL/rejected_KL_mean": -18.91412353515625, "KL/std": 13.768760681152344, "epoch": 0.22320117474302498, "fcm_dpo/beta": 0.03628038614988327, "fcm_dpo/delta": -0.1284693032503128, "fcm_dpo/margin": 14.37417984008789, "fcm_dpo/q_t": 0.380440354347229, "grad_norm": 12.698140144348145, "learning_rate": 4.78177092112495e-07, "logits/chosen": -1.591780662536621, "logits/rejected": -1.2121691703796387, "logps/chosen": -102.13068389892578, "logps/ref_chosen": -97.59074401855469, "logps/ref_rejected": -110.49913024902344, "logps/rejected": -129.4132537841797, "loss": 1.0057, "margin_dpo/margin_mean": 14.37417984008789, "margin_dpo/margin_std": 15.97136116027832, "step": 152 }, { "KL/chosen_KL_mean": -7.0359649658203125, "KL/mean": -12.211174011230469, "KL/rejected_KL_mean": -17.386390686035156, "KL/std": 13.36141586303711, "epoch": 0.22466960352422907, "fcm_dpo/beta": 0.036238543689250946, "fcm_dpo/delta": 0.025889476761221886, "fcm_dpo/margin": 10.350433349609375, "fcm_dpo/q_t": 0.41438400745391846, "grad_norm": 12.577719688415527, "learning_rate": 4.776497044244016e-07, "logits/chosen": -1.801414132118225, "logits/rejected": -1.484548568725586, "logps/chosen": -94.06156921386719, "logps/ref_chosen": -87.02560424804688, "logps/ref_rejected": -91.06854248046875, "logps/rejected": -108.45492553710938, "loss": 1.1196, "margin_dpo/margin_mean": 10.350433349609375, "margin_dpo/margin_std": 15.322272300720215, "step": 153 }, { "KL/chosen_KL_mean": -5.361888885498047, "KL/mean": -13.792267799377441, "KL/rejected_KL_mean": -22.22264862060547, "KL/std": 14.728988647460938, "epoch": 0.2261380323054332, "fcm_dpo/beta": 0.03542676195502281, "fcm_dpo/delta": -0.20938807725906372, "fcm_dpo/margin": 16.860761642456055, "fcm_dpo/q_t": 0.36348289251327515, "grad_norm": 12.334086418151855, "learning_rate": 4.771163179548808e-07, "logits/chosen": -1.3828412294387817, "logits/rejected": -1.056118130683899, "logps/chosen": -96.46755981445312, "logps/ref_chosen": -91.10566711425781, "logps/ref_rejected": -111.37238311767578, "logps/rejected": -133.59503173828125, "loss": 0.9479, "margin_dpo/margin_mean": 16.860761642456055, "margin_dpo/margin_std": 15.506657600402832, "step": 154 }, { "KL/chosen_KL_mean": -7.401176452636719, "KL/mean": -13.71750259399414, "KL/rejected_KL_mean": -20.03382110595703, "KL/std": 15.158482551574707, "epoch": 0.2276064610866373, "fcm_dpo/beta": 0.034447215497493744, "fcm_dpo/delta": -0.0373886413872242, "fcm_dpo/margin": 12.632644653320312, "fcm_dpo/q_t": 0.3984300494194031, "grad_norm": 14.7758207321167, "learning_rate": 4.7657694675916247e-07, "logits/chosen": -1.2129627466201782, "logits/rejected": -0.8201339244842529, "logps/chosen": -101.96215057373047, "logps/ref_chosen": -94.56097412109375, "logps/ref_rejected": -84.77359008789062, "logps/rejected": -104.80741119384766, "loss": 1.0921, "margin_dpo/margin_mean": 12.632644653320312, "margin_dpo/margin_std": 18.63851547241211, "step": 155 }, { "KL/chosen_KL_mean": -9.110862731933594, "KL/mean": -14.732440948486328, "KL/rejected_KL_mean": -20.354019165039062, "KL/std": 13.479048728942871, "epoch": 0.2290748898678414, "fcm_dpo/beta": 0.03445587307214737, "fcm_dpo/delta": 0.012766610831022263, "fcm_dpo/margin": 11.243158340454102, "fcm_dpo/q_t": 0.4130709171295166, "grad_norm": 13.148513793945312, "learning_rate": 4.7603160505017893e-07, "logits/chosen": -1.6528770923614502, "logits/rejected": -1.1561336517333984, "logps/chosen": -103.76625061035156, "logps/ref_chosen": -94.65538787841797, "logps/ref_rejected": -84.41934204101562, "logps/rejected": -104.77336120605469, "loss": 1.1121, "margin_dpo/margin_mean": 11.243158340454102, "margin_dpo/margin_std": 16.401500701904297, "step": 156 }, { "KL/chosen_KL_mean": -8.642799377441406, "KL/mean": -16.464439392089844, "KL/rejected_KL_mean": -24.286083221435547, "KL/std": 14.467855453491211, "epoch": 0.2305433186490455, "fcm_dpo/beta": 0.033964671194553375, "fcm_dpo/delta": -0.13855916261672974, "fcm_dpo/margin": 15.643275260925293, "fcm_dpo/q_t": 0.3766776919364929, "grad_norm": 13.348742485046387, "learning_rate": 4.7548030719819154e-07, "logits/chosen": -1.8689866065979004, "logits/rejected": -1.403817892074585, "logps/chosen": -105.65518951416016, "logps/ref_chosen": -97.01239013671875, "logps/ref_rejected": -111.70133972167969, "logps/rejected": -135.9874267578125, "loss": 0.985, "margin_dpo/margin_mean": 15.64327621459961, "margin_dpo/margin_std": 15.14262580871582, "step": 157 }, { "KL/chosen_KL_mean": -7.932018280029297, "KL/mean": -16.193653106689453, "KL/rejected_KL_mean": -24.455291748046875, "KL/std": 15.697269439697266, "epoch": 0.23201174743024963, "fcm_dpo/beta": 0.0329747274518013, "fcm_dpo/delta": -0.1530257910490036, "fcm_dpo/margin": 16.523273468017578, "fcm_dpo/q_t": 0.3755152225494385, "grad_norm": 12.30068588256836, "learning_rate": 4.7492306773041136e-07, "logits/chosen": -1.5118037462234497, "logits/rejected": -1.2434831857681274, "logps/chosen": -102.64498138427734, "logps/ref_chosen": -94.71296691894531, "logps/ref_rejected": -120.93733978271484, "logps/rejected": -145.39263916015625, "loss": 0.982, "margin_dpo/margin_mean": 16.523273468017578, "margin_dpo/margin_std": 16.8155517578125, "step": 158 }, { "KL/chosen_KL_mean": -7.457000732421875, "KL/mean": -14.765888214111328, "KL/rejected_KL_mean": -22.07476806640625, "KL/std": 15.155786514282227, "epoch": 0.23348017621145375, "fcm_dpo/beta": 0.03229406103491783, "fcm_dpo/delta": -0.07566466927528381, "fcm_dpo/margin": 14.617778778076172, "fcm_dpo/q_t": 0.391844242811203, "grad_norm": 11.161052703857422, "learning_rate": 4.743599013306165e-07, "logits/chosen": -2.170686721801758, "logits/rejected": -1.7007007598876953, "logps/chosen": -115.67523956298828, "logps/ref_chosen": -108.2182388305664, "logps/ref_rejected": -94.83901977539062, "logps/rejected": -116.91378784179688, "loss": 1.04, "margin_dpo/margin_mean": 14.617779731750488, "margin_dpo/margin_std": 17.310461044311523, "step": 159 }, { "KL/chosen_KL_mean": -6.272430419921875, "KL/mean": -13.562728881835938, "KL/rejected_KL_mean": -20.853031158447266, "KL/std": 14.143604278564453, "epoch": 0.23494860499265785, "fcm_dpo/beta": 0.03179869428277016, "fcm_dpo/delta": -0.06694494187831879, "fcm_dpo/margin": 14.58059310913086, "fcm_dpo/q_t": 0.39149439334869385, "grad_norm": 11.524096488952637, "learning_rate": 4.737908228387656e-07, "logits/chosen": -1.8693783283233643, "logits/rejected": -1.3392395973205566, "logps/chosen": -104.43583679199219, "logps/ref_chosen": -98.16340637207031, "logps/ref_rejected": -102.28097534179688, "logps/rejected": -123.1340103149414, "loss": 1.0289, "margin_dpo/margin_mean": 14.580594062805176, "margin_dpo/margin_std": 15.101428031921387, "step": 160 }, { "KL/chosen_KL_mean": -8.430809020996094, "KL/mean": -14.989182472229004, "KL/rejected_KL_mean": -21.54754638671875, "KL/std": 13.022780418395996, "epoch": 0.23641703377386197, "fcm_dpo/beta": 0.03156358003616333, "fcm_dpo/delta": -0.01495346799492836, "fcm_dpo/margin": 13.11674976348877, "fcm_dpo/q_t": 0.4045618772506714, "grad_norm": 11.468718528747559, "learning_rate": 4.7321584725060594e-07, "logits/chosen": -1.8448528051376343, "logits/rejected": -1.3534014225006104, "logps/chosen": -95.02267456054688, "logps/ref_chosen": -86.59187316894531, "logps/ref_rejected": -89.01643371582031, "logps/rejected": -110.56398010253906, "loss": 1.0822, "margin_dpo/margin_mean": 13.116750717163086, "margin_dpo/margin_std": 16.861217498779297, "step": 161 }, { "KL/chosen_KL_mean": -7.500347137451172, "KL/mean": -14.770840644836426, "KL/rejected_KL_mean": -22.041336059570312, "KL/std": 15.046878814697266, "epoch": 0.23788546255506607, "fcm_dpo/beta": 0.031307023018598557, "fcm_dpo/delta": -0.058134980499744415, "fcm_dpo/margin": 14.540987014770508, "fcm_dpo/q_t": 0.3938494026660919, "grad_norm": 11.069710731506348, "learning_rate": 4.7263498971727905e-07, "logits/chosen": -1.3527427911758423, "logits/rejected": -0.9557307958602905, "logps/chosen": -104.90322875976562, "logps/ref_chosen": -97.40288543701172, "logps/ref_rejected": -95.86923217773438, "logps/rejected": -117.91056823730469, "loss": 1.0408, "margin_dpo/margin_mean": 14.540987014770508, "margin_dpo/margin_std": 16.120601654052734, "step": 162 }, { "KL/chosen_KL_mean": -7.379367828369141, "KL/mean": -16.087940216064453, "KL/rejected_KL_mean": -24.79651641845703, "KL/std": 17.236690521240234, "epoch": 0.2393538913362702, "fcm_dpo/beta": 0.030569162219762802, "fcm_dpo/delta": -0.1403488665819168, "fcm_dpo/margin": 17.417146682739258, "fcm_dpo/q_t": 0.37913113832473755, "grad_norm": 9.892509460449219, "learning_rate": 4.720482655449212e-07, "logits/chosen": -1.5070016384124756, "logits/rejected": -1.0983291864395142, "logps/chosen": -89.55476379394531, "logps/ref_chosen": -82.1753921508789, "logps/ref_rejected": -82.86473846435547, "logps/rejected": -107.6612548828125, "loss": 0.9927, "margin_dpo/margin_mean": 17.417146682739258, "margin_dpo/margin_std": 18.49786376953125, "step": 163 }, { "KL/chosen_KL_mean": -11.06167221069336, "KL/mean": -16.953601837158203, "KL/rejected_KL_mean": -22.84552764892578, "KL/std": 13.660167694091797, "epoch": 0.24082232011747431, "fcm_dpo/beta": 0.030404653400182724, "fcm_dpo/delta": 0.04255591332912445, "fcm_dpo/margin": 11.783851623535156, "fcm_dpo/q_t": 0.41618168354034424, "grad_norm": 11.670961380004883, "learning_rate": 4.714556901942599e-07, "logits/chosen": -1.5513638257980347, "logits/rejected": -1.0949259996414185, "logps/chosen": -101.53594207763672, "logps/ref_chosen": -90.47427368164062, "logps/ref_rejected": -87.02809143066406, "logps/rejected": -109.87361907958984, "loss": 1.1146, "margin_dpo/margin_mean": 11.783851623535156, "margin_dpo/margin_std": 15.049238204956055, "step": 164 }, { "KL/chosen_KL_mean": -7.628330230712891, "KL/mean": -16.240192413330078, "KL/rejected_KL_mean": -24.852054595947266, "KL/std": 16.3148136138916, "epoch": 0.2422907488986784, "fcm_dpo/beta": 0.03018486127257347, "fcm_dpo/delta": -0.12628839910030365, "fcm_dpo/margin": 17.22372055053711, "fcm_dpo/q_t": 0.38260453939437866, "grad_norm": 9.607232093811035, "learning_rate": 4.708572792802069e-07, "logits/chosen": -1.7736711502075195, "logits/rejected": -1.353489637374878, "logps/chosen": -94.94155883789062, "logps/ref_chosen": -87.313232421875, "logps/ref_rejected": -81.902587890625, "logps/rejected": -106.754638671875, "loss": 0.9993, "margin_dpo/margin_mean": 17.22372055053711, "margin_dpo/margin_std": 18.517019271850586, "step": 165 }, { "KL/chosen_KL_mean": -12.258926391601562, "KL/mean": -20.66061782836914, "KL/rejected_KL_mean": -29.06230926513672, "KL/std": 17.62511444091797, "epoch": 0.24375917767988253, "fcm_dpo/beta": 0.02947160229086876, "fcm_dpo/delta": -0.10020294040441513, "fcm_dpo/margin": 16.803375244140625, "fcm_dpo/q_t": 0.39099574089050293, "grad_norm": 10.955010414123535, "learning_rate": 4.702530485714461e-07, "logits/chosen": -1.630630373954773, "logits/rejected": -1.4273345470428467, "logps/chosen": -93.8449935913086, "logps/ref_chosen": -81.58606719970703, "logps/ref_rejected": -103.16517639160156, "logps/rejected": -132.22747802734375, "loss": 1.0324, "margin_dpo/margin_mean": 16.80337905883789, "margin_dpo/margin_std": 21.17521095275879, "step": 166 }, { "KL/chosen_KL_mean": -10.784908294677734, "KL/mean": -19.800373077392578, "KL/rejected_KL_mean": -28.815834045410156, "KL/std": 16.327632904052734, "epoch": 0.24522760646108663, "fcm_dpo/beta": 0.028640426695346832, "fcm_dpo/delta": -0.12395722419023514, "fcm_dpo/margin": 18.030925750732422, "fcm_dpo/q_t": 0.37993520498275757, "grad_norm": 10.143755912780762, "learning_rate": 4.6964301399001877e-07, "logits/chosen": -0.997816801071167, "logits/rejected": -0.8215293288230896, "logps/chosen": -94.3465805053711, "logps/ref_chosen": -83.56167602539062, "logps/ref_rejected": -99.05305480957031, "logps/rejected": -127.86888122558594, "loss": 0.9884, "margin_dpo/margin_mean": 18.03092384338379, "margin_dpo/margin_std": 16.70894432067871, "step": 167 }, { "KL/chosen_KL_mean": -10.434764862060547, "KL/mean": -18.226167678833008, "KL/rejected_KL_mean": -26.017574310302734, "KL/std": 16.21035385131836, "epoch": 0.24669603524229075, "fcm_dpo/beta": 0.02845989167690277, "fcm_dpo/delta": -0.04548220336437225, "fcm_dpo/margin": 15.582802772521973, "fcm_dpo/q_t": 0.39653563499450684, "grad_norm": 10.963912010192871, "learning_rate": 4.690271916109034e-07, "logits/chosen": -1.6443315744400024, "logits/rejected": -1.304487943649292, "logps/chosen": -90.36599731445312, "logps/ref_chosen": -79.93123626708984, "logps/ref_rejected": -80.10216522216797, "logps/rejected": -106.11973571777344, "loss": 1.0415, "margin_dpo/margin_mean": 15.582803726196289, "margin_dpo/margin_std": 16.265296936035156, "step": 168 }, { "KL/chosen_KL_mean": -8.978813171386719, "KL/mean": -17.928028106689453, "KL/rejected_KL_mean": -26.877243041992188, "KL/std": 17.50518035888672, "epoch": 0.24816446402349487, "fcm_dpo/beta": 0.027897316962480545, "fcm_dpo/delta": -0.10459548234939575, "fcm_dpo/margin": 17.898433685302734, "fcm_dpo/q_t": 0.3854382336139679, "grad_norm": 10.683631896972656, "learning_rate": 4.6840559766159235e-07, "logits/chosen": -1.5616600513458252, "logits/rejected": -1.3373570442199707, "logps/chosen": -91.88252258300781, "logps/ref_chosen": -82.90370178222656, "logps/ref_rejected": -90.12191009521484, "logps/rejected": -116.99915313720703, "loss": 1.0098, "margin_dpo/margin_mean": 17.8984317779541, "margin_dpo/margin_std": 18.769298553466797, "step": 169 }, { "KL/chosen_KL_mean": -10.64669418334961, "KL/mean": -18.446090698242188, "KL/rejected_KL_mean": -26.245494842529297, "KL/std": 14.645050048828125, "epoch": 0.24963289280469897, "fcm_dpo/beta": 0.027510786429047585, "fcm_dpo/delta": -0.03098824992775917, "fcm_dpo/margin": 15.59880542755127, "fcm_dpo/q_t": 0.398211270570755, "grad_norm": 10.988161087036133, "learning_rate": 4.6777824852166437e-07, "logits/chosen": -1.3103442192077637, "logits/rejected": -1.087742567062378, "logps/chosen": -85.628662109375, "logps/ref_chosen": -74.98197174072266, "logps/ref_rejected": -82.16560363769531, "logps/rejected": -108.41110229492188, "loss": 1.0425, "margin_dpo/margin_mean": 15.59880542755127, "margin_dpo/margin_std": 14.625118255615234, "step": 170 }, { "KL/chosen_KL_mean": -13.508792877197266, "KL/mean": -22.186763763427734, "KL/rejected_KL_mean": -30.864742279052734, "KL/std": 21.014923095703125, "epoch": 0.2511013215859031, "fcm_dpo/beta": 0.02730223909020424, "fcm_dpo/delta": -0.07747206836938858, "fcm_dpo/margin": 17.355945587158203, "fcm_dpo/q_t": 0.39762622117996216, "grad_norm": 12.436711311340332, "learning_rate": 4.6714516072235273e-07, "logits/chosen": -1.612771987915039, "logits/rejected": -1.0566449165344238, "logps/chosen": -132.48731994628906, "logps/ref_chosen": -118.97853088378906, "logps/ref_rejected": -115.76930236816406, "logps/rejected": -146.63404846191406, "loss": 1.0659, "margin_dpo/margin_mean": 17.355945587158203, "margin_dpo/margin_std": 25.149932861328125, "step": 171 }, { "KL/chosen_KL_mean": -14.472297668457031, "KL/mean": -22.21991729736328, "KL/rejected_KL_mean": -29.96753692626953, "KL/std": 15.089700698852539, "epoch": 0.2525697503671072, "fcm_dpo/beta": 0.02701481804251671, "fcm_dpo/delta": -0.019610438495874405, "fcm_dpo/margin": 15.495243072509766, "fcm_dpo/q_t": 0.39959830045700073, "grad_norm": 10.235724449157715, "learning_rate": 4.6650635094610966e-07, "logits/chosen": -2.227363109588623, "logits/rejected": -1.4574682712554932, "logps/chosen": -115.62608337402344, "logps/ref_chosen": -101.15379333496094, "logps/ref_rejected": -91.00568389892578, "logps/rejected": -120.97322082519531, "loss": 1.0391, "margin_dpo/margin_mean": 15.495243072509766, "margin_dpo/margin_std": 12.502436637878418, "step": 172 }, { "KL/chosen_KL_mean": -12.949691772460938, "KL/mean": -19.736547470092773, "KL/rejected_KL_mean": -26.523399353027344, "KL/std": 14.301332473754883, "epoch": 0.2540381791483113, "fcm_dpo/beta": 0.027224641293287277, "fcm_dpo/delta": 0.03154721111059189, "fcm_dpo/margin": 13.573709487915039, "fcm_dpo/q_t": 0.41275954246520996, "grad_norm": 9.87925910949707, "learning_rate": 4.6586183602616687e-07, "logits/chosen": -1.3797621726989746, "logits/rejected": -0.8793134689331055, "logps/chosen": -112.00637817382812, "logps/ref_chosen": -99.05668640136719, "logps/ref_rejected": -83.8507308959961, "logps/rejected": -110.37413024902344, "loss": 1.0909, "margin_dpo/margin_mean": 13.573709487915039, "margin_dpo/margin_std": 15.109273910522461, "step": 173 }, { "KL/chosen_KL_mean": -12.552255630493164, "KL/mean": -21.91756820678711, "KL/rejected_KL_mean": -31.282875061035156, "KL/std": 19.526504516601562, "epoch": 0.2555066079295154, "fcm_dpo/beta": 0.026833459734916687, "fcm_dpo/delta": -0.10793031752109528, "fcm_dpo/margin": 18.730621337890625, "fcm_dpo/q_t": 0.3838348090648651, "grad_norm": 11.543603897094727, "learning_rate": 4.652116329460919e-07, "logits/chosen": -1.7279245853424072, "logits/rejected": -1.684494137763977, "logps/chosen": -92.96116638183594, "logps/ref_chosen": -80.40890502929688, "logps/ref_rejected": -108.20171356201172, "logps/rejected": -139.48458862304688, "loss": 1.0076, "margin_dpo/margin_mean": 18.730621337890625, "margin_dpo/margin_std": 19.532512664794922, "step": 174 }, { "KL/chosen_KL_mean": -16.063671112060547, "KL/mean": -25.041492462158203, "KL/rejected_KL_mean": -34.019309997558594, "KL/std": 17.418283462524414, "epoch": 0.25697503671071953, "fcm_dpo/beta": 0.02640717476606369, "fcm_dpo/delta": -0.07778085768222809, "fcm_dpo/margin": 17.95564079284668, "fcm_dpo/q_t": 0.3881799578666687, "grad_norm": 11.220284461975098, "learning_rate": 4.645557588393406e-07, "logits/chosen": -1.2639981508255005, "logits/rejected": -0.999302089214325, "logps/chosen": -98.78016662597656, "logps/ref_chosen": -82.71649169921875, "logps/ref_rejected": -96.66712951660156, "logps/rejected": -130.68643188476562, "loss": 1.0137, "margin_dpo/margin_mean": 17.95564079284668, "margin_dpo/margin_std": 16.53908920288086, "step": 175 }, { "KL/chosen_KL_mean": -13.272895812988281, "KL/mean": -22.423259735107422, "KL/rejected_KL_mean": -31.573619842529297, "KL/std": 18.908517837524414, "epoch": 0.25844346549192365, "fcm_dpo/beta": 0.02592502161860466, "fcm_dpo/delta": -0.07818669080734253, "fcm_dpo/margin": 18.300722122192383, "fcm_dpo/q_t": 0.3917316794395447, "grad_norm": 10.996040344238281, "learning_rate": 4.638942309888058e-07, "logits/chosen": -1.8402767181396484, "logits/rejected": -1.6770572662353516, "logps/chosen": -92.9021987915039, "logps/ref_chosen": -79.62930297851562, "logps/ref_rejected": -102.6885986328125, "logps/rejected": -134.26220703125, "loss": 1.0396, "margin_dpo/margin_mean": 18.300724029541016, "margin_dpo/margin_std": 21.731956481933594, "step": 176 }, { "KL/chosen_KL_mean": -14.718318939208984, "KL/mean": -25.48288345336914, "KL/rejected_KL_mean": -36.24745559692383, "KL/std": 21.691837310791016, "epoch": 0.2599118942731278, "fcm_dpo/beta": 0.025350552052259445, "fcm_dpo/delta": -0.15390421450138092, "fcm_dpo/margin": 21.529129028320312, "fcm_dpo/q_t": 0.37775957584381104, "grad_norm": 9.846456527709961, "learning_rate": 4.6322706682636137e-07, "logits/chosen": -1.767153263092041, "logits/rejected": -1.4347813129425049, "logps/chosen": -107.2794189453125, "logps/ref_chosen": -92.56109619140625, "logps/ref_rejected": -103.33839416503906, "logps/rejected": -139.58584594726562, "loss": 0.9851, "margin_dpo/margin_mean": 21.529129028320312, "margin_dpo/margin_std": 22.610218048095703, "step": 177 }, { "KL/chosen_KL_mean": -13.632190704345703, "KL/mean": -26.36572265625, "KL/rejected_KL_mean": -39.09925079345703, "KL/std": 21.0909481048584, "epoch": 0.26138032305433184, "fcm_dpo/beta": 0.02418706938624382, "fcm_dpo/delta": -0.23035897314548492, "fcm_dpo/margin": 25.467058181762695, "fcm_dpo/q_t": 0.3588945269584656, "grad_norm": 10.92495059967041, "learning_rate": 4.6255428393240354e-07, "logits/chosen": -1.5901134014129639, "logits/rejected": -1.3684968948364258, "logps/chosen": -97.89798736572266, "logps/ref_chosen": -84.26579284667969, "logps/ref_rejected": -110.79730987548828, "logps/rejected": -149.8965606689453, "loss": 0.9276, "margin_dpo/margin_mean": 25.467056274414062, "margin_dpo/margin_std": 22.77474594116211, "step": 178 }, { "KL/chosen_KL_mean": -17.438541412353516, "KL/mean": -25.677539825439453, "KL/rejected_KL_mean": -33.916534423828125, "KL/std": 20.47817611694336, "epoch": 0.26284875183553597, "fcm_dpo/beta": 0.02386603131890297, "fcm_dpo/delta": 0.006813723593950272, "fcm_dpo/margin": 16.477989196777344, "fcm_dpo/q_t": 0.4107162058353424, "grad_norm": 10.42714786529541, "learning_rate": 4.6187590003538724e-07, "logits/chosen": -1.5469441413879395, "logits/rejected": -1.118253469467163, "logps/chosen": -110.88652038574219, "logps/ref_chosen": -93.4479751586914, "logps/ref_rejected": -94.25541687011719, "logps/rejected": -128.1719512939453, "loss": 1.107, "margin_dpo/margin_mean": 16.477989196777344, "margin_dpo/margin_std": 23.78270721435547, "step": 179 }, { "KL/chosen_KL_mean": -14.69903564453125, "KL/mean": -25.23954963684082, "KL/rejected_KL_mean": -35.780067443847656, "KL/std": 19.73122787475586, "epoch": 0.2643171806167401, "fcm_dpo/beta": 0.02352871373295784, "fcm_dpo/delta": -0.10145914554595947, "fcm_dpo/margin": 21.081026077270508, "fcm_dpo/q_t": 0.3847147524356842, "grad_norm": 12.551189422607422, "learning_rate": 4.611919330113591e-07, "logits/chosen": -1.3281564712524414, "logits/rejected": -1.0874998569488525, "logps/chosen": -103.53457641601562, "logps/ref_chosen": -88.83554077148438, "logps/ref_rejected": -103.90696716308594, "logps/rejected": -139.68704223632812, "loss": 1.0009, "margin_dpo/margin_mean": 21.081024169921875, "margin_dpo/margin_std": 20.26422119140625, "step": 180 }, { "KL/chosen_KL_mean": -14.44692611694336, "KL/mean": -22.602191925048828, "KL/rejected_KL_mean": -30.75745391845703, "KL/std": 18.833786010742188, "epoch": 0.2657856093979442, "fcm_dpo/beta": 0.0235487949103117, "fcm_dpo/delta": 0.016514623537659645, "fcm_dpo/margin": 16.310527801513672, "fcm_dpo/q_t": 0.4115299582481384, "grad_norm": 9.514108657836914, "learning_rate": 4.605024008834863e-07, "logits/chosen": -1.7391119003295898, "logits/rejected": -1.289538025856018, "logps/chosen": -93.68089294433594, "logps/ref_chosen": -79.23396301269531, "logps/ref_rejected": -69.17442321777344, "logps/rejected": -99.93187713623047, "loss": 1.0897, "margin_dpo/margin_mean": 16.310527801513672, "margin_dpo/margin_std": 20.101951599121094, "step": 181 }, { "KL/chosen_KL_mean": -18.265716552734375, "KL/mean": -28.868030548095703, "KL/rejected_KL_mean": -39.47034454345703, "KL/std": 21.815288543701172, "epoch": 0.26725403817914833, "fcm_dpo/beta": 0.023138057440519333, "fcm_dpo/delta": -0.09586258977651596, "fcm_dpo/margin": 21.204618453979492, "fcm_dpo/q_t": 0.3878190517425537, "grad_norm": 10.097704887390137, "learning_rate": 4.598073218215817e-07, "logits/chosen": -1.2371292114257812, "logits/rejected": -1.0156357288360596, "logps/chosen": -94.347900390625, "logps/ref_chosen": -76.08218383789062, "logps/ref_rejected": -94.9686050415039, "logps/rejected": -134.43894958496094, "loss": 1.0249, "margin_dpo/margin_mean": 21.204618453979492, "margin_dpo/margin_std": 24.06843376159668, "step": 182 }, { "KL/chosen_KL_mean": -19.13947296142578, "KL/mean": -29.47510528564453, "KL/rejected_KL_mean": -39.81074142456055, "KL/std": 19.90219497680664, "epoch": 0.2687224669603524, "fcm_dpo/beta": 0.022764312103390694, "fcm_dpo/delta": -0.0745917558670044, "fcm_dpo/margin": 20.671260833740234, "fcm_dpo/q_t": 0.3900007903575897, "grad_norm": 10.54615306854248, "learning_rate": 4.5910671414162484e-07, "logits/chosen": -1.8418855667114258, "logits/rejected": -1.2634952068328857, "logps/chosen": -113.04737854003906, "logps/ref_chosen": -93.90790557861328, "logps/ref_rejected": -83.56061553955078, "logps/rejected": -123.37135314941406, "loss": 1.0218, "margin_dpo/margin_mean": 20.671262741088867, "margin_dpo/margin_std": 20.524635314941406, "step": 183 }, { "KL/chosen_KL_mean": -17.223876953125, "KL/mean": -24.208553314208984, "KL/rejected_KL_mean": -31.193225860595703, "KL/std": 20.56912612915039, "epoch": 0.2701908957415565, "fcm_dpo/beta": 0.023053349927067757, "fcm_dpo/delta": 0.08047661185264587, "fcm_dpo/margin": 13.96934986114502, "fcm_dpo/q_t": 0.4245620369911194, "grad_norm": 9.78708267211914, "learning_rate": 4.5840059630527985e-07, "logits/chosen": -1.6431834697723389, "logits/rejected": -1.365020990371704, "logps/chosen": -104.09178161621094, "logps/ref_chosen": -86.86790466308594, "logps/ref_rejected": -84.28456115722656, "logps/rejected": -115.477783203125, "loss": 1.1529, "margin_dpo/margin_mean": 13.969348907470703, "margin_dpo/margin_std": 22.358993530273438, "step": 184 }, { "KL/chosen_KL_mean": -18.10415267944336, "KL/mean": -26.577861785888672, "KL/rejected_KL_mean": -35.05157470703125, "KL/std": 20.19632911682129, "epoch": 0.27165932452276065, "fcm_dpo/beta": 0.023215875029563904, "fcm_dpo/delta": 0.006646869704127312, "fcm_dpo/margin": 16.947423934936523, "fcm_dpo/q_t": 0.4101752042770386, "grad_norm": 10.907907485961914, "learning_rate": 4.5768898691940836e-07, "logits/chosen": -2.0594029426574707, "logits/rejected": -1.431849718093872, "logps/chosen": -112.12557983398438, "logps/ref_chosen": -94.02142333984375, "logps/ref_rejected": -81.45668029785156, "logps/rejected": -116.50825500488281, "loss": 1.0988, "margin_dpo/margin_mean": 16.947425842285156, "margin_dpo/margin_std": 23.298566818237305, "step": 185 }, { "KL/chosen_KL_mean": -22.690967559814453, "KL/mean": -31.867900848388672, "KL/rejected_KL_mean": -41.04484176635742, "KL/std": 19.511262893676758, "epoch": 0.27312775330396477, "fcm_dpo/beta": 0.02300911396741867, "fcm_dpo/delta": -0.023549221456050873, "fcm_dpo/margin": 18.3538818359375, "fcm_dpo/q_t": 0.40151679515838623, "grad_norm": 11.547987937927246, "learning_rate": 4.5697190473557947e-07, "logits/chosen": -2.2386269569396973, "logits/rejected": -1.3188517093658447, "logps/chosen": -136.88265991210938, "logps/ref_chosen": -114.19168090820312, "logps/ref_rejected": -93.33912658691406, "logps/rejected": -134.38397216796875, "loss": 1.0642, "margin_dpo/margin_mean": 18.353883743286133, "margin_dpo/margin_std": 21.181140899658203, "step": 186 }, { "KL/chosen_KL_mean": -19.331409454345703, "KL/mean": -28.141225814819336, "KL/rejected_KL_mean": -36.9510383605957, "KL/std": 21.912456512451172, "epoch": 0.2745961820851689, "fcm_dpo/beta": 0.0230933278799057, "fcm_dpo/delta": -0.007340654730796814, "fcm_dpo/margin": 17.619632720947266, "fcm_dpo/q_t": 0.40705031156539917, "grad_norm": 10.247783660888672, "learning_rate": 4.5624936864957555e-07, "logits/chosen": -1.6343810558319092, "logits/rejected": -1.3935773372650146, "logps/chosen": -96.20095825195312, "logps/ref_chosen": -76.86955261230469, "logps/ref_rejected": -88.16494750976562, "logps/rejected": -125.11598205566406, "loss": 1.0912, "margin_dpo/margin_mean": 17.619632720947266, "margin_dpo/margin_std": 23.924713134765625, "step": 187 }, { "KL/chosen_KL_mean": -14.763961791992188, "KL/mean": -24.81519889831543, "KL/rejected_KL_mean": -34.86643981933594, "KL/std": 21.60088539123535, "epoch": 0.27606461086637296, "fcm_dpo/beta": 0.02273169532418251, "fcm_dpo/delta": -0.06015936657786369, "fcm_dpo/margin": 20.10247230529785, "fcm_dpo/q_t": 0.39219599962234497, "grad_norm": 10.581904411315918, "learning_rate": 4.5552139770089454e-07, "logits/chosen": -2.0301551818847656, "logits/rejected": -1.7869179248809814, "logps/chosen": -93.9587631225586, "logps/ref_chosen": -79.1948013305664, "logps/ref_rejected": -96.82889556884766, "logps/rejected": -131.69534301757812, "loss": 1.0342, "margin_dpo/margin_mean": 20.10247230529785, "margin_dpo/margin_std": 20.44036102294922, "step": 188 }, { "KL/chosen_KL_mean": -19.49332046508789, "KL/mean": -29.830921173095703, "KL/rejected_KL_mean": -40.168521881103516, "KL/std": 21.700077056884766, "epoch": 0.2775330396475771, "fcm_dpo/beta": 0.022628169506788254, "fcm_dpo/delta": -0.07140851020812988, "fcm_dpo/margin": 20.675209045410156, "fcm_dpo/q_t": 0.39424267411231995, "grad_norm": 9.42530632019043, "learning_rate": 4.5478801107224794e-07, "logits/chosen": -1.832093358039856, "logits/rejected": -1.1984375715255737, "logps/chosen": -103.38612365722656, "logps/ref_chosen": -83.8927993774414, "logps/ref_rejected": -79.46589660644531, "logps/rejected": -119.63441467285156, "loss": 1.0418, "margin_dpo/margin_mean": 20.675209045410156, "margin_dpo/margin_std": 24.332874298095703, "step": 189 }, { "KL/chosen_KL_mean": -15.719375610351562, "KL/mean": -28.10742950439453, "KL/rejected_KL_mean": -40.495487213134766, "KL/std": 23.99024200439453, "epoch": 0.2790014684287812, "fcm_dpo/beta": 0.02190946415066719, "fcm_dpo/delta": -0.1511247754096985, "fcm_dpo/margin": 24.776111602783203, "fcm_dpo/q_t": 0.37361854314804077, "grad_norm": 10.192390441894531, "learning_rate": 4.5404922808905543e-07, "logits/chosen": -1.7690606117248535, "logits/rejected": -1.1645863056182861, "logps/chosen": -119.16084289550781, "logps/ref_chosen": -103.44146728515625, "logps/ref_rejected": -109.21502685546875, "logps/rejected": -149.71051025390625, "loss": 0.972, "margin_dpo/margin_mean": 24.776111602783203, "margin_dpo/margin_std": 22.701156616210938, "step": 190 }, { "KL/chosen_KL_mean": -14.93801498413086, "KL/mean": -28.920734405517578, "KL/rejected_KL_mean": -42.90345764160156, "KL/std": 25.98645782470703, "epoch": 0.28046989720998533, "fcm_dpo/beta": 0.021139226853847504, "fcm_dpo/delta": -0.2029237151145935, "fcm_dpo/margin": 27.96544647216797, "fcm_dpo/q_t": 0.36511901021003723, "grad_norm": 10.978670120239258, "learning_rate": 4.5330506821893565e-07, "logits/chosen": -1.4164198637008667, "logits/rejected": -1.0936098098754883, "logps/chosen": -114.72352600097656, "logps/ref_chosen": -99.78551483154297, "logps/ref_rejected": -116.00028228759766, "logps/rejected": -158.90374755859375, "loss": 0.9499, "margin_dpo/margin_mean": 27.96544647216797, "margin_dpo/margin_std": 26.608383178710938, "step": 191 }, { "KL/chosen_KL_mean": -24.26626968383789, "KL/mean": -34.041934967041016, "KL/rejected_KL_mean": -43.81760025024414, "KL/std": 20.774959564208984, "epoch": 0.28193832599118945, "fcm_dpo/beta": 0.02083655819296837, "fcm_dpo/delta": -0.007738408632576466, "fcm_dpo/margin": 19.551334381103516, "fcm_dpo/q_t": 0.4061148464679718, "grad_norm": 11.384042739868164, "learning_rate": 4.5255555107119336e-07, "logits/chosen": -1.5559508800506592, "logits/rejected": -1.3025813102722168, "logps/chosen": -125.12659454345703, "logps/ref_chosen": -100.86032104492188, "logps/ref_rejected": -109.44500732421875, "logps/rejected": -153.26260375976562, "loss": 1.0786, "margin_dpo/margin_mean": 19.551334381103516, "margin_dpo/margin_std": 24.199932098388672, "step": 192 }, { "KL/chosen_KL_mean": -20.133132934570312, "KL/mean": -28.178543090820312, "KL/rejected_KL_mean": -36.22394943237305, "KL/std": 22.017080307006836, "epoch": 0.2834067547723935, "fcm_dpo/beta": 0.021108925342559814, "fcm_dpo/delta": 0.062118690460920334, "fcm_dpo/margin": 16.09081268310547, "fcm_dpo/q_t": 0.42173948884010315, "grad_norm": 10.740169525146484, "learning_rate": 4.5180069639630236e-07, "logits/chosen": -2.015076160430908, "logits/rejected": -1.4234066009521484, "logps/chosen": -113.07144927978516, "logps/ref_chosen": -92.93831634521484, "logps/ref_rejected": -83.10917663574219, "logps/rejected": -119.3331298828125, "loss": 1.1515, "margin_dpo/margin_mean": 16.0908145904541, "margin_dpo/margin_std": 26.60179901123047, "step": 193 }, { "KL/chosen_KL_mean": -20.697654724121094, "KL/mean": -30.044784545898438, "KL/rejected_KL_mean": -39.39191436767578, "KL/std": 22.731914520263672, "epoch": 0.28487518355359764, "fcm_dpo/beta": 0.02119286358356476, "fcm_dpo/delta": 0.003764871507883072, "fcm_dpo/margin": 18.694259643554688, "fcm_dpo/q_t": 0.40884268283843994, "grad_norm": 9.072481155395508, "learning_rate": 4.510405240853854e-07, "logits/chosen": -2.217127561569214, "logits/rejected": -1.8274306058883667, "logps/chosen": -84.48922729492188, "logps/ref_chosen": -63.79157257080078, "logps/ref_rejected": -68.72665405273438, "logps/rejected": -108.11856842041016, "loss": 1.0899, "margin_dpo/margin_mean": 18.694259643554688, "margin_dpo/margin_std": 24.07916259765625, "step": 194 }, { "KL/chosen_KL_mean": -19.735191345214844, "KL/mean": -32.239219665527344, "KL/rejected_KL_mean": -44.74325180053711, "KL/std": 23.160146713256836, "epoch": 0.28634361233480177, "fcm_dpo/beta": 0.0207916758954525, "fcm_dpo/delta": -0.126350536942482, "fcm_dpo/margin": 25.00806427001953, "fcm_dpo/q_t": 0.3796185851097107, "grad_norm": 9.468811988830566, "learning_rate": 4.5027505416968985e-07, "logits/chosen": -1.4475696086883545, "logits/rejected": -1.3281883001327515, "logps/chosen": -103.62065124511719, "logps/ref_chosen": -83.88545989990234, "logps/ref_rejected": -104.00125122070312, "logps/rejected": -148.7445068359375, "loss": 1.0029, "margin_dpo/margin_mean": 25.00806427001953, "margin_dpo/margin_std": 26.489301681518555, "step": 195 }, { "KL/chosen_KL_mean": -23.260173797607422, "KL/mean": -34.94920349121094, "KL/rejected_KL_mean": -46.63823318481445, "KL/std": 24.75688934326172, "epoch": 0.2878120411160059, "fcm_dpo/beta": 0.020357917994260788, "fcm_dpo/delta": -0.07973592728376389, "fcm_dpo/margin": 23.378055572509766, "fcm_dpo/q_t": 0.3876831531524658, "grad_norm": 9.031380653381348, "learning_rate": 4.495043068200599e-07, "logits/chosen": -1.4926586151123047, "logits/rejected": -1.0519791841506958, "logps/chosen": -112.66153717041016, "logps/ref_chosen": -89.4013671875, "logps/ref_rejected": -83.34735107421875, "logps/rejected": -129.98558044433594, "loss": 1.0144, "margin_dpo/margin_mean": 23.378055572509766, "margin_dpo/margin_std": 22.274261474609375, "step": 196 }, { "KL/chosen_KL_mean": -23.508270263671875, "KL/mean": -30.71670150756836, "KL/rejected_KL_mean": -37.92512893676758, "KL/std": 20.083301544189453, "epoch": 0.28928046989721, "fcm_dpo/beta": 0.020603254437446594, "fcm_dpo/delta": 0.10611146688461304, "fcm_dpo/margin": 14.416849136352539, "fcm_dpo/q_t": 0.4300667643547058, "grad_norm": 9.478358268737793, "learning_rate": 4.4872830234640493e-07, "logits/chosen": -1.5702838897705078, "logits/rejected": -1.2472870349884033, "logps/chosen": -110.3489761352539, "logps/ref_chosen": -86.84070587158203, "logps/ref_rejected": -82.79289245605469, "logps/rejected": -120.71802520751953, "loss": 1.1572, "margin_dpo/margin_mean": 14.416850090026855, "margin_dpo/margin_std": 21.39665412902832, "step": 197 }, { "KL/chosen_KL_mean": -27.154850006103516, "KL/mean": -38.79590606689453, "KL/rejected_KL_mean": -50.43695831298828, "KL/std": 23.541431427001953, "epoch": 0.2907488986784141, "fcm_dpo/beta": 0.02047930657863617, "fcm_dpo/delta": -0.08060362935066223, "fcm_dpo/margin": 23.282108306884766, "fcm_dpo/q_t": 0.38967156410217285, "grad_norm": 11.172621726989746, "learning_rate": 4.479470611971645e-07, "logits/chosen": -1.7387126684188843, "logits/rejected": -1.5650384426116943, "logps/chosen": -112.20309448242188, "logps/ref_chosen": -85.04824829101562, "logps/ref_rejected": -104.10365295410156, "logps/rejected": -154.54061889648438, "loss": 1.0219, "margin_dpo/margin_mean": 23.282108306884766, "margin_dpo/margin_std": 24.403831481933594, "step": 198 }, { "KL/chosen_KL_mean": -26.532012939453125, "KL/mean": -36.09756851196289, "KL/rejected_KL_mean": -45.66312789916992, "KL/std": 22.940486907958984, "epoch": 0.2922173274596182, "fcm_dpo/beta": 0.020430803298950195, "fcm_dpo/delta": 0.009478235617280006, "fcm_dpo/margin": 19.131107330322266, "fcm_dpo/q_t": 0.4091432988643646, "grad_norm": 9.791498184204102, "learning_rate": 4.471606039587695e-07, "logits/chosen": -1.675721287727356, "logits/rejected": -1.4163570404052734, "logps/chosen": -108.31871032714844, "logps/ref_chosen": -81.78669738769531, "logps/ref_rejected": -90.790771484375, "logps/rejected": -136.4539031982422, "loss": 1.0904, "margin_dpo/margin_mean": 19.131105422973633, "margin_dpo/margin_std": 24.091829299926758, "step": 199 }, { "KL/chosen_KL_mean": -26.071044921875, "KL/mean": -37.47578430175781, "KL/rejected_KL_mean": -48.88051986694336, "KL/std": 24.595252990722656, "epoch": 0.2936857562408223, "fcm_dpo/beta": 0.020144090056419373, "fcm_dpo/delta": -0.06297807395458221, "fcm_dpo/margin": 22.809467315673828, "fcm_dpo/q_t": 0.39492619037628174, "grad_norm": 9.758691787719727, "learning_rate": 4.4636895135509966e-07, "logits/chosen": -1.8676292896270752, "logits/rejected": -1.3456683158874512, "logps/chosen": -110.33914184570312, "logps/ref_chosen": -84.26809692382812, "logps/ref_rejected": -86.96786499023438, "logps/rejected": -135.848388671875, "loss": 1.0565, "margin_dpo/margin_mean": 22.809467315673828, "margin_dpo/margin_std": 28.493427276611328, "step": 200 }, { "KL/chosen_KL_mean": -27.926815032958984, "KL/mean": -38.90440368652344, "KL/rejected_KL_mean": -49.881996154785156, "KL/std": 26.645179748535156, "epoch": 0.29515418502202645, "fcm_dpo/beta": 0.020086470991373062, "fcm_dpo/delta": -0.042913518846035004, "fcm_dpo/margin": 21.955181121826172, "fcm_dpo/q_t": 0.39924177527427673, "grad_norm": 10.003375053405762, "learning_rate": 4.455721242469372e-07, "logits/chosen": -1.4469943046569824, "logits/rejected": -1.0615458488464355, "logps/chosen": -135.69931030273438, "logps/ref_chosen": -107.77249145507812, "logps/ref_rejected": -119.79248046875, "logps/rejected": -169.67446899414062, "loss": 1.0722, "margin_dpo/margin_mean": 21.955181121826172, "margin_dpo/margin_std": 29.179582595825195, "step": 201 }, { "KL/chosen_KL_mean": -26.26013946533203, "KL/mean": -36.46544647216797, "KL/rejected_KL_mean": -46.67076110839844, "KL/std": 22.05629539489746, "epoch": 0.2966226138032305, "fcm_dpo/beta": 0.019991470500826836, "fcm_dpo/delta": -0.008399421349167824, "fcm_dpo/margin": 20.410612106323242, "fcm_dpo/q_t": 0.4059469699859619, "grad_norm": 10.302306175231934, "learning_rate": 4.4477014363141755e-07, "logits/chosen": -1.6098666191101074, "logits/rejected": -1.4822697639465332, "logps/chosen": -102.23258972167969, "logps/ref_chosen": -75.97245788574219, "logps/ref_rejected": -94.4599838256836, "logps/rejected": -141.1307373046875, "loss": 1.0797, "margin_dpo/margin_mean": 20.410612106323242, "margin_dpo/margin_std": 24.968143463134766, "step": 202 }, { "KL/chosen_KL_mean": -23.449317932128906, "KL/mean": -35.02262496948242, "KL/rejected_KL_mean": -46.59593963623047, "KL/std": 23.070955276489258, "epoch": 0.29809104258443464, "fcm_dpo/beta": 0.019808560609817505, "fcm_dpo/delta": -0.06126658618450165, "fcm_dpo/margin": 23.146617889404297, "fcm_dpo/q_t": 0.3927616477012634, "grad_norm": 9.253658294677734, "learning_rate": 4.439630306414758e-07, "logits/chosen": -1.7696117162704468, "logits/rejected": -1.306767463684082, "logps/chosen": -118.41647338867188, "logps/ref_chosen": -94.96715545654297, "logps/ref_rejected": -92.8876724243164, "logps/rejected": -139.48361206054688, "loss": 1.0329, "margin_dpo/margin_mean": 23.146617889404297, "margin_dpo/margin_std": 24.110424041748047, "step": 203 }, { "KL/chosen_KL_mean": -29.416732788085938, "KL/mean": -39.408409118652344, "KL/rejected_KL_mean": -49.40008544921875, "KL/std": 23.036495208740234, "epoch": 0.29955947136563876, "fcm_dpo/beta": 0.019786227494478226, "fcm_dpo/delta": 0.00461270846426487, "fcm_dpo/margin": 19.98335075378418, "fcm_dpo/q_t": 0.409085214138031, "grad_norm": 10.091337203979492, "learning_rate": 4.431508065452897e-07, "logits/chosen": -1.7380378246307373, "logits/rejected": -1.0611560344696045, "logps/chosen": -145.77392578125, "logps/ref_chosen": -116.35719299316406, "logps/ref_rejected": -93.39759063720703, "logps/rejected": -142.79766845703125, "loss": 1.0976, "margin_dpo/margin_mean": 19.983352661132812, "margin_dpo/margin_std": 26.7713623046875, "step": 204 }, { "KL/chosen_KL_mean": -27.381458282470703, "KL/mean": -40.31005859375, "KL/rejected_KL_mean": -53.23866271972656, "KL/std": 25.47120475769043, "epoch": 0.3010279001468429, "fcm_dpo/beta": 0.019382575526833534, "fcm_dpo/delta": -0.1068301647901535, "fcm_dpo/margin": 25.857196807861328, "fcm_dpo/q_t": 0.3810048997402191, "grad_norm": 8.986839294433594, "learning_rate": 4.4233349274571974e-07, "logits/chosen": -2.105185031890869, "logits/rejected": -1.6590595245361328, "logps/chosen": -116.24080657958984, "logps/ref_chosen": -88.85934448242188, "logps/ref_rejected": -91.8544921875, "logps/rejected": -145.09315490722656, "loss": 0.9886, "margin_dpo/margin_mean": 25.857196807861328, "margin_dpo/margin_std": 21.800201416015625, "step": 205 }, { "KL/chosen_KL_mean": -27.806926727294922, "KL/mean": -39.61248016357422, "KL/rejected_KL_mean": -51.41804122924805, "KL/std": 24.83611297607422, "epoch": 0.302496328928047, "fcm_dpo/beta": 0.019097376614809036, "fcm_dpo/delta": -0.05373923107981682, "fcm_dpo/margin": 23.61111068725586, "fcm_dpo/q_t": 0.39285486936569214, "grad_norm": 10.534939765930176, "learning_rate": 4.415111107797445e-07, "logits/chosen": -1.690922737121582, "logits/rejected": -1.5431774854660034, "logps/chosen": -104.353271484375, "logps/ref_chosen": -76.54634857177734, "logps/ref_rejected": -102.95314025878906, "logps/rejected": -154.37118530273438, "loss": 1.0255, "margin_dpo/margin_mean": 23.61111068725586, "margin_dpo/margin_std": 21.280559539794922, "step": 206 }, { "KL/chosen_KL_mean": -29.11838150024414, "KL/mean": -42.390838623046875, "KL/rejected_KL_mean": -55.66330337524414, "KL/std": 30.269773483276367, "epoch": 0.3039647577092511, "fcm_dpo/beta": 0.018822699785232544, "fcm_dpo/delta": -0.10484839975833893, "fcm_dpo/margin": 26.544925689697266, "fcm_dpo/q_t": 0.3889656662940979, "grad_norm": 10.510990142822266, "learning_rate": 4.4068368231789365e-07, "logits/chosen": -1.880220890045166, "logits/rejected": -1.440982460975647, "logps/chosen": -115.35002899169922, "logps/ref_chosen": -86.23164367675781, "logps/ref_rejected": -90.65512084960938, "logps/rejected": -146.31842041015625, "loss": 1.0422, "margin_dpo/margin_mean": 26.544925689697266, "margin_dpo/margin_std": 34.792259216308594, "step": 207 }, { "KL/chosen_KL_mean": -30.124591827392578, "KL/mean": -43.70378112792969, "KL/rejected_KL_mean": -57.282958984375, "KL/std": 27.608057022094727, "epoch": 0.3054331864904552, "fcm_dpo/beta": 0.018493659794330597, "fcm_dpo/delta": -0.10757000744342804, "fcm_dpo/margin": 27.158363342285156, "fcm_dpo/q_t": 0.38162073493003845, "grad_norm": 10.291736602783203, "learning_rate": 4.398512291636768e-07, "logits/chosen": -1.8993340730667114, "logits/rejected": -1.5495939254760742, "logps/chosen": -124.28413391113281, "logps/ref_chosen": -94.1595458984375, "logps/ref_rejected": -100.96233367919922, "logps/rejected": -158.24530029296875, "loss": 0.9984, "margin_dpo/margin_mean": 27.15836524963379, "margin_dpo/margin_std": 25.236501693725586, "step": 208 }, { "KL/chosen_KL_mean": -25.538318634033203, "KL/mean": -37.59413146972656, "KL/rejected_KL_mean": -49.64993667602539, "KL/std": 24.777786254882812, "epoch": 0.3069016152716593, "fcm_dpo/beta": 0.018140096217393875, "fcm_dpo/delta": -0.03953101485967636, "fcm_dpo/margin": 24.11162567138672, "fcm_dpo/q_t": 0.3987649083137512, "grad_norm": 9.123018264770508, "learning_rate": 4.3901377325300857e-07, "logits/chosen": -2.0401103496551514, "logits/rejected": -1.56000554561615, "logps/chosen": -109.7088851928711, "logps/ref_chosen": -84.17056274414062, "logps/ref_rejected": -87.61955261230469, "logps/rejected": -137.2694854736328, "loss": 1.0578, "margin_dpo/margin_mean": 24.11162757873535, "margin_dpo/margin_std": 28.087017059326172, "step": 209 }, { "KL/chosen_KL_mean": -29.13782501220703, "KL/mean": -40.50688552856445, "KL/rejected_KL_mean": -51.87594223022461, "KL/std": 26.160972595214844, "epoch": 0.30837004405286345, "fcm_dpo/beta": 0.018106302246451378, "fcm_dpo/delta": -0.012289915233850479, "fcm_dpo/margin": 22.738117218017578, "fcm_dpo/q_t": 0.4043741822242737, "grad_norm": 8.53181266784668, "learning_rate": 4.381713366536311e-07, "logits/chosen": -1.8092029094696045, "logits/rejected": -1.4135223627090454, "logps/chosen": -110.30899810791016, "logps/ref_chosen": -81.17117309570312, "logps/ref_rejected": -84.17478942871094, "logps/rejected": -136.05072021484375, "loss": 1.0698, "margin_dpo/margin_mean": 22.738117218017578, "margin_dpo/margin_std": 26.364755630493164, "step": 210 }, { "KL/chosen_KL_mean": -32.44643020629883, "KL/mean": -43.59911346435547, "KL/rejected_KL_mean": -54.75179672241211, "KL/std": 27.73133087158203, "epoch": 0.30983847283406757, "fcm_dpo/beta": 0.018148761242628098, "fcm_dpo/delta": -0.005087519064545631, "fcm_dpo/margin": 22.305368423461914, "fcm_dpo/q_t": 0.4065849483013153, "grad_norm": 12.406351089477539, "learning_rate": 4.373239415645323e-07, "logits/chosen": -1.946195125579834, "logits/rejected": -1.305687665939331, "logps/chosen": -141.15914916992188, "logps/ref_chosen": -108.71270751953125, "logps/ref_rejected": -93.55564880371094, "logps/rejected": -148.3074493408203, "loss": 1.0819, "margin_dpo/margin_mean": 22.30536651611328, "margin_dpo/margin_std": 28.048786163330078, "step": 211 }, { "KL/chosen_KL_mean": -22.685386657714844, "KL/mean": -39.07157897949219, "KL/rejected_KL_mean": -55.45777893066406, "KL/std": 31.4821720123291, "epoch": 0.31130690161527164, "fcm_dpo/beta": 0.017571106553077698, "fcm_dpo/delta": -0.18714313209056854, "fcm_dpo/margin": 32.77239227294922, "fcm_dpo/q_t": 0.36779311299324036, "grad_norm": 9.006429672241211, "learning_rate": 4.3647161031536086e-07, "logits/chosen": -1.9155206680297852, "logits/rejected": -1.505225658416748, "logps/chosen": -121.04733276367188, "logps/ref_chosen": -98.36195373535156, "logps/ref_rejected": -109.88999938964844, "logps/rejected": -165.3477783203125, "loss": 0.9591, "margin_dpo/margin_mean": 32.77239227294922, "margin_dpo/margin_std": 31.458105087280273, "step": 212 }, { "KL/chosen_KL_mean": -26.68935775756836, "KL/mean": -38.932594299316406, "KL/rejected_KL_mean": -51.17582702636719, "KL/std": 28.607389450073242, "epoch": 0.31277533039647576, "fcm_dpo/beta": 0.017338991165161133, "fcm_dpo/delta": -0.025808624923229218, "fcm_dpo/margin": 24.486473083496094, "fcm_dpo/q_t": 0.40154188871383667, "grad_norm": 11.486190795898438, "learning_rate": 4.3561436536583774e-07, "logits/chosen": -1.9941582679748535, "logits/rejected": -1.21329927444458, "logps/chosen": -134.74465942382812, "logps/ref_chosen": -108.05531311035156, "logps/ref_rejected": -100.14414978027344, "logps/rejected": -151.31997680664062, "loss": 1.0751, "margin_dpo/margin_mean": 24.48647117614746, "margin_dpo/margin_std": 31.313743591308594, "step": 213 }, { "KL/chosen_KL_mean": -28.40774154663086, "KL/mean": -40.04389953613281, "KL/rejected_KL_mean": -51.68006896972656, "KL/std": 30.76535415649414, "epoch": 0.3142437591776799, "fcm_dpo/beta": 0.0173606276512146, "fcm_dpo/delta": -0.0042098285630345345, "fcm_dpo/margin": 23.272327423095703, "fcm_dpo/q_t": 0.40854400396347046, "grad_norm": 8.88479232788086, "learning_rate": 4.3475222930516473e-07, "logits/chosen": -1.790917992591858, "logits/rejected": -1.668558955192566, "logps/chosen": -106.21247863769531, "logps/ref_chosen": -77.80473327636719, "logps/ref_rejected": -84.27578735351562, "logps/rejected": -135.9558563232422, "loss": 1.1019, "margin_dpo/margin_mean": 23.272327423095703, "margin_dpo/margin_std": 33.80026626586914, "step": 214 }, { "KL/chosen_KL_mean": -24.703697204589844, "KL/mean": -40.422752380371094, "KL/rejected_KL_mean": -56.14181137084961, "KL/std": 30.33739471435547, "epoch": 0.315712187958884, "fcm_dpo/beta": 0.01698577031493187, "fcm_dpo/delta": -0.14150086045265198, "fcm_dpo/margin": 31.43810272216797, "fcm_dpo/q_t": 0.37655872106552124, "grad_norm": 8.77225399017334, "learning_rate": 4.3388522485142885e-07, "logits/chosen": -1.6902751922607422, "logits/rejected": -1.2118675708770752, "logps/chosen": -109.81759643554688, "logps/ref_chosen": -85.1138916015625, "logps/ref_rejected": -96.86151123046875, "logps/rejected": -153.00332641601562, "loss": 0.9803, "margin_dpo/margin_mean": 31.438106536865234, "margin_dpo/margin_std": 30.043346405029297, "step": 215 }, { "KL/chosen_KL_mean": -26.54071044921875, "KL/mean": -40.23850631713867, "KL/rejected_KL_mean": -53.936302185058594, "KL/std": 27.348447799682617, "epoch": 0.31718061674008813, "fcm_dpo/beta": 0.0167689211666584, "fcm_dpo/delta": -0.06234792247414589, "fcm_dpo/margin": 27.395593643188477, "fcm_dpo/q_t": 0.39330339431762695, "grad_norm": 8.334915161132812, "learning_rate": 4.330133748510036e-07, "logits/chosen": -1.677945613861084, "logits/rejected": -1.2599682807922363, "logps/chosen": -107.13301086425781, "logps/ref_chosen": -80.59230041503906, "logps/ref_rejected": -81.41983795166016, "logps/rejected": -135.35614013671875, "loss": 1.0321, "margin_dpo/margin_mean": 27.395591735839844, "margin_dpo/margin_std": 28.218055725097656, "step": 216 }, { "KL/chosen_KL_mean": -32.48088073730469, "KL/mean": -43.73289108276367, "KL/rejected_KL_mean": -54.98491287231445, "KL/std": 28.130788803100586, "epoch": 0.3186490455212922, "fcm_dpo/beta": 0.01673309877514839, "fcm_dpo/delta": 0.024263232946395874, "fcm_dpo/margin": 22.504024505615234, "fcm_dpo/q_t": 0.41414111852645874, "grad_norm": 10.248025894165039, "learning_rate": 4.3213670227794757e-07, "logits/chosen": -2.0343308448791504, "logits/rejected": -1.727921485900879, "logps/chosen": -125.95346069335938, "logps/ref_chosen": -93.47257995605469, "logps/ref_rejected": -103.488525390625, "logps/rejected": -158.4734344482422, "loss": 1.1115, "margin_dpo/margin_mean": 22.504024505615234, "margin_dpo/margin_std": 32.14043045043945, "step": 217 }, { "KL/chosen_KL_mean": -31.87409210205078, "KL/mean": -45.39189147949219, "KL/rejected_KL_mean": -58.909706115722656, "KL/std": 28.43138313293457, "epoch": 0.3201174743024963, "fcm_dpo/beta": 0.016557011753320694, "fcm_dpo/delta": -0.05015214532613754, "fcm_dpo/margin": 27.03561019897461, "fcm_dpo/q_t": 0.39724159240722656, "grad_norm": 8.858884811401367, "learning_rate": 4.3125523023339815e-07, "logits/chosen": -1.392014741897583, "logits/rejected": -1.009111762046814, "logps/chosen": -120.93292236328125, "logps/ref_chosen": -89.05883026123047, "logps/ref_rejected": -94.30680847167969, "logps/rejected": -153.21652221679688, "loss": 1.0623, "margin_dpo/margin_mean": 27.03561019897461, "margin_dpo/margin_std": 34.11198425292969, "step": 218 }, { "KL/chosen_KL_mean": -28.407535552978516, "KL/mean": -40.62030029296875, "KL/rejected_KL_mean": -52.83307647705078, "KL/std": 33.11445236206055, "epoch": 0.32158590308370044, "fcm_dpo/beta": 0.01663254201412201, "fcm_dpo/delta": -0.00708409771323204, "fcm_dpo/margin": 24.425540924072266, "fcm_dpo/q_t": 0.40718722343444824, "grad_norm": 9.904301643371582, "learning_rate": 4.303689819449636e-07, "logits/chosen": -1.8689491748809814, "logits/rejected": -1.4209094047546387, "logps/chosen": -129.4148712158203, "logps/ref_chosen": -101.00733947753906, "logps/ref_rejected": -92.46794128417969, "logps/rejected": -145.301025390625, "loss": 1.1021, "margin_dpo/margin_mean": 24.425540924072266, "margin_dpo/margin_std": 35.33160400390625, "step": 219 }, { "KL/chosen_KL_mean": -29.472442626953125, "KL/mean": -40.872840881347656, "KL/rejected_KL_mean": -52.27324676513672, "KL/std": 27.503631591796875, "epoch": 0.32305433186490456, "fcm_dpo/beta": 0.016602829098701477, "fcm_dpo/delta": 0.02226090244948864, "fcm_dpo/margin": 22.800806045532227, "fcm_dpo/q_t": 0.4112911522388458, "grad_norm": 10.389994621276855, "learning_rate": 4.2947798076611047e-07, "logits/chosen": -1.3132164478302002, "logits/rejected": -0.9558252096176147, "logps/chosen": -125.00965881347656, "logps/ref_chosen": -95.53721618652344, "logps/ref_rejected": -94.30703735351562, "logps/rejected": -146.58029174804688, "loss": 1.0921, "margin_dpo/margin_mean": 22.800806045532227, "margin_dpo/margin_std": 27.134029388427734, "step": 220 }, { "KL/chosen_KL_mean": -27.60589599609375, "KL/mean": -45.146339416503906, "KL/rejected_KL_mean": -62.6867790222168, "KL/std": 34.223506927490234, "epoch": 0.3245227606461087, "fcm_dpo/beta": 0.016273923218250275, "fcm_dpo/delta": -0.1809280514717102, "fcm_dpo/margin": 35.08088302612305, "fcm_dpo/q_t": 0.36800310015678406, "grad_norm": 9.935919761657715, "learning_rate": 4.285822501755485e-07, "logits/chosen": -1.3770164251327515, "logits/rejected": -1.240946888923645, "logps/chosen": -110.45075988769531, "logps/ref_chosen": -82.84486389160156, "logps/ref_rejected": -110.81179809570312, "logps/rejected": -173.4985809326172, "loss": 0.9546, "margin_dpo/margin_mean": 35.08088302612305, "margin_dpo/margin_std": 29.985408782958984, "step": 221 }, { "KL/chosen_KL_mean": -35.47964859008789, "KL/mean": -49.76268768310547, "KL/rejected_KL_mean": -64.04573059082031, "KL/std": 33.002288818359375, "epoch": 0.32599118942731276, "fcm_dpo/beta": 0.015947973355650902, "fcm_dpo/delta": -0.05833457410335541, "fcm_dpo/margin": 28.566089630126953, "fcm_dpo/q_t": 0.3960283696651459, "grad_norm": 10.089146614074707, "learning_rate": 4.276818137766118e-07, "logits/chosen": -1.9512139558792114, "logits/rejected": -1.519282341003418, "logps/chosen": -130.6216278076172, "logps/ref_chosen": -95.14198303222656, "logps/ref_rejected": -106.80441284179688, "logps/rejected": -170.8501434326172, "loss": 1.0719, "margin_dpo/margin_mean": 28.566091537475586, "margin_dpo/margin_std": 38.6176872253418, "step": 222 }, { "KL/chosen_KL_mean": -34.617347717285156, "KL/mean": -44.575653076171875, "KL/rejected_KL_mean": -54.53396987915039, "KL/std": 29.037792205810547, "epoch": 0.3274596182085169, "fcm_dpo/beta": 0.01601281762123108, "fcm_dpo/delta": 0.08382445573806763, "fcm_dpo/margin": 19.916614532470703, "fcm_dpo/q_t": 0.42625313997268677, "grad_norm": 9.741445541381836, "learning_rate": 4.2677669529663686e-07, "logits/chosen": -2.338665246963501, "logits/rejected": -1.958406686782837, "logps/chosen": -120.19245910644531, "logps/ref_chosen": -85.57511138916016, "logps/ref_rejected": -86.45237731933594, "logps/rejected": -140.98635864257812, "loss": 1.1545, "margin_dpo/margin_mean": 19.916614532470703, "margin_dpo/margin_std": 31.80205535888672, "step": 223 }, { "KL/chosen_KL_mean": -31.606170654296875, "KL/mean": -46.03142547607422, "KL/rejected_KL_mean": -60.456687927246094, "KL/std": 32.223419189453125, "epoch": 0.328928046989721, "fcm_dpo/beta": 0.015940003097057343, "fcm_dpo/delta": -0.06282280385494232, "fcm_dpo/margin": 28.850515365600586, "fcm_dpo/q_t": 0.3941079378128052, "grad_norm": 9.348638534545898, "learning_rate": 4.2586691858633747e-07, "logits/chosen": -1.761162519454956, "logits/rejected": -1.3015685081481934, "logps/chosen": -114.32997131347656, "logps/ref_chosen": -82.72380065917969, "logps/ref_rejected": -82.59538269042969, "logps/rejected": -143.05206298828125, "loss": 1.0451, "margin_dpo/margin_mean": 28.850515365600586, "margin_dpo/margin_std": 33.700008392333984, "step": 224 }, { "KL/chosen_KL_mean": -35.68672561645508, "KL/mean": -50.52009582519531, "KL/rejected_KL_mean": -65.35346221923828, "KL/std": 33.91242980957031, "epoch": 0.3303964757709251, "fcm_dpo/beta": 0.015788692981004715, "fcm_dpo/delta": -0.07176551967859268, "fcm_dpo/margin": 29.666730880737305, "fcm_dpo/q_t": 0.3939414620399475, "grad_norm": 10.13900089263916, "learning_rate": 4.249525076191759e-07, "logits/chosen": -2.0239028930664062, "logits/rejected": -1.549783706665039, "logps/chosen": -131.36441040039062, "logps/ref_chosen": -95.67768859863281, "logps/ref_rejected": -105.09687805175781, "logps/rejected": -170.45034790039062, "loss": 1.0537, "margin_dpo/margin_mean": 29.666730880737305, "margin_dpo/margin_std": 38.10256576538086, "step": 225 }, { "KL/chosen_KL_mean": -29.28963851928711, "KL/mean": -43.64720153808594, "KL/rejected_KL_mean": -58.00476837158203, "KL/std": 33.11974334716797, "epoch": 0.33186490455212925, "fcm_dpo/beta": 0.015484048053622246, "fcm_dpo/delta": -0.04748788848519325, "fcm_dpo/margin": 28.71513557434082, "fcm_dpo/q_t": 0.39612627029418945, "grad_norm": 8.397480010986328, "learning_rate": 4.2403348649073167e-07, "logits/chosen": -1.7018557786941528, "logits/rejected": -1.1514077186584473, "logps/chosen": -122.75056457519531, "logps/ref_chosen": -93.46092224121094, "logps/ref_rejected": -86.7017593383789, "logps/rejected": -144.70652770996094, "loss": 1.0413, "margin_dpo/margin_mean": 28.715137481689453, "margin_dpo/margin_std": 29.570858001708984, "step": 226 }, { "KL/chosen_KL_mean": -39.73588562011719, "KL/mean": -54.5280876159668, "KL/rejected_KL_mean": -69.3202896118164, "KL/std": 31.901851654052734, "epoch": 0.3333333333333333, "fcm_dpo/beta": 0.015399668365716934, "fcm_dpo/delta": -0.05821998044848442, "fcm_dpo/margin": 29.584400177001953, "fcm_dpo/q_t": 0.39603230357170105, "grad_norm": 9.336464881896973, "learning_rate": 4.2310987941806615e-07, "logits/chosen": -1.4634356498718262, "logits/rejected": -1.1119801998138428, "logps/chosen": -132.55015563964844, "logps/ref_chosen": -92.81427001953125, "logps/ref_rejected": -104.73693084716797, "logps/rejected": -174.05722045898438, "loss": 1.0609, "margin_dpo/margin_mean": 29.584400177001953, "margin_dpo/margin_std": 37.857730865478516, "step": 227 }, { "KL/chosen_KL_mean": -37.173683166503906, "KL/mean": -48.369468688964844, "KL/rejected_KL_mean": -59.565242767333984, "KL/std": 29.936826705932617, "epoch": 0.33480176211453744, "fcm_dpo/beta": 0.015463687479496002, "fcm_dpo/delta": 0.05570037662982941, "fcm_dpo/margin": 22.391559600830078, "fcm_dpo/q_t": 0.419523686170578, "grad_norm": 9.622178077697754, "learning_rate": 4.2218171073908463e-07, "logits/chosen": -1.9535956382751465, "logits/rejected": -1.5260062217712402, "logps/chosen": -131.2108154296875, "logps/ref_chosen": -94.03712463378906, "logps/ref_rejected": -96.02151489257812, "logps/rejected": -155.58676147460938, "loss": 1.1359, "margin_dpo/margin_mean": 22.391559600830078, "margin_dpo/margin_std": 34.00199508666992, "step": 228 }, { "KL/chosen_KL_mean": -33.15263366699219, "KL/mean": -51.63897705078125, "KL/rejected_KL_mean": -70.12532043457031, "KL/std": 36.35321044921875, "epoch": 0.33627019089574156, "fcm_dpo/beta": 0.015141281299293041, "fcm_dpo/delta": -0.16921040415763855, "fcm_dpo/margin": 36.97268295288086, "fcm_dpo/q_t": 0.37347644567489624, "grad_norm": 9.325298309326172, "learning_rate": 4.212490049118951e-07, "logits/chosen": -1.7200881242752075, "logits/rejected": -1.07284414768219, "logps/chosen": -128.7303009033203, "logps/ref_chosen": -95.57766723632812, "logps/ref_rejected": -89.17379760742188, "logps/rejected": -159.2991180419922, "loss": 0.983, "margin_dpo/margin_mean": 36.97268295288086, "margin_dpo/margin_std": 39.79274368286133, "step": 229 }, { "KL/chosen_KL_mean": -34.472755432128906, "KL/mean": -53.44895935058594, "KL/rejected_KL_mean": -72.42515563964844, "KL/std": 36.047637939453125, "epoch": 0.3377386196769457, "fcm_dpo/beta": 0.014635426923632622, "fcm_dpo/delta": -0.1646428108215332, "fcm_dpo/margin": 37.95240020751953, "fcm_dpo/q_t": 0.3710649907588959, "grad_norm": 10.58165168762207, "learning_rate": 4.203117865141635e-07, "logits/chosen": -1.7172642946243286, "logits/rejected": -1.6551059484481812, "logps/chosen": -98.18638610839844, "logps/ref_chosen": -63.713626861572266, "logps/ref_rejected": -91.90870666503906, "logps/rejected": -164.3338623046875, "loss": 0.9696, "margin_dpo/margin_mean": 37.95240020751953, "margin_dpo/margin_std": 36.09141540527344, "step": 230 }, { "KL/chosen_KL_mean": -38.07428741455078, "KL/mean": -54.739036560058594, "KL/rejected_KL_mean": -71.4037857055664, "KL/std": 35.03746032714844, "epoch": 0.3392070484581498, "fcm_dpo/beta": 0.014278186485171318, "fcm_dpo/delta": -0.080464206635952, "fcm_dpo/margin": 33.329505920410156, "fcm_dpo/q_t": 0.3878564238548279, "grad_norm": 8.192068099975586, "learning_rate": 4.1937008024246625e-07, "logits/chosen": -1.7149224281311035, "logits/rejected": -1.147202491760254, "logps/chosen": -133.52996826171875, "logps/ref_chosen": -95.45567321777344, "logps/ref_rejected": -80.95568084716797, "logps/rejected": -152.35946655273438, "loss": 1.0107, "margin_dpo/margin_mean": 33.329505920410156, "margin_dpo/margin_std": 30.13718032836914, "step": 231 }, { "KL/chosen_KL_mean": -40.231407165527344, "KL/mean": -53.94416809082031, "KL/rejected_KL_mean": -67.65693664550781, "KL/std": 32.17742156982422, "epoch": 0.3406754772393539, "fcm_dpo/beta": 0.01429024152457714, "fcm_dpo/delta": 0.00840519368648529, "fcm_dpo/margin": 27.4255313873291, "fcm_dpo/q_t": 0.40822291374206543, "grad_norm": 9.362154960632324, "learning_rate": 4.1842391091163933e-07, "logits/chosen": -1.465308666229248, "logits/rejected": -0.9777147769927979, "logps/chosen": -137.128662109375, "logps/ref_chosen": -96.89726257324219, "logps/ref_rejected": -89.76461791992188, "logps/rejected": -157.4215545654297, "loss": 1.0833, "margin_dpo/margin_mean": 27.4255313873291, "margin_dpo/margin_std": 32.273902893066406, "step": 232 }, { "KL/chosen_KL_mean": -42.0622673034668, "KL/mean": -61.23181915283203, "KL/rejected_KL_mean": -80.40135955810547, "KL/std": 40.70005798339844, "epoch": 0.342143906020558, "fcm_dpo/beta": 0.013980589807033539, "fcm_dpo/delta": -0.14402565360069275, "fcm_dpo/margin": 38.33909606933594, "fcm_dpo/q_t": 0.37638208270072937, "grad_norm": 9.780364036560059, "learning_rate": 4.174733034541245e-07, "logits/chosen": -1.8855468034744263, "logits/rejected": -1.6860301494598389, "logps/chosen": -131.1125946044922, "logps/ref_chosen": -89.05032348632812, "logps/ref_rejected": -112.75917053222656, "logps/rejected": -193.1605224609375, "loss": 0.9894, "margin_dpo/margin_mean": 38.33909606933594, "margin_dpo/margin_std": 39.270713806152344, "step": 233 }, { "KL/chosen_KL_mean": -41.455177307128906, "KL/mean": -61.94432067871094, "KL/rejected_KL_mean": -82.4334716796875, "KL/std": 38.823890686035156, "epoch": 0.3436123348017621, "fcm_dpo/beta": 0.013470085337758064, "fcm_dpo/delta": -0.16315418481826782, "fcm_dpo/margin": 40.97828674316406, "fcm_dpo/q_t": 0.37390878796577454, "grad_norm": 8.586908340454102, "learning_rate": 4.165182829193126e-07, "logits/chosen": -1.3222860097885132, "logits/rejected": -1.2541083097457886, "logps/chosen": -115.77354431152344, "logps/ref_chosen": -74.318359375, "logps/ref_rejected": -106.38758850097656, "logps/rejected": -188.82106018066406, "loss": 0.9854, "margin_dpo/margin_mean": 40.97828674316406, "margin_dpo/margin_std": 42.53706359863281, "step": 234 }, { "KL/chosen_KL_mean": -49.8904914855957, "KL/mean": -65.36975860595703, "KL/rejected_KL_mean": -80.8490219116211, "KL/std": 35.03124237060547, "epoch": 0.34508076358296624, "fcm_dpo/beta": 0.013437163084745407, "fcm_dpo/delta": -0.01669587939977646, "fcm_dpo/margin": 30.958538055419922, "fcm_dpo/q_t": 0.40230244398117065, "grad_norm": 8.795056343078613, "learning_rate": 4.1555887447288255e-07, "logits/chosen": -1.8069056272506714, "logits/rejected": -1.2679738998413086, "logps/chosen": -148.1075439453125, "logps/ref_chosen": -98.217041015625, "logps/ref_rejected": -97.24677276611328, "logps/rejected": -178.09579467773438, "loss": 1.0595, "margin_dpo/margin_mean": 30.958538055419922, "margin_dpo/margin_std": 33.03438186645508, "step": 235 }, { "KL/chosen_KL_mean": -41.40312194824219, "KL/mean": -61.42710876464844, "KL/rejected_KL_mean": -81.45109558105469, "KL/std": 43.86043167114258, "epoch": 0.3465491923641703, "fcm_dpo/beta": 0.013170724734663963, "fcm_dpo/delta": -0.13437847793102264, "fcm_dpo/margin": 40.04798126220703, "fcm_dpo/q_t": 0.3838500380516052, "grad_norm": 9.57690143585205, "learning_rate": 4.1459510339613946e-07, "logits/chosen": -1.373631238937378, "logits/rejected": -1.3757076263427734, "logps/chosen": -120.24085235595703, "logps/ref_chosen": -78.83773040771484, "logps/ref_rejected": -109.06343078613281, "logps/rejected": -190.5145263671875, "loss": 1.0094, "margin_dpo/margin_mean": 40.04798126220703, "margin_dpo/margin_std": 47.5125732421875, "step": 236 }, { "KL/chosen_KL_mean": -49.884090423583984, "KL/mean": -64.48992156982422, "KL/rejected_KL_mean": -79.09575653076172, "KL/std": 38.73450469970703, "epoch": 0.34801762114537443, "fcm_dpo/beta": 0.013091443106532097, "fcm_dpo/delta": 0.018275106325745583, "fcm_dpo/margin": 29.211658477783203, "fcm_dpo/q_t": 0.41567617654800415, "grad_norm": 8.747806549072266, "learning_rate": 4.136269950853473e-07, "logits/chosen": -1.4771971702575684, "logits/rejected": -1.1215885877609253, "logps/chosen": -135.0953826904297, "logps/ref_chosen": -85.21128845214844, "logps/ref_rejected": -99.90999603271484, "logps/rejected": -179.00575256347656, "loss": 1.1375, "margin_dpo/margin_mean": 29.211658477783203, "margin_dpo/margin_std": 50.04640197753906, "step": 237 }, { "KL/chosen_KL_mean": -54.736942291259766, "KL/mean": -71.89501953125, "KL/rejected_KL_mean": -89.05308532714844, "KL/std": 40.52821350097656, "epoch": 0.34948604992657856, "fcm_dpo/beta": 0.01299445889890194, "fcm_dpo/delta": -0.04814508557319641, "fcm_dpo/margin": 34.316139221191406, "fcm_dpo/q_t": 0.39806628227233887, "grad_norm": 8.210455894470215, "learning_rate": 4.126545750510605e-07, "logits/chosen": -1.3698650598526, "logits/rejected": -1.221639633178711, "logps/chosen": -133.46817016601562, "logps/ref_chosen": -78.73123168945312, "logps/ref_rejected": -95.41840362548828, "logps/rejected": -184.47149658203125, "loss": 1.0571, "margin_dpo/margin_mean": 34.316139221191406, "margin_dpo/margin_std": 41.95155334472656, "step": 238 }, { "KL/chosen_KL_mean": -49.4133415222168, "KL/mean": -65.30250549316406, "KL/rejected_KL_mean": -81.19168090820312, "KL/std": 36.027400970458984, "epoch": 0.3509544787077827, "fcm_dpo/beta": 0.01287921704351902, "fcm_dpo/delta": -0.010486140847206116, "fcm_dpo/margin": 31.778337478637695, "fcm_dpo/q_t": 0.4054362177848816, "grad_norm": 9.076879501342773, "learning_rate": 4.116778689174514e-07, "logits/chosen": -1.8060572147369385, "logits/rejected": -1.3277091979980469, "logps/chosen": -142.0142822265625, "logps/ref_chosen": -92.60093688964844, "logps/ref_rejected": -100.51769256591797, "logps/rejected": -181.70938110351562, "loss": 1.0841, "margin_dpo/margin_mean": 31.778339385986328, "margin_dpo/margin_std": 39.84032440185547, "step": 239 }, { "KL/chosen_KL_mean": -51.6435661315918, "KL/mean": -65.69683837890625, "KL/rejected_KL_mean": -79.7501220703125, "KL/std": 36.285400390625, "epoch": 0.3524229074889868, "fcm_dpo/beta": 0.012994194403290749, "fcm_dpo/delta": 0.03608718886971474, "fcm_dpo/margin": 28.10655403137207, "fcm_dpo/q_t": 0.41531968116760254, "grad_norm": 10.340274810791016, "learning_rate": 4.106969024216348e-07, "logits/chosen": -1.9386857748031616, "logits/rejected": -1.5682348012924194, "logps/chosen": -137.8033447265625, "logps/ref_chosen": -86.15977478027344, "logps/ref_rejected": -80.45567321777344, "logps/rejected": -160.20578002929688, "loss": 1.1159, "margin_dpo/margin_mean": 28.106555938720703, "margin_dpo/margin_std": 39.13177490234375, "step": 240 }, { "KL/chosen_KL_mean": -47.43107223510742, "KL/mean": -72.43565368652344, "KL/rejected_KL_mean": -97.44023132324219, "KL/std": 41.25971221923828, "epoch": 0.35389133627019087, "fcm_dpo/beta": 0.012638799846172333, "fcm_dpo/delta": -0.2469959855079651, "fcm_dpo/margin": 50.0091667175293, "fcm_dpo/q_t": 0.35611748695373535, "grad_norm": 8.822714805603027, "learning_rate": 4.097117014129903e-07, "logits/chosen": -1.5388983488082886, "logits/rejected": -0.8363416194915771, "logps/chosen": -148.47702026367188, "logps/ref_chosen": -101.04594421386719, "logps/ref_rejected": -94.04934692382812, "logps/rejected": -191.48959350585938, "loss": 0.9286, "margin_dpo/margin_mean": 50.00917053222656, "margin_dpo/margin_std": 46.128807067871094, "step": 241 }, { "KL/chosen_KL_mean": -56.151676177978516, "KL/mean": -74.1827392578125, "KL/rejected_KL_mean": -92.21379852294922, "KL/std": 37.1671257019043, "epoch": 0.355359765051395, "fcm_dpo/beta": 0.012362953275442123, "fcm_dpo/delta": -0.048006571829319, "fcm_dpo/margin": 36.0621223449707, "fcm_dpo/q_t": 0.3958459496498108, "grad_norm": 9.104336738586426, "learning_rate": 4.087222918524807e-07, "logits/chosen": -2.061933755874634, "logits/rejected": -1.4207961559295654, "logps/chosen": -151.8243408203125, "logps/ref_chosen": -95.67266845703125, "logps/ref_rejected": -90.65454864501953, "logps/rejected": -182.86834716796875, "loss": 1.0513, "margin_dpo/margin_mean": 36.06211853027344, "margin_dpo/margin_std": 41.17481231689453, "step": 242 }, { "KL/chosen_KL_mean": -56.09545135498047, "KL/mean": -73.864501953125, "KL/rejected_KL_mean": -91.63353729248047, "KL/std": 39.685401916503906, "epoch": 0.3568281938325991, "fcm_dpo/beta": 0.012318115681409836, "fcm_dpo/delta": -0.040395744144916534, "fcm_dpo/margin": 35.5380973815918, "fcm_dpo/q_t": 0.40031713247299194, "grad_norm": 8.73207950592041, "learning_rate": 4.07728699811968e-07, "logits/chosen": -1.7568073272705078, "logits/rejected": -0.910798192024231, "logps/chosen": -154.12686157226562, "logps/ref_chosen": -98.03140258789062, "logps/ref_rejected": -83.18806457519531, "logps/rejected": -174.82159423828125, "loss": 1.0802, "margin_dpo/margin_mean": 35.5380973815918, "margin_dpo/margin_std": 47.2676887512207, "step": 243 }, { "KL/chosen_KL_mean": -58.320762634277344, "KL/mean": -77.29695892333984, "KL/rejected_KL_mean": -96.27316284179688, "KL/std": 43.78956604003906, "epoch": 0.35829662261380324, "fcm_dpo/beta": 0.012067940086126328, "fcm_dpo/delta": -0.06094461679458618, "fcm_dpo/margin": 37.95237731933594, "fcm_dpo/q_t": 0.39431750774383545, "grad_norm": 8.589831352233887, "learning_rate": 4.067309514735267e-07, "logits/chosen": -2.0359673500061035, "logits/rejected": -1.6989645957946777, "logps/chosen": -147.21469116210938, "logps/ref_chosen": -88.8939208984375, "logps/ref_rejected": -102.57278442382812, "logps/rejected": -198.845947265625, "loss": 1.0436, "margin_dpo/margin_mean": 37.95237731933594, "margin_dpo/margin_std": 43.64624786376953, "step": 244 }, { "KL/chosen_KL_mean": -50.111698150634766, "KL/mean": -70.05577087402344, "KL/rejected_KL_mean": -89.99984741210938, "KL/std": 40.363319396972656, "epoch": 0.35976505139500736, "fcm_dpo/beta": 0.011926956474781036, "fcm_dpo/delta": -0.07945388555526733, "fcm_dpo/margin": 39.888153076171875, "fcm_dpo/q_t": 0.3893914520740509, "grad_norm": 8.152714729309082, "learning_rate": 4.057290731287531e-07, "logits/chosen": -2.371279716491699, "logits/rejected": -1.5508568286895752, "logps/chosen": -154.30569458007812, "logps/ref_chosen": -104.19400024414062, "logps/ref_rejected": -92.65645599365234, "logps/rejected": -182.65631103515625, "loss": 1.0229, "margin_dpo/margin_mean": 39.888153076171875, "margin_dpo/margin_std": 41.41615295410156, "step": 245 }, { "KL/chosen_KL_mean": -64.91271209716797, "KL/mean": -80.6871337890625, "KL/rejected_KL_mean": -96.4615478515625, "KL/std": 39.65472412109375, "epoch": 0.36123348017621143, "fcm_dpo/beta": 0.011866935528814793, "fcm_dpo/delta": 0.02646595798432827, "fcm_dpo/margin": 31.548839569091797, "fcm_dpo/q_t": 0.4136810302734375, "grad_norm": 8.333203315734863, "learning_rate": 4.047230911780736e-07, "logits/chosen": -1.7728571891784668, "logits/rejected": -1.221175193786621, "logps/chosen": -168.1317596435547, "logps/ref_chosen": -103.21904754638672, "logps/ref_rejected": -90.99221801757812, "logps/rejected": -187.45376586914062, "loss": 1.1085, "margin_dpo/margin_mean": 31.548837661743164, "margin_dpo/margin_std": 43.0427131652832, "step": 246 }, { "KL/chosen_KL_mean": -58.820045471191406, "KL/mean": -84.69198608398438, "KL/rejected_KL_mean": -110.56390380859375, "KL/std": 44.322593688964844, "epoch": 0.36270190895741555, "fcm_dpo/beta": 0.01150570809841156, "fcm_dpo/delta": -0.20911051332950592, "fcm_dpo/margin": 51.74386215209961, "fcm_dpo/q_t": 0.3645823001861572, "grad_norm": 8.458048820495605, "learning_rate": 4.0371303213004814e-07, "logits/chosen": -1.8682951927185059, "logits/rejected": -1.683955430984497, "logps/chosen": -145.81442260742188, "logps/ref_chosen": -86.99437713623047, "logps/ref_rejected": -111.33802795410156, "logps/rejected": -221.9019317626953, "loss": 0.9531, "margin_dpo/margin_mean": 51.743865966796875, "margin_dpo/margin_std": 50.284793853759766, "step": 247 }, { "KL/chosen_KL_mean": -67.72038269042969, "KL/mean": -84.95086669921875, "KL/rejected_KL_mean": -102.18134307861328, "KL/std": 39.13410186767578, "epoch": 0.3641703377386197, "fcm_dpo/beta": 0.011476580053567886, "fcm_dpo/delta": 0.004495013505220413, "fcm_dpo/margin": 34.46095275878906, "fcm_dpo/q_t": 0.4075547158718109, "grad_norm": 8.877899169921875, "learning_rate": 4.0269892260067197e-07, "logits/chosen": -1.6155591011047363, "logits/rejected": -1.589552640914917, "logps/chosen": -142.5059051513672, "logps/ref_chosen": -74.7855224609375, "logps/ref_rejected": -98.27689361572266, "logps/rejected": -200.45823669433594, "loss": 1.0802, "margin_dpo/margin_mean": 34.46095275878906, "margin_dpo/margin_std": 40.156829833984375, "step": 248 }, { "KL/chosen_KL_mean": -75.35456848144531, "KL/mean": -91.05412292480469, "KL/rejected_KL_mean": -106.75369262695312, "KL/std": 41.995384216308594, "epoch": 0.3656387665198238, "fcm_dpo/beta": 0.011470707133412361, "fcm_dpo/delta": 0.04116884618997574, "fcm_dpo/margin": 31.399133682250977, "fcm_dpo/q_t": 0.4174729287624359, "grad_norm": 10.088454246520996, "learning_rate": 4.0168078931267426e-07, "logits/chosen": -2.144710063934326, "logits/rejected": -1.6005651950836182, "logps/chosen": -171.058349609375, "logps/ref_chosen": -95.70379638671875, "logps/ref_rejected": -87.14646911621094, "logps/rejected": -193.90017700195312, "loss": 1.1321, "margin_dpo/margin_mean": 31.399133682250977, "margin_dpo/margin_std": 48.28464126586914, "step": 249 }, { "KL/chosen_KL_mean": -67.25942993164062, "KL/mean": -84.98207092285156, "KL/rejected_KL_mean": -102.7047119140625, "KL/std": 42.618160247802734, "epoch": 0.3671071953010279, "fcm_dpo/beta": 0.01146991178393364, "fcm_dpo/delta": -0.007370438426733017, "fcm_dpo/margin": 35.445274353027344, "fcm_dpo/q_t": 0.4053910970687866, "grad_norm": 9.236113548278809, "learning_rate": 4.006586590948141e-07, "logits/chosen": -1.9479122161865234, "logits/rejected": -1.0106346607208252, "logps/chosen": -181.3116455078125, "logps/ref_chosen": -114.05221557617188, "logps/ref_rejected": -81.08767700195312, "logps/rejected": -183.79238891601562, "loss": 1.0909, "margin_dpo/margin_mean": 35.44527816772461, "margin_dpo/margin_std": 45.9248046875, "step": 250 }, { "KL/chosen_KL_mean": -67.23397827148438, "KL/mean": -82.95054626464844, "KL/rejected_KL_mean": -98.66709899902344, "KL/std": 39.586795806884766, "epoch": 0.368575624082232, "fcm_dpo/beta": 0.011526285670697689, "fcm_dpo/delta": 0.03871246427297592, "fcm_dpo/margin": 31.433120727539062, "fcm_dpo/q_t": 0.41749024391174316, "grad_norm": 10.645429611206055, "learning_rate": 3.9963255888117325e-07, "logits/chosen": -1.9285283088684082, "logits/rejected": -1.2350648641586304, "logps/chosen": -164.9452667236328, "logps/ref_chosen": -97.71128845214844, "logps/ref_rejected": -83.52742004394531, "logps/rejected": -182.19451904296875, "loss": 1.1302, "margin_dpo/margin_mean": 31.433120727539062, "margin_dpo/margin_std": 47.392086029052734, "step": 251 }, { "KL/chosen_KL_mean": -67.73063659667969, "KL/mean": -83.39911651611328, "KL/rejected_KL_mean": -99.06759643554688, "KL/std": 42.18002700805664, "epoch": 0.3700440528634361, "fcm_dpo/beta": 0.011692160740494728, "fcm_dpo/delta": 0.03476369380950928, "fcm_dpo/margin": 31.33696174621582, "fcm_dpo/q_t": 0.41485434770584106, "grad_norm": 9.804216384887695, "learning_rate": 3.9860251571044666e-07, "logits/chosen": -2.1997249126434326, "logits/rejected": -1.6761187314987183, "logps/chosen": -176.71673583984375, "logps/ref_chosen": -108.9861068725586, "logps/ref_rejected": -91.56423950195312, "logps/rejected": -190.6318359375, "loss": 1.1206, "margin_dpo/margin_mean": 31.336959838867188, "margin_dpo/margin_std": 44.979400634765625, "step": 252 }, { "KL/chosen_KL_mean": -74.38153076171875, "KL/mean": -88.86967468261719, "KL/rejected_KL_mean": -103.3578109741211, "KL/std": 45.63330841064453, "epoch": 0.37151248164464024, "fcm_dpo/beta": 0.011864080093801022, "fcm_dpo/delta": 0.057150699198246, "fcm_dpo/margin": 28.97627067565918, "fcm_dpo/q_t": 0.4192999601364136, "grad_norm": 9.911102294921875, "learning_rate": 3.9756855672522986e-07, "logits/chosen": -2.173192024230957, "logits/rejected": -1.9697760343551636, "logps/chosen": -174.59783935546875, "logps/ref_chosen": -100.21630096435547, "logps/ref_rejected": -105.67670440673828, "logps/rejected": -209.03451538085938, "loss": 1.1278, "margin_dpo/margin_mean": 28.976272583007812, "margin_dpo/margin_std": 39.78358459472656, "step": 253 }, { "KL/chosen_KL_mean": -71.67863464355469, "KL/mean": -86.93233489990234, "KL/rejected_KL_mean": -102.18603515625, "KL/std": 43.05427551269531, "epoch": 0.37298091042584436, "fcm_dpo/beta": 0.011850297451019287, "fcm_dpo/delta": 0.03975265100598335, "fcm_dpo/margin": 30.507396697998047, "fcm_dpo/q_t": 0.41578859090805054, "grad_norm": 9.625710487365723, "learning_rate": 3.965307091713037e-07, "logits/chosen": -2.080414295196533, "logits/rejected": -1.5540627241134644, "logps/chosen": -170.413818359375, "logps/ref_chosen": -98.73518371582031, "logps/ref_rejected": -93.73825073242188, "logps/rejected": -195.92428588867188, "loss": 1.1299, "margin_dpo/margin_mean": 30.507396697998047, "margin_dpo/margin_std": 46.05572509765625, "step": 254 }, { "KL/chosen_KL_mean": -72.20733642578125, "KL/mean": -95.62421417236328, "KL/rejected_KL_mean": -119.04110717773438, "KL/std": 48.78860092163086, "epoch": 0.3744493392070485, "fcm_dpo/beta": 0.011663969606161118, "fcm_dpo/delta": -0.15457873046398163, "fcm_dpo/margin": 46.833778381347656, "fcm_dpo/q_t": 0.3745570182800293, "grad_norm": 9.4197416305542, "learning_rate": 3.954890003969163e-07, "logits/chosen": -2.077897071838379, "logits/rejected": -1.746326208114624, "logps/chosen": -162.58990478515625, "logps/ref_chosen": -90.382568359375, "logps/ref_rejected": -97.07624816894531, "logps/rejected": -216.1173553466797, "loss": 0.9903, "margin_dpo/margin_mean": 46.833778381347656, "margin_dpo/margin_std": 49.80805206298828, "step": 255 }, { "KL/chosen_KL_mean": -70.64230346679688, "KL/mean": -91.27308654785156, "KL/rejected_KL_mean": -111.90386962890625, "KL/std": 50.66539001464844, "epoch": 0.37591776798825255, "fcm_dpo/beta": 0.01145664881914854, "fcm_dpo/delta": -0.07626346498727798, "fcm_dpo/margin": 41.26158142089844, "fcm_dpo/q_t": 0.3939765989780426, "grad_norm": 9.820229530334473, "learning_rate": 3.944434578520628e-07, "logits/chosen": -1.8952274322509766, "logits/rejected": -1.557331919670105, "logps/chosen": -159.39511108398438, "logps/ref_chosen": -88.7528076171875, "logps/ref_rejected": -98.49383544921875, "logps/rejected": -210.397705078125, "loss": 1.0747, "margin_dpo/margin_mean": 41.26158142089844, "margin_dpo/margin_std": 59.442039489746094, "step": 256 }, { "KL/chosen_KL_mean": -72.33631896972656, "KL/mean": -93.30062103271484, "KL/rejected_KL_mean": -114.26492309570312, "KL/std": 52.06700897216797, "epoch": 0.37738619676945667, "fcm_dpo/beta": 0.011158918030560017, "fcm_dpo/delta": -0.07354697585105896, "fcm_dpo/margin": 41.92859649658203, "fcm_dpo/q_t": 0.39290672540664673, "grad_norm": 8.849942207336426, "learning_rate": 3.933941090877615e-07, "logits/chosen": -1.7903369665145874, "logits/rejected": -1.3825359344482422, "logps/chosen": -155.13983154296875, "logps/ref_chosen": -82.80352020263672, "logps/ref_rejected": -85.8677978515625, "logps/rejected": -200.13272094726562, "loss": 1.0525, "margin_dpo/margin_mean": 41.9286003112793, "margin_dpo/margin_std": 50.41793441772461, "step": 257 }, { "KL/chosen_KL_mean": -75.58670806884766, "KL/mean": -94.62666320800781, "KL/rejected_KL_mean": -113.66661071777344, "KL/std": 49.517578125, "epoch": 0.3788546255506608, "fcm_dpo/beta": 0.0112393107265234, "fcm_dpo/delta": -0.029875658452510834, "fcm_dpo/margin": 38.079898834228516, "fcm_dpo/q_t": 0.40077000856399536, "grad_norm": 9.6758394241333, "learning_rate": 3.923409817553284e-07, "logits/chosen": -2.0687413215637207, "logits/rejected": -1.7093684673309326, "logps/chosen": -165.77444458007812, "logps/ref_chosen": -90.187744140625, "logps/ref_rejected": -103.47068786621094, "logps/rejected": -217.13729858398438, "loss": 1.0744, "margin_dpo/margin_mean": 38.079898834228516, "margin_dpo/margin_std": 47.45009994506836, "step": 258 }, { "KL/chosen_KL_mean": -81.70648193359375, "KL/mean": -97.93675231933594, "KL/rejected_KL_mean": -114.16703796386719, "KL/std": 49.55610275268555, "epoch": 0.3803230543318649, "fcm_dpo/beta": 0.011150555685162544, "fcm_dpo/delta": 0.0390009768307209, "fcm_dpo/margin": 32.46056365966797, "fcm_dpo/q_t": 0.4168040156364441, "grad_norm": 9.390748977661133, "learning_rate": 3.9128410360564793e-07, "logits/chosen": -1.288635492324829, "logits/rejected": -1.0335092544555664, "logps/chosen": -172.47903442382812, "logps/ref_chosen": -90.77254486083984, "logps/ref_rejected": -94.58816528320312, "logps/rejected": -208.75518798828125, "loss": 1.1511, "margin_dpo/margin_mean": 32.4605598449707, "margin_dpo/margin_std": 56.19084930419922, "step": 259 }, { "KL/chosen_KL_mean": -83.81351470947266, "KL/mean": -97.7538070678711, "KL/rejected_KL_mean": -111.69407653808594, "KL/std": 50.275901794433594, "epoch": 0.38179148311306904, "fcm_dpo/beta": 0.011374548077583313, "fcm_dpo/delta": 0.08563709259033203, "fcm_dpo/margin": 27.880565643310547, "fcm_dpo/q_t": 0.4273487329483032, "grad_norm": 11.298893928527832, "learning_rate": 3.9022350248844246e-07, "logits/chosen": -1.6252734661102295, "logits/rejected": -1.6555004119873047, "logps/chosen": -159.40621948242188, "logps/ref_chosen": -75.59269714355469, "logps/ref_rejected": -100.84554290771484, "logps/rejected": -212.53961181640625, "loss": 1.1869, "margin_dpo/margin_mean": 27.880565643310547, "margin_dpo/margin_std": 55.072208404541016, "step": 260 }, { "KL/chosen_KL_mean": -64.14804077148438, "KL/mean": -88.40974426269531, "KL/rejected_KL_mean": -112.67144012451172, "KL/std": 50.80274963378906, "epoch": 0.3832599118942731, "fcm_dpo/beta": 0.011117441579699516, "fcm_dpo/delta": -0.14896252751350403, "fcm_dpo/margin": 48.523406982421875, "fcm_dpo/q_t": 0.376009464263916, "grad_norm": 8.7848482131958, "learning_rate": 3.891592063515376e-07, "logits/chosen": -1.8122601509094238, "logits/rejected": -1.4337575435638428, "logps/chosen": -158.4029541015625, "logps/ref_chosen": -94.25491333007812, "logps/ref_rejected": -93.65699005126953, "logps/rejected": -206.32843017578125, "loss": 0.9882, "margin_dpo/margin_mean": 48.523406982421875, "margin_dpo/margin_std": 49.55324172973633, "step": 261 }, { "KL/chosen_KL_mean": -69.96524810791016, "KL/mean": -92.67085266113281, "KL/rejected_KL_mean": -115.37644958496094, "KL/std": 53.453800201416016, "epoch": 0.38472834067547723, "fcm_dpo/beta": 0.010893210768699646, "fcm_dpo/delta": -0.10022999346256256, "fcm_dpo/margin": 45.41120529174805, "fcm_dpo/q_t": 0.38662827014923096, "grad_norm": 8.782097816467285, "learning_rate": 3.880912432401264e-07, "logits/chosen": -1.5024497509002686, "logits/rejected": -0.94440758228302, "logps/chosen": -155.2325439453125, "logps/ref_chosen": -85.26730346679688, "logps/ref_rejected": -90.82609558105469, "logps/rejected": -206.20254516601562, "loss": 1.0294, "margin_dpo/margin_mean": 45.41120529174805, "margin_dpo/margin_std": 53.28410339355469, "step": 262 }, { "KL/chosen_KL_mean": -56.32964324951172, "KL/mean": -87.00668334960938, "KL/rejected_KL_mean": -117.6837158203125, "KL/std": 58.169456481933594, "epoch": 0.38619676945668135, "fcm_dpo/beta": 0.01048569567501545, "fcm_dpo/delta": -0.25966909527778625, "fcm_dpo/margin": 61.35406494140625, "fcm_dpo/q_t": 0.3539855480194092, "grad_norm": 8.541414260864258, "learning_rate": 3.870196412960302e-07, "logits/chosen": -1.90781569480896, "logits/rejected": -1.3328871726989746, "logps/chosen": -152.08755493164062, "logps/ref_chosen": -95.75790405273438, "logps/ref_rejected": -101.83377075195312, "logps/rejected": -219.51748657226562, "loss": 0.9389, "margin_dpo/margin_mean": 61.35406494140625, "margin_dpo/margin_std": 62.21375274658203, "step": 263 }, { "KL/chosen_KL_mean": -65.29212951660156, "KL/mean": -91.18324279785156, "KL/rejected_KL_mean": -117.07435607910156, "KL/std": 54.85941696166992, "epoch": 0.3876651982378855, "fcm_dpo/beta": 0.010110534727573395, "fcm_dpo/delta": -0.13065072894096375, "fcm_dpo/margin": 51.782230377197266, "fcm_dpo/q_t": 0.3794293999671936, "grad_norm": 8.447540283203125, "learning_rate": 3.8594442875695665e-07, "logits/chosen": -1.908103585243225, "logits/rejected": -1.6155619621276855, "logps/chosen": -155.914794921875, "logps/ref_chosen": -90.6226577758789, "logps/ref_rejected": -100.32554626464844, "logps/rejected": -217.39990234375, "loss": 1.0041, "margin_dpo/margin_mean": 51.782230377197266, "margin_dpo/margin_std": 55.9980354309082, "step": 264 }, { "KL/chosen_KL_mean": -68.76493835449219, "KL/mean": -91.58894348144531, "KL/rejected_KL_mean": -114.4129638671875, "KL/std": 56.07514572143555, "epoch": 0.3891336270190896, "fcm_dpo/beta": 0.009931758046150208, "fcm_dpo/delta": -0.056543126702308655, "fcm_dpo/margin": 45.64802932739258, "fcm_dpo/q_t": 0.39733201265335083, "grad_norm": 8.681195259094238, "learning_rate": 3.848656339557562e-07, "logits/chosen": -1.7762231826782227, "logits/rejected": -1.3487505912780762, "logps/chosen": -161.13726806640625, "logps/ref_chosen": -92.37232971191406, "logps/ref_rejected": -94.62757873535156, "logps/rejected": -209.04052734375, "loss": 1.0686, "margin_dpo/margin_mean": 45.64802932739258, "margin_dpo/margin_std": 60.5543212890625, "step": 265 }, { "KL/chosen_KL_mean": -79.43736267089844, "KL/mean": -100.08321380615234, "KL/rejected_KL_mean": -120.72904968261719, "KL/std": 53.83013153076172, "epoch": 0.39060205580029367, "fcm_dpo/beta": 0.009913979098200798, "fcm_dpo/delta": -0.009850640781223774, "fcm_dpo/margin": 41.29167938232422, "fcm_dpo/q_t": 0.40648722648620605, "grad_norm": 10.616071701049805, "learning_rate": 3.8378328531967507e-07, "logits/chosen": -2.3424954414367676, "logits/rejected": -1.5312023162841797, "logps/chosen": -181.63739013671875, "logps/ref_chosen": -102.20002746582031, "logps/ref_rejected": -74.36642456054688, "logps/rejected": -195.09547424316406, "loss": 1.0952, "margin_dpo/margin_mean": 41.29167938232422, "margin_dpo/margin_std": 57.17870330810547, "step": 266 }, { "KL/chosen_KL_mean": -70.18959045410156, "KL/mean": -94.31575012207031, "KL/rejected_KL_mean": -118.44192504882812, "KL/std": 54.498008728027344, "epoch": 0.3920704845814978, "fcm_dpo/beta": 0.009777444414794445, "fcm_dpo/delta": -0.07594425231218338, "fcm_dpo/margin": 48.25233459472656, "fcm_dpo/q_t": 0.3907311260700226, "grad_norm": 9.000791549682617, "learning_rate": 3.8269741136960646e-07, "logits/chosen": -1.6187171936035156, "logits/rejected": -0.9765427112579346, "logps/chosen": -174.4755859375, "logps/ref_chosen": -104.28599548339844, "logps/ref_rejected": -95.98719024658203, "logps/rejected": -214.42910766601562, "loss": 1.0304, "margin_dpo/margin_mean": 48.25233459472656, "margin_dpo/margin_std": 51.87737274169922, "step": 267 }, { "KL/chosen_KL_mean": -69.17428588867188, "KL/mean": -97.68125915527344, "KL/rejected_KL_mean": -126.18822479248047, "KL/std": 54.84250259399414, "epoch": 0.3935389133627019, "fcm_dpo/beta": 0.009514464996755123, "fcm_dpo/delta": -0.15179967880249023, "fcm_dpo/margin": 57.013938903808594, "fcm_dpo/q_t": 0.3736265003681183, "grad_norm": 9.013367652893066, "learning_rate": 3.8160804071933894e-07, "logits/chosen": -1.5971605777740479, "logits/rejected": -1.440614938735962, "logps/chosen": -155.87051391601562, "logps/ref_chosen": -86.69622039794922, "logps/ref_rejected": -109.19184112548828, "logps/rejected": -235.38006591796875, "loss": 0.9784, "margin_dpo/margin_mean": 57.013938903808594, "margin_dpo/margin_std": 54.197654724121094, "step": 268 }, { "KL/chosen_KL_mean": -73.63774108886719, "KL/mean": -101.22775268554688, "KL/rejected_KL_mean": -128.8177490234375, "KL/std": 58.63764953613281, "epoch": 0.39500734214390604, "fcm_dpo/beta": 0.00933960173279047, "fcm_dpo/delta": -0.12145110964775085, "fcm_dpo/margin": 55.180015563964844, "fcm_dpo/q_t": 0.38056087493896484, "grad_norm": 8.388808250427246, "learning_rate": 3.8051520207480204e-07, "logits/chosen": -1.7027852535247803, "logits/rejected": -1.238226294517517, "logps/chosen": -178.60955810546875, "logps/ref_chosen": -104.97181701660156, "logps/ref_rejected": -112.47647094726562, "logps/rejected": -241.29421997070312, "loss": 0.9993, "margin_dpo/margin_mean": 55.180015563964844, "margin_dpo/margin_std": 55.78822326660156, "step": 269 }, { "KL/chosen_KL_mean": -82.46034240722656, "KL/mean": -105.67640686035156, "KL/rejected_KL_mean": -128.89248657226562, "KL/std": 57.93647766113281, "epoch": 0.3964757709251101, "fcm_dpo/beta": 0.009243566542863846, "fcm_dpo/delta": -0.030604396015405655, "fcm_dpo/margin": 46.43213653564453, "fcm_dpo/q_t": 0.4013768136501312, "grad_norm": 8.815256118774414, "learning_rate": 3.794189242333106e-07, "logits/chosen": -2.4643261432647705, "logits/rejected": -2.0539543628692627, "logps/chosen": -183.5341796875, "logps/ref_chosen": -101.07383728027344, "logps/ref_rejected": -117.75289154052734, "logps/rejected": -246.64537048339844, "loss": 1.0901, "margin_dpo/margin_mean": 46.43213653564453, "margin_dpo/margin_std": 65.63973236083984, "step": 270 }, { "KL/chosen_KL_mean": -68.19467163085938, "KL/mean": -97.00505065917969, "KL/rejected_KL_mean": -125.8154296875, "KL/std": 54.33827209472656, "epoch": 0.39794419970631423, "fcm_dpo/beta": 0.009018335491418839, "fcm_dpo/delta": -0.12639491260051727, "fcm_dpo/margin": 57.620750427246094, "fcm_dpo/q_t": 0.37857508659362793, "grad_norm": 8.725642204284668, "learning_rate": 3.7831923608280514e-07, "logits/chosen": -1.52085280418396, "logits/rejected": -1.0682916641235352, "logps/chosen": -164.91928100585938, "logps/ref_chosen": -96.72459411621094, "logps/ref_rejected": -98.5244140625, "logps/rejected": -224.33984375, "loss": 1.0077, "margin_dpo/margin_mean": 57.620750427246094, "margin_dpo/margin_std": 62.450218200683594, "step": 271 }, { "KL/chosen_KL_mean": -58.34852600097656, "KL/mean": -85.76690673828125, "KL/rejected_KL_mean": -113.18529510498047, "KL/std": 59.8043327331543, "epoch": 0.39941262848751835, "fcm_dpo/beta": 0.008849442936480045, "fcm_dpo/delta": -0.08965744823217392, "fcm_dpo/margin": 54.836761474609375, "fcm_dpo/q_t": 0.3888952434062958, "grad_norm": 8.433688163757324, "learning_rate": 3.772161666010912e-07, "logits/chosen": -1.6201162338256836, "logits/rejected": -1.508727788925171, "logps/chosen": -139.32574462890625, "logps/ref_chosen": -80.97721862792969, "logps/ref_rejected": -108.55535888671875, "logps/rejected": -221.74066162109375, "loss": 1.036, "margin_dpo/margin_mean": 54.836761474609375, "margin_dpo/margin_std": 65.11723327636719, "step": 272 }, { "KL/chosen_KL_mean": -80.06129455566406, "KL/mean": -108.20361328125, "KL/rejected_KL_mean": -136.345947265625, "KL/std": 61.56871795654297, "epoch": 0.4008810572687225, "fcm_dpo/beta": 0.008626678958535194, "fcm_dpo/delta": -0.09179206192493439, "fcm_dpo/margin": 56.284645080566406, "fcm_dpo/q_t": 0.3881162405014038, "grad_norm": 8.25420093536377, "learning_rate": 3.761097448550755e-07, "logits/chosen": -1.662980079650879, "logits/rejected": -1.2549288272857666, "logps/chosen": -172.285888671875, "logps/ref_chosen": -92.22460174560547, "logps/ref_rejected": -97.3630599975586, "logps/rejected": -233.70899963378906, "loss": 1.0477, "margin_dpo/margin_mean": 56.284645080566406, "margin_dpo/margin_std": 69.96568298339844, "step": 273 }, { "KL/chosen_KL_mean": -82.17292785644531, "KL/mean": -104.04536437988281, "KL/rejected_KL_mean": -125.91780090332031, "KL/std": 60.16001892089844, "epoch": 0.4023494860499266, "fcm_dpo/beta": 0.008659105747938156, "fcm_dpo/delta": 0.021978136152029037, "fcm_dpo/margin": 43.744869232177734, "fcm_dpo/q_t": 0.4120224714279175, "grad_norm": 12.805569648742676, "learning_rate": 3.75e-07, "logits/chosen": -2.005481004714966, "logits/rejected": -1.6128761768341064, "logps/chosen": -164.43380737304688, "logps/ref_chosen": -82.26089477539062, "logps/ref_rejected": -83.87699890136719, "logps/rejected": -209.7947998046875, "loss": 1.0982, "margin_dpo/margin_mean": 43.744869232177734, "margin_dpo/margin_std": 55.616249084472656, "step": 274 }, { "KL/chosen_KL_mean": -69.74342346191406, "KL/mean": -94.57157135009766, "KL/rejected_KL_mean": -119.39972686767578, "KL/std": 59.278602600097656, "epoch": 0.40381791483113066, "fcm_dpo/beta": 0.008692773059010506, "fcm_dpo/delta": -0.03347904607653618, "fcm_dpo/margin": 49.65629577636719, "fcm_dpo/q_t": 0.4008089303970337, "grad_norm": 9.076638221740723, "learning_rate": 3.738869612786737e-07, "logits/chosen": -2.254915952682495, "logits/rejected": -1.9389665126800537, "logps/chosen": -149.43038940429688, "logps/ref_chosen": -79.68696594238281, "logps/ref_rejected": -98.7509765625, "logps/rejected": -218.15069580078125, "loss": 1.0922, "margin_dpo/margin_mean": 49.65629577636719, "margin_dpo/margin_std": 70.50784301757812, "step": 275 }, { "KL/chosen_KL_mean": -83.38230895996094, "KL/mean": -109.9288101196289, "KL/rejected_KL_mean": -136.47531127929688, "KL/std": 59.75408935546875, "epoch": 0.4052863436123348, "fcm_dpo/beta": 0.008504962548613548, "fcm_dpo/delta": -0.055239662528038025, "fcm_dpo/margin": 53.0930061340332, "fcm_dpo/q_t": 0.3967442512512207, "grad_norm": 9.568376541137695, "learning_rate": 3.7277065802070204e-07, "logits/chosen": -1.9277946949005127, "logits/rejected": -1.336341142654419, "logps/chosen": -169.92202758789062, "logps/ref_chosen": -86.53970336914062, "logps/ref_rejected": -77.85394287109375, "logps/rejected": -214.32925415039062, "loss": 1.0649, "margin_dpo/margin_mean": 53.0930061340332, "margin_dpo/margin_std": 67.81797790527344, "step": 276 }, { "KL/chosen_KL_mean": -90.15188598632812, "KL/mean": -109.43403625488281, "KL/rejected_KL_mean": -128.7161865234375, "KL/std": 57.99913787841797, "epoch": 0.4067547723935389, "fcm_dpo/beta": 0.008596517145633698, "fcm_dpo/delta": 0.07081888616085052, "fcm_dpo/margin": 38.56431198120117, "fcm_dpo/q_t": 0.4233102798461914, "grad_norm": 9.395383834838867, "learning_rate": 3.71651119641714e-07, "logits/chosen": -2.024069309234619, "logits/rejected": -1.6068099737167358, "logps/chosen": -174.39599609375, "logps/ref_chosen": -84.24411010742188, "logps/ref_rejected": -98.83421325683594, "logps/rejected": -227.55039978027344, "loss": 1.1458, "margin_dpo/margin_mean": 38.564308166503906, "margin_dpo/margin_std": 59.540489196777344, "step": 277 }, { "KL/chosen_KL_mean": -86.2275161743164, "KL/mean": -110.80245971679688, "KL/rejected_KL_mean": -135.37741088867188, "KL/std": 62.160240173339844, "epoch": 0.40822320117474303, "fcm_dpo/beta": 0.008609242737293243, "fcm_dpo/delta": -0.024306947365403175, "fcm_dpo/margin": 49.1498908996582, "fcm_dpo/q_t": 0.4011944830417633, "grad_norm": 11.119629859924316, "learning_rate": 3.705283756425872e-07, "logits/chosen": -2.0411388874053955, "logits/rejected": -1.8247040510177612, "logps/chosen": -168.65939331054688, "logps/ref_chosen": -82.431884765625, "logps/ref_rejected": -97.85691833496094, "logps/rejected": -233.2343292236328, "loss": 1.0772, "margin_dpo/margin_mean": 49.14989471435547, "margin_dpo/margin_std": 62.38859176635742, "step": 278 }, { "KL/chosen_KL_mean": -88.58573150634766, "KL/mean": -119.23694610595703, "KL/rejected_KL_mean": -149.88815307617188, "KL/std": 65.74015808105469, "epoch": 0.40969162995594716, "fcm_dpo/beta": 0.008432027883827686, "fcm_dpo/delta": -0.12389300018548965, "fcm_dpo/margin": 61.30241775512695, "fcm_dpo/q_t": 0.38086533546447754, "grad_norm": 10.231287002563477, "learning_rate": 3.6940245560867e-07, "logits/chosen": -1.8119239807128906, "logits/rejected": -1.5989265441894531, "logps/chosen": -173.75372314453125, "logps/ref_chosen": -85.16799926757812, "logps/ref_rejected": -94.12664794921875, "logps/rejected": -244.01480102539062, "loss": 1.0053, "margin_dpo/margin_mean": 61.30241394042969, "margin_dpo/margin_std": 65.72706604003906, "step": 279 }, { "KL/chosen_KL_mean": -92.36898040771484, "KL/mean": -122.72671508789062, "KL/rejected_KL_mean": -153.08444213867188, "KL/std": 60.17634582519531, "epoch": 0.4111600587371512, "fcm_dpo/beta": 0.008288409560918808, "fcm_dpo/delta": -0.10856837034225464, "fcm_dpo/margin": 60.7154541015625, "fcm_dpo/q_t": 0.38302576541900635, "grad_norm": 8.873209953308105, "learning_rate": 3.6827338920900253e-07, "logits/chosen": -1.6613609790802002, "logits/rejected": -1.4586639404296875, "logps/chosen": -178.22540283203125, "logps/ref_chosen": -85.85641479492188, "logps/ref_rejected": -104.11859130859375, "logps/rejected": -257.2030334472656, "loss": 1.0088, "margin_dpo/margin_mean": 60.7154541015625, "margin_dpo/margin_std": 62.59893798828125, "step": 280 }, { "KL/chosen_KL_mean": -98.33480834960938, "KL/mean": -128.55044555664062, "KL/rejected_KL_mean": -158.76609802246094, "KL/std": 70.43211364746094, "epoch": 0.41262848751835535, "fcm_dpo/beta": 0.008093010634183884, "fcm_dpo/delta": -0.09396214783191681, "fcm_dpo/margin": 60.43128967285156, "fcm_dpo/q_t": 0.38818925619125366, "grad_norm": 7.325157165527344, "learning_rate": 3.6714120619553435e-07, "logits/chosen": -2.118122100830078, "logits/rejected": -1.3820266723632812, "logps/chosen": -184.8856201171875, "logps/ref_chosen": -86.55082702636719, "logps/ref_rejected": -88.62866973876953, "logps/rejected": -247.394775390625, "loss": 1.0321, "margin_dpo/margin_mean": 60.43128967285156, "margin_dpo/margin_std": 71.22540283203125, "step": 281 }, { "KL/chosen_KL_mean": -96.30721282958984, "KL/mean": -134.07363891601562, "KL/rejected_KL_mean": -171.84005737304688, "KL/std": 68.0960693359375, "epoch": 0.41409691629955947, "fcm_dpo/beta": 0.007823405787348747, "fcm_dpo/delta": -0.20317885279655457, "fcm_dpo/margin": 75.53285217285156, "fcm_dpo/q_t": 0.3628065586090088, "grad_norm": 8.731741905212402, "learning_rate": 3.660059364023408e-07, "logits/chosen": -1.950559377670288, "logits/rejected": -1.5796027183532715, "logps/chosen": -201.41232299804688, "logps/ref_chosen": -105.10511016845703, "logps/ref_rejected": -102.85336303710938, "logps/rejected": -274.69342041015625, "loss": 0.9432, "margin_dpo/margin_mean": 75.53285217285156, "margin_dpo/margin_std": 67.68347930908203, "step": 282 }, { "KL/chosen_KL_mean": -111.34022521972656, "KL/mean": -137.95974731445312, "KL/rejected_KL_mean": -164.57928466796875, "KL/std": 67.3225326538086, "epoch": 0.4155653450807636, "fcm_dpo/beta": 0.0077342987060546875, "fcm_dpo/delta": -0.012277229689061642, "fcm_dpo/margin": 53.239070892333984, "fcm_dpo/q_t": 0.40546029806137085, "grad_norm": 10.066877365112305, "learning_rate": 3.6486760974483685e-07, "logits/chosen": -2.4790327548980713, "logits/rejected": -2.159942150115967, "logps/chosen": -206.392822265625, "logps/ref_chosen": -95.05259704589844, "logps/ref_rejected": -103.54454803466797, "logps/rejected": -268.12384033203125, "loss": 1.0948, "margin_dpo/margin_mean": 53.239070892333984, "margin_dpo/margin_std": 74.80327606201172, "step": 283 }, { "KL/chosen_KL_mean": -100.40809631347656, "KL/mean": -134.46926879882812, "KL/rejected_KL_mean": -168.53042602539062, "KL/std": 66.63973999023438, "epoch": 0.4170337738619677, "fcm_dpo/beta": 0.007560379803180695, "fcm_dpo/delta": -0.12216213345527649, "fcm_dpo/margin": 68.122314453125, "fcm_dpo/q_t": 0.37858593463897705, "grad_norm": 9.819082260131836, "learning_rate": 3.6372625621898863e-07, "logits/chosen": -1.6678447723388672, "logits/rejected": -1.3255014419555664, "logps/chosen": -188.0745849609375, "logps/ref_chosen": -87.66647338867188, "logps/ref_rejected": -98.75103759765625, "logps/rejected": -267.2814636230469, "loss": 0.9845, "margin_dpo/margin_mean": 68.122314453125, "margin_dpo/margin_std": 60.033966064453125, "step": 284 }, { "KL/chosen_KL_mean": -112.99932861328125, "KL/mean": -136.78817749023438, "KL/rejected_KL_mean": -160.5770263671875, "KL/std": 61.12230682373047, "epoch": 0.4185022026431718, "fcm_dpo/beta": 0.007606650702655315, "fcm_dpo/delta": 0.039328526705503464, "fcm_dpo/margin": 47.57769012451172, "fcm_dpo/q_t": 0.4143468141555786, "grad_norm": 9.825729370117188, "learning_rate": 3.625819059005228e-07, "logits/chosen": -2.511685371398926, "logits/rejected": -2.240790843963623, "logps/chosen": -207.432373046875, "logps/ref_chosen": -94.43303680419922, "logps/ref_rejected": -104.07194519042969, "logps/rejected": -264.64898681640625, "loss": 1.1039, "margin_dpo/margin_mean": 47.57769012451172, "margin_dpo/margin_std": 57.93160629272461, "step": 285 }, { "KL/chosen_KL_mean": -116.50370025634766, "KL/mean": -148.73658752441406, "KL/rejected_KL_mean": -180.969482421875, "KL/std": 65.21870422363281, "epoch": 0.4199706314243759, "fcm_dpo/beta": 0.007494920399039984, "fcm_dpo/delta": -0.08756605535745621, "fcm_dpo/margin": 64.46578216552734, "fcm_dpo/q_t": 0.3872363865375519, "grad_norm": 8.346090316772461, "learning_rate": 3.614345889441346e-07, "logits/chosen": -1.9204883575439453, "logits/rejected": -1.454888105392456, "logps/chosen": -220.22409057617188, "logps/ref_chosen": -103.72039794921875, "logps/ref_rejected": -96.25775909423828, "logps/rejected": -277.22723388671875, "loss": 1.0226, "margin_dpo/margin_mean": 64.46578216552734, "margin_dpo/margin_std": 68.00028991699219, "step": 286 }, { "KL/chosen_KL_mean": -134.32696533203125, "KL/mean": -152.8894500732422, "KL/rejected_KL_mean": -171.45193481445312, "KL/std": 59.541168212890625, "epoch": 0.42143906020558003, "fcm_dpo/beta": 0.007641005329787731, "fcm_dpo/delta": 0.11875972896814346, "fcm_dpo/margin": 37.124977111816406, "fcm_dpo/q_t": 0.43547385931015015, "grad_norm": 8.232430458068848, "learning_rate": 3.6028433558269275e-07, "logits/chosen": -2.3431167602539062, "logits/rejected": -1.7603269815444946, "logps/chosen": -228.21685791015625, "logps/ref_chosen": -93.88988494873047, "logps/ref_rejected": -83.33365631103516, "logps/rejected": -254.78558349609375, "loss": 1.1924, "margin_dpo/margin_mean": 37.124977111816406, "margin_dpo/margin_std": 69.37480926513672, "step": 287 }, { "KL/chosen_KL_mean": -126.54493713378906, "KL/mean": -160.3050537109375, "KL/rejected_KL_mean": -194.06515502929688, "KL/std": 67.60092163085938, "epoch": 0.42290748898678415, "fcm_dpo/beta": 0.007510521914809942, "fcm_dpo/delta": -0.11311750113964081, "fcm_dpo/margin": 67.52021789550781, "fcm_dpo/q_t": 0.38448822498321533, "grad_norm": 7.803260803222656, "learning_rate": 3.5913117612644327e-07, "logits/chosen": -1.772340178489685, "logits/rejected": -1.354698896408081, "logps/chosen": -214.70095825195312, "logps/ref_chosen": -88.15602111816406, "logps/ref_rejected": -93.28195190429688, "logps/rejected": -287.34710693359375, "loss": 1.0152, "margin_dpo/margin_mean": 67.52021789550781, "margin_dpo/margin_std": 75.62127685546875, "step": 288 }, { "KL/chosen_KL_mean": -122.99992370605469, "KL/mean": -157.12820434570312, "KL/rejected_KL_mean": -191.25648498535156, "KL/std": 70.7049560546875, "epoch": 0.4243759177679883, "fcm_dpo/beta": 0.007376356981694698, "fcm_dpo/delta": -0.10883389413356781, "fcm_dpo/margin": 68.25656127929688, "fcm_dpo/q_t": 0.3869898319244385, "grad_norm": 7.493872165679932, "learning_rate": 3.5797514096221024e-07, "logits/chosen": -1.988786220550537, "logits/rejected": -1.7871909141540527, "logps/chosen": -198.39285278320312, "logps/ref_chosen": -75.39292907714844, "logps/ref_rejected": -93.15428161621094, "logps/rejected": -284.4107666015625, "loss": 1.0264, "margin_dpo/margin_mean": 68.25656127929688, "margin_dpo/margin_std": 81.30477905273438, "step": 289 }, { "KL/chosen_KL_mean": -132.27047729492188, "KL/mean": -167.71914672851562, "KL/rejected_KL_mean": -203.16783142089844, "KL/std": 74.82049560546875, "epoch": 0.42584434654919234, "fcm_dpo/beta": 0.007237486075609922, "fcm_dpo/delta": -0.11930535733699799, "fcm_dpo/margin": 70.8973617553711, "fcm_dpo/q_t": 0.3836299180984497, "grad_norm": 8.120606422424316, "learning_rate": 3.568162605525952e-07, "logits/chosen": -1.3985788822174072, "logits/rejected": -1.4427635669708252, "logps/chosen": -220.3124542236328, "logps/ref_chosen": -88.04197692871094, "logps/ref_rejected": -123.212158203125, "logps/rejected": -326.3800048828125, "loss": 1.0136, "margin_dpo/margin_mean": 70.89735412597656, "margin_dpo/margin_std": 78.59803009033203, "step": 290 }, { "KL/chosen_KL_mean": -125.97178649902344, "KL/mean": -156.45558166503906, "KL/rejected_KL_mean": -186.9393768310547, "KL/std": 71.01144409179688, "epoch": 0.42731277533039647, "fcm_dpo/beta": 0.00711851567029953, "fcm_dpo/delta": -0.035528771579265594, "fcm_dpo/margin": 60.967594146728516, "fcm_dpo/q_t": 0.398427277803421, "grad_norm": 9.580745697021484, "learning_rate": 3.5565456543517485e-07, "logits/chosen": -2.290384292602539, "logits/rejected": -1.8227465152740479, "logps/chosen": -220.06703186035156, "logps/ref_chosen": -94.09524536132812, "logps/ref_rejected": -96.05006408691406, "logps/rejected": -282.98944091796875, "loss": 1.0587, "margin_dpo/margin_mean": 60.96759796142578, "margin_dpo/margin_std": 70.82112121582031, "step": 291 }, { "KL/chosen_KL_mean": -129.9403076171875, "KL/mean": -161.02955627441406, "KL/rejected_KL_mean": -192.11880493164062, "KL/std": 68.55718994140625, "epoch": 0.4287812041116006, "fcm_dpo/beta": 0.0070516085252165794, "fcm_dpo/delta": -0.04024067893624306, "fcm_dpo/margin": 62.17851257324219, "fcm_dpo/q_t": 0.39719507098197937, "grad_norm": 8.493743896484375, "learning_rate": 3.5449008622169583e-07, "logits/chosen": -2.0268194675445557, "logits/rejected": -1.5925397872924805, "logps/chosen": -218.19070434570312, "logps/ref_chosen": -88.25041198730469, "logps/ref_rejected": -96.41764068603516, "logps/rejected": -288.53643798828125, "loss": 1.0427, "margin_dpo/margin_mean": 62.17851257324219, "margin_dpo/margin_std": 64.17147064208984, "step": 292 }, { "KL/chosen_KL_mean": -129.24388122558594, "KL/mean": -154.35092163085938, "KL/rejected_KL_mean": -179.45794677734375, "KL/std": 72.99819946289062, "epoch": 0.4302496328928047, "fcm_dpo/beta": 0.007114714942872524, "fcm_dpo/delta": 0.04381367564201355, "fcm_dpo/margin": 50.21406173706055, "fcm_dpo/q_t": 0.4188409447669983, "grad_norm": 7.958837032318115, "learning_rate": 3.5332285359726846e-07, "logits/chosen": -2.3155980110168457, "logits/rejected": -1.7433830499649048, "logps/chosen": -216.62042236328125, "logps/ref_chosen": -87.37654113769531, "logps/ref_rejected": -85.75579833984375, "logps/rejected": -265.2137451171875, "loss": 1.1307, "margin_dpo/margin_mean": 50.21406173706055, "margin_dpo/margin_std": 76.37503051757812, "step": 293 }, { "KL/chosen_KL_mean": -132.36477661132812, "KL/mean": -153.15823364257812, "KL/rejected_KL_mean": -173.95172119140625, "KL/std": 68.6073226928711, "epoch": 0.43171806167400884, "fcm_dpo/beta": 0.007231607101857662, "fcm_dpo/delta": 0.1018688902258873, "fcm_dpo/margin": 41.586936950683594, "fcm_dpo/q_t": 0.43155238032341003, "grad_norm": 8.460532188415527, "learning_rate": 3.5215289831955786e-07, "logits/chosen": -1.61783766746521, "logits/rejected": -1.4491127729415894, "logps/chosen": -205.87274169921875, "logps/ref_chosen": -73.50794982910156, "logps/ref_rejected": -88.08877563476562, "logps/rejected": -262.0404968261719, "loss": 1.1899, "margin_dpo/margin_mean": 41.58693313598633, "margin_dpo/margin_std": 80.09100341796875, "step": 294 }, { "KL/chosen_KL_mean": -125.109375, "KL/mean": -156.964111328125, "KL/rejected_KL_mean": -188.81884765625, "KL/std": 70.66976928710938, "epoch": 0.4331864904552129, "fcm_dpo/beta": 0.007155354134738445, "fcm_dpo/delta": -0.05911008268594742, "fcm_dpo/margin": 63.70947265625, "fcm_dpo/q_t": 0.3951072692871094, "grad_norm": 8.203312873840332, "learning_rate": 3.509802512179737e-07, "logits/chosen": -1.6105291843414307, "logits/rejected": -1.4538731575012207, "logps/chosen": -202.87486267089844, "logps/ref_chosen": -77.76548767089844, "logps/ref_rejected": -94.24726867675781, "logps/rejected": -283.06610107421875, "loss": 1.0505, "margin_dpo/margin_mean": 63.70948028564453, "margin_dpo/margin_std": 75.96926879882812, "step": 295 }, { "KL/chosen_KL_mean": -135.63143920898438, "KL/mean": -169.474609375, "KL/rejected_KL_mean": -203.31781005859375, "KL/std": 70.63226318359375, "epoch": 0.434654919236417, "fcm_dpo/beta": 0.007052103988826275, "fcm_dpo/delta": -0.08172537386417389, "fcm_dpo/margin": 67.68637084960938, "fcm_dpo/q_t": 0.3883482813835144, "grad_norm": 11.446629524230957, "learning_rate": 3.498049431928577e-07, "logits/chosen": -2.5962634086608887, "logits/rejected": -2.121018886566162, "logps/chosen": -233.48785400390625, "logps/ref_chosen": -97.85641479492188, "logps/ref_rejected": -100.81631469726562, "logps/rejected": -304.1341247558594, "loss": 1.0243, "margin_dpo/margin_mean": 67.68636322021484, "margin_dpo/margin_std": 70.95008850097656, "step": 296 }, { "KL/chosen_KL_mean": -136.91798400878906, "KL/mean": -165.3471221923828, "KL/rejected_KL_mean": -193.77627563476562, "KL/std": 70.63783264160156, "epoch": 0.43612334801762115, "fcm_dpo/beta": 0.0070589687675237656, "fcm_dpo/delta": -0.0014937762171030045, "fcm_dpo/margin": 56.8582878112793, "fcm_dpo/q_t": 0.40824609994888306, "grad_norm": 11.429444313049316, "learning_rate": 3.486270052146694e-07, "logits/chosen": -1.079359769821167, "logits/rejected": -0.7649163007736206, "logps/chosen": -225.48382568359375, "logps/ref_chosen": -88.56583404541016, "logps/ref_rejected": -101.55656433105469, "logps/rejected": -295.33282470703125, "loss": 1.0908, "margin_dpo/margin_mean": 56.8582878112793, "margin_dpo/margin_std": 75.65048217773438, "step": 297 }, { "KL/chosen_KL_mean": -116.39627075195312, "KL/mean": -152.83682250976562, "KL/rejected_KL_mean": -189.2773895263672, "KL/std": 77.52164459228516, "epoch": 0.43759177679882527, "fcm_dpo/beta": 0.006945013999938965, "fcm_dpo/delta": -0.11167125403881073, "fcm_dpo/margin": 72.88111877441406, "fcm_dpo/q_t": 0.38477081060409546, "grad_norm": 12.134675025939941, "learning_rate": 3.474464683231698e-07, "logits/chosen": -2.104311466217041, "logits/rejected": -2.0794267654418945, "logps/chosen": -211.27670288085938, "logps/ref_chosen": -94.88043975830078, "logps/ref_rejected": -122.31101989746094, "logps/rejected": -311.5884094238281, "loss": 1.0116, "margin_dpo/margin_mean": 72.88111877441406, "margin_dpo/margin_std": 79.29714965820312, "step": 298 }, { "KL/chosen_KL_mean": -119.75212097167969, "KL/mean": -155.98001098632812, "KL/rejected_KL_mean": -192.2078857421875, "KL/std": 74.18081665039062, "epoch": 0.4390602055800294, "fcm_dpo/beta": 0.006785230711102486, "fcm_dpo/delta": -0.09646116942167282, "fcm_dpo/margin": 72.45574951171875, "fcm_dpo/q_t": 0.38615018129348755, "grad_norm": 10.285080909729004, "learning_rate": 3.462633636266041e-07, "logits/chosen": -1.1107854843139648, "logits/rejected": -0.888792872428894, "logps/chosen": -200.1604766845703, "logps/ref_chosen": -80.40835571289062, "logps/ref_rejected": -89.53717041015625, "logps/rejected": -281.74505615234375, "loss": 1.0176, "margin_dpo/margin_mean": 72.45574951171875, "margin_dpo/margin_std": 77.30664825439453, "step": 299 }, { "KL/chosen_KL_mean": -126.189208984375, "KL/mean": -156.3235626220703, "KL/rejected_KL_mean": -186.4578857421875, "KL/std": 78.00128173828125, "epoch": 0.44052863436123346, "fcm_dpo/beta": 0.006725644692778587, "fcm_dpo/delta": -0.005893506109714508, "fcm_dpo/margin": 60.26869201660156, "fcm_dpo/q_t": 0.40646183490753174, "grad_norm": 8.347450256347656, "learning_rate": 3.4507772230088147e-07, "logits/chosen": -1.6317514181137085, "logits/rejected": -1.421809196472168, "logps/chosen": -214.34811401367188, "logps/ref_chosen": -88.15890502929688, "logps/ref_rejected": -100.93919372558594, "logps/rejected": -287.3970947265625, "loss": 1.0961, "margin_dpo/margin_mean": 60.26869201660156, "margin_dpo/margin_std": 82.96455383300781, "step": 300 }, { "KL/chosen_KL_mean": -115.54048919677734, "KL/mean": -153.27952575683594, "KL/rejected_KL_mean": -191.0185546875, "KL/std": 75.96161651611328, "epoch": 0.4419970631424376, "fcm_dpo/beta": 0.0066088540479540825, "fcm_dpo/delta": -0.10517693310976028, "fcm_dpo/margin": 75.4780502319336, "fcm_dpo/q_t": 0.3853745758533478, "grad_norm": 9.649535179138184, "learning_rate": 3.4388957558875316e-07, "logits/chosen": -1.5251753330230713, "logits/rejected": -1.0786831378936768, "logps/chosen": -201.04541015625, "logps/ref_chosen": -85.50491333007812, "logps/ref_rejected": -98.91683959960938, "logps/rejected": -289.9353942871094, "loss": 1.0233, "margin_dpo/margin_mean": 75.47805786132812, "margin_dpo/margin_std": 85.46128845214844, "step": 301 }, { "KL/chosen_KL_mean": -112.21430206298828, "KL/mean": -144.54495239257812, "KL/rejected_KL_mean": -176.87559509277344, "KL/std": 78.56361389160156, "epoch": 0.4434654919236417, "fcm_dpo/beta": 0.006570211611688137, "fcm_dpo/delta": -0.02607264369726181, "fcm_dpo/margin": 64.6612777709961, "fcm_dpo/q_t": 0.40064749121665955, "grad_norm": 8.487879753112793, "learning_rate": 3.426989547989902e-07, "logits/chosen": -1.3129959106445312, "logits/rejected": -1.136385440826416, "logps/chosen": -191.66470336914062, "logps/ref_chosen": -79.45040893554688, "logps/ref_rejected": -103.0591049194336, "logps/rejected": -279.9346923828125, "loss": 1.0659, "margin_dpo/margin_mean": 64.6612777709961, "margin_dpo/margin_std": 76.64193725585938, "step": 302 }, { "KL/chosen_KL_mean": -104.95634460449219, "KL/mean": -140.41067504882812, "KL/rejected_KL_mean": -175.86502075195312, "KL/std": 73.0411605834961, "epoch": 0.44493392070484583, "fcm_dpo/beta": 0.006527240388095379, "fcm_dpo/delta": -0.06585341691970825, "fcm_dpo/margin": 70.90867614746094, "fcm_dpo/q_t": 0.3918619155883789, "grad_norm": 8.243387222290039, "learning_rate": 3.4150589130555773e-07, "logits/chosen": -1.984487533569336, "logits/rejected": -1.536018967628479, "logps/chosen": -200.5546417236328, "logps/ref_chosen": -95.59829711914062, "logps/ref_rejected": -92.0645751953125, "logps/rejected": -267.9295959472656, "loss": 1.0377, "margin_dpo/margin_mean": 70.90867614746094, "margin_dpo/margin_std": 77.53782653808594, "step": 303 }, { "KL/chosen_KL_mean": -104.6500015258789, "KL/mean": -141.21600341796875, "KL/rejected_KL_mean": -177.781982421875, "KL/std": 75.79883575439453, "epoch": 0.44640234948604995, "fcm_dpo/beta": 0.0064209504052996635, "fcm_dpo/delta": -0.07297976315021515, "fcm_dpo/margin": 73.13198852539062, "fcm_dpo/q_t": 0.3902961015701294, "grad_norm": 8.929335594177246, "learning_rate": 3.403104165467883e-07, "logits/chosen": -2.0816545486450195, "logits/rejected": -1.6583287715911865, "logps/chosen": -204.1031494140625, "logps/ref_chosen": -99.4531478881836, "logps/ref_rejected": -98.08226013183594, "logps/rejected": -275.8642578125, "loss": 1.0302, "margin_dpo/margin_mean": 73.13198852539062, "margin_dpo/margin_std": 78.54046630859375, "step": 304 }, { "KL/chosen_KL_mean": -113.73192596435547, "KL/mean": -148.34182739257812, "KL/rejected_KL_mean": -182.95172119140625, "KL/std": 78.6681900024414, "epoch": 0.447870778267254, "fcm_dpo/beta": 0.006321310065686703, "fcm_dpo/delta": -0.040127988904714584, "fcm_dpo/margin": 69.21980285644531, "fcm_dpo/q_t": 0.3981311321258545, "grad_norm": 7.7749152183532715, "learning_rate": 3.391125620245535e-07, "logits/chosen": -1.988163709640503, "logits/rejected": -1.4907026290893555, "logps/chosen": -216.99533081054688, "logps/ref_chosen": -103.26339721679688, "logps/ref_rejected": -97.82897186279297, "logps/rejected": -280.78070068359375, "loss": 1.0676, "margin_dpo/margin_mean": 69.21980285644531, "margin_dpo/margin_std": 84.89730834960938, "step": 305 }, { "KL/chosen_KL_mean": -102.22395324707031, "KL/mean": -142.90106201171875, "KL/rejected_KL_mean": -183.57818603515625, "KL/std": 74.71578979492188, "epoch": 0.44933920704845814, "fcm_dpo/beta": 0.006222227588295937, "fcm_dpo/delta": -0.11229479312896729, "fcm_dpo/margin": 81.35423278808594, "fcm_dpo/q_t": 0.3798482418060303, "grad_norm": 9.87553882598877, "learning_rate": 3.3791235930343417e-07, "logits/chosen": -2.107722759246826, "logits/rejected": -1.5386652946472168, "logps/chosen": -202.5277862548828, "logps/ref_chosen": -100.3038330078125, "logps/ref_rejected": -92.23086547851562, "logps/rejected": -275.8090515136719, "loss": 0.9872, "margin_dpo/margin_mean": 81.35423278808594, "margin_dpo/margin_std": 69.09818267822266, "step": 306 }, { "KL/chosen_KL_mean": -107.63002014160156, "KL/mean": -138.406005859375, "KL/rejected_KL_mean": -169.18199157714844, "KL/std": 75.46446990966797, "epoch": 0.45080763582966227, "fcm_dpo/beta": 0.006199567578732967, "fcm_dpo/delta": 0.018974458798766136, "fcm_dpo/margin": 61.55196762084961, "fcm_dpo/q_t": 0.4122597575187683, "grad_norm": 9.448859214782715, "learning_rate": 3.367098400098881e-07, "logits/chosen": -2.0353684425354004, "logits/rejected": -1.693960428237915, "logps/chosen": -204.27978515625, "logps/ref_chosen": -96.64976501464844, "logps/ref_rejected": -93.34860229492188, "logps/rejected": -262.53057861328125, "loss": 1.1133, "margin_dpo/margin_mean": 61.55196762084961, "margin_dpo/margin_std": 89.02847290039062, "step": 307 }, { "KL/chosen_KL_mean": -112.44061279296875, "KL/mean": -143.23040771484375, "KL/rejected_KL_mean": -174.02020263671875, "KL/std": 77.04743957519531, "epoch": 0.4522760646108664, "fcm_dpo/beta": 0.006205511279404163, "fcm_dpo/delta": 0.017949584871530533, "fcm_dpo/margin": 61.57958221435547, "fcm_dpo/q_t": 0.41267380118370056, "grad_norm": 10.331384658813477, "learning_rate": 3.355050358314172e-07, "logits/chosen": -1.7937939167022705, "logits/rejected": -1.5163633823394775, "logps/chosen": -198.165283203125, "logps/ref_chosen": -85.72467041015625, "logps/ref_rejected": -86.42317962646484, "logps/rejected": -260.4433898925781, "loss": 1.1177, "margin_dpo/margin_mean": 61.57958221435547, "margin_dpo/margin_std": 89.75591278076172, "step": 308 }, { "KL/chosen_KL_mean": -126.922607421875, "KL/mean": -154.89108276367188, "KL/rejected_KL_mean": -182.85958862304688, "KL/std": 82.50601196289062, "epoch": 0.45374449339207046, "fcm_dpo/beta": 0.006298947148025036, "fcm_dpo/delta": 0.04935268312692642, "fcm_dpo/margin": 55.936973571777344, "fcm_dpo/q_t": 0.418404757976532, "grad_norm": 12.34549331665039, "learning_rate": 3.3429797851573183e-07, "logits/chosen": -1.7677497863769531, "logits/rejected": -1.339789867401123, "logps/chosen": -210.07406616210938, "logps/ref_chosen": -83.15145874023438, "logps/ref_rejected": -86.58602142333984, "logps/rejected": -269.44561767578125, "loss": 1.1311, "margin_dpo/margin_mean": 55.936973571777344, "margin_dpo/margin_std": 82.15354919433594, "step": 309 }, { "KL/chosen_KL_mean": -114.83667755126953, "KL/mean": -145.59567260742188, "KL/rejected_KL_mean": -176.35470581054688, "KL/std": 74.91990661621094, "epoch": 0.4552129221732746, "fcm_dpo/beta": 0.006317371502518654, "fcm_dpo/delta": 0.01182640716433525, "fcm_dpo/margin": 61.518009185791016, "fcm_dpo/q_t": 0.40947502851486206, "grad_norm": 8.960644721984863, "learning_rate": 3.3308869986991487e-07, "logits/chosen": -2.2129263877868652, "logits/rejected": -1.7958842515945435, "logps/chosen": -204.57467651367188, "logps/ref_chosen": -89.73799133300781, "logps/ref_rejected": -87.07083129882812, "logps/rejected": -263.425537109375, "loss": 1.0932, "margin_dpo/margin_mean": 61.51800537109375, "margin_dpo/margin_std": 78.45738983154297, "step": 310 }, { "KL/chosen_KL_mean": -134.3755645751953, "KL/mean": -174.03924560546875, "KL/rejected_KL_mean": -213.7029266357422, "KL/std": 75.95979309082031, "epoch": 0.4566813509544787, "fcm_dpo/beta": 0.006234113126993179, "fcm_dpo/delta": -0.09943139553070068, "fcm_dpo/margin": 79.327392578125, "fcm_dpo/q_t": 0.3849089741706848, "grad_norm": 9.209837913513184, "learning_rate": 3.3187723175958346e-07, "logits/chosen": -1.5854158401489258, "logits/rejected": -0.8472986817359924, "logps/chosen": -222.90200805664062, "logps/ref_chosen": -88.52644348144531, "logps/ref_rejected": -82.26608276367188, "logps/rejected": -295.968994140625, "loss": 1.0046, "margin_dpo/margin_mean": 79.327392578125, "margin_dpo/margin_std": 77.30213928222656, "step": 311 }, { "KL/chosen_KL_mean": -129.7441864013672, "KL/mean": -162.77256774902344, "KL/rejected_KL_mean": -195.8009490966797, "KL/std": 83.3153076171875, "epoch": 0.4581497797356828, "fcm_dpo/beta": 0.0062103113159537315, "fcm_dpo/delta": -0.010837860405445099, "fcm_dpo/margin": 66.05677795410156, "fcm_dpo/q_t": 0.4043039083480835, "grad_norm": 9.065444946289062, "learning_rate": 3.306636061080487e-07, "logits/chosen": -1.5602099895477295, "logits/rejected": -1.0316680669784546, "logps/chosen": -210.14488220214844, "logps/ref_chosen": -80.40069580078125, "logps/ref_rejected": -82.4471435546875, "logps/rejected": -278.24810791015625, "loss": 1.0767, "margin_dpo/margin_mean": 66.05677795410156, "margin_dpo/margin_std": 80.07574462890625, "step": 312 }, { "KL/chosen_KL_mean": -123.891845703125, "KL/mean": -156.16903686523438, "KL/rejected_KL_mean": -188.44625854492188, "KL/std": 81.96527099609375, "epoch": 0.45961820851688695, "fcm_dpo/beta": 0.006205535493791103, "fcm_dpo/delta": -0.000805957242846489, "fcm_dpo/margin": 64.55440521240234, "fcm_dpo/q_t": 0.4074169397354126, "grad_norm": 9.81668758392334, "learning_rate": 3.2944785489547537e-07, "logits/chosen": -2.358447551727295, "logits/rejected": -1.9496670961380005, "logps/chosen": -208.03375244140625, "logps/ref_chosen": -84.14190673828125, "logps/ref_rejected": -93.42684173583984, "logps/rejected": -281.87310791015625, "loss": 1.0974, "margin_dpo/margin_mean": 64.55440521240234, "margin_dpo/margin_std": 88.79119110107422, "step": 313 }, { "KL/chosen_KL_mean": -125.8941879272461, "KL/mean": -157.61489868164062, "KL/rejected_KL_mean": -189.3356170654297, "KL/std": 82.35855102539062, "epoch": 0.461086637298091, "fcm_dpo/beta": 0.006176512222737074, "fcm_dpo/delta": 0.008343830704689026, "fcm_dpo/margin": 63.441444396972656, "fcm_dpo/q_t": 0.40994417667388916, "grad_norm": 10.614872932434082, "learning_rate": 3.2823001015803857e-07, "logits/chosen": -1.946634292602539, "logits/rejected": -1.7289049625396729, "logps/chosen": -214.70391845703125, "logps/ref_chosen": -88.80972290039062, "logps/ref_rejected": -104.15892028808594, "logps/rejected": -293.4945373535156, "loss": 1.0956, "margin_dpo/margin_mean": 63.441444396972656, "margin_dpo/margin_std": 83.60945129394531, "step": 314 }, { "KL/chosen_KL_mean": -121.49390411376953, "KL/mean": -153.9754180908203, "KL/rejected_KL_mean": -186.45693969726562, "KL/std": 86.40809631347656, "epoch": 0.46255506607929514, "fcm_dpo/beta": 0.006189560517668724, "fcm_dpo/delta": -0.002196177374571562, "fcm_dpo/margin": 64.96302795410156, "fcm_dpo/q_t": 0.40865981578826904, "grad_norm": 9.033608436584473, "learning_rate": 3.270101039870797e-07, "logits/chosen": -1.3006126880645752, "logits/rejected": -1.1215593814849854, "logps/chosen": -202.28759765625, "logps/ref_chosen": -80.793701171875, "logps/ref_rejected": -92.13711547851562, "logps/rejected": -278.59405517578125, "loss": 1.113, "margin_dpo/margin_mean": 64.96302795410156, "margin_dpo/margin_std": 99.74339294433594, "step": 315 }, { "KL/chosen_KL_mean": -123.95455932617188, "KL/mean": -159.21331787109375, "KL/rejected_KL_mean": -194.47210693359375, "KL/std": 84.68959045410156, "epoch": 0.46402349486049926, "fcm_dpo/beta": 0.006167247891426086, "fcm_dpo/delta": -0.036470551043748856, "fcm_dpo/margin": 70.51754760742188, "fcm_dpo/q_t": 0.40030843019485474, "grad_norm": 9.124999046325684, "learning_rate": 3.2578816852826086e-07, "logits/chosen": -1.7123621702194214, "logits/rejected": -1.7129731178283691, "logps/chosen": -208.13880920410156, "logps/ref_chosen": -84.18424987792969, "logps/ref_rejected": -105.6908187866211, "logps/rejected": -300.16290283203125, "loss": 1.0667, "margin_dpo/margin_mean": 70.51754760742188, "margin_dpo/margin_std": 88.71147155761719, "step": 316 }, { "KL/chosen_KL_mean": -128.98146057128906, "KL/mean": -172.23904418945312, "KL/rejected_KL_mean": -215.49658203125, "KL/std": 93.64411926269531, "epoch": 0.4654919236417034, "fcm_dpo/beta": 0.006051095202565193, "fcm_dpo/delta": -0.13012844324111938, "fcm_dpo/margin": 86.51513671875, "fcm_dpo/q_t": 0.38108256459236145, "grad_norm": 10.449542999267578, "learning_rate": 3.2456423598071783e-07, "logits/chosen": -1.8087862730026245, "logits/rejected": -1.3236925601959229, "logps/chosen": -216.85494995117188, "logps/ref_chosen": -87.87348937988281, "logps/ref_rejected": -108.26519775390625, "logps/rejected": -323.76177978515625, "loss": 1.0162, "margin_dpo/margin_mean": 86.51513671875, "margin_dpo/margin_std": 100.49026489257812, "step": 317 }, { "KL/chosen_KL_mean": -125.18738555908203, "KL/mean": -163.5219268798828, "KL/rejected_KL_mean": -201.8564453125, "KL/std": 78.46136474609375, "epoch": 0.4669603524229075, "fcm_dpo/beta": 0.005933158099651337, "fcm_dpo/delta": -0.05759740620851517, "fcm_dpo/margin": 76.6690673828125, "fcm_dpo/q_t": 0.39416980743408203, "grad_norm": 10.36240291595459, "learning_rate": 3.233383385962115e-07, "logits/chosen": -2.283783197402954, "logits/rejected": -1.5803554058074951, "logps/chosen": -223.39291381835938, "logps/ref_chosen": -98.20553588867188, "logps/ref_rejected": -88.13629150390625, "logps/rejected": -289.99273681640625, "loss": 1.0438, "margin_dpo/margin_mean": 76.6690673828125, "margin_dpo/margin_std": 86.57158660888672, "step": 318 }, { "KL/chosen_KL_mean": -108.65480041503906, "KL/mean": -150.66128540039062, "KL/rejected_KL_mean": -192.6677703857422, "KL/std": 81.78934478759766, "epoch": 0.4684287812041116, "fcm_dpo/beta": 0.005829245783388615, "fcm_dpo/delta": -0.09458824247121811, "fcm_dpo/margin": 84.01296997070312, "fcm_dpo/q_t": 0.3865634799003601, "grad_norm": 9.794469833374023, "learning_rate": 3.2211050867827805e-07, "logits/chosen": -1.5781879425048828, "logits/rejected": -1.509718894958496, "logps/chosen": -191.15817260742188, "logps/ref_chosen": -82.50337219238281, "logps/ref_rejected": -118.65068054199219, "logps/rejected": -311.3184509277344, "loss": 1.0233, "margin_dpo/margin_mean": 84.01296997070312, "margin_dpo/margin_std": 92.87960815429688, "step": 319 }, { "KL/chosen_KL_mean": -110.72135925292969, "KL/mean": -145.1161651611328, "KL/rejected_KL_mean": -179.510986328125, "KL/std": 83.51746368408203, "epoch": 0.4698972099853157, "fcm_dpo/beta": 0.005802720785140991, "fcm_dpo/delta": 0.0008152760565280914, "fcm_dpo/margin": 68.78961944580078, "fcm_dpo/q_t": 0.4069502055644989, "grad_norm": 8.678085327148438, "learning_rate": 3.208807785813777e-07, "logits/chosen": -1.9781862497329712, "logits/rejected": -1.7877423763275146, "logps/chosen": -197.85858154296875, "logps/ref_chosen": -87.13722229003906, "logps/ref_rejected": -103.22412109375, "logps/rejected": -282.735107421875, "loss": 1.0782, "margin_dpo/margin_mean": 68.78961944580078, "margin_dpo/margin_std": 80.85707092285156, "step": 320 }, { "KL/chosen_KL_mean": -123.40225982666016, "KL/mean": -163.32908630371094, "KL/rejected_KL_mean": -203.25592041015625, "KL/std": 85.43701171875, "epoch": 0.4713656387665198, "fcm_dpo/beta": 0.005739005282521248, "fcm_dpo/delta": -0.0615546740591526, "fcm_dpo/margin": 79.85365295410156, "fcm_dpo/q_t": 0.3920021057128906, "grad_norm": 8.840250015258789, "learning_rate": 3.1964918071004217e-07, "logits/chosen": -1.7601863145828247, "logits/rejected": -1.3567407131195068, "logps/chosen": -216.50692749023438, "logps/ref_chosen": -93.10466766357422, "logps/ref_rejected": -97.4681396484375, "logps/rejected": -300.72406005859375, "loss": 1.0371, "margin_dpo/margin_mean": 79.85365295410156, "margin_dpo/margin_std": 85.41590881347656, "step": 321 }, { "KL/chosen_KL_mean": -115.46151733398438, "KL/mean": -158.67465209960938, "KL/rejected_KL_mean": -201.88778686523438, "KL/std": 85.09862518310547, "epoch": 0.47283406754772395, "fcm_dpo/beta": 0.005626651458442211, "fcm_dpo/delta": -0.09215117990970612, "fcm_dpo/margin": 86.42625427246094, "fcm_dpo/q_t": 0.3867570161819458, "grad_norm": 13.06740951538086, "learning_rate": 3.184157475180207e-07, "logits/chosen": -1.7424213886260986, "logits/rejected": -1.3848967552185059, "logps/chosen": -207.65443420410156, "logps/ref_chosen": -92.19291687011719, "logps/ref_rejected": -100.89321899414062, "logps/rejected": -302.781005859375, "loss": 1.0241, "margin_dpo/margin_mean": 86.42625427246094, "margin_dpo/margin_std": 92.06568908691406, "step": 322 }, { "KL/chosen_KL_mean": -132.735107421875, "KL/mean": -168.98065185546875, "KL/rejected_KL_mean": -205.22618103027344, "KL/std": 85.6290283203125, "epoch": 0.47430249632892807, "fcm_dpo/beta": 0.005619035568088293, "fcm_dpo/delta": -0.00773264653980732, "fcm_dpo/margin": 72.49104309082031, "fcm_dpo/q_t": 0.40597328543663025, "grad_norm": 10.823671340942383, "learning_rate": 3.171805115074251e-07, "logits/chosen": -1.9703731536865234, "logits/rejected": -1.5882459878921509, "logps/chosen": -213.00721740722656, "logps/ref_chosen": -80.27210235595703, "logps/ref_rejected": -80.3033447265625, "logps/rejected": -285.529541015625, "loss": 1.0788, "margin_dpo/margin_mean": 72.49104309082031, "margin_dpo/margin_std": 88.7784652709961, "step": 323 }, { "KL/chosen_KL_mean": -131.9173583984375, "KL/mean": -169.50033569335938, "KL/rejected_KL_mean": -207.08331298828125, "KL/std": 85.6475830078125, "epoch": 0.47577092511013214, "fcm_dpo/beta": 0.005616380833089352, "fcm_dpo/delta": -0.023145999759435654, "fcm_dpo/margin": 75.16593933105469, "fcm_dpo/q_t": 0.4004562497138977, "grad_norm": 8.714640617370605, "learning_rate": 3.1594350522787295e-07, "logits/chosen": -1.6716153621673584, "logits/rejected": -1.0188782215118408, "logps/chosen": -227.64671325683594, "logps/ref_chosen": -95.72935485839844, "logps/ref_rejected": -94.16924285888672, "logps/rejected": -301.2525634765625, "loss": 1.0574, "margin_dpo/margin_mean": 75.16593933105469, "margin_dpo/margin_std": 81.05679321289062, "step": 324 }, { "KL/chosen_KL_mean": -120.71771240234375, "KL/mean": -161.6676025390625, "KL/rejected_KL_mean": -202.61749267578125, "KL/std": 81.90916442871094, "epoch": 0.47723935389133626, "fcm_dpo/beta": 0.005559112876653671, "fcm_dpo/delta": -0.057890474796295166, "fcm_dpo/margin": 81.8997802734375, "fcm_dpo/q_t": 0.3929550051689148, "grad_norm": 7.78330659866333, "learning_rate": 3.147047612756302e-07, "logits/chosen": -1.792612910270691, "logits/rejected": -1.0598881244659424, "logps/chosen": -219.42459106445312, "logps/ref_chosen": -98.70686340332031, "logps/ref_rejected": -76.40809631347656, "logps/rejected": -279.02557373046875, "loss": 1.0349, "margin_dpo/margin_mean": 81.8997802734375, "margin_dpo/margin_std": 84.59835815429688, "step": 325 }, { "KL/chosen_KL_mean": -135.0028839111328, "KL/mean": -171.5477752685547, "KL/rejected_KL_mean": -208.0926513671875, "KL/std": 81.25959777832031, "epoch": 0.4787077826725404, "fcm_dpo/beta": 0.0055312179028987885, "fcm_dpo/delta": -0.004461154341697693, "fcm_dpo/margin": 73.08978271484375, "fcm_dpo/q_t": 0.4050220847129822, "grad_norm": 7.506682872772217, "learning_rate": 3.134643122927519e-07, "logits/chosen": -2.443956136703491, "logits/rejected": -1.7722558975219727, "logps/chosen": -239.6641845703125, "logps/ref_chosen": -104.66130828857422, "logps/ref_rejected": -86.82537841796875, "logps/rejected": -294.91802978515625, "loss": 1.0787, "margin_dpo/margin_mean": 73.08977508544922, "margin_dpo/margin_std": 87.8311767578125, "step": 326 }, { "KL/chosen_KL_mean": -130.874267578125, "KL/mean": -166.00027465820312, "KL/rejected_KL_mean": -201.12625122070312, "KL/std": 74.62130737304688, "epoch": 0.4801762114537445, "fcm_dpo/beta": 0.005527630914002657, "fcm_dpo/delta": 0.012033982202410698, "fcm_dpo/margin": 70.25199127197266, "fcm_dpo/q_t": 0.40982377529144287, "grad_norm": 8.535112380981445, "learning_rate": 3.1222219096622264e-07, "logits/chosen": -1.7414215803146362, "logits/rejected": -1.1496832370758057, "logps/chosen": -238.31451416015625, "logps/ref_chosen": -107.44023895263672, "logps/ref_rejected": -117.05323028564453, "logps/rejected": -318.17950439453125, "loss": 1.0865, "margin_dpo/margin_mean": 70.25199127197266, "margin_dpo/margin_std": 84.48846435546875, "step": 327 }, { "KL/chosen_KL_mean": -127.02562713623047, "KL/mean": -173.68069458007812, "KL/rejected_KL_mean": -220.33578491210938, "KL/std": 88.95399475097656, "epoch": 0.48164464023494863, "fcm_dpo/beta": 0.005454832687973976, "fcm_dpo/delta": -0.11478032171726227, "fcm_dpo/margin": 93.31015014648438, "fcm_dpo/q_t": 0.3804827630519867, "grad_norm": 9.99223518371582, "learning_rate": 3.1097843002709427e-07, "logits/chosen": -1.8083605766296387, "logits/rejected": -1.5159180164337158, "logps/chosen": -213.63632202148438, "logps/ref_chosen": -86.6106948852539, "logps/ref_rejected": -97.40188598632812, "logps/rejected": -317.7376708984375, "loss": 0.9919, "margin_dpo/margin_mean": 93.31015014648438, "margin_dpo/margin_std": 85.85917663574219, "step": 328 }, { "KL/chosen_KL_mean": -133.16836547851562, "KL/mean": -169.61550903320312, "KL/rejected_KL_mean": -206.06263732910156, "KL/std": 85.34268188476562, "epoch": 0.4831130690161527, "fcm_dpo/beta": 0.005404187366366386, "fcm_dpo/delta": 0.006044376641511917, "fcm_dpo/margin": 72.89430236816406, "fcm_dpo/q_t": 0.40867507457733154, "grad_norm": 9.990004539489746, "learning_rate": 3.0973306224962437e-07, "logits/chosen": -1.6754481792449951, "logits/rejected": -1.187462568283081, "logps/chosen": -233.78143310546875, "logps/ref_chosen": -100.61308288574219, "logps/ref_rejected": -104.95370483398438, "logps/rejected": -311.016357421875, "loss": 1.0893, "margin_dpo/margin_mean": 72.89430236816406, "margin_dpo/margin_std": 91.37336730957031, "step": 329 }, { "KL/chosen_KL_mean": -114.26388549804688, "KL/mean": -153.30062866210938, "KL/rejected_KL_mean": -192.33738708496094, "KL/std": 83.400146484375, "epoch": 0.4845814977973568, "fcm_dpo/beta": 0.005400915630161762, "fcm_dpo/delta": -0.022687481716275215, "fcm_dpo/margin": 78.0735092163086, "fcm_dpo/q_t": 0.39916732907295227, "grad_norm": 8.058147430419922, "learning_rate": 3.084861204504122e-07, "logits/chosen": -1.6987576484680176, "logits/rejected": -1.6189851760864258, "logps/chosen": -191.22232055664062, "logps/ref_chosen": -76.95843505859375, "logps/ref_rejected": -92.81494140625, "logps/rejected": -285.15234375, "loss": 1.0477, "margin_dpo/margin_mean": 78.0735092163086, "margin_dpo/margin_std": 73.57077026367188, "step": 330 }, { "KL/chosen_KL_mean": -122.99425506591797, "KL/mean": -166.6390380859375, "KL/rejected_KL_mean": -210.28382873535156, "KL/std": 81.52969360351562, "epoch": 0.48604992657856094, "fcm_dpo/beta": 0.005363701842725277, "fcm_dpo/delta": -0.07154914736747742, "fcm_dpo/margin": 87.28956604003906, "fcm_dpo/q_t": 0.389539897441864, "grad_norm": 9.554699897766113, "learning_rate": 3.072376374875335e-07, "logits/chosen": -1.8390285968780518, "logits/rejected": -1.5338919162750244, "logps/chosen": -206.05368041992188, "logps/ref_chosen": -83.05941772460938, "logps/ref_rejected": -94.13113403320312, "logps/rejected": -304.41497802734375, "loss": 1.0175, "margin_dpo/margin_mean": 87.28958129882812, "margin_dpo/margin_std": 80.81610870361328, "step": 331 }, { "KL/chosen_KL_mean": -130.1404571533203, "KL/mean": -160.1261749267578, "KL/rejected_KL_mean": -190.11190795898438, "KL/std": 89.97642517089844, "epoch": 0.48751835535976507, "fcm_dpo/beta": 0.005394460167735815, "fcm_dpo/delta": 0.07892205566167831, "fcm_dpo/margin": 59.97144317626953, "fcm_dpo/q_t": 0.4242321252822876, "grad_norm": 7.64940071105957, "learning_rate": 3.059876462596758e-07, "logits/chosen": -1.9703017473220825, "logits/rejected": -1.5313513278961182, "logps/chosen": -212.07135009765625, "logps/ref_chosen": -81.93089294433594, "logps/ref_rejected": -82.84590148925781, "logps/rejected": -272.95782470703125, "loss": 1.1422, "margin_dpo/margin_mean": 59.97144317626953, "margin_dpo/margin_std": 87.68479919433594, "step": 332 }, { "KL/chosen_KL_mean": -150.71536254882812, "KL/mean": -193.34939575195312, "KL/rejected_KL_mean": -235.98342895507812, "KL/std": 94.12724304199219, "epoch": 0.4889867841409692, "fcm_dpo/beta": 0.0053335814736783504, "fcm_dpo/delta": -0.05833350867033005, "fcm_dpo/margin": 85.26805114746094, "fcm_dpo/q_t": 0.39502716064453125, "grad_norm": 8.062228202819824, "learning_rate": 3.0473617970527015e-07, "logits/chosen": -1.5730019807815552, "logits/rejected": -1.234466552734375, "logps/chosen": -239.24795532226562, "logps/ref_chosen": -88.53257751464844, "logps/ref_rejected": -102.46601867675781, "logps/rejected": -338.449462890625, "loss": 1.0532, "margin_dpo/margin_mean": 85.2680435180664, "margin_dpo/margin_std": 101.24537658691406, "step": 333 }, { "KL/chosen_KL_mean": -136.96920776367188, "KL/mean": -174.57571411132812, "KL/rejected_KL_mean": -212.18222045898438, "KL/std": 88.89678955078125, "epoch": 0.49045521292217326, "fcm_dpo/beta": 0.005317076109349728, "fcm_dpo/delta": -0.0005399920046329498, "fcm_dpo/margin": 75.21300506591797, "fcm_dpo/q_t": 0.40622419118881226, "grad_norm": 8.990278244018555, "learning_rate": 3.034832708016243e-07, "logits/chosen": -1.2941745519638062, "logits/rejected": -0.9912099838256836, "logps/chosen": -234.72698974609375, "logps/ref_chosen": -97.75778198242188, "logps/ref_rejected": -100.0058364868164, "logps/rejected": -312.18804931640625, "loss": 1.0949, "margin_dpo/margin_mean": 75.21299743652344, "margin_dpo/margin_std": 98.96910858154297, "step": 334 }, { "KL/chosen_KL_mean": -136.19244384765625, "KL/mean": -168.79574584960938, "KL/rejected_KL_mean": -201.39907836914062, "KL/std": 88.71186828613281, "epoch": 0.4919236417033774, "fcm_dpo/beta": 0.005387984216213226, "fcm_dpo/delta": 0.05042743682861328, "fcm_dpo/margin": 65.20662689208984, "fcm_dpo/q_t": 0.4185336232185364, "grad_norm": 11.8197021484375, "learning_rate": 3.022289525640531e-07, "logits/chosen": -2.26584529876709, "logits/rejected": -1.8387019634246826, "logps/chosen": -232.5684814453125, "logps/ref_chosen": -96.37603759765625, "logps/ref_rejected": -94.92401123046875, "logps/rejected": -296.32305908203125, "loss": 1.1307, "margin_dpo/margin_mean": 65.20661926269531, "margin_dpo/margin_std": 97.06861877441406, "step": 335 }, { "KL/chosen_KL_mean": -138.96295166015625, "KL/mean": -180.85568237304688, "KL/rejected_KL_mean": -222.74839782714844, "KL/std": 91.02042388916016, "epoch": 0.4933920704845815, "fcm_dpo/beta": 0.005370216444134712, "fcm_dpo/delta": -0.0522807277739048, "fcm_dpo/margin": 83.78546142578125, "fcm_dpo/q_t": 0.3954579830169678, "grad_norm": 10.180022239685059, "learning_rate": 3.009732580450086e-07, "logits/chosen": -1.647291660308838, "logits/rejected": -1.4957342147827148, "logps/chosen": -227.49249267578125, "logps/ref_chosen": -88.52952575683594, "logps/ref_rejected": -109.5557861328125, "logps/rejected": -332.30419921875, "loss": 1.0465, "margin_dpo/margin_mean": 83.78546142578125, "margin_dpo/margin_std": 95.53353118896484, "step": 336 }, { "KL/chosen_KL_mean": -137.9287109375, "KL/mean": -177.61476135253906, "KL/rejected_KL_mean": -217.3008270263672, "KL/std": 91.79724884033203, "epoch": 0.4948604992657856, "fcm_dpo/beta": 0.00533271674066782, "fcm_dpo/delta": -0.024304304271936417, "fcm_dpo/margin": 79.37212371826172, "fcm_dpo/q_t": 0.4029567539691925, "grad_norm": 8.952764511108398, "learning_rate": 2.9971622033320914e-07, "logits/chosen": -2.411144495010376, "logits/rejected": -2.130584239959717, "logps/chosen": -235.2410888671875, "logps/ref_chosen": -97.3123779296875, "logps/ref_rejected": -110.13134765625, "logps/rejected": -327.43218994140625, "loss": 1.084, "margin_dpo/margin_mean": 79.37213134765625, "margin_dpo/margin_std": 107.55244445800781, "step": 337 }, { "KL/chosen_KL_mean": -142.20059204101562, "KL/mean": -178.76014709472656, "KL/rejected_KL_mean": -215.3197021484375, "KL/std": 93.02198028564453, "epoch": 0.49632892804698975, "fcm_dpo/beta": 0.005322361830621958, "fcm_dpo/delta": 0.011264639906585217, "fcm_dpo/margin": 73.11907958984375, "fcm_dpo/q_t": 0.40848663449287415, "grad_norm": 8.722229957580566, "learning_rate": 2.984578725527675e-07, "logits/chosen": -1.8243250846862793, "logits/rejected": -1.6700026988983154, "logps/chosen": -225.83071899414062, "logps/ref_chosen": -83.63011169433594, "logps/ref_rejected": -93.69235229492188, "logps/rejected": -309.0120544433594, "loss": 1.0909, "margin_dpo/margin_mean": 73.11907958984375, "margin_dpo/margin_std": 91.21495056152344, "step": 338 }, { "KL/chosen_KL_mean": -151.6955108642578, "KL/mean": -190.7566375732422, "KL/rejected_KL_mean": -229.8177490234375, "KL/std": 91.9947509765625, "epoch": 0.4977973568281938, "fcm_dpo/beta": 0.005350924097001553, "fcm_dpo/delta": -0.019602008163928986, "fcm_dpo/margin": 78.12224578857422, "fcm_dpo/q_t": 0.4030928611755371, "grad_norm": 8.072296142578125, "learning_rate": 2.9719824786231796e-07, "logits/chosen": -2.8388102054595947, "logits/rejected": -2.4054007530212402, "logps/chosen": -251.54733276367188, "logps/ref_chosen": -99.85182189941406, "logps/ref_rejected": -109.0743637084961, "logps/rejected": -338.8921203613281, "loss": 1.086, "margin_dpo/margin_mean": 78.12224578857422, "margin_dpo/margin_std": 102.2228775024414, "step": 339 }, { "KL/chosen_KL_mean": -163.107666015625, "KL/mean": -196.69796752929688, "KL/rejected_KL_mean": -230.28823852539062, "KL/std": 84.25697326660156, "epoch": 0.49926578560939794, "fcm_dpo/beta": 0.005345257930457592, "fcm_dpo/delta": 0.042408183217048645, "fcm_dpo/margin": 67.18055725097656, "fcm_dpo/q_t": 0.4158833920955658, "grad_norm": 9.673835754394531, "learning_rate": 2.959373794541426e-07, "logits/chosen": -1.3282344341278076, "logits/rejected": -0.8373836278915405, "logps/chosen": -248.08163452148438, "logps/ref_chosen": -84.97396850585938, "logps/ref_rejected": -78.80033874511719, "logps/rejected": -309.08856201171875, "loss": 1.1164, "margin_dpo/margin_mean": 67.18055725097656, "margin_dpo/margin_std": 92.64830017089844, "step": 340 }, { "KL/chosen_KL_mean": -170.20541381835938, "KL/mean": -203.14044189453125, "KL/rejected_KL_mean": -236.07545471191406, "KL/std": 88.54129028320312, "epoch": 0.5007342143906021, "fcm_dpo/beta": 0.005381280090659857, "fcm_dpo/delta": 0.04721618816256523, "fcm_dpo/margin": 65.87007141113281, "fcm_dpo/q_t": 0.4185011088848114, "grad_norm": 9.223365783691406, "learning_rate": 2.946753005532965e-07, "logits/chosen": -1.2060065269470215, "logits/rejected": -0.9475265741348267, "logps/chosen": -255.46939086914062, "logps/ref_chosen": -85.26399230957031, "logps/ref_rejected": -106.08595275878906, "logps/rejected": -342.1614074707031, "loss": 1.1355, "margin_dpo/margin_mean": 65.87007141113281, "margin_dpo/margin_std": 101.74478149414062, "step": 341 }, { "KL/chosen_KL_mean": -136.68931579589844, "KL/mean": -184.7283935546875, "KL/rejected_KL_mean": -232.76748657226562, "KL/std": 89.182861328125, "epoch": 0.5022026431718062, "fcm_dpo/beta": 0.00528441509231925, "fcm_dpo/delta": -0.11477241665124893, "fcm_dpo/margin": 96.07818603515625, "fcm_dpo/q_t": 0.3799949288368225, "grad_norm": 7.601553440093994, "learning_rate": 2.934120444167326e-07, "logits/chosen": -1.5630240440368652, "logits/rejected": -0.8176259994506836, "logps/chosen": -237.7279052734375, "logps/ref_chosen": -101.03860473632812, "logps/ref_rejected": -98.89851379394531, "logps/rejected": -331.666015625, "loss": 1.0064, "margin_dpo/margin_mean": 96.07818603515625, "margin_dpo/margin_std": 95.18651580810547, "step": 342 }, { "KL/chosen_KL_mean": -148.65756225585938, "KL/mean": -188.5728759765625, "KL/rejected_KL_mean": -228.4882049560547, "KL/std": 92.72538757324219, "epoch": 0.5036710719530103, "fcm_dpo/beta": 0.0052969674579799175, "fcm_dpo/delta": -0.024517640471458435, "fcm_dpo/margin": 79.83065032958984, "fcm_dpo/q_t": 0.40168386697769165, "grad_norm": 10.07509708404541, "learning_rate": 2.9214764433242476e-07, "logits/chosen": -2.0177361965179443, "logits/rejected": -1.9063098430633545, "logps/chosen": -232.40492248535156, "logps/ref_chosen": -83.74736022949219, "logps/ref_rejected": -116.57908630371094, "logps/rejected": -345.0672912597656, "loss": 1.0732, "margin_dpo/margin_mean": 79.83065032958984, "margin_dpo/margin_std": 96.46855163574219, "step": 343 }, { "KL/chosen_KL_mean": -148.3113250732422, "KL/mean": -190.6796875, "KL/rejected_KL_mean": -233.04803466796875, "KL/std": 98.50932312011719, "epoch": 0.5051395007342144, "fcm_dpo/beta": 0.0052472627721726894, "fcm_dpo/delta": -0.047002628445625305, "fcm_dpo/margin": 84.73670959472656, "fcm_dpo/q_t": 0.3975580036640167, "grad_norm": 9.563158988952637, "learning_rate": 2.9088213361849126e-07, "logits/chosen": -1.4217859506607056, "logits/rejected": -1.1645240783691406, "logps/chosen": -226.56529235839844, "logps/ref_chosen": -78.25396728515625, "logps/ref_rejected": -95.42870330810547, "logps/rejected": -328.47674560546875, "loss": 1.062, "margin_dpo/margin_mean": 84.73670959472656, "margin_dpo/margin_std": 103.69817352294922, "step": 344 }, { "KL/chosen_KL_mean": -152.54119873046875, "KL/mean": -197.28778076171875, "KL/rejected_KL_mean": -242.03436279296875, "KL/std": 98.54505157470703, "epoch": 0.5066079295154186, "fcm_dpo/beta": 0.005150836426764727, "fcm_dpo/delta": -0.06405314803123474, "fcm_dpo/margin": 89.49317932128906, "fcm_dpo/q_t": 0.393313467502594, "grad_norm": 9.231491088867188, "learning_rate": 2.896155456223163e-07, "logits/chosen": -1.3496811389923096, "logits/rejected": -1.0455434322357178, "logps/chosen": -250.6206817626953, "logps/ref_chosen": -98.0794906616211, "logps/ref_rejected": -104.03477478027344, "logps/rejected": -346.06915283203125, "loss": 1.0417, "margin_dpo/margin_mean": 89.49317932128906, "margin_dpo/margin_std": 102.03256225585938, "step": 345 }, { "KL/chosen_KL_mean": -169.85646057128906, "KL/mean": -204.7012481689453, "KL/rejected_KL_mean": -239.54603576660156, "KL/std": 91.71568298339844, "epoch": 0.5080763582966226, "fcm_dpo/beta": 0.00517165195196867, "fcm_dpo/delta": 0.04108089953660965, "fcm_dpo/margin": 69.68959045410156, "fcm_dpo/q_t": 0.4170967936515808, "grad_norm": 7.8569159507751465, "learning_rate": 2.883479137196714e-07, "logits/chosen": -1.782008171081543, "logits/rejected": -1.4670684337615967, "logps/chosen": -250.53994750976562, "logps/ref_chosen": -80.68348693847656, "logps/ref_rejected": -85.89260864257812, "logps/rejected": -325.43865966796875, "loss": 1.127, "margin_dpo/margin_mean": 69.68959045410156, "margin_dpo/margin_std": 104.76182556152344, "step": 346 }, { "KL/chosen_KL_mean": -160.25259399414062, "KL/mean": -193.49758911132812, "KL/rejected_KL_mean": -226.7425994873047, "KL/std": 91.04843139648438, "epoch": 0.5095447870778267, "fcm_dpo/beta": 0.005242632701992989, "fcm_dpo/delta": 0.05287010222673416, "fcm_dpo/margin": 66.49000549316406, "fcm_dpo/q_t": 0.4196370244026184, "grad_norm": 8.28531551361084, "learning_rate": 2.8707927131383614e-07, "logits/chosen": -1.823677897453308, "logits/rejected": -1.4991012811660767, "logps/chosen": -253.93197631835938, "logps/ref_chosen": -93.67938232421875, "logps/ref_rejected": -97.54251098632812, "logps/rejected": -324.28509521484375, "loss": 1.1317, "margin_dpo/margin_mean": 66.49000549316406, "margin_dpo/margin_std": 98.50048828125, "step": 347 }, { "KL/chosen_KL_mean": -138.22390747070312, "KL/mean": -176.05760192871094, "KL/rejected_KL_mean": -213.89129638671875, "KL/std": 86.05085754394531, "epoch": 0.5110132158590308, "fcm_dpo/beta": 0.005200096406042576, "fcm_dpo/delta": 0.005415529012680054, "fcm_dpo/margin": 75.66741180419922, "fcm_dpo/q_t": 0.40743619203567505, "grad_norm": 7.802852153778076, "learning_rate": 2.858096518347179e-07, "logits/chosen": -2.070651054382324, "logits/rejected": -1.8016595840454102, "logps/chosen": -228.4761505126953, "logps/ref_chosen": -90.25224304199219, "logps/ref_rejected": -95.77027893066406, "logps/rejected": -309.66156005859375, "loss": 1.0881, "margin_dpo/margin_mean": 75.66740417480469, "margin_dpo/margin_std": 87.98596954345703, "step": 348 }, { "KL/chosen_KL_mean": -148.9186553955078, "KL/mean": -184.7706298828125, "KL/rejected_KL_mean": -220.6226348876953, "KL/std": 94.21434783935547, "epoch": 0.5124816446402349, "fcm_dpo/beta": 0.0052825938910245895, "fcm_dpo/delta": 0.02165193296968937, "fcm_dpo/margin": 71.70397186279297, "fcm_dpo/q_t": 0.4125690460205078, "grad_norm": 9.777017593383789, "learning_rate": 2.845390887379706e-07, "logits/chosen": -2.01778507232666, "logits/rejected": -1.7916297912597656, "logps/chosen": -240.97674560546875, "logps/ref_chosen": -92.05809020996094, "logps/ref_rejected": -103.96527099609375, "logps/rejected": -324.587890625, "loss": 1.1068, "margin_dpo/margin_mean": 71.70397186279297, "margin_dpo/margin_std": 97.03965759277344, "step": 349 }, { "KL/chosen_KL_mean": -158.0184326171875, "KL/mean": -187.99649047851562, "KL/rejected_KL_mean": -217.97454833984375, "KL/std": 86.782958984375, "epoch": 0.5139500734214391, "fcm_dpo/beta": 0.005327719263732433, "fcm_dpo/delta": 0.08329633623361588, "fcm_dpo/margin": 59.95611572265625, "fcm_dpo/q_t": 0.42591628432273865, "grad_norm": 10.249733924865723, "learning_rate": 2.8326761550411346e-07, "logits/chosen": -1.8043642044067383, "logits/rejected": -1.5764849185943604, "logps/chosen": -245.0726318359375, "logps/ref_chosen": -87.05419921875, "logps/ref_rejected": -95.72193145751953, "logps/rejected": -313.69647216796875, "loss": 1.1531, "margin_dpo/margin_mean": 59.95611572265625, "margin_dpo/margin_std": 94.26403045654297, "step": 350 }, { "KL/chosen_KL_mean": -157.28900146484375, "KL/mean": -199.970947265625, "KL/rejected_KL_mean": -242.6529083251953, "KL/std": 96.5973129272461, "epoch": 0.5154185022026432, "fcm_dpo/beta": 0.0053214505314826965, "fcm_dpo/delta": -0.05680886656045914, "fcm_dpo/margin": 85.36390686035156, "fcm_dpo/q_t": 0.39450904726982117, "grad_norm": 8.475173950195312, "learning_rate": 2.819952656376487e-07, "logits/chosen": -1.5962677001953125, "logits/rejected": -1.2879164218902588, "logps/chosen": -246.45803833007812, "logps/ref_chosen": -89.16903686523438, "logps/ref_rejected": -108.29801940917969, "logps/rejected": -350.950927734375, "loss": 1.0534, "margin_dpo/margin_mean": 85.36390686035156, "margin_dpo/margin_std": 102.795166015625, "step": 351 }, { "KL/chosen_KL_mean": -144.54551696777344, "KL/mean": -184.79405212402344, "KL/rejected_KL_mean": -225.0426025390625, "KL/std": 95.195556640625, "epoch": 0.5168869309838473, "fcm_dpo/beta": 0.005275387316942215, "fcm_dpo/delta": -0.02574649080634117, "fcm_dpo/margin": 80.49707794189453, "fcm_dpo/q_t": 0.4031776785850525, "grad_norm": 8.512436866760254, "learning_rate": 2.8072207266617854e-07, "logits/chosen": -1.8377041816711426, "logits/rejected": -1.283332109451294, "logps/chosen": -241.35699462890625, "logps/ref_chosen": -96.81147766113281, "logps/ref_rejected": -84.40483093261719, "logps/rejected": -309.4474182128906, "loss": 1.0758, "margin_dpo/margin_mean": 80.49707794189453, "margin_dpo/margin_std": 102.7906494140625, "step": 352 }, { "KL/chosen_KL_mean": -164.73947143554688, "KL/mean": -190.3006591796875, "KL/rejected_KL_mean": -215.86184692382812, "KL/std": 84.70457458496094, "epoch": 0.5183553597650514, "fcm_dpo/beta": 0.005355102941393852, "fcm_dpo/delta": 0.129995658993721, "fcm_dpo/margin": 51.12238311767578, "fcm_dpo/q_t": 0.4361839294433594, "grad_norm": 9.822903633117676, "learning_rate": 2.794480701395219e-07, "logits/chosen": -2.083961248397827, "logits/rejected": -1.6949265003204346, "logps/chosen": -250.10195922851562, "logps/ref_chosen": -85.36248779296875, "logps/ref_rejected": -87.36732482910156, "logps/rejected": -303.22918701171875, "loss": 1.1954, "margin_dpo/margin_mean": 51.12238311767578, "margin_dpo/margin_std": 94.99560546875, "step": 353 }, { "KL/chosen_KL_mean": -144.2899932861328, "KL/mean": -184.491455078125, "KL/rejected_KL_mean": -224.6929168701172, "KL/std": 86.07633972167969, "epoch": 0.5198237885462555, "fcm_dpo/beta": 0.00539117306470871, "fcm_dpo/delta": -0.03500629961490631, "fcm_dpo/margin": 80.40291595458984, "fcm_dpo/q_t": 0.39802688360214233, "grad_norm": 8.374780654907227, "learning_rate": 2.781732916288303e-07, "logits/chosen": -1.9189400672912598, "logits/rejected": -1.4527201652526855, "logps/chosen": -240.09115600585938, "logps/ref_chosen": -95.8011703491211, "logps/ref_rejected": -94.86614227294922, "logps/rejected": -319.5590515136719, "loss": 1.0422, "margin_dpo/margin_mean": 80.40291595458984, "margin_dpo/margin_std": 79.29198455810547, "step": 354 }, { "KL/chosen_KL_mean": -141.72293090820312, "KL/mean": -179.54232788085938, "KL/rejected_KL_mean": -217.36172485351562, "KL/std": 87.66195678710938, "epoch": 0.5212922173274597, "fcm_dpo/beta": 0.005346843507140875, "fcm_dpo/delta": -0.004883013665676117, "fcm_dpo/margin": 75.63878631591797, "fcm_dpo/q_t": 0.40546560287475586, "grad_norm": 9.587749481201172, "learning_rate": 2.7689777072570284e-07, "logits/chosen": -2.3642358779907227, "logits/rejected": -1.8737528324127197, "logps/chosen": -227.90847778320312, "logps/ref_chosen": -86.185546875, "logps/ref_rejected": -88.4615707397461, "logps/rejected": -305.82330322265625, "loss": 1.0845, "margin_dpo/margin_mean": 75.63878631591797, "margin_dpo/margin_std": 93.0059814453125, "step": 355 }, { "KL/chosen_KL_mean": -167.07469177246094, "KL/mean": -196.17752075195312, "KL/rejected_KL_mean": -225.2803497314453, "KL/std": 86.08345031738281, "epoch": 0.5227606461086637, "fcm_dpo/beta": 0.005449127405881882, "fcm_dpo/delta": 0.08510417491197586, "fcm_dpo/margin": 58.205657958984375, "fcm_dpo/q_t": 0.4265674948692322, "grad_norm": 9.30924129486084, "learning_rate": 2.7562154104130176e-07, "logits/chosen": -1.8423707485198975, "logits/rejected": -1.3742547035217285, "logps/chosen": -257.8927917480469, "logps/ref_chosen": -90.818115234375, "logps/ref_rejected": -82.1273193359375, "logps/rejected": -307.40765380859375, "loss": 1.1636, "margin_dpo/margin_mean": 58.205657958984375, "margin_dpo/margin_std": 98.74101257324219, "step": 356 }, { "KL/chosen_KL_mean": -139.47706604003906, "KL/mean": -186.3784637451172, "KL/rejected_KL_mean": -233.27987670898438, "KL/std": 94.34734344482422, "epoch": 0.5242290748898678, "fcm_dpo/beta": 0.005332863423973322, "fcm_dpo/delta": -0.10724751651287079, "fcm_dpo/margin": 93.80278015136719, "fcm_dpo/q_t": 0.3824051022529602, "grad_norm": 8.545256614685059, "learning_rate": 2.7434463620546594e-07, "logits/chosen": -1.8580143451690674, "logits/rejected": -1.4828578233718872, "logps/chosen": -226.98182678222656, "logps/ref_chosen": -87.5047607421875, "logps/ref_rejected": -92.24937438964844, "logps/rejected": -325.52923583984375, "loss": 1.0019, "margin_dpo/margin_mean": 93.80278015136719, "margin_dpo/margin_std": 87.20972442626953, "step": 357 }, { "KL/chosen_KL_mean": -150.90765380859375, "KL/mean": -185.48580932617188, "KL/rejected_KL_mean": -220.06393432617188, "KL/std": 88.06880950927734, "epoch": 0.5256975036710719, "fcm_dpo/beta": 0.005362528841942549, "fcm_dpo/delta": 0.0302790068089962, "fcm_dpo/margin": 69.15626525878906, "fcm_dpo/q_t": 0.41520389914512634, "grad_norm": 8.945636749267578, "learning_rate": 2.730670898658255e-07, "logits/chosen": -1.6291003227233887, "logits/rejected": -1.1567466259002686, "logps/chosen": -245.96311950683594, "logps/ref_chosen": -95.05546569824219, "logps/ref_rejected": -96.47611999511719, "logps/rejected": -316.5400390625, "loss": 1.1091, "margin_dpo/margin_mean": 69.15626525878906, "margin_dpo/margin_std": 94.54415893554688, "step": 358 }, { "KL/chosen_KL_mean": -167.6346893310547, "KL/mean": -199.76876831054688, "KL/rejected_KL_mean": -231.90283203125, "KL/std": 83.03228759765625, "epoch": 0.527165932452276, "fcm_dpo/beta": 0.0053904419764876366, "fcm_dpo/delta": 0.05525226891040802, "fcm_dpo/margin": 64.26815032958984, "fcm_dpo/q_t": 0.42040812969207764, "grad_norm": 9.140324592590332, "learning_rate": 2.717889356869146e-07, "logits/chosen": -1.4964267015457153, "logits/rejected": -0.9199814200401306, "logps/chosen": -255.00802612304688, "logps/ref_chosen": -87.37332916259766, "logps/ref_rejected": -88.77188110351562, "logps/rejected": -320.6747131347656, "loss": 1.1351, "margin_dpo/margin_mean": 64.26814270019531, "margin_dpo/margin_std": 95.560546875, "step": 359 }, { "KL/chosen_KL_mean": -153.234130859375, "KL/mean": -194.63311767578125, "KL/rejected_KL_mean": -236.03208923339844, "KL/std": 83.87837219238281, "epoch": 0.5286343612334802, "fcm_dpo/beta": 0.005387818906456232, "fcm_dpo/delta": -0.04833075404167175, "fcm_dpo/margin": 82.7979736328125, "fcm_dpo/q_t": 0.39470797777175903, "grad_norm": 11.412223815917969, "learning_rate": 2.7051020734928443e-07, "logits/chosen": -1.4258179664611816, "logits/rejected": -1.0705134868621826, "logps/chosen": -228.45758056640625, "logps/ref_chosen": -75.22344970703125, "logps/ref_rejected": -76.09400177001953, "logps/rejected": -312.1260986328125, "loss": 1.0414, "margin_dpo/margin_mean": 82.79796600341797, "margin_dpo/margin_std": 87.13102722167969, "step": 360 }, { "KL/chosen_KL_mean": -157.49366760253906, "KL/mean": -193.69223022460938, "KL/rejected_KL_mean": -229.89077758789062, "KL/std": 94.52250671386719, "epoch": 0.5301027900146843, "fcm_dpo/beta": 0.00539267435669899, "fcm_dpo/delta": 0.009954613633453846, "fcm_dpo/margin": 72.39712524414062, "fcm_dpo/q_t": 0.41063615679740906, "grad_norm": 10.186558723449707, "learning_rate": 2.6923093854861593e-07, "logits/chosen": -1.7251112461090088, "logits/rejected": -1.4804906845092773, "logps/chosen": -239.44699096679688, "logps/ref_chosen": -81.95333099365234, "logps/ref_rejected": -96.67919921875, "logps/rejected": -326.5699768066406, "loss": 1.102, "margin_dpo/margin_mean": 72.39713287353516, "margin_dpo/margin_std": 100.34922790527344, "step": 361 }, { "KL/chosen_KL_mean": -163.6489715576172, "KL/mean": -201.3729248046875, "KL/rejected_KL_mean": -239.0968780517578, "KL/std": 89.62762451171875, "epoch": 0.5315712187958884, "fcm_dpo/beta": 0.005375918000936508, "fcm_dpo/delta": -0.0059273988008499146, "fcm_dpo/margin": 75.44790649414062, "fcm_dpo/q_t": 0.40532955527305603, "grad_norm": 9.284049987792969, "learning_rate": 2.679511629948319e-07, "logits/chosen": -1.9747636318206787, "logits/rejected": -1.775768518447876, "logps/chosen": -252.98867797851562, "logps/ref_chosen": -89.33971405029297, "logps/ref_rejected": -110.50196838378906, "logps/rejected": -349.5988464355469, "loss": 1.0841, "margin_dpo/margin_mean": 75.44790649414062, "margin_dpo/margin_std": 94.416259765625, "step": 362 }, { "KL/chosen_KL_mean": -152.61293029785156, "KL/mean": -201.7706298828125, "KL/rejected_KL_mean": -250.9283447265625, "KL/std": 94.21994018554688, "epoch": 0.5330396475770925, "fcm_dpo/beta": 0.005305218044668436, "fcm_dpo/delta": -0.12809552252292633, "fcm_dpo/margin": 98.31539916992188, "fcm_dpo/q_t": 0.37988364696502686, "grad_norm": 10.135531425476074, "learning_rate": 2.6667091441120816e-07, "logits/chosen": -1.8009496927261353, "logits/rejected": -1.3658455610275269, "logps/chosen": -233.38792419433594, "logps/ref_chosen": -80.77499389648438, "logps/ref_rejected": -79.43267822265625, "logps/rejected": -330.36102294921875, "loss": 0.9992, "margin_dpo/margin_mean": 98.31539916992188, "margin_dpo/margin_std": 101.54324340820312, "step": 363 }, { "KL/chosen_KL_mean": -150.95285034179688, "KL/mean": -190.74490356445312, "KL/rejected_KL_mean": -230.53689575195312, "KL/std": 87.7202377319336, "epoch": 0.5345080763582967, "fcm_dpo/beta": 0.0052646249532699585, "fcm_dpo/delta": -0.02044062316417694, "fcm_dpo/margin": 79.58405303955078, "fcm_dpo/q_t": 0.40109869837760925, "grad_norm": 11.583788871765137, "learning_rate": 2.6539022653348575e-07, "logits/chosen": -1.9678800106048584, "logits/rejected": -1.8237006664276123, "logps/chosen": -227.48245239257812, "logps/ref_chosen": -76.52960205078125, "logps/ref_rejected": -97.56173706054688, "logps/rejected": -328.0986328125, "loss": 1.0602, "margin_dpo/margin_mean": 79.58405303955078, "margin_dpo/margin_std": 83.42306518554688, "step": 364 }, { "KL/chosen_KL_mean": -147.66452026367188, "KL/mean": -190.9501190185547, "KL/rejected_KL_mean": -234.23573303222656, "KL/std": 93.64146423339844, "epoch": 0.5359765051395007, "fcm_dpo/beta": 0.005204772111028433, "fcm_dpo/delta": -0.05304336175322533, "fcm_dpo/margin": 86.57122039794922, "fcm_dpo/q_t": 0.3970166742801666, "grad_norm": 8.863701820373535, "learning_rate": 2.641091331089811e-07, "logits/chosen": -1.7680310010910034, "logits/rejected": -1.641089916229248, "logps/chosen": -228.20941162109375, "logps/ref_chosen": -80.54489135742188, "logps/ref_rejected": -99.32316589355469, "logps/rejected": -333.55889892578125, "loss": 1.0577, "margin_dpo/margin_mean": 86.57121276855469, "margin_dpo/margin_std": 106.39285278320312, "step": 365 }, { "KL/chosen_KL_mean": -149.32733154296875, "KL/mean": -185.19911193847656, "KL/rejected_KL_mean": -221.07089233398438, "KL/std": 97.117431640625, "epoch": 0.5374449339207048, "fcm_dpo/beta": 0.005193917080760002, "fcm_dpo/delta": 0.028429482132196426, "fcm_dpo/margin": 71.7435531616211, "fcm_dpo/q_t": 0.41512954235076904, "grad_norm": 8.329041481018066, "learning_rate": 2.6282766789569736e-07, "logits/chosen": -1.9326473474502563, "logits/rejected": -1.9646201133728027, "logps/chosen": -221.56539916992188, "logps/ref_chosen": -72.23806762695312, "logps/ref_rejected": -91.67182922363281, "logps/rejected": -312.74273681640625, "loss": 1.1031, "margin_dpo/margin_mean": 71.74354553222656, "margin_dpo/margin_std": 94.58836364746094, "step": 366 }, { "KL/chosen_KL_mean": -135.5565185546875, "KL/mean": -178.5682373046875, "KL/rejected_KL_mean": -221.5799560546875, "KL/std": 85.60757446289062, "epoch": 0.5389133627019089, "fcm_dpo/beta": 0.005168301984667778, "fcm_dpo/delta": -0.04665067046880722, "fcm_dpo/margin": 86.02342224121094, "fcm_dpo/q_t": 0.3957254886627197, "grad_norm": 8.32479190826416, "learning_rate": 2.615458646614349e-07, "logits/chosen": -1.8714748620986938, "logits/rejected": -1.5303019285202026, "logps/chosen": -220.978515625, "logps/ref_chosen": -85.42201232910156, "logps/ref_rejected": -83.51779174804688, "logps/rejected": -305.0977478027344, "loss": 1.0393, "margin_dpo/margin_mean": 86.02342224121094, "margin_dpo/margin_std": 88.7979736328125, "step": 367 }, { "KL/chosen_KL_mean": -127.16500854492188, "KL/mean": -174.32020568847656, "KL/rejected_KL_mean": -221.47540283203125, "KL/std": 94.36270904541016, "epoch": 0.540381791483113, "fcm_dpo/beta": 0.005108362063765526, "fcm_dpo/delta": -0.08589579910039902, "fcm_dpo/margin": 94.31039428710938, "fcm_dpo/q_t": 0.38680511713027954, "grad_norm": 9.08455753326416, "learning_rate": 2.6026375718290083e-07, "logits/chosen": -1.3821510076522827, "logits/rejected": -1.3483476638793945, "logps/chosen": -201.00613403320312, "logps/ref_chosen": -73.84112548828125, "logps/ref_rejected": -104.69528198242188, "logps/rejected": -326.1706848144531, "loss": 1.0206, "margin_dpo/margin_mean": 94.31039428710938, "margin_dpo/margin_std": 95.70059967041016, "step": 368 }, { "KL/chosen_KL_mean": -140.31944274902344, "KL/mean": -184.54110717773438, "KL/rejected_KL_mean": -228.76278686523438, "KL/std": 87.055908203125, "epoch": 0.5418502202643172, "fcm_dpo/beta": 0.005039115436375141, "fcm_dpo/delta": -0.047789957374334335, "fcm_dpo/margin": 88.4433364868164, "fcm_dpo/q_t": 0.39524587988853455, "grad_norm": 8.945512771606445, "learning_rate": 2.589813792448196e-07, "logits/chosen": -1.8400635719299316, "logits/rejected": -1.480085849761963, "logps/chosen": -238.78140258789062, "logps/ref_chosen": -98.46195983886719, "logps/ref_rejected": -99.15672302246094, "logps/rejected": -327.91949462890625, "loss": 1.0351, "margin_dpo/margin_mean": 88.4433364868164, "margin_dpo/margin_std": 87.24639892578125, "step": 369 }, { "KL/chosen_KL_mean": -156.91098022460938, "KL/mean": -186.57937622070312, "KL/rejected_KL_mean": -216.24778747558594, "KL/std": 84.00354766845703, "epoch": 0.5433186490455213, "fcm_dpo/beta": 0.005095846019685268, "fcm_dpo/delta": 0.10061434656381607, "fcm_dpo/margin": 59.336788177490234, "fcm_dpo/q_t": 0.4290916919708252, "grad_norm": 8.560161590576172, "learning_rate": 2.5769876463904263e-07, "logits/chosen": -2.002098321914673, "logits/rejected": -1.8049036264419556, "logps/chosen": -253.6867218017578, "logps/ref_chosen": -96.77574157714844, "logps/ref_rejected": -102.74411010742188, "logps/rejected": -318.99188232421875, "loss": 1.1656, "margin_dpo/margin_mean": 59.3367919921875, "margin_dpo/margin_std": 97.11265563964844, "step": 370 }, { "KL/chosen_KL_mean": -148.52569580078125, "KL/mean": -181.70806884765625, "KL/rejected_KL_mean": -214.89047241210938, "KL/std": 84.48982238769531, "epoch": 0.5447870778267254, "fcm_dpo/beta": 0.0051774801686406136, "fcm_dpo/delta": 0.05810657516121864, "fcm_dpo/margin": 66.36475372314453, "fcm_dpo/q_t": 0.4192705750465393, "grad_norm": 11.066864013671875, "learning_rate": 2.5641594716365744e-07, "logits/chosen": -1.7561756372451782, "logits/rejected": -1.3831920623779297, "logps/chosen": -251.51589965820312, "logps/ref_chosen": -102.99020385742188, "logps/ref_rejected": -105.35062408447266, "logps/rejected": -320.2410888671875, "loss": 1.1273, "margin_dpo/margin_mean": 66.36475372314453, "margin_dpo/margin_std": 92.12724304199219, "step": 371 }, { "KL/chosen_KL_mean": -126.91694641113281, "KL/mean": -176.1567840576172, "KL/rejected_KL_mean": -225.39662170410156, "KL/std": 98.13107299804688, "epoch": 0.5462555066079295, "fcm_dpo/beta": 0.005101591348648071, "fcm_dpo/delta": -0.10773710906505585, "fcm_dpo/margin": 98.47967529296875, "fcm_dpo/q_t": 0.3835327923297882, "grad_norm": 7.525407314300537, "learning_rate": 2.551329606220976e-07, "logits/chosen": -1.7830212116241455, "logits/rejected": -1.1372520923614502, "logps/chosen": -217.94317626953125, "logps/ref_chosen": -91.02623748779297, "logps/ref_rejected": -85.6102294921875, "logps/rejected": -311.0068359375, "loss": 1.0067, "margin_dpo/margin_mean": 98.47967529296875, "margin_dpo/margin_std": 100.09083557128906, "step": 372 }, { "KL/chosen_KL_mean": -146.71539306640625, "KL/mean": -183.12155151367188, "KL/rejected_KL_mean": -219.52767944335938, "KL/std": 93.6815414428711, "epoch": 0.5477239353891337, "fcm_dpo/beta": 0.0050873481668531895, "fcm_dpo/delta": 0.030727455392479897, "fcm_dpo/margin": 72.81230163574219, "fcm_dpo/q_t": 0.4139280319213867, "grad_norm": 10.22166919708252, "learning_rate": 2.538498388222517e-07, "logits/chosen": -2.1616246700286865, "logits/rejected": -1.6437251567840576, "logps/chosen": -246.78846740722656, "logps/ref_chosen": -100.07307434082031, "logps/ref_rejected": -91.11634826660156, "logps/rejected": -310.64404296875, "loss": 1.1121, "margin_dpo/margin_mean": 72.81230163574219, "margin_dpo/margin_std": 100.50997924804688, "step": 373 }, { "KL/chosen_KL_mean": -128.02476501464844, "KL/mean": -168.38232421875, "KL/rejected_KL_mean": -208.7399139404297, "KL/std": 88.89723205566406, "epoch": 0.5491923641703378, "fcm_dpo/beta": 0.005080570466816425, "fcm_dpo/delta": -0.010633300989866257, "fcm_dpo/margin": 80.71514892578125, "fcm_dpo/q_t": 0.40380898118019104, "grad_norm": 8.068751335144043, "learning_rate": 2.525666155755725e-07, "logits/chosen": -2.122737407684326, "logits/rejected": -1.7238510847091675, "logps/chosen": -226.037109375, "logps/ref_chosen": -98.01234436035156, "logps/ref_rejected": -100.75224304199219, "logps/rejected": -309.4921569824219, "loss": 1.0622, "margin_dpo/margin_mean": 80.71514892578125, "margin_dpo/margin_std": 85.29051208496094, "step": 374 }, { "KL/chosen_KL_mean": -129.74090576171875, "KL/mean": -175.42880249023438, "KL/rejected_KL_mean": -221.11669921875, "KL/std": 85.85121154785156, "epoch": 0.5506607929515418, "fcm_dpo/beta": 0.005012167617678642, "fcm_dpo/delta": -0.06188717484474182, "fcm_dpo/margin": 91.37579345703125, "fcm_dpo/q_t": 0.3918214440345764, "grad_norm": 9.415217399597168, "learning_rate": 2.512833246961859e-07, "logits/chosen": -1.6516311168670654, "logits/rejected": -1.4192759990692139, "logps/chosen": -214.13719177246094, "logps/ref_chosen": -84.39627838134766, "logps/ref_rejected": -95.7188491821289, "logps/rejected": -316.8355712890625, "loss": 1.0322, "margin_dpo/margin_mean": 91.37579345703125, "margin_dpo/margin_std": 91.16218566894531, "step": 375 }, { "KL/chosen_KL_mean": -125.13233184814453, "KL/mean": -175.6561279296875, "KL/rejected_KL_mean": -226.179931640625, "KL/std": 98.2298583984375, "epoch": 0.5521292217327459, "fcm_dpo/beta": 0.004943961277604103, "fcm_dpo/delta": -0.10492968559265137, "fcm_dpo/margin": 101.04759979248047, "fcm_dpo/q_t": 0.3837137222290039, "grad_norm": 9.656167984008789, "learning_rate": 2.5e-07, "logits/chosen": -1.4719998836517334, "logits/rejected": -1.1430370807647705, "logps/chosen": -220.87002563476562, "logps/ref_chosen": -95.73770141601562, "logps/ref_rejected": -111.97331237792969, "logps/rejected": -338.15325927734375, "loss": 1.0094, "margin_dpo/margin_mean": 101.04759216308594, "margin_dpo/margin_std": 104.1722412109375, "step": 376 }, { "KL/chosen_KL_mean": -133.2311553955078, "KL/mean": -185.32313537597656, "KL/rejected_KL_mean": -237.41510009765625, "KL/std": 98.94145202636719, "epoch": 0.55359765051395, "fcm_dpo/beta": 0.004824788309633732, "fcm_dpo/delta": -0.1088038757443428, "fcm_dpo/margin": 104.18394470214844, "fcm_dpo/q_t": 0.3818095922470093, "grad_norm": 7.88934326171875, "learning_rate": 2.487166753038141e-07, "logits/chosen": -1.4121242761611938, "logits/rejected": -1.294377326965332, "logps/chosen": -212.15811157226562, "logps/ref_chosen": -78.92694854736328, "logps/ref_rejected": -105.89892578125, "logps/rejected": -343.31402587890625, "loss": 0.9989, "margin_dpo/margin_mean": 104.18394470214844, "margin_dpo/margin_std": 98.33584594726562, "step": 377 }, { "KL/chosen_KL_mean": -129.8414764404297, "KL/mean": -171.4268798828125, "KL/rejected_KL_mean": -213.01229858398438, "KL/std": 94.24797058105469, "epoch": 0.5550660792951542, "fcm_dpo/beta": 0.004835226573050022, "fcm_dpo/delta": -0.0026134587824344635, "fcm_dpo/margin": 83.17083740234375, "fcm_dpo/q_t": 0.40480339527130127, "grad_norm": 9.472088813781738, "learning_rate": 2.4743338442442754e-07, "logits/chosen": -2.058898448944092, "logits/rejected": -1.781050205230713, "logps/chosen": -205.2138671875, "logps/ref_chosen": -75.37240600585938, "logps/ref_rejected": -93.70409393310547, "logps/rejected": -306.7164001464844, "loss": 1.0786, "margin_dpo/margin_mean": 83.17082977294922, "margin_dpo/margin_std": 97.11559295654297, "step": 378 }, { "KL/chosen_KL_mean": -152.46328735351562, "KL/mean": -189.37307739257812, "KL/rejected_KL_mean": -226.28286743164062, "KL/std": 97.5453109741211, "epoch": 0.5565345080763583, "fcm_dpo/beta": 0.0048427889123559, "fcm_dpo/delta": 0.044100962579250336, "fcm_dpo/margin": 73.819580078125, "fcm_dpo/q_t": 0.41824716329574585, "grad_norm": 8.71568489074707, "learning_rate": 2.461501611777483e-07, "logits/chosen": -1.8444311618804932, "logits/rejected": -1.7671618461608887, "logps/chosen": -241.3189697265625, "logps/ref_chosen": -88.85568237304688, "logps/ref_rejected": -121.38114166259766, "logps/rejected": -347.66400146484375, "loss": 1.1296, "margin_dpo/margin_mean": 73.819580078125, "margin_dpo/margin_std": 111.70608520507812, "step": 379 }, { "KL/chosen_KL_mean": -167.66091918945312, "KL/mean": -212.29371643066406, "KL/rejected_KL_mean": -256.926513671875, "KL/std": 99.86642456054688, "epoch": 0.5580029368575624, "fcm_dpo/beta": 0.004831093829125166, "fcm_dpo/delta": -0.032671816647052765, "fcm_dpo/margin": 89.26560974121094, "fcm_dpo/q_t": 0.40140682458877563, "grad_norm": 12.010602951049805, "learning_rate": 2.4486703937790243e-07, "logits/chosen": -1.3914977312088013, "logits/rejected": -1.3648738861083984, "logps/chosen": -247.49412536621094, "logps/ref_chosen": -79.83321380615234, "logps/ref_rejected": -110.7447509765625, "logps/rejected": -367.6712646484375, "loss": 1.0692, "margin_dpo/margin_mean": 89.26561737060547, "margin_dpo/margin_std": 113.45790100097656, "step": 380 }, { "KL/chosen_KL_mean": -147.15130615234375, "KL/mean": -185.87960815429688, "KL/rejected_KL_mean": -224.60791015625, "KL/std": 95.64622497558594, "epoch": 0.5594713656387665, "fcm_dpo/beta": 0.004844530485570431, "fcm_dpo/delta": 0.02571532130241394, "fcm_dpo/margin": 77.45659637451172, "fcm_dpo/q_t": 0.4118459224700928, "grad_norm": 9.567269325256348, "learning_rate": 2.435840528363426e-07, "logits/chosen": -1.8670192956924438, "logits/rejected": -1.3251140117645264, "logps/chosen": -233.7181396484375, "logps/ref_chosen": -86.56683349609375, "logps/ref_rejected": -84.416748046875, "logps/rejected": -309.024658203125, "loss": 1.1006, "margin_dpo/margin_mean": 77.45659637451172, "margin_dpo/margin_std": 98.81370544433594, "step": 381 }, { "KL/chosen_KL_mean": -144.52587890625, "KL/mean": -189.03732299804688, "KL/rejected_KL_mean": -233.54879760742188, "KL/std": 93.94854736328125, "epoch": 0.5609397944199707, "fcm_dpo/beta": 0.0048013608902692795, "fcm_dpo/delta": -0.02946055307984352, "fcm_dpo/margin": 89.0229263305664, "fcm_dpo/q_t": 0.3997684419155121, "grad_norm": 8.85572624206543, "learning_rate": 2.4230123536095745e-07, "logits/chosen": -1.951839804649353, "logits/rejected": -1.6882745027542114, "logps/chosen": -248.9876708984375, "logps/ref_chosen": -104.46180725097656, "logps/ref_rejected": -114.60223388671875, "logps/rejected": -348.1510314941406, "loss": 1.0568, "margin_dpo/margin_mean": 89.0229263305664, "margin_dpo/margin_std": 95.94386291503906, "step": 382 }, { "KL/chosen_KL_mean": -157.48068237304688, "KL/mean": -195.09051513671875, "KL/rejected_KL_mean": -232.70033264160156, "KL/std": 96.7877197265625, "epoch": 0.5624082232011748, "fcm_dpo/beta": 0.0048394761979579926, "fcm_dpo/delta": 0.03733859583735466, "fcm_dpo/margin": 75.21961975097656, "fcm_dpo/q_t": 0.41475772857666016, "grad_norm": 9.064055442810059, "learning_rate": 2.4101862075518037e-07, "logits/chosen": -1.74147367477417, "logits/rejected": -1.621586799621582, "logps/chosen": -239.5967559814453, "logps/ref_chosen": -82.11607360839844, "logps/ref_rejected": -98.77716064453125, "logps/rejected": -331.47747802734375, "loss": 1.1067, "margin_dpo/margin_mean": 75.2196273803711, "margin_dpo/margin_std": 94.78604125976562, "step": 383 }, { "KL/chosen_KL_mean": -168.5755615234375, "KL/mean": -206.14352416992188, "KL/rejected_KL_mean": -243.71153259277344, "KL/std": 101.11558532714844, "epoch": 0.5638766519823789, "fcm_dpo/beta": 0.0049112094566226006, "fcm_dpo/delta": 0.031138919293880463, "fcm_dpo/margin": 75.135986328125, "fcm_dpo/q_t": 0.4140623211860657, "grad_norm": 11.031511306762695, "learning_rate": 2.397362428170992e-07, "logits/chosen": -1.7411856651306152, "logits/rejected": -1.5146982669830322, "logps/chosen": -249.16079711914062, "logps/ref_chosen": -80.58525085449219, "logps/ref_rejected": -91.44789123535156, "logps/rejected": -335.159423828125, "loss": 1.1285, "margin_dpo/margin_mean": 75.13597869873047, "margin_dpo/margin_std": 111.42028045654297, "step": 384 }, { "KL/chosen_KL_mean": -139.665283203125, "KL/mean": -188.2332000732422, "KL/rejected_KL_mean": -236.80111694335938, "KL/std": 90.89952087402344, "epoch": 0.5653450807635829, "fcm_dpo/beta": 0.004812178667634726, "fcm_dpo/delta": -0.07157735526561737, "fcm_dpo/margin": 97.13581848144531, "fcm_dpo/q_t": 0.39141756296157837, "grad_norm": 10.002744674682617, "learning_rate": 2.3845413533856514e-07, "logits/chosen": -2.064868211746216, "logits/rejected": -1.4383587837219238, "logps/chosen": -232.42544555664062, "logps/ref_chosen": -92.76016235351562, "logps/ref_rejected": -84.45997619628906, "logps/rejected": -321.2611083984375, "loss": 1.0335, "margin_dpo/margin_mean": 97.13581085205078, "margin_dpo/margin_std": 104.595458984375, "step": 385 }, { "KL/chosen_KL_mean": -148.46829223632812, "KL/mean": -195.04586791992188, "KL/rejected_KL_mean": -241.62347412109375, "KL/std": 92.96345520019531, "epoch": 0.566813509544787, "fcm_dpo/beta": 0.0047836098819971085, "fcm_dpo/delta": -0.04774869978427887, "fcm_dpo/margin": 93.15520477294922, "fcm_dpo/q_t": 0.3943381905555725, "grad_norm": 7.758502006530762, "learning_rate": 2.3717233210430254e-07, "logits/chosen": -1.7004544734954834, "logits/rejected": -1.4167189598083496, "logps/chosen": -233.82723999023438, "logps/ref_chosen": -85.35896301269531, "logps/ref_rejected": -96.99065399169922, "logps/rejected": -338.6141357421875, "loss": 1.0452, "margin_dpo/margin_mean": 93.15519714355469, "margin_dpo/margin_std": 100.85755920410156, "step": 386 }, { "KL/chosen_KL_mean": -172.2111358642578, "KL/mean": -214.04965209960938, "KL/rejected_KL_mean": -255.88818359375, "KL/std": 90.3999252319336, "epoch": 0.5682819383259912, "fcm_dpo/beta": 0.004779561422765255, "fcm_dpo/delta": 1.473166048526764e-05, "fcm_dpo/margin": 83.67704772949219, "fcm_dpo/q_t": 0.40584662556648254, "grad_norm": 10.350507736206055, "learning_rate": 2.3589086689101889e-07, "logits/chosen": -2.3950657844543457, "logits/rejected": -1.7065882682800293, "logps/chosen": -274.8992919921875, "logps/ref_chosen": -102.68814086914062, "logps/ref_rejected": -99.42858123779297, "logps/rejected": -355.3167724609375, "loss": 1.0727, "margin_dpo/margin_mean": 83.67704772949219, "margin_dpo/margin_std": 92.84671783447266, "step": 387 }, { "KL/chosen_KL_mean": -173.95590209960938, "KL/mean": -219.8619384765625, "KL/rejected_KL_mean": -265.7679443359375, "KL/std": 99.94122314453125, "epoch": 0.5697503671071953, "fcm_dpo/beta": 0.004741578362882137, "fcm_dpo/delta": -0.036994777619838715, "fcm_dpo/margin": 91.81205749511719, "fcm_dpo/q_t": 0.3978651165962219, "grad_norm": 8.8153715133667, "learning_rate": 2.3460977346651428e-07, "logits/chosen": -1.9559533596038818, "logits/rejected": -1.839688777923584, "logps/chosen": -257.5411376953125, "logps/ref_chosen": -83.58523559570312, "logps/ref_rejected": -110.10037231445312, "logps/rejected": -375.8683166503906, "loss": 1.0539, "margin_dpo/margin_mean": 91.81205749511719, "margin_dpo/margin_std": 102.70115661621094, "step": 388 }, { "KL/chosen_KL_mean": -179.04478454589844, "KL/mean": -218.5767364501953, "KL/rejected_KL_mean": -258.1086730957031, "KL/std": 101.88360595703125, "epoch": 0.5712187958883994, "fcm_dpo/beta": 0.004752439446747303, "fcm_dpo/delta": 0.025208082050085068, "fcm_dpo/margin": 79.06390380859375, "fcm_dpo/q_t": 0.41277527809143066, "grad_norm": 10.859795570373535, "learning_rate": 2.3332908558879177e-07, "logits/chosen": -2.582777261734009, "logits/rejected": -2.1513166427612305, "logps/chosen": -268.8670654296875, "logps/ref_chosen": -89.822265625, "logps/ref_rejected": -84.85376739501953, "logps/rejected": -342.96246337890625, "loss": 1.1094, "margin_dpo/margin_mean": 79.06390380859375, "margin_dpo/margin_std": 109.668212890625, "step": 389 }, { "KL/chosen_KL_mean": -180.07965087890625, "KL/mean": -217.7396240234375, "KL/rejected_KL_mean": -255.3995819091797, "KL/std": 99.57670593261719, "epoch": 0.5726872246696035, "fcm_dpo/beta": 0.004768001846969128, "fcm_dpo/delta": 0.0421249158680439, "fcm_dpo/margin": 75.31990051269531, "fcm_dpo/q_t": 0.41875743865966797, "grad_norm": 8.654621124267578, "learning_rate": 2.320488370051681e-07, "logits/chosen": -1.5758507251739502, "logits/rejected": -1.3685557842254639, "logps/chosen": -257.6120910644531, "logps/ref_chosen": -77.53244018554688, "logps/ref_rejected": -92.58392333984375, "logps/rejected": -347.9835205078125, "loss": 1.1303, "margin_dpo/margin_mean": 75.31990814208984, "margin_dpo/margin_std": 114.56854248046875, "step": 390 }, { "KL/chosen_KL_mean": -162.88204956054688, "KL/mean": -203.8579864501953, "KL/rejected_KL_mean": -244.83392333984375, "KL/std": 95.37869262695312, "epoch": 0.5741556534508077, "fcm_dpo/beta": 0.004793091677129269, "fcm_dpo/delta": 0.007351510226726532, "fcm_dpo/margin": 81.95186614990234, "fcm_dpo/q_t": 0.408242791891098, "grad_norm": 8.766377449035645, "learning_rate": 2.3076906145138405e-07, "logits/chosen": -1.85637629032135, "logits/rejected": -1.4000773429870605, "logps/chosen": -251.114990234375, "logps/ref_chosen": -88.2329330444336, "logps/ref_rejected": -93.00756072998047, "logps/rejected": -337.84149169921875, "loss": 1.0879, "margin_dpo/margin_mean": 81.95187377929688, "margin_dpo/margin_std": 100.46595001220703, "step": 391 }, { "KL/chosen_KL_mean": -164.63592529296875, "KL/mean": -214.9346923828125, "KL/rejected_KL_mean": -265.23345947265625, "KL/std": 107.60557556152344, "epoch": 0.5756240822320118, "fcm_dpo/beta": 0.004765060730278492, "fcm_dpo/delta": -0.08329806476831436, "fcm_dpo/margin": 100.59749603271484, "fcm_dpo/q_t": 0.38864779472351074, "grad_norm": 9.282988548278809, "learning_rate": 2.294897926507156e-07, "logits/chosen": -1.4620879888534546, "logits/rejected": -1.3438429832458496, "logps/chosen": -252.38035583496094, "logps/ref_chosen": -87.74442291259766, "logps/ref_rejected": -108.57071685791016, "logps/rejected": -373.80413818359375, "loss": 1.0344, "margin_dpo/margin_mean": 100.59750366210938, "margin_dpo/margin_std": 114.80479431152344, "step": 392 }, { "KL/chosen_KL_mean": -145.29449462890625, "KL/mean": -190.38160705566406, "KL/rejected_KL_mean": -235.46868896484375, "KL/std": 101.34538269042969, "epoch": 0.5770925110132159, "fcm_dpo/beta": 0.004707988351583481, "fcm_dpo/delta": -0.0256502665579319, "fcm_dpo/margin": 90.17418670654297, "fcm_dpo/q_t": 0.4004923105239868, "grad_norm": 7.896361827850342, "learning_rate": 2.2821106431308543e-07, "logits/chosen": -1.8128411769866943, "logits/rejected": -1.4961481094360352, "logps/chosen": -221.85830688476562, "logps/ref_chosen": -76.56381225585938, "logps/ref_rejected": -79.72618103027344, "logps/rejected": -315.19488525390625, "loss": 1.0572, "margin_dpo/margin_mean": 90.17418670654297, "margin_dpo/margin_std": 98.73701477050781, "step": 393 }, { "KL/chosen_KL_mean": -155.27406311035156, "KL/mean": -204.72610473632812, "KL/rejected_KL_mean": -254.17813110351562, "KL/std": 96.45612335205078, "epoch": 0.57856093979442, "fcm_dpo/beta": 0.004660965874791145, "fcm_dpo/delta": -0.06392569839954376, "fcm_dpo/margin": 98.90409088134766, "fcm_dpo/q_t": 0.39087218046188354, "grad_norm": 38.80651092529297, "learning_rate": 2.2693291013417452e-07, "logits/chosen": -1.38722562789917, "logits/rejected": -1.1494613885879517, "logps/chosen": -239.97271728515625, "logps/ref_chosen": -84.69866180419922, "logps/ref_rejected": -97.457763671875, "logps/rejected": -351.6358947753906, "loss": 1.0237, "margin_dpo/margin_mean": 98.90408325195312, "margin_dpo/margin_std": 94.56683349609375, "step": 394 }, { "KL/chosen_KL_mean": -161.4971923828125, "KL/mean": -199.0489044189453, "KL/rejected_KL_mean": -236.6005859375, "KL/std": 101.01194763183594, "epoch": 0.580029368575624, "fcm_dpo/beta": 0.004660163074731827, "fcm_dpo/delta": 0.051645971834659576, "fcm_dpo/margin": 75.1033935546875, "fcm_dpo/q_t": 0.421477735042572, "grad_norm": 8.861296653747559, "learning_rate": 2.2565536379453404e-07, "logits/chosen": -2.608971118927002, "logits/rejected": -2.213060140609741, "logps/chosen": -250.3848419189453, "logps/ref_chosen": -88.88763427734375, "logps/ref_rejected": -91.0455322265625, "logps/rejected": -327.6461181640625, "loss": 1.1634, "margin_dpo/margin_mean": 75.1033935546875, "margin_dpo/margin_std": 136.51522827148438, "step": 395 }, { "KL/chosen_KL_mean": -169.39981079101562, "KL/mean": -203.4297637939453, "KL/rejected_KL_mean": -237.45970153808594, "KL/std": 89.18807983398438, "epoch": 0.5814977973568282, "fcm_dpo/beta": 0.004748208448290825, "fcm_dpo/delta": 0.07941794395446777, "fcm_dpo/margin": 68.05992126464844, "fcm_dpo/q_t": 0.4247896671295166, "grad_norm": 10.958311080932617, "learning_rate": 2.2437845895869825e-07, "logits/chosen": -1.8654687404632568, "logits/rejected": -1.3699885606765747, "logps/chosen": -271.87115478515625, "logps/ref_chosen": -102.47132873535156, "logps/ref_rejected": -93.23478698730469, "logps/rejected": -330.6944885253906, "loss": 1.1472, "margin_dpo/margin_mean": 68.05992126464844, "margin_dpo/margin_std": 103.806640625, "step": 396 }, { "KL/chosen_KL_mean": -152.5701904296875, "KL/mean": -212.58587646484375, "KL/rejected_KL_mean": -272.6015319824219, "KL/std": 103.95213317871094, "epoch": 0.5829662261380323, "fcm_dpo/beta": 0.004643572494387627, "fcm_dpo/delta": -0.1670106053352356, "fcm_dpo/margin": 120.03134155273438, "fcm_dpo/q_t": 0.37108665704727173, "grad_norm": 8.510375022888184, "learning_rate": 2.2310222927429716e-07, "logits/chosen": -2.180654525756836, "logits/rejected": -1.573169469833374, "logps/chosen": -253.6146240234375, "logps/ref_chosen": -101.04441833496094, "logps/ref_rejected": -108.8511734008789, "logps/rejected": -381.45269775390625, "loss": 0.982, "margin_dpo/margin_mean": 120.0313491821289, "margin_dpo/margin_std": 124.32489013671875, "step": 397 }, { "KL/chosen_KL_mean": -161.62478637695312, "KL/mean": -205.41476440429688, "KL/rejected_KL_mean": -249.2047576904297, "KL/std": 102.17959594726562, "epoch": 0.5844346549192364, "fcm_dpo/beta": 0.004603114910423756, "fcm_dpo/delta": -0.0033147865906357765, "fcm_dpo/margin": 87.57994842529297, "fcm_dpo/q_t": 0.407081663608551, "grad_norm": 8.125770568847656, "learning_rate": 2.2182670837116972e-07, "logits/chosen": -2.1205811500549316, "logits/rejected": -1.8388051986694336, "logps/chosen": -256.67401123046875, "logps/ref_chosen": -95.04922485351562, "logps/ref_rejected": -114.00563049316406, "logps/rejected": -363.21038818359375, "loss": 1.0953, "margin_dpo/margin_mean": 87.5799560546875, "margin_dpo/margin_std": 120.67646026611328, "step": 398 }, { "KL/chosen_KL_mean": -141.64822387695312, "KL/mean": -196.94744873046875, "KL/rejected_KL_mean": -252.24667358398438, "KL/std": 92.30752563476562, "epoch": 0.5859030837004405, "fcm_dpo/beta": 0.004543212708085775, "fcm_dpo/delta": -0.10776926577091217, "fcm_dpo/margin": 110.59844970703125, "fcm_dpo/q_t": 0.38157543540000916, "grad_norm": 8.470447540283203, "learning_rate": 2.2055192986047804e-07, "logits/chosen": -2.1580612659454346, "logits/rejected": -1.3645155429840088, "logps/chosen": -238.28677368164062, "logps/ref_chosen": -96.63854217529297, "logps/ref_rejected": -83.53645324707031, "logps/rejected": -335.78314208984375, "loss": 0.9933, "margin_dpo/margin_mean": 110.59844970703125, "margin_dpo/margin_std": 99.02703857421875, "step": 399 }, { "KL/chosen_KL_mean": -147.7364501953125, "KL/mean": -204.21615600585938, "KL/rejected_KL_mean": -260.6958923339844, "KL/std": 101.56245422363281, "epoch": 0.5873715124816447, "fcm_dpo/beta": 0.0044386135414242744, "fcm_dpo/delta": -0.10675300657749176, "fcm_dpo/margin": 112.95943450927734, "fcm_dpo/q_t": 0.3831738233566284, "grad_norm": 12.24710750579834, "learning_rate": 2.192779273338215e-07, "logits/chosen": -2.05845308303833, "logits/rejected": -1.5658423900604248, "logps/chosen": -253.0395965576172, "logps/ref_chosen": -105.30314636230469, "logps/ref_rejected": -108.6555404663086, "logps/rejected": -369.3514404296875, "loss": 1.0105, "margin_dpo/margin_mean": 112.95942687988281, "margin_dpo/margin_std": 116.37683868408203, "step": 400 }, { "KL/chosen_KL_mean": -163.4637451171875, "KL/mean": -198.381591796875, "KL/rejected_KL_mean": -233.2994384765625, "KL/std": 102.88335418701172, "epoch": 0.5888399412628488, "fcm_dpo/beta": 0.004463404882699251, "fcm_dpo/delta": 0.091233029961586, "fcm_dpo/margin": 69.835693359375, "fcm_dpo/q_t": 0.4302549958229065, "grad_norm": 10.84851360321045, "learning_rate": 2.1800473436235136e-07, "logits/chosen": -1.5348196029663086, "logits/rejected": -1.291013240814209, "logps/chosen": -247.75291442871094, "logps/ref_chosen": -84.28916931152344, "logps/ref_rejected": -90.943115234375, "logps/rejected": -324.2425537109375, "loss": 1.1721, "margin_dpo/margin_mean": 69.835693359375, "margin_dpo/margin_std": 125.79289245605469, "step": 401 }, { "KL/chosen_KL_mean": -144.9648895263672, "KL/mean": -210.73675537109375, "KL/rejected_KL_mean": -276.50860595703125, "KL/std": 109.6194839477539, "epoch": 0.5903083700440529, "fcm_dpo/beta": 0.004380761180073023, "fcm_dpo/delta": -0.18678948283195496, "fcm_dpo/margin": 131.5437469482422, "fcm_dpo/q_t": 0.3639332354068756, "grad_norm": 8.066213607788086, "learning_rate": 2.1673238449588665e-07, "logits/chosen": -1.7152506113052368, "logits/rejected": -1.2308242321014404, "logps/chosen": -228.55801391601562, "logps/ref_chosen": -83.59312438964844, "logps/ref_rejected": -87.81027221679688, "logps/rejected": -364.3188781738281, "loss": 0.9432, "margin_dpo/margin_mean": 131.5437469482422, "margin_dpo/margin_std": 106.96871948242188, "step": 402 }, { "KL/chosen_KL_mean": -147.63211059570312, "KL/mean": -196.5526885986328, "KL/rejected_KL_mean": -245.47329711914062, "KL/std": 105.8347396850586, "epoch": 0.591776798825257, "fcm_dpo/beta": 0.004308074712753296, "fcm_dpo/delta": -0.022586064413189888, "fcm_dpo/margin": 97.84117126464844, "fcm_dpo/q_t": 0.4012760519981384, "grad_norm": 8.59002685546875, "learning_rate": 2.154609112620295e-07, "logits/chosen": -2.2919535636901855, "logits/rejected": -1.8949791193008423, "logps/chosen": -221.38519287109375, "logps/ref_chosen": -73.75308227539062, "logps/ref_rejected": -83.92012786865234, "logps/rejected": -329.3934326171875, "loss": 1.0655, "margin_dpo/margin_mean": 97.84117126464844, "margin_dpo/margin_std": 113.22218322753906, "step": 403 }, { "KL/chosen_KL_mean": -176.33816528320312, "KL/mean": -218.0425262451172, "KL/rejected_KL_mean": -259.74688720703125, "KL/std": 102.04798889160156, "epoch": 0.593245227606461, "fcm_dpo/beta": 0.0043287696316838264, "fcm_dpo/delta": 0.0403948538005352, "fcm_dpo/margin": 83.40870666503906, "fcm_dpo/q_t": 0.4165842533111572, "grad_norm": 12.13433837890625, "learning_rate": 2.1419034816528218e-07, "logits/chosen": -1.8630778789520264, "logits/rejected": -1.5164891481399536, "logps/chosen": -256.01434326171875, "logps/ref_chosen": -79.67617797851562, "logps/ref_rejected": -84.280517578125, "logps/rejected": -344.02740478515625, "loss": 1.1268, "margin_dpo/margin_mean": 83.40870666503906, "margin_dpo/margin_std": 124.99861907958984, "step": 404 }, { "KL/chosen_KL_mean": -180.105712890625, "KL/mean": -222.22166442871094, "KL/rejected_KL_mean": -264.33758544921875, "KL/std": 95.65190887451172, "epoch": 0.5947136563876652, "fcm_dpo/beta": 0.004387823864817619, "fcm_dpo/delta": 0.03096681833267212, "fcm_dpo/margin": 84.23188018798828, "fcm_dpo/q_t": 0.4135516285896301, "grad_norm": 8.131792068481445, "learning_rate": 2.129207286861638e-07, "logits/chosen": -1.6474275588989258, "logits/rejected": -1.1774401664733887, "logps/chosen": -276.56768798828125, "logps/ref_chosen": -96.46195220947266, "logps/ref_rejected": -92.87071228027344, "logps/rejected": -357.20831298828125, "loss": 1.0984, "margin_dpo/margin_mean": 84.23187255859375, "margin_dpo/margin_std": 100.60556030273438, "step": 405 }, { "KL/chosen_KL_mean": -171.2765655517578, "KL/mean": -209.51084899902344, "KL/rejected_KL_mean": -247.7451171875, "KL/std": 102.41468811035156, "epoch": 0.5961820851688693, "fcm_dpo/beta": 0.0044171009212732315, "fcm_dpo/delta": 0.06439150869846344, "fcm_dpo/margin": 76.46855926513672, "fcm_dpo/q_t": 0.42172834277153015, "grad_norm": 13.802324295043945, "learning_rate": 2.1165208628032861e-07, "logits/chosen": -1.879835605621338, "logits/rejected": -1.7107501029968262, "logps/chosen": -249.4105224609375, "logps/ref_chosen": -78.13396453857422, "logps/ref_rejected": -98.28359985351562, "logps/rejected": -346.0287170410156, "loss": 1.1435, "margin_dpo/margin_mean": 76.46855926513672, "margin_dpo/margin_std": 119.75570678710938, "step": 406 }, { "KL/chosen_KL_mean": -177.90869140625, "KL/mean": -211.5441436767578, "KL/rejected_KL_mean": -245.17959594726562, "KL/std": 100.73377990722656, "epoch": 0.5976505139500734, "fcm_dpo/beta": 0.004497930407524109, "fcm_dpo/delta": 0.1004524976015091, "fcm_dpo/margin": 67.27088928222656, "fcm_dpo/q_t": 0.42943549156188965, "grad_norm": 9.671611785888672, "learning_rate": 2.1038445437768375e-07, "logits/chosen": -2.317671298980713, "logits/rejected": -1.812265396118164, "logps/chosen": -261.9215393066406, "logps/ref_chosen": -84.01283264160156, "logps/ref_rejected": -82.78103637695312, "logps/rejected": -327.96063232421875, "loss": 1.162, "margin_dpo/margin_mean": 67.27088928222656, "margin_dpo/margin_std": 107.75228118896484, "step": 407 }, { "KL/chosen_KL_mean": -170.44894409179688, "KL/mean": -204.17022705078125, "KL/rejected_KL_mean": -237.89154052734375, "KL/std": 96.15237426757812, "epoch": 0.5991189427312775, "fcm_dpo/beta": 0.004579237662255764, "fcm_dpo/delta": 0.09415175765752792, "fcm_dpo/margin": 67.44257354736328, "fcm_dpo/q_t": 0.4293147921562195, "grad_norm": 9.34108829498291, "learning_rate": 2.0911786638150872e-07, "logits/chosen": -2.3941664695739746, "logits/rejected": -1.7890679836273193, "logps/chosen": -274.91070556640625, "logps/ref_chosen": -104.46174621582031, "logps/ref_rejected": -96.37217712402344, "logps/rejected": -334.26373291015625, "loss": 1.1727, "margin_dpo/margin_mean": 67.44256591796875, "margin_dpo/margin_std": 119.92399597167969, "step": 408 }, { "KL/chosen_KL_mean": -180.74114990234375, "KL/mean": -216.11959838867188, "KL/rejected_KL_mean": -251.498046875, "KL/std": 98.67672729492188, "epoch": 0.6005873715124816, "fcm_dpo/beta": 0.004657105077058077, "fcm_dpo/delta": 0.07287521660327911, "fcm_dpo/margin": 70.75688171386719, "fcm_dpo/q_t": 0.42199981212615967, "grad_norm": 8.716501235961914, "learning_rate": 2.0785235566757517e-07, "logits/chosen": -1.4818743467330933, "logits/rejected": -1.0387271642684937, "logps/chosen": -278.40948486328125, "logps/ref_chosen": -97.66830444335938, "logps/ref_rejected": -90.04583740234375, "logps/rejected": -341.54388427734375, "loss": 1.1475, "margin_dpo/margin_mean": 70.75688171386719, "margin_dpo/margin_std": 108.87568664550781, "step": 409 }, { "KL/chosen_KL_mean": -175.37799072265625, "KL/mean": -218.5403289794922, "KL/rejected_KL_mean": -261.7026672363281, "KL/std": 104.52674102783203, "epoch": 0.6020558002936858, "fcm_dpo/beta": 0.00467026187106967, "fcm_dpo/delta": -0.0032948169391602278, "fcm_dpo/margin": 86.32466125488281, "fcm_dpo/q_t": 0.4048500657081604, "grad_norm": 8.333985328674316, "learning_rate": 2.065879555832674e-07, "logits/chosen": -1.7712609767913818, "logits/rejected": -1.6410791873931885, "logps/chosen": -251.84722900390625, "logps/ref_chosen": -76.46923828125, "logps/ref_rejected": -88.64064025878906, "logps/rejected": -350.34332275390625, "loss": 1.0802, "margin_dpo/margin_mean": 86.32466125488281, "margin_dpo/margin_std": 104.95033264160156, "step": 410 }, { "KL/chosen_KL_mean": -187.11279296875, "KL/mean": -227.45846557617188, "KL/rejected_KL_mean": -267.8041687011719, "KL/std": 97.15614318847656, "epoch": 0.6035242290748899, "fcm_dpo/beta": 0.004671902395784855, "fcm_dpo/delta": 0.023768678307533264, "fcm_dpo/margin": 80.69136047363281, "fcm_dpo/q_t": 0.41313672065734863, "grad_norm": 8.546412467956543, "learning_rate": 2.0532469944670343e-07, "logits/chosen": -2.2369627952575684, "logits/rejected": -1.823148250579834, "logps/chosen": -274.27911376953125, "logps/ref_chosen": -87.16630554199219, "logps/ref_rejected": -87.09603118896484, "logps/rejected": -354.90020751953125, "loss": 1.1023, "margin_dpo/margin_mean": 80.69136047363281, "margin_dpo/margin_std": 104.68389892578125, "step": 411 }, { "KL/chosen_KL_mean": -177.09315490722656, "KL/mean": -216.38818359375, "KL/rejected_KL_mean": -255.68325805664062, "KL/std": 97.79566192626953, "epoch": 0.604992657856094, "fcm_dpo/beta": 0.004703844897449017, "fcm_dpo/delta": 0.0314432829618454, "fcm_dpo/margin": 78.59010314941406, "fcm_dpo/q_t": 0.4162543714046478, "grad_norm": 10.165987014770508, "learning_rate": 2.0406262054585738e-07, "logits/chosen": -2.3015284538269043, "logits/rejected": -2.081984519958496, "logps/chosen": -256.0404968261719, "logps/ref_chosen": -78.94734191894531, "logps/ref_rejected": -106.10554504394531, "logps/rejected": -361.788818359375, "loss": 1.134, "margin_dpo/margin_mean": 78.59010314941406, "margin_dpo/margin_std": 126.52278900146484, "step": 412 }, { "KL/chosen_KL_mean": -191.408203125, "KL/mean": -234.6050262451172, "KL/rejected_KL_mean": -277.8018493652344, "KL/std": 103.73475646972656, "epoch": 0.6064610866372981, "fcm_dpo/beta": 0.004702753387391567, "fcm_dpo/delta": -0.006794095039367676, "fcm_dpo/margin": 86.39363098144531, "fcm_dpo/q_t": 0.40529611706733704, "grad_norm": 9.821824073791504, "learning_rate": 2.0280175213768205e-07, "logits/chosen": -1.3946678638458252, "logits/rejected": -1.1435439586639404, "logps/chosen": -287.1029052734375, "logps/ref_chosen": -95.69470977783203, "logps/ref_rejected": -107.96085357666016, "logps/rejected": -385.7626953125, "loss": 1.1, "margin_dpo/margin_mean": 86.39363861083984, "margin_dpo/margin_std": 123.67024230957031, "step": 413 }, { "KL/chosen_KL_mean": -175.42660522460938, "KL/mean": -216.73126220703125, "KL/rejected_KL_mean": -258.03594970703125, "KL/std": 100.31507873535156, "epoch": 0.6079295154185022, "fcm_dpo/beta": 0.0047295354306697845, "fcm_dpo/delta": 0.009625216946005821, "fcm_dpo/margin": 82.60934448242188, "fcm_dpo/q_t": 0.4108094573020935, "grad_norm": 12.460843086242676, "learning_rate": 2.0154212744723247e-07, "logits/chosen": -1.6047205924987793, "logits/rejected": -1.2488441467285156, "logps/chosen": -263.7032775878906, "logps/ref_chosen": -88.27667236328125, "logps/ref_rejected": -92.87004089355469, "logps/rejected": -350.906005859375, "loss": 1.1188, "margin_dpo/margin_mean": 82.60934448242188, "margin_dpo/margin_std": 128.03842163085938, "step": 414 }, { "KL/chosen_KL_mean": -191.16400146484375, "KL/mean": -225.69607543945312, "KL/rejected_KL_mean": -260.2281494140625, "KL/std": 91.4719009399414, "epoch": 0.6093979441997063, "fcm_dpo/beta": 0.004780493211001158, "fcm_dpo/delta": 0.07215416431427002, "fcm_dpo/margin": 69.06413269042969, "fcm_dpo/q_t": 0.4239092469215393, "grad_norm": 10.98272705078125, "learning_rate": 2.002837796667909e-07, "logits/chosen": -1.6034247875213623, "logits/rejected": -1.2234405279159546, "logps/chosen": -300.07989501953125, "logps/ref_chosen": -108.91590118408203, "logps/ref_rejected": -107.47135925292969, "logps/rejected": -367.69952392578125, "loss": 1.1521, "margin_dpo/margin_mean": 69.06413269042969, "margin_dpo/margin_std": 112.0533447265625, "step": 415 }, { "KL/chosen_KL_mean": -190.90484619140625, "KL/mean": -238.57345581054688, "KL/rejected_KL_mean": -286.2420349121094, "KL/std": 105.56501770019531, "epoch": 0.6108663729809104, "fcm_dpo/beta": 0.004787248559296131, "fcm_dpo/delta": -0.059774283319711685, "fcm_dpo/margin": 95.33720397949219, "fcm_dpo/q_t": 0.39473649859428406, "grad_norm": 9.096339225769043, "learning_rate": 1.990267419549914e-07, "logits/chosen": -1.9815460443496704, "logits/rejected": -1.4922939538955688, "logps/chosen": -284.3037414550781, "logps/ref_chosen": -93.39888000488281, "logps/ref_rejected": -97.6729736328125, "logps/rejected": -383.9150390625, "loss": 1.0535, "margin_dpo/margin_mean": 95.33720397949219, "margin_dpo/margin_std": 111.15675354003906, "step": 416 }, { "KL/chosen_KL_mean": -177.103271484375, "KL/mean": -221.63092041015625, "KL/rejected_KL_mean": -266.1585693359375, "KL/std": 98.60084533691406, "epoch": 0.6123348017621145, "fcm_dpo/beta": 0.0047265347093343735, "fcm_dpo/delta": -0.021840443834662437, "fcm_dpo/margin": 89.05528259277344, "fcm_dpo/q_t": 0.40146446228027344, "grad_norm": 8.924833297729492, "learning_rate": 1.9777104743594686e-07, "logits/chosen": -2.1439530849456787, "logits/rejected": -1.2152490615844727, "logps/chosen": -260.63861083984375, "logps/ref_chosen": -83.53533172607422, "logps/ref_rejected": -74.44184875488281, "logps/rejected": -340.60040283203125, "loss": 1.0628, "margin_dpo/margin_mean": 89.05526733398438, "margin_dpo/margin_std": 100.57928466796875, "step": 417 }, { "KL/chosen_KL_mean": -169.47756958007812, "KL/mean": -225.71487426757812, "KL/rejected_KL_mean": -281.9521789550781, "KL/std": 105.68417358398438, "epoch": 0.6138032305433186, "fcm_dpo/beta": 0.004667467903345823, "fcm_dpo/delta": -0.13238637149333954, "fcm_dpo/margin": 112.47457885742188, "fcm_dpo/q_t": 0.3781015872955322, "grad_norm": 10.708224296569824, "learning_rate": 1.965167291983757e-07, "logits/chosen": -2.171066999435425, "logits/rejected": -1.5124069452285767, "logps/chosen": -277.6990966796875, "logps/ref_chosen": -108.22152709960938, "logps/ref_rejected": -111.8646469116211, "logps/rejected": -393.81683349609375, "loss": 0.9906, "margin_dpo/margin_mean": 112.47457885742188, "margin_dpo/margin_std": 103.07366943359375, "step": 418 }, { "KL/chosen_KL_mean": -175.8536376953125, "KL/mean": -224.85279846191406, "KL/rejected_KL_mean": -273.8519287109375, "KL/std": 93.91587829589844, "epoch": 0.6152716593245228, "fcm_dpo/beta": 0.004578390158712864, "fcm_dpo/delta": -0.05112157762050629, "fcm_dpo/margin": 97.99830627441406, "fcm_dpo/q_t": 0.39496368169784546, "grad_norm": 10.895743370056152, "learning_rate": 1.9526382029472988e-07, "logits/chosen": -1.5334709882736206, "logits/rejected": -1.091383695602417, "logps/chosen": -273.03692626953125, "logps/ref_chosen": -97.18328094482422, "logps/ref_rejected": -98.18531799316406, "logps/rejected": -372.0372619628906, "loss": 1.0415, "margin_dpo/margin_mean": 97.99830627441406, "margin_dpo/margin_std": 103.14251708984375, "step": 419 }, { "KL/chosen_KL_mean": -179.98837280273438, "KL/mean": -223.7959442138672, "KL/rejected_KL_mean": -267.603515625, "KL/std": 99.4555892944336, "epoch": 0.6167400881057269, "fcm_dpo/beta": 0.004560777451843023, "fcm_dpo/delta": 0.0002319682389497757, "fcm_dpo/margin": 87.61510467529297, "fcm_dpo/q_t": 0.40743955969810486, "grad_norm": 14.228276252746582, "learning_rate": 1.9401235374032425e-07, "logits/chosen": -2.125715970993042, "logits/rejected": -1.0029550790786743, "logps/chosen": -294.296875, "logps/ref_chosen": -114.30848693847656, "logps/ref_rejected": -75.68355560302734, "logps/rejected": -343.28704833984375, "loss": 1.091, "margin_dpo/margin_mean": 87.6151123046875, "margin_dpo/margin_std": 114.40043640136719, "step": 420 }, { "KL/chosen_KL_mean": -171.89007568359375, "KL/mean": -220.2574462890625, "KL/rejected_KL_mean": -268.62481689453125, "KL/std": 102.25340270996094, "epoch": 0.618208516886931, "fcm_dpo/beta": 0.0045146881602704525, "fcm_dpo/delta": -0.038495756685733795, "fcm_dpo/margin": 96.7347412109375, "fcm_dpo/q_t": 0.3978639841079712, "grad_norm": 8.217570304870605, "learning_rate": 1.9276236251246653e-07, "logits/chosen": -1.9371953010559082, "logits/rejected": -1.5686148405075073, "logps/chosen": -257.7699279785156, "logps/ref_chosen": -85.87985229492188, "logps/ref_rejected": -96.33648681640625, "logps/rejected": -364.9613037109375, "loss": 1.058, "margin_dpo/margin_mean": 96.7347412109375, "margin_dpo/margin_std": 112.34698486328125, "step": 421 }, { "KL/chosen_KL_mean": -169.70358276367188, "KL/mean": -215.88485717773438, "KL/rejected_KL_mean": -262.066162109375, "KL/std": 94.60616302490234, "epoch": 0.6196769456681351, "fcm_dpo/beta": 0.004505760967731476, "fcm_dpo/delta": -0.016872413456439972, "fcm_dpo/margin": 92.36255645751953, "fcm_dpo/q_t": 0.4029741883277893, "grad_norm": 11.991118431091309, "learning_rate": 1.9151387954958792e-07, "logits/chosen": -2.4875924587249756, "logits/rejected": -2.127922534942627, "logps/chosen": -270.1842041015625, "logps/ref_chosen": -100.48060607910156, "logps/ref_rejected": -94.40821838378906, "logps/rejected": -356.474365234375, "loss": 1.067, "margin_dpo/margin_mean": 92.362548828125, "margin_dpo/margin_std": 106.64411926269531, "step": 422 }, { "KL/chosen_KL_mean": -158.3551025390625, "KL/mean": -204.87356567382812, "KL/rejected_KL_mean": -251.39205932617188, "KL/std": 113.11082458496094, "epoch": 0.6211453744493393, "fcm_dpo/beta": 0.004501148127019405, "fcm_dpo/delta": -0.019828110933303833, "fcm_dpo/margin": 93.03694152832031, "fcm_dpo/q_t": 0.4047386944293976, "grad_norm": 10.097667694091797, "learning_rate": 1.902669377503756e-07, "logits/chosen": -1.875777006149292, "logits/rejected": -1.7562098503112793, "logps/chosen": -236.80503845214844, "logps/ref_chosen": -78.44993591308594, "logps/ref_rejected": -92.04652404785156, "logps/rejected": -343.4385986328125, "loss": 1.0902, "margin_dpo/margin_mean": 93.03694152832031, "margin_dpo/margin_std": 128.62796020507812, "step": 423 }, { "KL/chosen_KL_mean": -177.8150634765625, "KL/mean": -223.44064331054688, "KL/rejected_KL_mean": -269.0661926269531, "KL/std": 100.35517883300781, "epoch": 0.6226138032305433, "fcm_dpo/beta": 0.004471376538276672, "fcm_dpo/delta": -0.008367246948182583, "fcm_dpo/margin": 91.25111389160156, "fcm_dpo/q_t": 0.4039370119571686, "grad_norm": 8.527449607849121, "learning_rate": 1.890215699729057e-07, "logits/chosen": -1.753572702407837, "logits/rejected": -1.158422827720642, "logps/chosen": -265.4574279785156, "logps/ref_chosen": -87.64236450195312, "logps/ref_rejected": -72.36566162109375, "logps/rejected": -341.4318542480469, "loss": 1.0764, "margin_dpo/margin_mean": 91.25111389160156, "margin_dpo/margin_std": 109.28106689453125, "step": 424 }, { "KL/chosen_KL_mean": -184.27423095703125, "KL/mean": -215.5604248046875, "KL/rejected_KL_mean": -246.84664916992188, "KL/std": 103.53453063964844, "epoch": 0.6240822320117474, "fcm_dpo/beta": 0.004543634131550789, "fcm_dpo/delta": 0.11922936886548996, "fcm_dpo/margin": 62.57243347167969, "fcm_dpo/q_t": 0.43667247891426086, "grad_norm": 10.237264633178711, "learning_rate": 1.8777780903377732e-07, "logits/chosen": -1.5960094928741455, "logits/rejected": -1.528268575668335, "logps/chosen": -262.79400634765625, "logps/ref_chosen": -78.51979064941406, "logps/ref_rejected": -102.74864196777344, "logps/rejected": -349.59527587890625, "loss": 1.2032, "margin_dpo/margin_mean": 62.572425842285156, "margin_dpo/margin_std": 129.049560546875, "step": 425 }, { "KL/chosen_KL_mean": -157.53213500976562, "KL/mean": -203.17276000976562, "KL/rejected_KL_mean": -248.81341552734375, "KL/std": 95.47308349609375, "epoch": 0.6255506607929515, "fcm_dpo/beta": 0.00457720085978508, "fcm_dpo/delta": -0.018653854727745056, "fcm_dpo/margin": 91.28130340576172, "fcm_dpo/q_t": 0.4024432599544525, "grad_norm": 10.744503021240234, "learning_rate": 1.8653568770724803e-07, "logits/chosen": -1.6125023365020752, "logits/rejected": -0.8609820008277893, "logps/chosen": -266.0379638671875, "logps/ref_chosen": -108.50582885742188, "logps/ref_rejected": -88.300048828125, "logps/rejected": -337.11346435546875, "loss": 1.0735, "margin_dpo/margin_mean": 91.28129577636719, "margin_dpo/margin_std": 111.07772827148438, "step": 426 }, { "KL/chosen_KL_mean": -130.6856231689453, "KL/mean": -172.5328369140625, "KL/rejected_KL_mean": -214.3800506591797, "KL/std": 91.2623062133789, "epoch": 0.6270190895741556, "fcm_dpo/beta": 0.004579422064125538, "fcm_dpo/delta": 0.01734790951013565, "fcm_dpo/margin": 83.69442749023438, "fcm_dpo/q_t": 0.40990543365478516, "grad_norm": 8.622220039367676, "learning_rate": 1.8529523872436977e-07, "logits/chosen": -2.22707200050354, "logits/rejected": -1.5730688571929932, "logps/chosen": -229.80609130859375, "logps/ref_chosen": -99.12046813964844, "logps/ref_rejected": -85.724609375, "logps/rejected": -300.10467529296875, "loss": 1.0879, "margin_dpo/margin_mean": 83.69442749023438, "margin_dpo/margin_std": 98.64570617675781, "step": 427 }, { "KL/chosen_KL_mean": -185.00865173339844, "KL/mean": -221.10183715820312, "KL/rejected_KL_mean": -257.19500732421875, "KL/std": 102.99807739257812, "epoch": 0.6284875183553598, "fcm_dpo/beta": 0.004606014117598534, "fcm_dpo/delta": 0.06974522024393082, "fcm_dpo/margin": 72.18636322021484, "fcm_dpo/q_t": 0.42333298921585083, "grad_norm": 9.928332328796387, "learning_rate": 1.8405649477212697e-07, "logits/chosen": -1.6796432733535767, "logits/rejected": -1.4184119701385498, "logps/chosen": -290.9779052734375, "logps/ref_chosen": -105.96925354003906, "logps/ref_rejected": -109.10218048095703, "logps/rejected": -366.29718017578125, "loss": 1.1488, "margin_dpo/margin_mean": 72.18636322021484, "margin_dpo/margin_std": 114.17198181152344, "step": 428 }, { "KL/chosen_KL_mean": -170.1451416015625, "KL/mean": -216.1502685546875, "KL/rejected_KL_mean": -262.1553955078125, "KL/std": 106.94287109375, "epoch": 0.6299559471365639, "fcm_dpo/beta": 0.004654177464544773, "fcm_dpo/delta": -0.030279342085123062, "fcm_dpo/margin": 92.01028442382812, "fcm_dpo/q_t": 0.4021182656288147, "grad_norm": 8.30920696258545, "learning_rate": 1.828194884925749e-07, "logits/chosen": -1.9182894229888916, "logits/rejected": -1.1986092329025269, "logps/chosen": -283.69000244140625, "logps/ref_chosen": -113.54487609863281, "logps/ref_rejected": -98.24201965332031, "logps/rejected": -360.3974304199219, "loss": 1.0827, "margin_dpo/margin_mean": 92.01028442382812, "margin_dpo/margin_std": 121.56130981445312, "step": 429 }, { "KL/chosen_KL_mean": -162.14920043945312, "KL/mean": -197.70428466796875, "KL/rejected_KL_mean": -233.2593994140625, "KL/std": 99.39602661132812, "epoch": 0.631424375917768, "fcm_dpo/beta": 0.004676306154578924, "fcm_dpo/delta": 0.06940542906522751, "fcm_dpo/margin": 71.1102066040039, "fcm_dpo/q_t": 0.42368191480636597, "grad_norm": 9.174887657165527, "learning_rate": 1.8158425248197928e-07, "logits/chosen": -1.3752224445343018, "logits/rejected": -1.338731050491333, "logps/chosen": -253.46856689453125, "logps/ref_chosen": -91.31936645507812, "logps/ref_rejected": -110.1096420288086, "logps/rejected": -343.3690490722656, "loss": 1.1512, "margin_dpo/margin_mean": 71.1102066040039, "margin_dpo/margin_std": 115.85983276367188, "step": 430 }, { "KL/chosen_KL_mean": -162.9004669189453, "KL/mean": -204.98194885253906, "KL/rejected_KL_mean": -247.06344604492188, "KL/std": 101.5677261352539, "epoch": 0.6328928046989721, "fcm_dpo/beta": 0.004677704535424709, "fcm_dpo/delta": 0.0065451571717858315, "fcm_dpo/margin": 84.16297912597656, "fcm_dpo/q_t": 0.40864941477775574, "grad_norm": 10.291074752807617, "learning_rate": 1.8035081928995788e-07, "logits/chosen": -1.905835509300232, "logits/rejected": -1.5202605724334717, "logps/chosen": -256.0816650390625, "logps/ref_chosen": -93.18122100830078, "logps/ref_rejected": -98.13226318359375, "logps/rejected": -345.1957092285156, "loss": 1.1001, "margin_dpo/margin_mean": 84.16297912597656, "margin_dpo/margin_std": 115.48672485351562, "step": 431 }, { "KL/chosen_KL_mean": -142.22576904296875, "KL/mean": -193.51620483398438, "KL/rejected_KL_mean": -244.80661010742188, "KL/std": 105.91761779785156, "epoch": 0.6343612334801763, "fcm_dpo/beta": 0.0046443454921245575, "fcm_dpo/delta": -0.08019526302814484, "fcm_dpo/margin": 102.58087158203125, "fcm_dpo/q_t": 0.3883529603481293, "grad_norm": 8.808527946472168, "learning_rate": 1.791192214186223e-07, "logits/chosen": -1.262732982635498, "logits/rejected": -0.6869934797286987, "logps/chosen": -246.66055297851562, "logps/ref_chosen": -104.43478393554688, "logps/ref_rejected": -105.08955383300781, "logps/rejected": -349.89617919921875, "loss": 1.0224, "margin_dpo/margin_mean": 102.58087158203125, "margin_dpo/margin_std": 102.51104736328125, "step": 432 }, { "KL/chosen_KL_mean": -170.4425048828125, "KL/mean": -210.44442749023438, "KL/rejected_KL_mean": -250.44631958007812, "KL/std": 99.75376892089844, "epoch": 0.6358296622613803, "fcm_dpo/beta": 0.004651183262467384, "fcm_dpo/delta": 0.02841937355697155, "fcm_dpo/margin": 80.00381469726562, "fcm_dpo/q_t": 0.4140810966491699, "grad_norm": 9.179040908813477, "learning_rate": 1.7788949132172193e-07, "logits/chosen": -1.5476620197296143, "logits/rejected": -1.2211322784423828, "logps/chosen": -260.2857360839844, "logps/ref_chosen": -89.84322357177734, "logps/ref_rejected": -101.73345947265625, "logps/rejected": -352.1797790527344, "loss": 1.1168, "margin_dpo/margin_mean": 80.00381469726562, "margin_dpo/margin_std": 113.90576171875, "step": 433 }, { "KL/chosen_KL_mean": -147.14837646484375, "KL/mean": -193.00115966796875, "KL/rejected_KL_mean": -238.85394287109375, "KL/std": 111.14796447753906, "epoch": 0.6372980910425844, "fcm_dpo/beta": 0.004623515531420708, "fcm_dpo/delta": -0.025058362632989883, "fcm_dpo/margin": 91.70557403564453, "fcm_dpo/q_t": 0.40513288974761963, "grad_norm": 12.068562507629395, "learning_rate": 1.7666166140378853e-07, "logits/chosen": -2.3639960289001465, "logits/rejected": -1.6832940578460693, "logps/chosen": -244.84091186523438, "logps/ref_chosen": -97.6925277709961, "logps/ref_rejected": -84.09129333496094, "logps/rejected": -322.94525146484375, "loss": 1.0866, "margin_dpo/margin_mean": 91.70557403564453, "margin_dpo/margin_std": 129.50987243652344, "step": 434 }, { "KL/chosen_KL_mean": -145.34385681152344, "KL/mean": -192.22213745117188, "KL/rejected_KL_mean": -239.10040283203125, "KL/std": 103.24746704101562, "epoch": 0.6387665198237885, "fcm_dpo/beta": 0.004597329534590244, "fcm_dpo/delta": -0.03241932764649391, "fcm_dpo/margin": 93.75656127929688, "fcm_dpo/q_t": 0.40081021189689636, "grad_norm": 9.877218246459961, "learning_rate": 1.7543576401928218e-07, "logits/chosen": -2.5195233821868896, "logits/rejected": -2.081613302230835, "logps/chosen": -231.51577758789062, "logps/ref_chosen": -86.17192077636719, "logps/ref_rejected": -93.751708984375, "logps/rejected": -332.85211181640625, "loss": 1.0656, "margin_dpo/margin_mean": 93.75656127929688, "margin_dpo/margin_std": 114.79005432128906, "step": 435 }, { "KL/chosen_KL_mean": -120.9896469116211, "KL/mean": -169.9293670654297, "KL/rejected_KL_mean": -218.86904907226562, "KL/std": 87.64741516113281, "epoch": 0.6402349486049926, "fcm_dpo/beta": 0.004547707736492157, "fcm_dpo/delta": -0.04728689044713974, "fcm_dpo/margin": 97.87940216064453, "fcm_dpo/q_t": 0.3944322168827057, "grad_norm": 10.02210807800293, "learning_rate": 1.742118314717391e-07, "logits/chosen": -1.6290252208709717, "logits/rejected": -0.7463002800941467, "logps/chosen": -226.77674865722656, "logps/ref_chosen": -105.78710174560547, "logps/ref_rejected": -88.62471008300781, "logps/rejected": -307.4937744140625, "loss": 1.0305, "margin_dpo/margin_mean": 97.87940216064453, "margin_dpo/margin_std": 90.94055938720703, "step": 436 }, { "KL/chosen_KL_mean": -141.02743530273438, "KL/mean": -181.14300537109375, "KL/rejected_KL_mean": -221.25857543945312, "KL/std": 93.98324584960938, "epoch": 0.6417033773861968, "fcm_dpo/beta": 0.004569545388221741, "fcm_dpo/delta": 0.034581996500492096, "fcm_dpo/margin": 80.23114013671875, "fcm_dpo/q_t": 0.4125591516494751, "grad_norm": 10.12276840209961, "learning_rate": 1.7298989601292036e-07, "logits/chosen": -1.9861946105957031, "logits/rejected": -1.3308312892913818, "logps/chosen": -237.0894775390625, "logps/ref_chosen": -96.06204223632812, "logps/ref_rejected": -89.01220703125, "logps/rejected": -310.2707824707031, "loss": 1.0963, "margin_dpo/margin_mean": 80.23114013671875, "margin_dpo/margin_std": 92.78572082519531, "step": 437 }, { "KL/chosen_KL_mean": -145.30038452148438, "KL/mean": -185.49411010742188, "KL/rejected_KL_mean": -225.6878662109375, "KL/std": 94.65817260742188, "epoch": 0.6431718061674009, "fcm_dpo/beta": 0.004576482810080051, "fcm_dpo/delta": 0.03313709795475006, "fcm_dpo/margin": 80.38749694824219, "fcm_dpo/q_t": 0.4136542081832886, "grad_norm": 9.757686614990234, "learning_rate": 1.7176998984196144e-07, "logits/chosen": -2.095676898956299, "logits/rejected": -1.2759695053100586, "logps/chosen": -247.15576171875, "logps/ref_chosen": -101.85537719726562, "logps/ref_rejected": -89.44766235351562, "logps/rejected": -315.1355285644531, "loss": 1.1057, "margin_dpo/margin_mean": 80.38749694824219, "margin_dpo/margin_std": 101.8877944946289, "step": 438 }, { "KL/chosen_KL_mean": -156.38632202148438, "KL/mean": -196.69296264648438, "KL/rejected_KL_mean": -236.99960327148438, "KL/std": 102.48863220214844, "epoch": 0.644640234948605, "fcm_dpo/beta": 0.004594483412802219, "fcm_dpo/delta": 0.03014417365193367, "fcm_dpo/margin": 80.61329650878906, "fcm_dpo/q_t": 0.4137040078639984, "grad_norm": 9.894194602966309, "learning_rate": 1.7055214510452458e-07, "logits/chosen": -1.5999832153320312, "logits/rejected": -1.3621375560760498, "logps/chosen": -238.14193725585938, "logps/ref_chosen": -81.75563049316406, "logps/ref_rejected": -90.58636474609375, "logps/rejected": -327.5859680175781, "loss": 1.1045, "margin_dpo/margin_mean": 80.61329650878906, "margin_dpo/margin_std": 100.47586059570312, "step": 439 }, { "KL/chosen_KL_mean": -140.2466278076172, "KL/mean": -190.2432861328125, "KL/rejected_KL_mean": -240.23995971679688, "KL/std": 103.79228210449219, "epoch": 0.6461086637298091, "fcm_dpo/beta": 0.004565776791423559, "fcm_dpo/delta": -0.060098931193351746, "fcm_dpo/margin": 99.99331665039062, "fcm_dpo/q_t": 0.39449551701545715, "grad_norm": 7.804450988769531, "learning_rate": 1.6933639389195134e-07, "logits/chosen": -2.466993808746338, "logits/rejected": -1.9724578857421875, "logps/chosen": -245.88771057128906, "logps/ref_chosen": -105.64108276367188, "logps/ref_rejected": -103.4010009765625, "logps/rejected": -343.6409606933594, "loss": 1.0465, "margin_dpo/margin_mean": 99.99331665039062, "margin_dpo/margin_std": 114.491455078125, "step": 440 }, { "KL/chosen_KL_mean": -180.41220092773438, "KL/mean": -222.82586669921875, "KL/rejected_KL_mean": -265.239501953125, "KL/std": 98.21589660644531, "epoch": 0.6475770925110133, "fcm_dpo/beta": 0.004577081184834242, "fcm_dpo/delta": 0.012189902365207672, "fcm_dpo/margin": 84.82731628417969, "fcm_dpo/q_t": 0.4107508957386017, "grad_norm": 8.74206829071045, "learning_rate": 1.681227682404166e-07, "logits/chosen": -1.6649823188781738, "logits/rejected": -1.300221562385559, "logps/chosen": -271.9417419433594, "logps/ref_chosen": -91.529541015625, "logps/ref_rejected": -103.619384765625, "logps/rejected": -368.85888671875, "loss": 1.09, "margin_dpo/margin_mean": 84.82731628417969, "margin_dpo/margin_std": 106.7401123046875, "step": 441 }, { "KL/chosen_KL_mean": -175.50149536132812, "KL/mean": -229.36190795898438, "KL/rejected_KL_mean": -283.2222900390625, "KL/std": 104.31333923339844, "epoch": 0.6490455212922174, "fcm_dpo/beta": 0.004526270553469658, "fcm_dpo/delta": -0.09202161431312561, "fcm_dpo/margin": 107.72080993652344, "fcm_dpo/q_t": 0.3855440318584442, "grad_norm": 10.38838005065918, "learning_rate": 1.669113001300851e-07, "logits/chosen": -1.2952864170074463, "logits/rejected": -0.7945897579193115, "logps/chosen": -260.279052734375, "logps/ref_chosen": -84.77755737304688, "logps/ref_rejected": -83.82415771484375, "logps/rejected": -367.04644775390625, "loss": 1.0096, "margin_dpo/margin_mean": 107.72080993652344, "margin_dpo/margin_std": 104.86602783203125, "step": 442 }, { "KL/chosen_KL_mean": -170.86907958984375, "KL/mean": -204.43536376953125, "KL/rejected_KL_mean": -238.00164794921875, "KL/std": 101.60333251953125, "epoch": 0.6505139500734214, "fcm_dpo/beta": 0.004552439786493778, "fcm_dpo/delta": 0.09744662791490555, "fcm_dpo/margin": 67.1325454711914, "fcm_dpo/q_t": 0.4285755157470703, "grad_norm": 9.9346923828125, "learning_rate": 1.6570202148426815e-07, "logits/chosen": -1.8769023418426514, "logits/rejected": -1.3223586082458496, "logps/chosen": -273.51837158203125, "logps/ref_chosen": -102.64927673339844, "logps/ref_rejected": -93.03807067871094, "logps/rejected": -331.03973388671875, "loss": 1.1546, "margin_dpo/margin_mean": 67.1325454711914, "margin_dpo/margin_std": 98.70500183105469, "step": 443 }, { "KL/chosen_KL_mean": -176.17404174804688, "KL/mean": -215.4727783203125, "KL/rejected_KL_mean": -254.771484375, "KL/std": 99.03073120117188, "epoch": 0.6519823788546255, "fcm_dpo/beta": 0.0046212333254516125, "fcm_dpo/delta": 0.03812466934323311, "fcm_dpo/margin": 78.5974349975586, "fcm_dpo/q_t": 0.41620129346847534, "grad_norm": 10.73589038848877, "learning_rate": 1.6449496416858282e-07, "logits/chosen": -1.6693949699401855, "logits/rejected": -1.5038468837738037, "logps/chosen": -264.09375, "logps/ref_chosen": -87.91971588134766, "logps/ref_rejected": -103.32345581054688, "logps/rejected": -358.0949401855469, "loss": 1.1223, "margin_dpo/margin_mean": 78.5974349975586, "margin_dpo/margin_std": 114.91938018798828, "step": 444 }, { "KL/chosen_KL_mean": -155.11688232421875, "KL/mean": -200.65089416503906, "KL/rejected_KL_mean": -246.18490600585938, "KL/std": 96.17469024658203, "epoch": 0.6534508076358296, "fcm_dpo/beta": 0.004597203806042671, "fcm_dpo/delta": -0.01977653056383133, "fcm_dpo/margin": 91.06803131103516, "fcm_dpo/q_t": 0.4017190933227539, "grad_norm": 8.61320686340332, "learning_rate": 1.6329015999011182e-07, "logits/chosen": -1.8391239643096924, "logits/rejected": -1.416989803314209, "logps/chosen": -256.51776123046875, "logps/ref_chosen": -101.40086364746094, "logps/ref_rejected": -99.03790283203125, "logps/rejected": -345.2228088378906, "loss": 1.0638, "margin_dpo/margin_mean": 91.06803131103516, "margin_dpo/margin_std": 102.09701538085938, "step": 445 }, { "KL/chosen_KL_mean": -149.54432678222656, "KL/mean": -190.4094696044922, "KL/rejected_KL_mean": -231.2745819091797, "KL/std": 102.02165222167969, "epoch": 0.6549192364170338, "fcm_dpo/beta": 0.0046312520280480385, "fcm_dpo/delta": 0.02223687618970871, "fcm_dpo/margin": 81.73025512695312, "fcm_dpo/q_t": 0.4113684296607971, "grad_norm": 12.853103637695312, "learning_rate": 1.6208764069656578e-07, "logits/chosen": -1.843031883239746, "logits/rejected": -1.701148271560669, "logps/chosen": -236.9666748046875, "logps/ref_chosen": -87.4223403930664, "logps/ref_rejected": -106.70075988769531, "logps/rejected": -337.975341796875, "loss": 1.0929, "margin_dpo/margin_mean": 81.73025512695312, "margin_dpo/margin_std": 98.36163330078125, "step": 446 }, { "KL/chosen_KL_mean": -146.74166870117188, "KL/mean": -200.84628295898438, "KL/rejected_KL_mean": -254.95086669921875, "KL/std": 105.91786193847656, "epoch": 0.6563876651982379, "fcm_dpo/beta": 0.004560584202408791, "fcm_dpo/delta": -0.09837216138839722, "fcm_dpo/margin": 108.20919799804688, "fcm_dpo/q_t": 0.3859099745750427, "grad_norm": 8.567708969116211, "learning_rate": 1.608874379754465e-07, "logits/chosen": -2.172044277191162, "logits/rejected": -2.1266567707061768, "logps/chosen": -230.35693359375, "logps/ref_chosen": -83.6152572631836, "logps/ref_rejected": -104.91239929199219, "logps/rejected": -359.86328125, "loss": 1.0244, "margin_dpo/margin_mean": 108.20919799804688, "margin_dpo/margin_std": 121.09281158447266, "step": 447 }, { "KL/chosen_KL_mean": -159.46607971191406, "KL/mean": -211.13763427734375, "KL/rejected_KL_mean": -262.8091735839844, "KL/std": 110.52723693847656, "epoch": 0.657856093979442, "fcm_dpo/beta": 0.004512270446866751, "fcm_dpo/delta": -0.06966008245944977, "fcm_dpo/margin": 103.34309387207031, "fcm_dpo/q_t": 0.39303940534591675, "grad_norm": 9.80539321899414, "learning_rate": 1.5968958345321177e-07, "logits/chosen": -1.8242204189300537, "logits/rejected": -1.558175802230835, "logps/chosen": -252.0418701171875, "logps/ref_chosen": -92.5757827758789, "logps/ref_rejected": -107.68977355957031, "logps/rejected": -370.49896240234375, "loss": 1.0559, "margin_dpo/margin_mean": 103.34309387207031, "margin_dpo/margin_std": 129.36798095703125, "step": 448 }, { "KL/chosen_KL_mean": -157.51156616210938, "KL/mean": -211.2559814453125, "KL/rejected_KL_mean": -265.0003662109375, "KL/std": 103.53204345703125, "epoch": 0.6593245227606461, "fcm_dpo/beta": 0.004422758240252733, "fcm_dpo/delta": -0.07917153835296631, "fcm_dpo/margin": 107.48880767822266, "fcm_dpo/q_t": 0.38916462659835815, "grad_norm": 8.940352439880371, "learning_rate": 1.584941086944423e-07, "logits/chosen": -1.4678890705108643, "logits/rejected": -1.02360999584198, "logps/chosen": -259.9104919433594, "logps/ref_chosen": -102.39893341064453, "logps/ref_rejected": -95.14886474609375, "logps/rejected": -360.1492614746094, "loss": 1.0306, "margin_dpo/margin_mean": 107.48880767822266, "margin_dpo/margin_std": 118.35263061523438, "step": 449 }, { "KL/chosen_KL_mean": -140.04486083984375, "KL/mean": -190.2281494140625, "KL/rejected_KL_mean": -240.41143798828125, "KL/std": 104.83326721191406, "epoch": 0.6607929515418502, "fcm_dpo/beta": 0.004374480340629816, "fcm_dpo/delta": -0.040903240442276, "fcm_dpo/margin": 100.36658477783203, "fcm_dpo/q_t": 0.39710623025894165, "grad_norm": 9.121597290039062, "learning_rate": 1.573010452010098e-07, "logits/chosen": -2.342593193054199, "logits/rejected": -2.14859676361084, "logps/chosen": -227.0377197265625, "logps/ref_chosen": -86.99285888671875, "logps/ref_rejected": -108.53203582763672, "logps/rejected": -348.9434814453125, "loss": 1.0546, "margin_dpo/margin_mean": 100.3665771484375, "margin_dpo/margin_std": 115.07530212402344, "step": 450 }, { "KL/chosen_KL_mean": -159.4221649169922, "KL/mean": -199.80389404296875, "KL/rejected_KL_mean": -240.18563842773438, "KL/std": 104.59080505371094, "epoch": 0.6622613803230544, "fcm_dpo/beta": 0.004406338557600975, "fcm_dpo/delta": 0.045633986592292786, "fcm_dpo/margin": 80.76348114013672, "fcm_dpo/q_t": 0.41864028573036194, "grad_norm": 13.788801193237305, "learning_rate": 1.5611042441124687e-07, "logits/chosen": -2.316223621368408, "logits/rejected": -1.8023138046264648, "logps/chosen": -246.2334442138672, "logps/ref_chosen": -86.811279296875, "logps/ref_rejected": -79.8555908203125, "logps/rejected": -320.0412292480469, "loss": 1.1392, "margin_dpo/margin_mean": 80.76348114013672, "margin_dpo/margin_std": 129.68798828125, "step": 451 }, { "KL/chosen_KL_mean": -139.53416442871094, "KL/mean": -184.9852752685547, "KL/rejected_KL_mean": -230.4364013671875, "KL/std": 99.98948669433594, "epoch": 0.6637298091042585, "fcm_dpo/beta": 0.004379909485578537, "fcm_dpo/delta": 0.0009612664580345154, "fcm_dpo/margin": 90.90223693847656, "fcm_dpo/q_t": 0.40604913234710693, "grad_norm": 8.32327651977539, "learning_rate": 1.549222776991186e-07, "logits/chosen": -1.8307929039001465, "logits/rejected": -1.7722365856170654, "logps/chosen": -218.91380310058594, "logps/ref_chosen": -79.379638671875, "logps/ref_rejected": -103.71539306640625, "logps/rejected": -334.15179443359375, "loss": 1.0721, "margin_dpo/margin_mean": 90.90223693847656, "margin_dpo/margin_std": 92.71813201904297, "step": 452 }, { "KL/chosen_KL_mean": -168.4999237060547, "KL/mean": -223.211181640625, "KL/rejected_KL_mean": -277.92242431640625, "KL/std": 102.00343322753906, "epoch": 0.6651982378854625, "fcm_dpo/beta": 0.004362079314887524, "fcm_dpo/delta": -0.08112306892871857, "fcm_dpo/margin": 109.42253112792969, "fcm_dpo/q_t": 0.38889414072036743, "grad_norm": 12.29583740234375, "learning_rate": 1.5373663637339584e-07, "logits/chosen": -2.0353407859802246, "logits/rejected": -1.5714672803878784, "logps/chosen": -256.19512939453125, "logps/ref_chosen": -87.6951904296875, "logps/ref_rejected": -90.0582275390625, "logps/rejected": -367.98065185546875, "loss": 1.0192, "margin_dpo/margin_mean": 109.42252349853516, "margin_dpo/margin_std": 109.66608428955078, "step": 453 }, { "KL/chosen_KL_mean": -176.35374450683594, "KL/mean": -220.59213256835938, "KL/rejected_KL_mean": -264.83050537109375, "KL/std": 104.35549926757812, "epoch": 0.6666666666666666, "fcm_dpo/beta": 0.004368453752249479, "fcm_dpo/delta": 0.01353529840707779, "fcm_dpo/margin": 88.47679901123047, "fcm_dpo/q_t": 0.4116078317165375, "grad_norm": 9.555173873901367, "learning_rate": 1.5255353167683017e-07, "logits/chosen": -2.034606695175171, "logits/rejected": -1.5896660089492798, "logps/chosen": -265.91998291015625, "logps/ref_chosen": -89.56623840332031, "logps/ref_rejected": -92.92105102539062, "logps/rejected": -357.7515869140625, "loss": 1.1218, "margin_dpo/margin_mean": 88.476806640625, "margin_dpo/margin_std": 135.97451782226562, "step": 454 }, { "KL/chosen_KL_mean": -165.71676635742188, "KL/mean": -207.7672576904297, "KL/rejected_KL_mean": -249.8177490234375, "KL/std": 102.75578308105469, "epoch": 0.6681350954478708, "fcm_dpo/beta": 0.004385477863252163, "fcm_dpo/delta": 0.03208241984248161, "fcm_dpo/margin": 84.10101318359375, "fcm_dpo/q_t": 0.4142566919326782, "grad_norm": 9.749544143676758, "learning_rate": 1.5137299478533064e-07, "logits/chosen": -1.9461827278137207, "logits/rejected": -1.9060146808624268, "logps/chosen": -243.34674072265625, "logps/ref_chosen": -77.62998962402344, "logps/ref_rejected": -118.97795104980469, "logps/rejected": -368.79571533203125, "loss": 1.1155, "margin_dpo/margin_mean": 84.10100555419922, "margin_dpo/margin_std": 117.47057342529297, "step": 455 }, { "KL/chosen_KL_mean": -168.12991333007812, "KL/mean": -210.22055053710938, "KL/rejected_KL_mean": -252.31118774414062, "KL/std": 104.33153533935547, "epoch": 0.6696035242290749, "fcm_dpo/beta": 0.004406563937664032, "fcm_dpo/delta": 0.030055031180381775, "fcm_dpo/margin": 84.18126678466797, "fcm_dpo/q_t": 0.41327929496765137, "grad_norm": 11.235763549804688, "learning_rate": 1.5019505680714232e-07, "logits/chosen": -2.0052504539489746, "logits/rejected": -1.8733961582183838, "logps/chosen": -257.74676513671875, "logps/ref_chosen": -89.61686706542969, "logps/ref_rejected": -109.5597152709961, "logps/rejected": -361.87091064453125, "loss": 1.1014, "margin_dpo/margin_mean": 84.1812744140625, "margin_dpo/margin_std": 106.1058578491211, "step": 456 }, { "KL/chosen_KL_mean": -136.98916625976562, "KL/mean": -192.67095947265625, "KL/rejected_KL_mean": -248.35272216796875, "KL/std": 104.07998657226562, "epoch": 0.671071953010279, "fcm_dpo/beta": 0.00431801937520504, "fcm_dpo/delta": -0.08664345741271973, "fcm_dpo/margin": 111.36355590820312, "fcm_dpo/q_t": 0.3856748938560486, "grad_norm": 10.150973320007324, "learning_rate": 1.4901974878202627e-07, "logits/chosen": -2.000450849533081, "logits/rejected": -1.6788246631622314, "logps/chosen": -224.31085205078125, "logps/ref_chosen": -87.32168579101562, "logps/ref_rejected": -90.76660919189453, "logps/rejected": -339.11932373046875, "loss": 1.0046, "margin_dpo/margin_mean": 111.36357116699219, "margin_dpo/margin_std": 93.1021728515625, "step": 457 }, { "KL/chosen_KL_mean": -159.2378692626953, "KL/mean": -199.57888793945312, "KL/rejected_KL_mean": -239.91989135742188, "KL/std": 103.58051300048828, "epoch": 0.6725403817914831, "fcm_dpo/beta": 0.00437154108658433, "fcm_dpo/delta": 0.04880714416503906, "fcm_dpo/margin": 80.68203735351562, "fcm_dpo/q_t": 0.4194113314151764, "grad_norm": 9.38282299041748, "learning_rate": 1.4784710168044212e-07, "logits/chosen": -1.6099436283111572, "logits/rejected": -1.3710919618606567, "logps/chosen": -252.75831604003906, "logps/ref_chosen": -93.52044677734375, "logps/ref_rejected": -103.36898803710938, "logps/rejected": -343.28887939453125, "loss": 1.1294, "margin_dpo/margin_mean": 80.68203735351562, "margin_dpo/margin_std": 120.49508666992188, "step": 458 }, { "KL/chosen_KL_mean": -151.99929809570312, "KL/mean": -202.41018676757812, "KL/rejected_KL_mean": -252.82107543945312, "KL/std": 108.24928283691406, "epoch": 0.6740088105726872, "fcm_dpo/beta": 0.004327808041125536, "fcm_dpo/delta": -0.03858676552772522, "fcm_dpo/margin": 100.82177734375, "fcm_dpo/q_t": 0.3990144431591034, "grad_norm": 7.30731725692749, "learning_rate": 1.466771464027316e-07, "logits/chosen": -1.783677101135254, "logits/rejected": -1.538218379020691, "logps/chosen": -227.6875, "logps/ref_chosen": -75.68820190429688, "logps/ref_rejected": -92.17048645019531, "logps/rejected": -344.9915771484375, "loss": 1.0578, "margin_dpo/margin_mean": 100.82177734375, "margin_dpo/margin_std": 116.35655212402344, "step": 459 }, { "KL/chosen_KL_mean": -155.2317352294922, "KL/mean": -201.94851684570312, "KL/rejected_KL_mean": -248.66531372070312, "KL/std": 104.95709228515625, "epoch": 0.6754772393538914, "fcm_dpo/beta": 0.004321034997701645, "fcm_dpo/delta": -0.0041506327688694, "fcm_dpo/margin": 93.43359375, "fcm_dpo/q_t": 0.40629369020462036, "grad_norm": 10.196805000305176, "learning_rate": 1.4550991377830423e-07, "logits/chosen": -1.9044305086135864, "logits/rejected": -1.9326595067977905, "logps/chosen": -236.34962463378906, "logps/ref_chosen": -81.11788940429688, "logps/ref_rejected": -110.31238555908203, "logps/rejected": -358.97772216796875, "loss": 1.0879, "margin_dpo/margin_mean": 93.43359375, "margin_dpo/margin_std": 120.20079803466797, "step": 460 }, { "KL/chosen_KL_mean": -191.6822052001953, "KL/mean": -229.56356811523438, "KL/rejected_KL_mean": -267.4449462890625, "KL/std": 92.26930236816406, "epoch": 0.6769456681350955, "fcm_dpo/beta": 0.004371312912553549, "fcm_dpo/delta": 0.07122945785522461, "fcm_dpo/margin": 75.76275634765625, "fcm_dpo/q_t": 0.42320647835731506, "grad_norm": 8.283832550048828, "learning_rate": 1.4434543456482518e-07, "logits/chosen": -1.441823124885559, "logits/rejected": -1.2454891204833984, "logps/chosen": -273.2657470703125, "logps/ref_chosen": -81.58352661132812, "logps/ref_rejected": -93.87710571289062, "logps/rejected": -361.32208251953125, "loss": 1.1385, "margin_dpo/margin_mean": 75.76275634765625, "margin_dpo/margin_std": 109.87903594970703, "step": 461 }, { "KL/chosen_KL_mean": -145.62591552734375, "KL/mean": -194.15530395507812, "KL/rejected_KL_mean": -242.68472290039062, "KL/std": 99.33123779296875, "epoch": 0.6784140969162996, "fcm_dpo/beta": 0.004386282525956631, "fcm_dpo/delta": -0.02687562257051468, "fcm_dpo/margin": 97.05878448486328, "fcm_dpo/q_t": 0.4005051851272583, "grad_norm": 8.773185729980469, "learning_rate": 1.4318373944740484e-07, "logits/chosen": -2.1755919456481934, "logits/rejected": -1.6656078100204468, "logps/chosen": -239.824462890625, "logps/ref_chosen": -94.19854736328125, "logps/ref_rejected": -85.63162994384766, "logps/rejected": -328.31634521484375, "loss": 1.0574, "margin_dpo/margin_mean": 97.05878448486328, "margin_dpo/margin_std": 107.20410919189453, "step": 462 }, { "KL/chosen_KL_mean": -141.45054626464844, "KL/mean": -199.62652587890625, "KL/rejected_KL_mean": -257.8025207519531, "KL/std": 106.71873474121094, "epoch": 0.6798825256975036, "fcm_dpo/beta": 0.0043309880420565605, "fcm_dpo/delta": -0.10956902801990509, "fcm_dpo/margin": 116.35195922851562, "fcm_dpo/q_t": 0.38164666295051575, "grad_norm": 13.257331848144531, "learning_rate": 1.4202485903778976e-07, "logits/chosen": -1.8005731105804443, "logits/rejected": -1.3703184127807617, "logps/chosen": -227.37530517578125, "logps/ref_chosen": -85.92474365234375, "logps/ref_rejected": -96.90184020996094, "logps/rejected": -354.704345703125, "loss": 1.0082, "margin_dpo/margin_mean": 116.35195922851562, "margin_dpo/margin_std": 113.35713195800781, "step": 463 }, { "KL/chosen_KL_mean": -174.3768768310547, "KL/mean": -221.88125610351562, "KL/rejected_KL_mean": -269.3856506347656, "KL/std": 106.38433837890625, "epoch": 0.6813509544787077, "fcm_dpo/beta": 0.004265697672963142, "fcm_dpo/delta": -0.005621535703539848, "fcm_dpo/margin": 95.0087890625, "fcm_dpo/q_t": 0.4059327244758606, "grad_norm": 12.329377174377441, "learning_rate": 1.4086882387355658e-07, "logits/chosen": -1.875147819519043, "logits/rejected": -1.9550440311431885, "logps/chosen": -254.06607055664062, "logps/ref_chosen": -79.68919372558594, "logps/ref_rejected": -107.29232025146484, "logps/rejected": -376.677978515625, "loss": 1.085, "margin_dpo/margin_mean": 95.0087890625, "margin_dpo/margin_std": 121.85136413574219, "step": 464 }, { "KL/chosen_KL_mean": -166.10015869140625, "KL/mean": -221.586669921875, "KL/rejected_KL_mean": -277.0732421875, "KL/std": 114.9747314453125, "epoch": 0.6828193832599119, "fcm_dpo/beta": 0.0042225755751132965, "fcm_dpo/delta": -0.07210975885391235, "fcm_dpo/margin": 110.97308349609375, "fcm_dpo/q_t": 0.3896597623825073, "grad_norm": 8.71872329711914, "learning_rate": 1.3971566441730714e-07, "logits/chosen": -1.5314579010009766, "logits/rejected": -1.224401593208313, "logps/chosen": -257.96038818359375, "logps/ref_chosen": -91.8602294921875, "logps/ref_rejected": -118.71001434326172, "logps/rejected": -395.7832336425781, "loss": 1.0216, "margin_dpo/margin_mean": 110.97308349609375, "margin_dpo/margin_std": 109.06581115722656, "step": 465 }, { "KL/chosen_KL_mean": -147.10629272460938, "KL/mean": -203.729736328125, "KL/rejected_KL_mean": -260.3531494140625, "KL/std": 109.51838684082031, "epoch": 0.684287812041116, "fcm_dpo/beta": 0.004161463119089603, "fcm_dpo/delta": -0.07491360604763031, "fcm_dpo/margin": 113.24684143066406, "fcm_dpo/q_t": 0.3894752860069275, "grad_norm": 6.989395618438721, "learning_rate": 1.3856541105586545e-07, "logits/chosen": -1.4670743942260742, "logits/rejected": -1.1436817646026611, "logps/chosen": -231.80770874023438, "logps/ref_chosen": -84.70140838623047, "logps/ref_rejected": -96.05084228515625, "logps/rejected": -356.40399169921875, "loss": 1.0229, "margin_dpo/margin_mean": 113.24684143066406, "margin_dpo/margin_std": 114.69720458984375, "step": 466 }, { "KL/chosen_KL_mean": -188.8863067626953, "KL/mean": -236.03106689453125, "KL/rejected_KL_mean": -283.17584228515625, "KL/std": 114.32596588134766, "epoch": 0.6857562408223201, "fcm_dpo/beta": 0.00413773488253355, "fcm_dpo/delta": 0.00987982377409935, "fcm_dpo/margin": 94.28953552246094, "fcm_dpo/q_t": 0.4092826843261719, "grad_norm": 11.707625389099121, "learning_rate": 1.3741809409947729e-07, "logits/chosen": -2.38200044631958, "logits/rejected": -1.809912085533142, "logps/chosen": -298.18463134765625, "logps/ref_chosen": -109.29832458496094, "logps/ref_rejected": -108.8436508178711, "logps/rejected": -392.01947021484375, "loss": 1.1039, "margin_dpo/margin_mean": 94.2895278930664, "margin_dpo/margin_std": 129.6176300048828, "step": 467 }, { "KL/chosen_KL_mean": -147.089599609375, "KL/mean": -190.4575653076172, "KL/rejected_KL_mean": -233.82553100585938, "KL/std": 111.98554992675781, "epoch": 0.6872246696035242, "fcm_dpo/beta": 0.0041780369356274605, "fcm_dpo/delta": 0.03904637694358826, "fcm_dpo/margin": 86.73590087890625, "fcm_dpo/q_t": 0.41717907786369324, "grad_norm": 9.456071853637695, "learning_rate": 1.362737437810114e-07, "logits/chosen": -1.8576431274414062, "logits/rejected": -1.654072880744934, "logps/chosen": -245.4112548828125, "logps/ref_chosen": -98.32164764404297, "logps/ref_rejected": -106.68048095703125, "logps/rejected": -340.5060119628906, "loss": 1.1241, "margin_dpo/margin_mean": 86.73590087890625, "margin_dpo/margin_std": 128.5650634765625, "step": 468 }, { "KL/chosen_KL_mean": -180.93460083007812, "KL/mean": -234.97314453125, "KL/rejected_KL_mean": -289.01165771484375, "KL/std": 112.00450897216797, "epoch": 0.6886930983847284, "fcm_dpo/beta": 0.00413902010768652, "fcm_dpo/delta": -0.05032285302877426, "fcm_dpo/margin": 108.07706451416016, "fcm_dpo/q_t": 0.39570537209510803, "grad_norm": 10.224261283874512, "learning_rate": 1.351323902551631e-07, "logits/chosen": -2.2089333534240723, "logits/rejected": -1.7875107526779175, "logps/chosen": -277.69879150390625, "logps/ref_chosen": -96.76420593261719, "logps/ref_rejected": -109.59500885009766, "logps/rejected": -398.606689453125, "loss": 1.0492, "margin_dpo/margin_mean": 108.07706451416016, "margin_dpo/margin_std": 121.17799377441406, "step": 469 }, { "KL/chosen_KL_mean": -166.94015502929688, "KL/mean": -216.27455139160156, "KL/rejected_KL_mean": -265.60894775390625, "KL/std": 104.40805053710938, "epoch": 0.6901615271659325, "fcm_dpo/beta": 0.004141214303672314, "fcm_dpo/delta": -0.00898781605064869, "fcm_dpo/margin": 98.66877746582031, "fcm_dpo/q_t": 0.4037138819694519, "grad_norm": 9.038516998291016, "learning_rate": 1.339940635976592e-07, "logits/chosen": -1.694380283355713, "logits/rejected": -1.276410460472107, "logps/chosen": -250.43682861328125, "logps/ref_chosen": -83.49665832519531, "logps/ref_rejected": -88.48578643798828, "logps/rejected": -354.0947265625, "loss": 1.0641, "margin_dpo/margin_mean": 98.66879272460938, "margin_dpo/margin_std": 105.29015350341797, "step": 470 }, { "KL/chosen_KL_mean": -190.96237182617188, "KL/mean": -237.6298828125, "KL/rejected_KL_mean": -284.29742431640625, "KL/std": 103.05272674560547, "epoch": 0.6916299559471366, "fcm_dpo/beta": 0.004148954525589943, "fcm_dpo/delta": 0.013273140415549278, "fcm_dpo/margin": 93.33503723144531, "fcm_dpo/q_t": 0.4086982309818268, "grad_norm": 8.817078590393066, "learning_rate": 1.3285879380446563e-07, "logits/chosen": -1.7130258083343506, "logits/rejected": -1.3059699535369873, "logps/chosen": -279.4366760253906, "logps/ref_chosen": -88.47430419921875, "logps/ref_rejected": -90.48171997070312, "logps/rejected": -374.77911376953125, "loss": 1.0799, "margin_dpo/margin_mean": 93.33503723144531, "margin_dpo/margin_std": 103.68357849121094, "step": 471 }, { "KL/chosen_KL_mean": -178.9311065673828, "KL/mean": -234.23001098632812, "KL/rejected_KL_mean": -289.52886962890625, "KL/std": 104.03006744384766, "epoch": 0.6930983847283406, "fcm_dpo/beta": 0.0041074855253100395, "fcm_dpo/delta": -0.057126980274915695, "fcm_dpo/margin": 110.59777069091797, "fcm_dpo/q_t": 0.3938341736793518, "grad_norm": 8.330387115478516, "learning_rate": 1.317266107909975e-07, "logits/chosen": -2.0469017028808594, "logits/rejected": -1.3350416421890259, "logps/chosen": -304.164794921875, "logps/ref_chosen": -125.23370361328125, "logps/ref_rejected": -121.05349731445312, "logps/rejected": -410.5823974609375, "loss": 1.0318, "margin_dpo/margin_mean": 110.59777069091797, "margin_dpo/margin_std": 110.41332244873047, "step": 472 }, { "KL/chosen_KL_mean": -202.18951416015625, "KL/mean": -236.78546142578125, "KL/rejected_KL_mean": -271.38140869140625, "KL/std": 105.14460754394531, "epoch": 0.6945668135095447, "fcm_dpo/beta": 0.0041816686280071735, "fcm_dpo/delta": 0.11376629769802094, "fcm_dpo/margin": 69.19190216064453, "fcm_dpo/q_t": 0.4338950514793396, "grad_norm": 9.605264663696289, "learning_rate": 1.3059754439133002e-07, "logits/chosen": -1.6872016191482544, "logits/rejected": -1.1825268268585205, "logps/chosen": -297.8009033203125, "logps/ref_chosen": -95.61137390136719, "logps/ref_rejected": -88.15115356445312, "logps/rejected": -359.5325622558594, "loss": 1.1791, "margin_dpo/margin_mean": 69.19190979003906, "margin_dpo/margin_std": 122.16900634765625, "step": 473 }, { "KL/chosen_KL_mean": -189.69647216796875, "KL/mean": -225.39956665039062, "KL/rejected_KL_mean": -261.1026916503906, "KL/std": 99.85199737548828, "epoch": 0.6960352422907489, "fcm_dpo/beta": 0.004289246164262295, "fcm_dpo/delta": 0.09509143978357315, "fcm_dpo/margin": 71.40618896484375, "fcm_dpo/q_t": 0.42929884791374207, "grad_norm": 13.86391544342041, "learning_rate": 1.2947162435741277e-07, "logits/chosen": -1.3879985809326172, "logits/rejected": -1.194183588027954, "logps/chosen": -271.17620849609375, "logps/ref_chosen": -81.47975158691406, "logps/ref_rejected": -96.46562957763672, "logps/rejected": -357.56829833984375, "loss": 1.1753, "margin_dpo/margin_mean": 71.40618896484375, "margin_dpo/margin_std": 122.19325256347656, "step": 474 }, { "KL/chosen_KL_mean": -143.13668823242188, "KL/mean": -198.124267578125, "KL/rejected_KL_mean": -253.1118621826172, "KL/std": 107.011962890625, "epoch": 0.697503671071953, "fcm_dpo/beta": 0.004239218309521675, "fcm_dpo/delta": -0.06941326707601547, "fcm_dpo/margin": 109.97517395019531, "fcm_dpo/q_t": 0.3920896053314209, "grad_norm": 8.250890731811523, "learning_rate": 1.2834888035828596e-07, "logits/chosen": -2.2223005294799805, "logits/rejected": -2.169830560684204, "logps/chosen": -217.33267211914062, "logps/ref_chosen": -74.19598388671875, "logps/ref_rejected": -94.69242095947266, "logps/rejected": -347.8042907714844, "loss": 1.0358, "margin_dpo/margin_mean": 109.97517395019531, "margin_dpo/margin_std": 122.1253433227539, "step": 475 }, { "KL/chosen_KL_mean": -167.28976440429688, "KL/mean": -215.92303466796875, "KL/rejected_KL_mean": -264.5562744140625, "KL/std": 108.85108947753906, "epoch": 0.6989720998531571, "fcm_dpo/beta": 0.004204140976071358, "fcm_dpo/delta": -0.009484604001045227, "fcm_dpo/margin": 97.26653289794922, "fcm_dpo/q_t": 0.40563684701919556, "grad_norm": 10.023250579833984, "learning_rate": 1.2722934197929802e-07, "logits/chosen": -1.865408182144165, "logits/rejected": -1.4813342094421387, "logps/chosen": -239.2608642578125, "logps/ref_chosen": -71.97109985351562, "logps/ref_rejected": -80.26224517822266, "logps/rejected": -344.81854248046875, "loss": 1.0776, "margin_dpo/margin_mean": 97.26654052734375, "margin_dpo/margin_std": 118.90120697021484, "step": 476 }, { "KL/chosen_KL_mean": -167.1858673095703, "KL/mean": -215.8837890625, "KL/rejected_KL_mean": -264.5816955566406, "KL/std": 95.95845031738281, "epoch": 0.7004405286343612, "fcm_dpo/beta": 0.004210454411804676, "fcm_dpo/delta": -0.010516196489334106, "fcm_dpo/margin": 97.39582061767578, "fcm_dpo/q_t": 0.40374594926834106, "grad_norm": 8.802793502807617, "learning_rate": 1.2611303872132631e-07, "logits/chosen": -1.8418238162994385, "logits/rejected": -0.867655336856842, "logps/chosen": -272.19140625, "logps/ref_chosen": -105.00555419921875, "logps/ref_rejected": -81.87843322753906, "logps/rejected": -346.4601135253906, "loss": 1.0776, "margin_dpo/margin_mean": 97.39582061767578, "margin_dpo/margin_std": 119.29344177246094, "step": 477 }, { "KL/chosen_KL_mean": -162.71475219726562, "KL/mean": -213.19015502929688, "KL/rejected_KL_mean": -263.66558837890625, "KL/std": 104.15471649169922, "epoch": 0.7019089574155654, "fcm_dpo/beta": 0.00418923981487751, "fcm_dpo/delta": -0.023923160508275032, "fcm_dpo/margin": 100.95083618164062, "fcm_dpo/q_t": 0.4009394943714142, "grad_norm": 8.504009246826172, "learning_rate": 1.2500000000000005e-07, "logits/chosen": -1.3972270488739014, "logits/rejected": -1.2154604196548462, "logps/chosen": -239.50294494628906, "logps/ref_chosen": -76.7882080078125, "logps/ref_rejected": -90.43994140625, "logps/rejected": -354.10552978515625, "loss": 1.0625, "margin_dpo/margin_mean": 100.95083618164062, "margin_dpo/margin_std": 115.06307983398438, "step": 478 }, { "KL/chosen_KL_mean": -165.75765991210938, "KL/mean": -215.889404296875, "KL/rejected_KL_mean": -266.0211486816406, "KL/std": 105.10525512695312, "epoch": 0.7033773861967695, "fcm_dpo/beta": 0.004187238402664661, "fcm_dpo/delta": -0.020896773785352707, "fcm_dpo/margin": 100.2635269165039, "fcm_dpo/q_t": 0.40095841884613037, "grad_norm": 11.566056251525879, "learning_rate": 1.2389025514492456e-07, "logits/chosen": -1.7468085289001465, "logits/rejected": -1.608776569366455, "logps/chosen": -247.11996459960938, "logps/ref_chosen": -81.3623046875, "logps/ref_rejected": -101.09114074707031, "logps/rejected": -367.1123046875, "loss": 1.0605, "margin_dpo/margin_mean": 100.2635269165039, "margin_dpo/margin_std": 108.38883972167969, "step": 479 }, { "KL/chosen_KL_mean": -191.5328369140625, "KL/mean": -230.63967895507812, "KL/rejected_KL_mean": -269.74652099609375, "KL/std": 100.10520935058594, "epoch": 0.7048458149779736, "fcm_dpo/beta": 0.004213536158204079, "fcm_dpo/delta": 0.07283270359039307, "fcm_dpo/margin": 78.21368408203125, "fcm_dpo/q_t": 0.42303138971328735, "grad_norm": 9.909329414367676, "learning_rate": 1.227838333989088e-07, "logits/chosen": -1.6748054027557373, "logits/rejected": -0.9865530133247375, "logps/chosen": -288.3067626953125, "logps/ref_chosen": -96.7739028930664, "logps/ref_rejected": -86.40473937988281, "logps/rejected": -356.1512451171875, "loss": 1.1358, "margin_dpo/margin_mean": 78.21369171142578, "margin_dpo/margin_std": 110.66896057128906, "step": 480 }, { "KL/chosen_KL_mean": -186.83792114257812, "KL/mean": -233.60037231445312, "KL/rejected_KL_mean": -280.36285400390625, "KL/std": 104.28231811523438, "epoch": 0.7063142437591777, "fcm_dpo/beta": 0.004228734411299229, "fcm_dpo/delta": 0.00468816515058279, "fcm_dpo/margin": 93.5249252319336, "fcm_dpo/q_t": 0.40736714005470276, "grad_norm": 11.558316230773926, "learning_rate": 1.2168076391719489e-07, "logits/chosen": -1.8934369087219238, "logits/rejected": -1.4527826309204102, "logps/chosen": -278.5080871582031, "logps/ref_chosen": -91.670166015625, "logps/ref_rejected": -98.69490051269531, "logps/rejected": -379.0577392578125, "loss": 1.0865, "margin_dpo/margin_mean": 93.5249252319336, "margin_dpo/margin_std": 116.08255004882812, "step": 481 }, { "KL/chosen_KL_mean": -181.28321838378906, "KL/mean": -226.77572631835938, "KL/rejected_KL_mean": -272.2682189941406, "KL/std": 102.97505187988281, "epoch": 0.7077826725403817, "fcm_dpo/beta": 0.004230765625834465, "fcm_dpo/delta": 0.015557918697595596, "fcm_dpo/margin": 90.98500061035156, "fcm_dpo/q_t": 0.4093700051307678, "grad_norm": 12.310957908630371, "learning_rate": 1.2058107576668938e-07, "logits/chosen": -1.519537329673767, "logits/rejected": -1.1587963104248047, "logps/chosen": -279.8033447265625, "logps/ref_chosen": -98.52011108398438, "logps/ref_rejected": -94.82945251464844, "logps/rejected": -367.09765625, "loss": 1.0949, "margin_dpo/margin_mean": 90.98500061035156, "margin_dpo/margin_std": 114.76008605957031, "step": 482 }, { "KL/chosen_KL_mean": -186.2301483154297, "KL/mean": -234.7520751953125, "KL/rejected_KL_mean": -283.27398681640625, "KL/std": 108.66519165039062, "epoch": 0.7092511013215859, "fcm_dpo/beta": 0.0042608436197042465, "fcm_dpo/delta": -0.014658600091934204, "fcm_dpo/margin": 97.04385375976562, "fcm_dpo/q_t": 0.40262287855148315, "grad_norm": 7.430368900299072, "learning_rate": 1.194847979251979e-07, "logits/chosen": -2.00972318649292, "logits/rejected": -1.36256742477417, "logps/chosen": -293.3487548828125, "logps/ref_chosen": -107.11860656738281, "logps/ref_rejected": -101.11499786376953, "logps/rejected": -384.38897705078125, "loss": 1.0683, "margin_dpo/margin_mean": 97.04385375976562, "margin_dpo/margin_std": 108.2557601928711, "step": 483 }, { "KL/chosen_KL_mean": -170.69711303710938, "KL/mean": -224.48233032226562, "KL/rejected_KL_mean": -278.267578125, "KL/std": 110.74266052246094, "epoch": 0.71071953010279, "fcm_dpo/beta": 0.004210704006254673, "fcm_dpo/delta": -0.055471114814281464, "fcm_dpo/margin": 107.5704345703125, "fcm_dpo/q_t": 0.39396029710769653, "grad_norm": 8.276098251342773, "learning_rate": 1.1839195928066101e-07, "logits/chosen": -2.4757258892059326, "logits/rejected": -2.003270149230957, "logps/chosen": -257.6770324707031, "logps/ref_chosen": -86.97991943359375, "logps/ref_rejected": -90.72367095947266, "logps/rejected": -368.9912109375, "loss": 1.0461, "margin_dpo/margin_mean": 107.5704345703125, "margin_dpo/margin_std": 121.27104187011719, "step": 484 }, { "KL/chosen_KL_mean": -161.6885986328125, "KL/mean": -220.36053466796875, "KL/rejected_KL_mean": -279.0324401855469, "KL/std": 114.90472412109375, "epoch": 0.7121879588839941, "fcm_dpo/beta": 0.004132692702114582, "fcm_dpo/delta": -0.08924850821495056, "fcm_dpo/margin": 117.34385681152344, "fcm_dpo/q_t": 0.3865705132484436, "grad_norm": 8.799509048461914, "learning_rate": 1.1730258863039347e-07, "logits/chosen": -1.7328630685806274, "logits/rejected": -1.3126494884490967, "logps/chosen": -255.74734497070312, "logps/ref_chosen": -94.05874633789062, "logps/ref_rejected": -108.56298828125, "logps/rejected": -387.5954284667969, "loss": 1.013, "margin_dpo/margin_mean": 117.34385681152344, "margin_dpo/margin_std": 116.30082702636719, "step": 485 }, { "KL/chosen_KL_mean": -186.04074096679688, "KL/mean": -236.86068725585938, "KL/rejected_KL_mean": -287.68060302734375, "KL/std": 110.8624267578125, "epoch": 0.7136563876651982, "fcm_dpo/beta": 0.004111514892429113, "fcm_dpo/delta": -0.018760444596409798, "fcm_dpo/margin": 101.63990783691406, "fcm_dpo/q_t": 0.4020587205886841, "grad_norm": 10.048659324645996, "learning_rate": 1.1621671468032493e-07, "logits/chosen": -1.8312005996704102, "logits/rejected": -1.0998258590698242, "logps/chosen": -279.78662109375, "logps/ref_chosen": -93.74588012695312, "logps/ref_rejected": -98.07064819335938, "logps/rejected": -385.75128173828125, "loss": 1.0696, "margin_dpo/margin_mean": 101.63990783691406, "margin_dpo/margin_std": 119.19694519042969, "step": 486 }, { "KL/chosen_KL_mean": -179.03953552246094, "KL/mean": -225.42352294921875, "KL/rejected_KL_mean": -271.8075256347656, "KL/std": 105.57036590576172, "epoch": 0.7151248164464024, "fcm_dpo/beta": 0.004107281100004911, "fcm_dpo/delta": 0.019732210785150528, "fcm_dpo/margin": 92.76798248291016, "fcm_dpo/q_t": 0.4106350541114807, "grad_norm": 9.83466911315918, "learning_rate": 1.1513436604424378e-07, "logits/chosen": -2.019442558288574, "logits/rejected": -1.7021853923797607, "logps/chosen": -267.0731201171875, "logps/ref_chosen": -88.0335693359375, "logps/ref_rejected": -98.47209930419922, "logps/rejected": -370.2796325683594, "loss": 1.0918, "margin_dpo/margin_mean": 92.76798248291016, "margin_dpo/margin_std": 112.29097747802734, "step": 487 }, { "KL/chosen_KL_mean": -162.5553436279297, "KL/mean": -214.2549285888672, "KL/rejected_KL_mean": -265.9544982910156, "KL/std": 112.14942932128906, "epoch": 0.7165932452276065, "fcm_dpo/beta": 0.004103820770978928, "fcm_dpo/delta": -0.025417225435376167, "fcm_dpo/margin": 103.3991470336914, "fcm_dpo/q_t": 0.3992619514465332, "grad_norm": 8.913911819458008, "learning_rate": 1.1405557124304335e-07, "logits/chosen": -1.4982420206069946, "logits/rejected": -1.2309049367904663, "logps/chosen": -247.34500122070312, "logps/ref_chosen": -84.78964233398438, "logps/ref_rejected": -90.2734603881836, "logps/rejected": -356.22796630859375, "loss": 1.0477, "margin_dpo/margin_mean": 103.3991470336914, "margin_dpo/margin_std": 99.01069641113281, "step": 488 }, { "KL/chosen_KL_mean": -182.739013671875, "KL/mean": -229.73736572265625, "KL/rejected_KL_mean": -276.7357482910156, "KL/std": 102.13237762451172, "epoch": 0.7180616740088106, "fcm_dpo/beta": 0.004102532751858234, "fcm_dpo/delta": 0.014946882613003254, "fcm_dpo/margin": 93.99673461914062, "fcm_dpo/q_t": 0.40981847047805786, "grad_norm": 9.139135360717773, "learning_rate": 1.1298035870396985e-07, "logits/chosen": -1.6019530296325684, "logits/rejected": -1.1069631576538086, "logps/chosen": -273.20831298828125, "logps/ref_chosen": -90.46929931640625, "logps/ref_rejected": -86.39761352539062, "logps/rejected": -363.13336181640625, "loss": 1.0885, "margin_dpo/margin_mean": 93.99673461914062, "margin_dpo/margin_std": 113.35433959960938, "step": 489 }, { "KL/chosen_KL_mean": -183.6405029296875, "KL/mean": -227.61453247070312, "KL/rejected_KL_mean": -271.5885314941406, "KL/std": 110.62335968017578, "epoch": 0.7195301027900147, "fcm_dpo/beta": 0.00415404886007309, "fcm_dpo/delta": 0.03480593487620354, "fcm_dpo/margin": 87.94801330566406, "fcm_dpo/q_t": 0.41698509454727173, "grad_norm": 8.293018341064453, "learning_rate": 1.1190875675987355e-07, "logits/chosen": -1.9676871299743652, "logits/rejected": -2.0501596927642822, "logps/chosen": -268.96063232421875, "logps/ref_chosen": -85.32012939453125, "logps/ref_rejected": -115.99385070800781, "logps/rejected": -387.5823974609375, "loss": 1.1448, "margin_dpo/margin_mean": 87.94801330566406, "margin_dpo/margin_std": 146.15811157226562, "step": 490 }, { "KL/chosen_KL_mean": -187.2249298095703, "KL/mean": -228.28598022460938, "KL/rejected_KL_mean": -269.3470153808594, "KL/std": 112.84669494628906, "epoch": 0.7209985315712188, "fcm_dpo/beta": 0.004168265964835882, "fcm_dpo/delta": 0.059762947261333466, "fcm_dpo/margin": 82.1220703125, "fcm_dpo/q_t": 0.41981106996536255, "grad_norm": 10.354742050170898, "learning_rate": 1.1084079364846241e-07, "logits/chosen": -1.6652309894561768, "logits/rejected": -1.1584563255310059, "logps/chosen": -273.36846923828125, "logps/ref_chosen": -86.14352416992188, "logps/ref_rejected": -80.67945861816406, "logps/rejected": -350.0264892578125, "loss": 1.1274, "margin_dpo/margin_mean": 82.1220703125, "margin_dpo/margin_std": 114.39408874511719, "step": 491 }, { "KL/chosen_KL_mean": -176.03094482421875, "KL/mean": -208.49180603027344, "KL/rejected_KL_mean": -240.95266723632812, "KL/std": 110.5134506225586, "epoch": 0.7224669603524229, "fcm_dpo/beta": 0.004259117878973484, "fcm_dpo/delta": 0.12710702419281006, "fcm_dpo/margin": 64.92172241210938, "fcm_dpo/q_t": 0.4367588758468628, "grad_norm": 13.730419158935547, "learning_rate": 1.097764975115576e-07, "logits/chosen": -1.7760796546936035, "logits/rejected": -1.3710782527923584, "logps/chosen": -257.1385192871094, "logps/ref_chosen": -81.10757446289062, "logps/ref_rejected": -80.75199890136719, "logps/rejected": -321.70465087890625, "loss": 1.2109, "margin_dpo/margin_mean": 64.92172241210938, "margin_dpo/margin_std": 137.99307250976562, "step": 492 }, { "KL/chosen_KL_mean": -172.85012817382812, "KL/mean": -217.06845092773438, "KL/rejected_KL_mean": -261.2867431640625, "KL/std": 100.18093872070312, "epoch": 0.723935389133627, "fcm_dpo/beta": 0.004285403992980719, "fcm_dpo/delta": 0.021531254053115845, "fcm_dpo/margin": 88.43656921386719, "fcm_dpo/q_t": 0.410750150680542, "grad_norm": 15.48604679107666, "learning_rate": 1.0871589639435203e-07, "logits/chosen": -2.5156140327453613, "logits/rejected": -1.7955269813537598, "logps/chosen": -285.0574951171875, "logps/ref_chosen": -112.20733642578125, "logps/ref_rejected": -93.60719299316406, "logps/rejected": -354.8939208984375, "loss": 1.095, "margin_dpo/margin_mean": 88.43656921386719, "margin_dpo/margin_std": 106.67979431152344, "step": 493 }, { "KL/chosen_KL_mean": -168.92860412597656, "KL/mean": -226.13827514648438, "KL/rejected_KL_mean": -283.347900390625, "KL/std": 110.72900390625, "epoch": 0.7254038179148311, "fcm_dpo/beta": 0.004229728598147631, "fcm_dpo/delta": -0.08929384499788284, "fcm_dpo/margin": 114.4193344116211, "fcm_dpo/q_t": 0.38685888051986694, "grad_norm": 10.036866188049316, "learning_rate": 1.0765901824467166e-07, "logits/chosen": -1.4146417379379272, "logits/rejected": -1.3378279209136963, "logps/chosen": -242.0435028076172, "logps/ref_chosen": -73.11489868164062, "logps/ref_rejected": -92.16300201416016, "logps/rejected": -375.51092529296875, "loss": 1.0223, "margin_dpo/margin_mean": 114.4193344116211, "margin_dpo/margin_std": 120.04611206054688, "step": 494 }, { "KL/chosen_KL_mean": -172.23629760742188, "KL/mean": -221.78680419921875, "KL/rejected_KL_mean": -271.3372802734375, "KL/std": 105.6611099243164, "epoch": 0.7268722466960352, "fcm_dpo/beta": 0.004218024201691151, "fcm_dpo/delta": -0.01882074773311615, "fcm_dpo/margin": 99.10099029541016, "fcm_dpo/q_t": 0.4030812680721283, "grad_norm": 10.692403793334961, "learning_rate": 1.0660589091223854e-07, "logits/chosen": -2.16510272026062, "logits/rejected": -1.693591594696045, "logps/chosen": -271.756591796875, "logps/ref_chosen": -99.52031707763672, "logps/ref_rejected": -97.93089294433594, "logps/rejected": -369.2681884765625, "loss": 1.0776, "margin_dpo/margin_mean": 99.10099029541016, "margin_dpo/margin_std": 126.4889907836914, "step": 495 }, { "KL/chosen_KL_mean": -192.02804565429688, "KL/mean": -234.2572021484375, "KL/rejected_KL_mean": -276.4863586425781, "KL/std": 106.07072448730469, "epoch": 0.7283406754772394, "fcm_dpo/beta": 0.004254691302776337, "fcm_dpo/delta": 0.04191937670111656, "fcm_dpo/margin": 84.45829772949219, "fcm_dpo/q_t": 0.4152357578277588, "grad_norm": 8.926499366760254, "learning_rate": 1.0555654214793722e-07, "logits/chosen": -2.25400710105896, "logits/rejected": -1.6204530000686646, "logps/chosen": -299.8847961425781, "logps/ref_chosen": -107.85675048828125, "logps/ref_rejected": -92.77056121826172, "logps/rejected": -369.25689697265625, "loss": 1.11, "margin_dpo/margin_mean": 84.45829772949219, "margin_dpo/margin_std": 106.83423614501953, "step": 496 }, { "KL/chosen_KL_mean": -205.92764282226562, "KL/mean": -233.37591552734375, "KL/rejected_KL_mean": -260.8241882324219, "KL/std": 107.04727172851562, "epoch": 0.7298091042584435, "fcm_dpo/beta": 0.004349136725068092, "fcm_dpo/delta": 0.16546620428562164, "fcm_dpo/margin": 54.89655303955078, "fcm_dpo/q_t": 0.44517213106155396, "grad_norm": 8.857268333435059, "learning_rate": 1.0451099960308374e-07, "logits/chosen": -1.661409616470337, "logits/rejected": -1.2055035829544067, "logps/chosen": -298.0108642578125, "logps/ref_chosen": -92.08322143554688, "logps/ref_rejected": -81.79503631591797, "logps/rejected": -342.61920166015625, "loss": 1.2241, "margin_dpo/margin_mean": 54.896549224853516, "margin_dpo/margin_std": 115.25721740722656, "step": 497 }, { "KL/chosen_KL_mean": -174.67718505859375, "KL/mean": -220.4405517578125, "KL/rejected_KL_mean": -266.20391845703125, "KL/std": 105.12934875488281, "epoch": 0.7312775330396476, "fcm_dpo/beta": 0.004391482099890709, "fcm_dpo/delta": -0.0020287479273974895, "fcm_dpo/margin": 91.52668762207031, "fcm_dpo/q_t": 0.40581366419792175, "grad_norm": 11.610233306884766, "learning_rate": 1.0346929082869641e-07, "logits/chosen": -1.71383798122406, "logits/rejected": -1.2065106630325317, "logps/chosen": -272.87158203125, "logps/ref_chosen": -98.19437408447266, "logps/ref_rejected": -90.68745422363281, "logps/rejected": -356.891357421875, "loss": 1.0853, "margin_dpo/margin_mean": 91.52668762207031, "margin_dpo/margin_std": 115.71757507324219, "step": 498 }, { "KL/chosen_KL_mean": -182.7064208984375, "KL/mean": -231.1018829345703, "KL/rejected_KL_mean": -279.497314453125, "KL/std": 114.11299133300781, "epoch": 0.7327459618208517, "fcm_dpo/beta": 0.0043817125260829926, "fcm_dpo/delta": -0.025172626599669456, "fcm_dpo/margin": 96.7908935546875, "fcm_dpo/q_t": 0.40175533294677734, "grad_norm": 9.653645515441895, "learning_rate": 1.0243144327477013e-07, "logits/chosen": -1.9752062559127808, "logits/rejected": -1.924021601676941, "logps/chosen": -263.746337890625, "logps/ref_chosen": -81.0399169921875, "logps/ref_rejected": -106.92170715332031, "logps/rejected": -386.4190368652344, "loss": 1.0723, "margin_dpo/margin_mean": 96.7908935546875, "margin_dpo/margin_std": 120.99515533447266, "step": 499 }, { "KL/chosen_KL_mean": -202.56689453125, "KL/mean": -243.93408203125, "KL/rejected_KL_mean": -285.30126953125, "KL/std": 109.66188049316406, "epoch": 0.7342143906020558, "fcm_dpo/beta": 0.00439080223441124, "fcm_dpo/delta": 0.038128774613142014, "fcm_dpo/margin": 82.73433685302734, "fcm_dpo/q_t": 0.4173325300216675, "grad_norm": 11.338047981262207, "learning_rate": 1.0139748428955333e-07, "logits/chosen": -1.5859615802764893, "logits/rejected": -1.4126412868499756, "logps/chosen": -291.81494140625, "logps/ref_chosen": -89.248046875, "logps/ref_rejected": -100.41021728515625, "logps/rejected": -385.71148681640625, "loss": 1.1258, "margin_dpo/margin_mean": 82.73433685302734, "margin_dpo/margin_std": 124.33133697509766, "step": 500 }, { "KL/chosen_KL_mean": -177.56747436523438, "KL/mean": -222.83395385742188, "KL/rejected_KL_mean": -268.10040283203125, "KL/std": 118.63863372802734, "epoch": 0.73568281938326, "fcm_dpo/beta": 0.004436301067471504, "fcm_dpo/delta": -0.0028001368045806885, "fcm_dpo/margin": 90.53295135498047, "fcm_dpo/q_t": 0.4082610607147217, "grad_norm": 9.574939727783203, "learning_rate": 1.0036744111882672e-07, "logits/chosen": -1.5185554027557373, "logits/rejected": -1.0766103267669678, "logps/chosen": -268.1643981933594, "logps/ref_chosen": -90.596923828125, "logps/ref_rejected": -85.2972640991211, "logps/rejected": -353.3976745605469, "loss": 1.1109, "margin_dpo/margin_mean": 90.532958984375, "margin_dpo/margin_std": 132.59976196289062, "step": 501 }, { "KL/chosen_KL_mean": -178.729736328125, "KL/mean": -219.24386596679688, "KL/rejected_KL_mean": -259.75799560546875, "KL/std": 98.63836669921875, "epoch": 0.737151248164464, "fcm_dpo/beta": 0.004447881132364273, "fcm_dpo/delta": 0.04055941104888916, "fcm_dpo/margin": 81.02822875976562, "fcm_dpo/q_t": 0.4161514341831207, "grad_norm": 10.576436042785645, "learning_rate": 9.934134090518592e-08, "logits/chosen": -1.9619543552398682, "logits/rejected": -1.231248140335083, "logps/chosen": -282.2004699707031, "logps/ref_chosen": -103.47073364257812, "logps/ref_rejected": -89.04415893554688, "logps/rejected": -348.8021240234375, "loss": 1.1312, "margin_dpo/margin_mean": 81.02822875976562, "margin_dpo/margin_std": 122.95236206054688, "step": 502 }, { "KL/chosen_KL_mean": -177.00762939453125, "KL/mean": -223.4568328857422, "KL/rejected_KL_mean": -269.906005859375, "KL/std": 106.0635986328125, "epoch": 0.7386196769456681, "fcm_dpo/beta": 0.004448665771633387, "fcm_dpo/delta": -0.014171328395605087, "fcm_dpo/margin": 92.89840698242188, "fcm_dpo/q_t": 0.4040161073207855, "grad_norm": 9.351664543151855, "learning_rate": 9.831921068732571e-08, "logits/chosen": -1.4262936115264893, "logits/rejected": -0.899752140045166, "logps/chosen": -270.73760986328125, "logps/ref_chosen": -93.72998809814453, "logps/ref_rejected": -87.60896301269531, "logps/rejected": -357.5149841308594, "loss": 1.0813, "margin_dpo/margin_mean": 92.89840698242188, "margin_dpo/margin_std": 117.13383483886719, "step": 503 }, { "KL/chosen_KL_mean": -179.8705291748047, "KL/mean": -238.86892700195312, "KL/rejected_KL_mean": -297.8673095703125, "KL/std": 110.18506622314453, "epoch": 0.7400881057268722, "fcm_dpo/beta": 0.004352197051048279, "fcm_dpo/delta": -0.11959915608167648, "fcm_dpo/margin": 117.99679565429688, "fcm_dpo/q_t": 0.37927427887916565, "grad_norm": 10.177372932434082, "learning_rate": 9.730107739932805e-08, "logits/chosen": -2.042445659637451, "logits/rejected": -1.706847071647644, "logps/chosen": -272.16241455078125, "logps/ref_chosen": -92.2918701171875, "logps/ref_rejected": -109.03765106201172, "logps/rejected": -406.90496826171875, "loss": 0.9927, "margin_dpo/margin_mean": 117.99679565429688, "margin_dpo/margin_std": 111.52045440673828, "step": 504 }, { "KL/chosen_KL_mean": -197.85537719726562, "KL/mean": -240.6873779296875, "KL/rejected_KL_mean": -283.5194091796875, "KL/std": 109.11072540283203, "epoch": 0.7415565345080763, "fcm_dpo/beta": 0.0043379999697208405, "fcm_dpo/delta": 0.029495451599359512, "fcm_dpo/margin": 85.66401672363281, "fcm_dpo/q_t": 0.41296038031578064, "grad_norm": 8.678653717041016, "learning_rate": 9.628696786995188e-08, "logits/chosen": -2.2985622882843018, "logits/rejected": -1.7311210632324219, "logps/chosen": -304.13232421875, "logps/ref_chosen": -106.27693176269531, "logps/ref_rejected": -94.20611572265625, "logps/rejected": -377.72552490234375, "loss": 1.0999, "margin_dpo/margin_mean": 85.66401672363281, "margin_dpo/margin_std": 106.06097412109375, "step": 505 }, { "KL/chosen_KL_mean": -192.6057891845703, "KL/mean": -240.49871826171875, "KL/rejected_KL_mean": -288.39166259765625, "KL/std": 102.69050598144531, "epoch": 0.7430249632892805, "fcm_dpo/beta": 0.004321200307458639, "fcm_dpo/delta": -0.014854539185762405, "fcm_dpo/margin": 95.78591918945312, "fcm_dpo/q_t": 0.4018400013446808, "grad_norm": 9.407258987426758, "learning_rate": 9.527690882192635e-08, "logits/chosen": -1.8202011585235596, "logits/rejected": -1.4497179985046387, "logps/chosen": -274.046630859375, "logps/ref_chosen": -81.44086456298828, "logps/ref_rejected": -85.66439819335938, "logps/rejected": -374.0560607910156, "loss": 1.0681, "margin_dpo/margin_mean": 95.7859115600586, "margin_dpo/margin_std": 108.21177673339844, "step": 506 }, { "KL/chosen_KL_mean": -198.88833618164062, "KL/mean": -247.51075744628906, "KL/rejected_KL_mean": -296.1331787109375, "KL/std": 107.88168334960938, "epoch": 0.7444933920704846, "fcm_dpo/beta": 0.004321376793086529, "fcm_dpo/delta": -0.02111583948135376, "fcm_dpo/margin": 97.24485778808594, "fcm_dpo/q_t": 0.40306657552719116, "grad_norm": 9.240011215209961, "learning_rate": 9.427092687124691e-08, "logits/chosen": -2.1469473838806152, "logits/rejected": -1.7033915519714355, "logps/chosen": -298.3152770996094, "logps/ref_chosen": -99.42694854736328, "logps/ref_rejected": -102.25296020507812, "logps/rejected": -398.38616943359375, "loss": 1.0829, "margin_dpo/margin_mean": 97.24486541748047, "margin_dpo/margin_std": 127.69987487792969, "step": 507 }, { "KL/chosen_KL_mean": -193.09701538085938, "KL/mean": -234.02938842773438, "KL/rejected_KL_mean": -274.9617919921875, "KL/std": 116.06913757324219, "epoch": 0.7459618208516887, "fcm_dpo/beta": 0.004345783498138189, "fcm_dpo/delta": 0.04586208984255791, "fcm_dpo/margin": 81.86478424072266, "fcm_dpo/q_t": 0.4191235303878784, "grad_norm": 15.551544189453125, "learning_rate": 9.326904852647344e-08, "logits/chosen": -2.0422074794769287, "logits/rejected": -1.5588324069976807, "logps/chosen": -291.0126647949219, "logps/ref_chosen": -97.9156494140625, "logps/ref_rejected": -103.79782104492188, "logps/rejected": -378.7596130371094, "loss": 1.1458, "margin_dpo/margin_mean": 81.86479187011719, "margin_dpo/margin_std": 134.82205200195312, "step": 508 }, { "KL/chosen_KL_mean": -180.90789794921875, "KL/mean": -218.70240783691406, "KL/rejected_KL_mean": -256.49688720703125, "KL/std": 90.41305541992188, "epoch": 0.7474302496328928, "fcm_dpo/beta": 0.004407496191561222, "fcm_dpo/delta": 0.06899089366197586, "fcm_dpo/margin": 75.58898162841797, "fcm_dpo/q_t": 0.4213552176952362, "grad_norm": 14.592718124389648, "learning_rate": 9.227130018803195e-08, "logits/chosen": -1.9822810888290405, "logits/rejected": -1.313735842704773, "logps/chosen": -281.63775634765625, "logps/ref_chosen": -100.72984313964844, "logps/ref_rejected": -86.943359375, "logps/rejected": -343.44024658203125, "loss": 1.1489, "margin_dpo/margin_mean": 75.58898162841797, "margin_dpo/margin_std": 120.22540283203125, "step": 509 }, { "KL/chosen_KL_mean": -213.26022338867188, "KL/mean": -259.2166442871094, "KL/rejected_KL_mean": -305.173095703125, "KL/std": 109.63390350341797, "epoch": 0.748898678414097, "fcm_dpo/beta": 0.004418557044118643, "fcm_dpo/delta": -0.0064174709841609, "fcm_dpo/margin": 91.9128646850586, "fcm_dpo/q_t": 0.40506818890571594, "grad_norm": 11.727859497070312, "learning_rate": 9.127770814751932e-08, "logits/chosen": -1.454599380493164, "logits/rejected": -1.2664161920547485, "logps/chosen": -302.4295959472656, "logps/ref_chosen": -89.16938781738281, "logps/ref_rejected": -107.91939544677734, "logps/rejected": -413.09246826171875, "loss": 1.0784, "margin_dpo/margin_mean": 91.9128646850586, "margin_dpo/margin_std": 109.64920806884766, "step": 510 }, { "KL/chosen_KL_mean": -194.58963012695312, "KL/mean": -244.0318145751953, "KL/rejected_KL_mean": -293.4739990234375, "KL/std": 107.83610534667969, "epoch": 0.750367107195301, "fcm_dpo/beta": 0.004395465366542339, "fcm_dpo/delta": -0.03622505068778992, "fcm_dpo/margin": 98.88436889648438, "fcm_dpo/q_t": 0.3997315764427185, "grad_norm": 11.376951217651367, "learning_rate": 9.028829858700973e-08, "logits/chosen": -2.0761289596557617, "logits/rejected": -1.5030100345611572, "logps/chosen": -284.33941650390625, "logps/ref_chosen": -89.74979400634766, "logps/ref_rejected": -97.73127746582031, "logps/rejected": -391.20526123046875, "loss": 1.064, "margin_dpo/margin_mean": 98.88436889648438, "margin_dpo/margin_std": 120.54209899902344, "step": 511 }, { "KL/chosen_KL_mean": -184.94110107421875, "KL/mean": -232.90672302246094, "KL/rejected_KL_mean": -280.87237548828125, "KL/std": 104.58906555175781, "epoch": 0.7518355359765051, "fcm_dpo/beta": 0.004386503249406815, "fcm_dpo/delta": -0.02212878316640854, "fcm_dpo/margin": 95.9312515258789, "fcm_dpo/q_t": 0.40208902955055237, "grad_norm": 14.000700950622559, "learning_rate": 8.930309757836516e-08, "logits/chosen": -1.8101218938827515, "logits/rejected": -1.3653795719146729, "logps/chosen": -270.6762390136719, "logps/ref_chosen": -85.73515319824219, "logps/ref_rejected": -87.6248779296875, "logps/rejected": -368.49725341796875, "loss": 1.0828, "margin_dpo/margin_mean": 95.9312515258789, "margin_dpo/margin_std": 124.9241714477539, "step": 512 }, { "KL/chosen_KL_mean": -197.28591918945312, "KL/mean": -237.035888671875, "KL/rejected_KL_mean": -276.7858581542969, "KL/std": 100.73687744140625, "epoch": 0.7533039647577092, "fcm_dpo/beta": 0.00437677837908268, "fcm_dpo/delta": 0.05380648374557495, "fcm_dpo/margin": 79.49991607666016, "fcm_dpo/q_t": 0.4178999662399292, "grad_norm": 11.505144119262695, "learning_rate": 8.832213108254863e-08, "logits/chosen": -2.4695053100585938, "logits/rejected": -1.7934657335281372, "logps/chosen": -293.4764404296875, "logps/ref_chosen": -96.19050598144531, "logps/ref_rejected": -82.91773986816406, "logps/rejected": -359.70361328125, "loss": 1.1302, "margin_dpo/margin_mean": 79.49991607666016, "margin_dpo/margin_std": 113.604248046875, "step": 513 }, { "KL/chosen_KL_mean": -190.5218505859375, "KL/mean": -234.18350219726562, "KL/rejected_KL_mean": -277.84515380859375, "KL/std": 101.11366271972656, "epoch": 0.7547723935389133, "fcm_dpo/beta": 0.004417263902723789, "fcm_dpo/delta": 0.014837978407740593, "fcm_dpo/margin": 87.32330322265625, "fcm_dpo/q_t": 0.4113604724407196, "grad_norm": 11.526236534118652, "learning_rate": 8.734542494893954e-08, "logits/chosen": -1.7030606269836426, "logits/rejected": -1.1298351287841797, "logps/chosen": -288.3454895019531, "logps/ref_chosen": -97.82363891601562, "logps/ref_rejected": -85.36982727050781, "logps/rejected": -363.2149658203125, "loss": 1.103, "margin_dpo/margin_mean": 87.32330322265625, "margin_dpo/margin_std": 119.74478149414062, "step": 514 }, { "KL/chosen_KL_mean": -183.12286376953125, "KL/mean": -225.93844604492188, "KL/rejected_KL_mean": -268.7540283203125, "KL/std": 105.05963134765625, "epoch": 0.7562408223201175, "fcm_dpo/beta": 0.00443321093916893, "fcm_dpo/delta": 0.0211846474558115, "fcm_dpo/margin": 85.63117980957031, "fcm_dpo/q_t": 0.41160672903060913, "grad_norm": 11.668662071228027, "learning_rate": 8.637300491465272e-08, "logits/chosen": -1.7530758380889893, "logits/rejected": -1.5200414657592773, "logps/chosen": -265.0564270019531, "logps/ref_chosen": -81.93355560302734, "logps/ref_rejected": -93.57656860351562, "logps/rejected": -362.3305969238281, "loss": 1.1056, "margin_dpo/margin_mean": 85.63117980957031, "margin_dpo/margin_std": 115.00526428222656, "step": 515 }, { "KL/chosen_KL_mean": -188.0991973876953, "KL/mean": -229.77914428710938, "KL/rejected_KL_mean": -271.4590759277344, "KL/std": 120.68377685546875, "epoch": 0.7577092511013216, "fcm_dpo/beta": 0.0044641559943556786, "fcm_dpo/delta": 0.02885700762271881, "fcm_dpo/margin": 83.35987091064453, "fcm_dpo/q_t": 0.41631439328193665, "grad_norm": 12.976597785949707, "learning_rate": 8.540489660386064e-08, "logits/chosen": -2.267245292663574, "logits/rejected": -2.0901594161987305, "logps/chosen": -285.19085693359375, "logps/ref_chosen": -97.09166717529297, "logps/ref_rejected": -118.55667114257812, "logps/rejected": -390.0157470703125, "loss": 1.1425, "margin_dpo/margin_mean": 83.35986328125, "margin_dpo/margin_std": 142.1790771484375, "step": 516 }, { "KL/chosen_KL_mean": -200.11038208007812, "KL/mean": -250.83828735351562, "KL/rejected_KL_mean": -301.56622314453125, "KL/std": 108.59696960449219, "epoch": 0.7591776798825257, "fcm_dpo/beta": 0.004401662386953831, "fcm_dpo/delta": -0.04998572915792465, "fcm_dpo/margin": 101.45581817626953, "fcm_dpo/q_t": 0.3969401717185974, "grad_norm": 9.312891960144043, "learning_rate": 8.444112552711752e-08, "logits/chosen": -1.9439467191696167, "logits/rejected": -1.5213518142700195, "logps/chosen": -293.29693603515625, "logps/ref_chosen": -93.18656921386719, "logps/ref_rejected": -95.5927963256836, "logps/rejected": -397.15899658203125, "loss": 1.0634, "margin_dpo/margin_mean": 101.45582580566406, "margin_dpo/margin_std": 123.90345001220703, "step": 517 }, { "KL/chosen_KL_mean": -208.23123168945312, "KL/mean": -246.13330078125, "KL/rejected_KL_mean": -284.035400390625, "KL/std": 102.28993225097656, "epoch": 0.7606461086637298, "fcm_dpo/beta": 0.004459921270608902, "fcm_dpo/delta": 0.06411272287368774, "fcm_dpo/margin": 75.80418395996094, "fcm_dpo/q_t": 0.4227701425552368, "grad_norm": 10.298627853393555, "learning_rate": 8.348171708068747e-08, "logits/chosen": -1.9349322319030762, "logits/rejected": -1.6066086292266846, "logps/chosen": -295.2525939941406, "logps/ref_chosen": -87.0213623046875, "logps/ref_rejected": -91.24456024169922, "logps/rejected": -375.27996826171875, "loss": 1.1523, "margin_dpo/margin_mean": 75.80419158935547, "margin_dpo/margin_std": 127.23600006103516, "step": 518 }, { "KL/chosen_KL_mean": -193.34893798828125, "KL/mean": -234.60690307617188, "KL/rejected_KL_mean": -275.8648986816406, "KL/std": 95.5638427734375, "epoch": 0.762114537444934, "fcm_dpo/beta": 0.0044878143817186356, "fcm_dpo/delta": 0.030799156054854393, "fcm_dpo/margin": 82.51598358154297, "fcm_dpo/q_t": 0.4136577844619751, "grad_norm": 10.373640060424805, "learning_rate": 8.25266965458755e-08, "logits/chosen": -1.7496259212493896, "logits/rejected": -1.362605333328247, "logps/chosen": -294.0364990234375, "logps/ref_chosen": -100.68755340576172, "logps/ref_rejected": -109.03042602539062, "logps/rejected": -384.89532470703125, "loss": 1.098, "margin_dpo/margin_mean": 82.51597595214844, "margin_dpo/margin_std": 99.99882507324219, "step": 519 }, { "KL/chosen_KL_mean": -179.33685302734375, "KL/mean": -228.9242706298828, "KL/rejected_KL_mean": -278.51165771484375, "KL/std": 107.62957000732422, "epoch": 0.7635829662261381, "fcm_dpo/beta": 0.004467126913368702, "fcm_dpo/delta": -0.04514292627573013, "fcm_dpo/margin": 99.17481994628906, "fcm_dpo/q_t": 0.3957825005054474, "grad_norm": 9.266866683959961, "learning_rate": 8.15760890883607e-08, "logits/chosen": -1.5378611087799072, "logits/rejected": -1.0951225757598877, "logps/chosen": -278.1144714355469, "logps/ref_chosen": -98.77762603759766, "logps/ref_rejected": -106.9457015991211, "logps/rejected": -385.4573669433594, "loss": 1.0443, "margin_dpo/margin_mean": 99.17481994628906, "margin_dpo/margin_std": 105.34744262695312, "step": 520 }, { "KL/chosen_KL_mean": -180.75860595703125, "KL/mean": -231.17333984375, "KL/rejected_KL_mean": -281.58807373046875, "KL/std": 100.75483703613281, "epoch": 0.7650513950073421, "fcm_dpo/beta": 0.004449758678674698, "fcm_dpo/delta": -0.05112721398472786, "fcm_dpo/margin": 100.82946014404297, "fcm_dpo/q_t": 0.394786536693573, "grad_norm": 10.373321533203125, "learning_rate": 8.062991975753378e-08, "logits/chosen": -1.9531986713409424, "logits/rejected": -1.6149442195892334, "logps/chosen": -267.10968017578125, "logps/ref_chosen": -86.35108947753906, "logps/ref_rejected": -90.39317321777344, "logps/rejected": -371.98126220703125, "loss": 1.0383, "margin_dpo/margin_mean": 100.82946014404297, "margin_dpo/margin_std": 102.85365295410156, "step": 521 }, { "KL/chosen_KL_mean": -188.46925354003906, "KL/mean": -233.85025024414062, "KL/rejected_KL_mean": -279.2312927246094, "KL/std": 109.94515228271484, "epoch": 0.7665198237885462, "fcm_dpo/beta": 0.00442523043602705, "fcm_dpo/delta": -0.001774543896317482, "fcm_dpo/margin": 90.76203918457031, "fcm_dpo/q_t": 0.40651342272758484, "grad_norm": 10.264397621154785, "learning_rate": 7.968821348583643e-08, "logits/chosen": -1.7708044052124023, "logits/rejected": -1.4054961204528809, "logps/chosen": -264.477294921875, "logps/ref_chosen": -76.008056640625, "logps/ref_rejected": -74.44125366210938, "logps/rejected": -353.67254638671875, "loss": 1.0864, "margin_dpo/margin_mean": 90.76203918457031, "margin_dpo/margin_std": 115.41966247558594, "step": 522 }, { "KL/chosen_KL_mean": -209.11566162109375, "KL/mean": -242.1001434326172, "KL/rejected_KL_mean": -275.0845947265625, "KL/std": 108.48616027832031, "epoch": 0.7679882525697503, "fcm_dpo/beta": 0.004483068827539682, "fcm_dpo/delta": 0.10753341019153595, "fcm_dpo/margin": 65.96895599365234, "fcm_dpo/q_t": 0.4327232539653778, "grad_norm": 11.171823501586914, "learning_rate": 7.875099508810484e-08, "logits/chosen": -1.9154253005981445, "logits/rejected": -1.203895926475525, "logps/chosen": -305.6119384765625, "logps/ref_chosen": -96.49627685546875, "logps/ref_rejected": -91.0152816772461, "logps/rejected": -366.0998840332031, "loss": 1.188, "margin_dpo/margin_mean": 65.96894836425781, "margin_dpo/margin_std": 126.03538513183594, "step": 523 }, { "KL/chosen_KL_mean": -190.57742309570312, "KL/mean": -237.455322265625, "KL/rejected_KL_mean": -284.3332214355469, "KL/std": 108.84417724609375, "epoch": 0.7694566813509545, "fcm_dpo/beta": 0.0044946325942873955, "fcm_dpo/delta": -0.022366058081388474, "fcm_dpo/margin": 93.75578308105469, "fcm_dpo/q_t": 0.4028388261795044, "grad_norm": 12.626813888549805, "learning_rate": 7.781828926091535e-08, "logits/chosen": -2.1018197536468506, "logits/rejected": -1.3975772857666016, "logps/chosen": -293.98291015625, "logps/ref_chosen": -103.40550994873047, "logps/ref_rejected": -87.91015625, "logps/rejected": -372.24334716796875, "loss": 1.0716, "margin_dpo/margin_mean": 93.75578308105469, "margin_dpo/margin_std": 115.51873779296875, "step": 524 }, { "KL/chosen_KL_mean": -178.91156005859375, "KL/mean": -231.97496032714844, "KL/rejected_KL_mean": -285.03839111328125, "KL/std": 108.86880493164062, "epoch": 0.7709251101321586, "fcm_dpo/beta": 0.004403066821396351, "fcm_dpo/delta": -0.07303775101900101, "fcm_dpo/margin": 106.12684631347656, "fcm_dpo/q_t": 0.39391109347343445, "grad_norm": 10.874677658081055, "learning_rate": 7.689012058193384e-08, "logits/chosen": -1.4949113130569458, "logits/rejected": -1.407296895980835, "logps/chosen": -263.1180419921875, "logps/ref_chosen": -84.20648193359375, "logps/ref_rejected": -106.26158905029297, "logps/rejected": -391.2999572753906, "loss": 1.0509, "margin_dpo/margin_mean": 106.12684631347656, "margin_dpo/margin_std": 126.00141906738281, "step": 525 }, { "KL/chosen_KL_mean": -185.08627319335938, "KL/mean": -232.97203063964844, "KL/rejected_KL_mean": -280.8577880859375, "KL/std": 103.37193298339844, "epoch": 0.7723935389133627, "fcm_dpo/beta": 0.0044127535074949265, "fcm_dpo/delta": -0.0236107986420393, "fcm_dpo/margin": 95.77149963378906, "fcm_dpo/q_t": 0.40203291177749634, "grad_norm": 8.5012845993042, "learning_rate": 7.596651350926836e-08, "logits/chosen": -1.7402470111846924, "logits/rejected": -1.0851659774780273, "logps/chosen": -287.94036865234375, "logps/ref_chosen": -102.85411071777344, "logps/ref_rejected": -93.65145874023438, "logps/rejected": -374.50921630859375, "loss": 1.076, "margin_dpo/margin_mean": 95.77149963378906, "margin_dpo/margin_std": 121.9001235961914, "step": 526 }, { "KL/chosen_KL_mean": -186.51031494140625, "KL/mean": -235.48590087890625, "KL/rejected_KL_mean": -284.46148681640625, "KL/std": 107.90658569335938, "epoch": 0.7738619676945668, "fcm_dpo/beta": 0.004385120701044798, "fcm_dpo/delta": -0.03086497262120247, "fcm_dpo/margin": 97.95115661621094, "fcm_dpo/q_t": 0.3981344997882843, "grad_norm": 10.557165145874023, "learning_rate": 7.504749238082414e-08, "logits/chosen": -3.06978178024292, "logits/rejected": -2.2450006008148193, "logps/chosen": -283.2757568359375, "logps/ref_chosen": -96.76543426513672, "logps/ref_rejected": -85.39518737792969, "logps/rejected": -369.856689453125, "loss": 1.0434, "margin_dpo/margin_mean": 97.95116424560547, "margin_dpo/margin_std": 95.04463958740234, "step": 527 }, { "KL/chosen_KL_mean": -174.7440948486328, "KL/mean": -222.70144653320312, "KL/rejected_KL_mean": -270.6588134765625, "KL/std": 109.72480773925781, "epoch": 0.775330396475771, "fcm_dpo/beta": 0.004371971357613802, "fcm_dpo/delta": -0.020189493894577026, "fcm_dpo/margin": 95.9146957397461, "fcm_dpo/q_t": 0.4028235673904419, "grad_norm": 9.738059997558594, "learning_rate": 7.413308141366254e-08, "logits/chosen": -2.101011276245117, "logits/rejected": -1.583996295928955, "logps/chosen": -268.90899658203125, "logps/ref_chosen": -94.1649169921875, "logps/ref_rejected": -100.372314453125, "logps/rejected": -371.0311279296875, "loss": 1.0797, "margin_dpo/margin_mean": 95.91470336914062, "margin_dpo/margin_std": 124.25143432617188, "step": 528 }, { "KL/chosen_KL_mean": -202.60260009765625, "KL/mean": -246.25973510742188, "KL/rejected_KL_mean": -289.9168701171875, "KL/std": 108.61520385742188, "epoch": 0.7767988252569751, "fcm_dpo/beta": 0.00438288226723671, "fcm_dpo/delta": 0.017871355637907982, "fcm_dpo/margin": 87.31427001953125, "fcm_dpo/q_t": 0.41246744990348816, "grad_norm": 11.07929801940918, "learning_rate": 7.322330470336313e-08, "logits/chosen": -1.8634397983551025, "logits/rejected": -1.688382625579834, "logps/chosen": -283.37939453125, "logps/ref_chosen": -80.77680206298828, "logps/ref_rejected": -97.191162109375, "logps/rejected": -387.1080322265625, "loss": 1.1144, "margin_dpo/margin_mean": 87.31427001953125, "margin_dpo/margin_std": 128.47286987304688, "step": 529 }, { "KL/chosen_KL_mean": -185.59027099609375, "KL/mean": -229.71343994140625, "KL/rejected_KL_mean": -273.83660888671875, "KL/std": 105.36119079589844, "epoch": 0.7782672540381792, "fcm_dpo/beta": 0.004374193958938122, "fcm_dpo/delta": 0.014422038570046425, "fcm_dpo/margin": 88.24635314941406, "fcm_dpo/q_t": 0.4098609387874603, "grad_norm": 9.579283714294434, "learning_rate": 7.231818622338822e-08, "logits/chosen": -1.587059497833252, "logits/rejected": -1.2667343616485596, "logps/chosen": -265.3659973144531, "logps/ref_chosen": -79.7757339477539, "logps/ref_rejected": -92.05390167236328, "logps/rejected": -365.8905029296875, "loss": 1.0838, "margin_dpo/margin_mean": 88.24635314941406, "margin_dpo/margin_std": 101.31544494628906, "step": 530 }, { "KL/chosen_KL_mean": -178.88958740234375, "KL/mean": -233.64175415039062, "KL/rejected_KL_mean": -288.3939208984375, "KL/std": 110.10360717773438, "epoch": 0.7797356828193832, "fcm_dpo/beta": 0.004334740340709686, "fcm_dpo/delta": -0.07849869132041931, "fcm_dpo/margin": 109.50434875488281, "fcm_dpo/q_t": 0.38788092136383057, "grad_norm": 11.46908950805664, "learning_rate": 7.141774982445147e-08, "logits/chosen": -2.0472640991210938, "logits/rejected": -1.4655787944793701, "logps/chosen": -258.2087707519531, "logps/ref_chosen": -79.31919860839844, "logps/ref_rejected": -78.10832214355469, "logps/rejected": -366.50225830078125, "loss": 1.0202, "margin_dpo/margin_mean": 109.50434875488281, "margin_dpo/margin_std": 109.61767578125, "step": 531 }, { "KL/chosen_KL_mean": -193.42477416992188, "KL/mean": -252.6409149169922, "KL/rejected_KL_mean": -311.8570556640625, "KL/std": 105.43724060058594, "epoch": 0.7812041116005873, "fcm_dpo/beta": 0.004261220805346966, "fcm_dpo/delta": -0.11008569598197937, "fcm_dpo/margin": 118.4322509765625, "fcm_dpo/q_t": 0.3821142911911011, "grad_norm": 11.007452964782715, "learning_rate": 7.052201923388953e-08, "logits/chosen": -1.5914090871810913, "logits/rejected": -0.956507682800293, "logps/chosen": -295.06170654296875, "logps/ref_chosen": -101.63691711425781, "logps/ref_rejected": -93.34539031982422, "logps/rejected": -405.20245361328125, "loss": 1.0039, "margin_dpo/margin_mean": 118.43226623535156, "margin_dpo/margin_std": 116.89279174804688, "step": 532 }, { "KL/chosen_KL_mean": -186.63168334960938, "KL/mean": -221.2418975830078, "KL/rejected_KL_mean": -255.85211181640625, "KL/std": 111.62553405761719, "epoch": 0.7826725403817915, "fcm_dpo/beta": 0.004300114698708057, "fcm_dpo/delta": 0.10532081127166748, "fcm_dpo/margin": 69.2204360961914, "fcm_dpo/q_t": 0.4323233962059021, "grad_norm": 10.59495735168457, "learning_rate": 6.963101805503646e-08, "logits/chosen": -2.45595121383667, "logits/rejected": -1.858580231666565, "logps/chosen": -277.3783264160156, "logps/ref_chosen": -90.74664306640625, "logps/ref_rejected": -82.88848876953125, "logps/rejected": -338.7406005859375, "loss": 1.1737, "margin_dpo/margin_mean": 69.2204360961914, "margin_dpo/margin_std": 120.114501953125, "step": 533 }, { "KL/chosen_KL_mean": -167.259521484375, "KL/mean": -217.863037109375, "KL/rejected_KL_mean": -268.466552734375, "KL/std": 104.90930938720703, "epoch": 0.7841409691629956, "fcm_dpo/beta": 0.004299655556678772, "fcm_dpo/delta": -0.03674093633890152, "fcm_dpo/margin": 101.20699310302734, "fcm_dpo/q_t": 0.39724352955818176, "grad_norm": 9.604201316833496, "learning_rate": 6.874476976660184e-08, "logits/chosen": -1.9472721815109253, "logits/rejected": -1.5493907928466797, "logps/chosen": -253.89035034179688, "logps/ref_chosen": -86.6308364868164, "logps/ref_rejected": -84.87196350097656, "logps/rejected": -353.3385009765625, "loss": 1.0446, "margin_dpo/margin_mean": 101.20699310302734, "margin_dpo/margin_std": 102.9453353881836, "step": 534 }, { "KL/chosen_KL_mean": -168.75717163085938, "KL/mean": -222.37673950195312, "KL/rejected_KL_mean": -275.996337890625, "KL/std": 105.69602966308594, "epoch": 0.7856093979441997, "fcm_dpo/beta": 0.004265861120074987, "fcm_dpo/delta": -0.06033168360590935, "fcm_dpo/margin": 107.23915100097656, "fcm_dpo/q_t": 0.39391085505485535, "grad_norm": 8.064408302307129, "learning_rate": 6.786329772205246e-08, "logits/chosen": -1.6743295192718506, "logits/rejected": -1.486403226852417, "logps/chosen": -259.1025390625, "logps/ref_chosen": -90.34539031982422, "logps/ref_rejected": -100.7779769897461, "logps/rejected": -376.7742919921875, "loss": 1.0439, "margin_dpo/margin_mean": 107.23915100097656, "margin_dpo/margin_std": 120.82598876953125, "step": 535 }, { "KL/chosen_KL_mean": -168.385498046875, "KL/mean": -221.67922973632812, "KL/rejected_KL_mean": -274.97296142578125, "KL/std": 106.84866333007812, "epoch": 0.7870778267254038, "fcm_dpo/beta": 0.004194909241050482, "fcm_dpo/delta": -0.04952087253332138, "fcm_dpo/margin": 106.58743286132812, "fcm_dpo/q_t": 0.39467746019363403, "grad_norm": 8.846919059753418, "learning_rate": 6.698662514899638e-08, "logits/chosen": -1.6700624227523804, "logits/rejected": -1.4123191833496094, "logps/chosen": -247.34506225585938, "logps/ref_chosen": -78.95956420898438, "logps/ref_rejected": -93.08779907226562, "logps/rejected": -368.06072998046875, "loss": 1.0409, "margin_dpo/margin_mean": 106.58743286132812, "margin_dpo/margin_std": 111.904296875, "step": 536 }, { "KL/chosen_KL_mean": -146.2003173828125, "KL/mean": -202.36572265625, "KL/rejected_KL_mean": -258.5310974121094, "KL/std": 110.12149047851562, "epoch": 0.788546255506608, "fcm_dpo/beta": 0.004172866698354483, "fcm_dpo/delta": -0.07241124659776688, "fcm_dpo/margin": 112.33076477050781, "fcm_dpo/q_t": 0.3895955979824066, "grad_norm": 9.220221519470215, "learning_rate": 6.611477514857114e-08, "logits/chosen": -1.4752066135406494, "logits/rejected": -0.7708967924118042, "logps/chosen": -232.9097137451172, "logps/ref_chosen": -86.70939636230469, "logps/ref_rejected": -78.90766906738281, "logps/rejected": -337.43878173828125, "loss": 1.0224, "margin_dpo/margin_mean": 112.33076477050781, "margin_dpo/margin_std": 106.85780334472656, "step": 537 }, { "KL/chosen_KL_mean": -199.89068603515625, "KL/mean": -245.57762145996094, "KL/rejected_KL_mean": -291.2645568847656, "KL/std": 98.5885009765625, "epoch": 0.7900146842878121, "fcm_dpo/beta": 0.004147420171648264, "fcm_dpo/delta": 0.021854396909475327, "fcm_dpo/margin": 91.37385559082031, "fcm_dpo/q_t": 0.41077721118927, "grad_norm": 8.785744667053223, "learning_rate": 6.524777069483525e-08, "logits/chosen": -1.6127369403839111, "logits/rejected": -1.0522353649139404, "logps/chosen": -298.33282470703125, "logps/ref_chosen": -98.44214630126953, "logps/ref_rejected": -91.09465026855469, "logps/rejected": -382.35919189453125, "loss": 1.0886, "margin_dpo/margin_mean": 91.37385559082031, "margin_dpo/margin_std": 105.71298217773438, "step": 538 }, { "KL/chosen_KL_mean": -174.08126831054688, "KL/mean": -222.22605895996094, "KL/rejected_KL_mean": -270.370849609375, "KL/std": 108.19784545898438, "epoch": 0.7914831130690162, "fcm_dpo/beta": 0.004152186214923859, "fcm_dpo/delta": 0.00019358634017407894, "fcm_dpo/margin": 96.28958129882812, "fcm_dpo/q_t": 0.40665799379348755, "grad_norm": 8.267725944519043, "learning_rate": 6.438563463416221e-08, "logits/chosen": -2.4267418384552, "logits/rejected": -1.946427822113037, "logps/chosen": -265.01336669921875, "logps/ref_chosen": -90.93211364746094, "logps/ref_rejected": -97.44546508789062, "logps/rejected": -367.8163146972656, "loss": 1.0878, "margin_dpo/margin_mean": 96.28958129882812, "margin_dpo/margin_std": 123.27275085449219, "step": 539 }, { "KL/chosen_KL_mean": -186.16783142089844, "KL/mean": -244.9136505126953, "KL/rejected_KL_mean": -303.65948486328125, "KL/std": 117.93635559082031, "epoch": 0.7929515418502202, "fcm_dpo/beta": 0.0041143083944916725, "fcm_dpo/delta": -0.08762075006961823, "fcm_dpo/margin": 117.49164581298828, "fcm_dpo/q_t": 0.3895614445209503, "grad_norm": 10.526578903198242, "learning_rate": 6.352838968463919e-08, "logits/chosen": -1.9619333744049072, "logits/rejected": -1.7225849628448486, "logps/chosen": -294.0377197265625, "logps/ref_chosen": -107.86990356445312, "logps/ref_rejected": -121.39350128173828, "logps/rejected": -425.052978515625, "loss": 1.0358, "margin_dpo/margin_mean": 117.49164581298828, "margin_dpo/margin_std": 138.30331420898438, "step": 540 }, { "KL/chosen_KL_mean": -206.0163116455078, "KL/mean": -242.61294555664062, "KL/rejected_KL_mean": -279.2096252441406, "KL/std": 97.84642028808594, "epoch": 0.7944199706314243, "fcm_dpo/beta": 0.004117786418646574, "fcm_dpo/delta": 0.10158645361661911, "fcm_dpo/margin": 73.19329071044922, "fcm_dpo/q_t": 0.429939866065979, "grad_norm": 9.508892059326172, "learning_rate": 6.267605843546767e-08, "logits/chosen": -2.3693134784698486, "logits/rejected": -1.9038939476013184, "logps/chosen": -312.58380126953125, "logps/ref_chosen": -106.56748962402344, "logps/ref_rejected": -109.06633758544922, "logps/rejected": -388.27593994140625, "loss": 1.1578, "margin_dpo/margin_mean": 73.19329833984375, "margin_dpo/margin_std": 105.11480712890625, "step": 541 }, { "KL/chosen_KL_mean": -175.96026611328125, "KL/mean": -231.47410583496094, "KL/rejected_KL_mean": -286.98797607421875, "KL/std": 109.6036605834961, "epoch": 0.7958883994126285, "fcm_dpo/beta": 0.00412379764020443, "fcm_dpo/delta": -0.06065434217453003, "fcm_dpo/margin": 111.02767944335938, "fcm_dpo/q_t": 0.39227452874183655, "grad_norm": 10.226316452026367, "learning_rate": 6.182866334636888e-08, "logits/chosen": -2.228135108947754, "logits/rejected": -2.1053619384765625, "logps/chosen": -263.3295593261719, "logps/ref_chosen": -87.36929321289062, "logps/ref_rejected": -102.82034301757812, "logps/rejected": -389.80828857421875, "loss": 1.0367, "margin_dpo/margin_mean": 111.02767944335938, "margin_dpo/margin_std": 118.6092529296875, "step": 542 }, { "KL/chosen_KL_mean": -201.052734375, "KL/mean": -230.72384643554688, "KL/rejected_KL_mean": -260.3949279785156, "KL/std": 115.84856414794922, "epoch": 0.7973568281938326, "fcm_dpo/beta": 0.004194458946585655, "fcm_dpo/delta": 0.15524333715438843, "fcm_dpo/margin": 59.342201232910156, "fcm_dpo/q_t": 0.44378897547721863, "grad_norm": 13.067957878112793, "learning_rate": 6.098622674699147e-08, "logits/chosen": -1.712776780128479, "logits/rejected": -1.758617639541626, "logps/chosen": -288.205322265625, "logps/ref_chosen": -87.152587890625, "logps/ref_rejected": -111.53172302246094, "logps/rejected": -371.9266662597656, "loss": 1.2367, "margin_dpo/margin_mean": 59.34219741821289, "margin_dpo/margin_std": 140.30624389648438, "step": 543 }, { "KL/chosen_KL_mean": -209.62869262695312, "KL/mean": -246.99368286132812, "KL/rejected_KL_mean": -284.358642578125, "KL/std": 108.78361511230469, "epoch": 0.7988252569750367, "fcm_dpo/beta": 0.00429158378392458, "fcm_dpo/delta": 0.08196108043193817, "fcm_dpo/margin": 74.72994995117188, "fcm_dpo/q_t": 0.4258914589881897, "grad_norm": 9.350296020507812, "learning_rate": 6.01487708363232e-08, "logits/chosen": -1.8148339986801147, "logits/rejected": -1.7211837768554688, "logps/chosen": -295.7756042480469, "logps/ref_chosen": -86.14691162109375, "logps/ref_rejected": -106.3939208984375, "logps/rejected": -390.7525634765625, "loss": 1.1466, "margin_dpo/margin_mean": 74.72994995117188, "margin_dpo/margin_std": 113.29115295410156, "step": 544 }, { "KL/chosen_KL_mean": -185.92526245117188, "KL/mean": -232.3931884765625, "KL/rejected_KL_mean": -278.861083984375, "KL/std": 108.36090087890625, "epoch": 0.8002936857562408, "fcm_dpo/beta": 0.004317790735512972, "fcm_dpo/delta": -0.001375081017613411, "fcm_dpo/margin": 92.93583679199219, "fcm_dpo/q_t": 0.4074089527130127, "grad_norm": 9.382490158081055, "learning_rate": 5.9316317682106294e-08, "logits/chosen": -1.4017585515975952, "logits/rejected": -0.9673062562942505, "logps/chosen": -282.3494873046875, "logps/ref_chosen": -96.42424011230469, "logps/ref_rejected": -100.73405456542969, "logps/rejected": -379.59515380859375, "loss": 1.1044, "margin_dpo/margin_mean": 92.93582153320312, "margin_dpo/margin_std": 134.23448181152344, "step": 545 }, { "KL/chosen_KL_mean": -192.50027465820312, "KL/mean": -233.96685791015625, "KL/rejected_KL_mean": -275.4334411621094, "KL/std": 110.21768188476562, "epoch": 0.801762114537445, "fcm_dpo/beta": 0.004329974763095379, "fcm_dpo/delta": 0.04243060201406479, "fcm_dpo/margin": 82.93315887451172, "fcm_dpo/q_t": 0.4157133102416992, "grad_norm": 9.266846656799316, "learning_rate": 5.848888922025552e-08, "logits/chosen": -1.6009818315505981, "logits/rejected": -1.250114917755127, "logps/chosen": -278.52142333984375, "logps/ref_chosen": -86.02112579345703, "logps/ref_rejected": -87.37263488769531, "logps/rejected": -362.80609130859375, "loss": 1.1248, "margin_dpo/margin_mean": 82.93316650390625, "margin_dpo/margin_std": 120.94393920898438, "step": 546 }, { "KL/chosen_KL_mean": -185.6078338623047, "KL/mean": -235.45370483398438, "KL/rejected_KL_mean": -285.2995910644531, "KL/std": 107.33078002929688, "epoch": 0.8032305433186491, "fcm_dpo/beta": 0.004337495192885399, "fcm_dpo/delta": -0.03393372893333435, "fcm_dpo/margin": 99.69174194335938, "fcm_dpo/q_t": 0.3975163698196411, "grad_norm": 9.415373802185059, "learning_rate": 5.7666507254280265e-08, "logits/chosen": -1.7992115020751953, "logits/rejected": -1.3324027061462402, "logps/chosen": -286.25885009765625, "logps/ref_chosen": -100.6510009765625, "logps/ref_rejected": -97.9117431640625, "logps/rejected": -383.2113342285156, "loss": 1.0418, "margin_dpo/margin_mean": 99.69174194335938, "margin_dpo/margin_std": 95.57373046875, "step": 547 }, { "KL/chosen_KL_mean": -181.62911987304688, "KL/mean": -222.6964111328125, "KL/rejected_KL_mean": -263.7637023925781, "KL/std": 109.63880920410156, "epoch": 0.8046989720998532, "fcm_dpo/beta": 0.0043485420756042, "fcm_dpo/delta": 0.04440275579690933, "fcm_dpo/margin": 82.13458251953125, "fcm_dpo/q_t": 0.41779372096061707, "grad_norm": 10.714794158935547, "learning_rate": 5.684919345471029e-08, "logits/chosen": -2.293891429901123, "logits/rejected": -1.9382702112197876, "logps/chosen": -277.65924072265625, "logps/ref_chosen": -96.03010559082031, "logps/ref_rejected": -98.41322326660156, "logps/rejected": -362.17694091796875, "loss": 1.1463, "margin_dpo/margin_mean": 82.13458251953125, "margin_dpo/margin_std": 137.5413055419922, "step": 548 }, { "KL/chosen_KL_mean": -172.27899169921875, "KL/mean": -214.41476440429688, "KL/rejected_KL_mean": -256.550537109375, "KL/std": 104.15974426269531, "epoch": 0.8061674008810573, "fcm_dpo/beta": 0.004397171549499035, "fcm_dpo/delta": 0.02997639775276184, "fcm_dpo/margin": 84.27154541015625, "fcm_dpo/q_t": 0.41377103328704834, "grad_norm": 10.98576831817627, "learning_rate": 5.603696935852426e-08, "logits/chosen": -1.8972722291946411, "logits/rejected": -1.4821691513061523, "logps/chosen": -255.72994995117188, "logps/ref_chosen": -83.45095825195312, "logps/ref_rejected": -81.38023376464844, "logps/rejected": -337.9307861328125, "loss": 1.1294, "margin_dpo/margin_mean": 84.27154541015625, "margin_dpo/margin_std": 128.57623291015625, "step": 549 }, { "KL/chosen_KL_mean": -191.3798065185547, "KL/mean": -232.61886596679688, "KL/rejected_KL_mean": -273.85791015625, "KL/std": 98.47868347167969, "epoch": 0.8076358296622613, "fcm_dpo/beta": 0.004415298346430063, "fcm_dpo/delta": 0.03705522045493126, "fcm_dpo/margin": 82.47809600830078, "fcm_dpo/q_t": 0.4152006208896637, "grad_norm": 9.2901611328125, "learning_rate": 5.5229856368582376e-08, "logits/chosen": -1.6797235012054443, "logits/rejected": -1.4659085273742676, "logps/chosen": -280.9741516113281, "logps/ref_chosen": -89.59434509277344, "logps/ref_rejected": -101.31549835205078, "logps/rejected": -375.17340087890625, "loss": 1.1097, "margin_dpo/margin_mean": 82.47809600830078, "margin_dpo/margin_std": 106.58848571777344, "step": 550 }, { "KL/chosen_KL_mean": -170.07052612304688, "KL/mean": -220.73728942871094, "KL/rejected_KL_mean": -271.4040832519531, "KL/std": 112.70904541015625, "epoch": 0.8091042584434655, "fcm_dpo/beta": 0.004365839995443821, "fcm_dpo/delta": -0.045019328594207764, "fcm_dpo/margin": 101.33354949951172, "fcm_dpo/q_t": 0.3982061743736267, "grad_norm": 11.297587394714355, "learning_rate": 5.4427875753062734e-08, "logits/chosen": -1.5907913446426392, "logits/rejected": -1.4489953517913818, "logps/chosen": -265.1526794433594, "logps/ref_chosen": -95.08216094970703, "logps/ref_rejected": -117.7192611694336, "logps/rejected": -389.12335205078125, "loss": 1.0512, "margin_dpo/margin_mean": 101.33354949951172, "margin_dpo/margin_std": 114.79612731933594, "step": 551 }, { "KL/chosen_KL_mean": -162.19448852539062, "KL/mean": -234.16094970703125, "KL/rejected_KL_mean": -306.12744140625, "KL/std": 121.77751159667969, "epoch": 0.8105726872246696, "fcm_dpo/beta": 0.004201897419989109, "fcm_dpo/delta": -0.22096319496631622, "fcm_dpo/margin": 143.93295288085938, "fcm_dpo/q_t": 0.35865312814712524, "grad_norm": 11.115509986877441, "learning_rate": 5.363104864490034e-08, "logits/chosen": -2.2102839946746826, "logits/rejected": -1.8569608926773071, "logps/chosen": -263.6036682128906, "logps/ref_chosen": -101.40918731689453, "logps/ref_rejected": -110.83843994140625, "logps/rejected": -416.96588134765625, "loss": 0.9387, "margin_dpo/margin_mean": 143.93295288085938, "margin_dpo/margin_std": 125.93617248535156, "step": 552 }, { "KL/chosen_KL_mean": -176.4202880859375, "KL/mean": -219.27212524414062, "KL/rejected_KL_mean": -262.12396240234375, "KL/std": 113.06398010253906, "epoch": 0.8120411160058737, "fcm_dpo/beta": 0.004217787645757198, "fcm_dpo/delta": 0.03986806422472, "fcm_dpo/margin": 85.70367431640625, "fcm_dpo/q_t": 0.41532623767852783, "grad_norm": 9.465404510498047, "learning_rate": 5.2839396041230415e-08, "logits/chosen": -1.514159083366394, "logits/rejected": -1.1841676235198975, "logps/chosen": -274.3841552734375, "logps/ref_chosen": -97.96388244628906, "logps/ref_rejected": -103.57719421386719, "logps/rejected": -365.7011413574219, "loss": 1.1206, "margin_dpo/margin_mean": 85.70367431640625, "margin_dpo/margin_std": 121.53829956054688, "step": 553 }, { "KL/chosen_KL_mean": -183.67279052734375, "KL/mean": -231.5159912109375, "KL/rejected_KL_mean": -279.35919189453125, "KL/std": 113.65313720703125, "epoch": 0.8135095447870778, "fcm_dpo/beta": 0.004232833161950111, "fcm_dpo/delta": -0.005503591150045395, "fcm_dpo/margin": 95.6864013671875, "fcm_dpo/q_t": 0.4070894420146942, "grad_norm": 9.835831642150879, "learning_rate": 5.205293880283551e-08, "logits/chosen": -1.8650469779968262, "logits/rejected": -1.0872533321380615, "logps/chosen": -287.359619140625, "logps/ref_chosen": -103.68680572509766, "logps/ref_rejected": -92.99210357666016, "logps/rejected": -372.351318359375, "loss": 1.1133, "margin_dpo/margin_mean": 95.6864013671875, "margin_dpo/margin_std": 146.53952026367188, "step": 554 }, { "KL/chosen_KL_mean": -192.05535888671875, "KL/mean": -236.15475463867188, "KL/rejected_KL_mean": -280.254150390625, "KL/std": 117.31645202636719, "epoch": 0.8149779735682819, "fcm_dpo/beta": 0.0042327167466282845, "fcm_dpo/delta": 0.02771700546145439, "fcm_dpo/margin": 88.19882202148438, "fcm_dpo/q_t": 0.4148179888725281, "grad_norm": 9.328210830688477, "learning_rate": 5.127169765359515e-08, "logits/chosen": -1.6450016498565674, "logits/rejected": -1.5520615577697754, "logps/chosen": -278.25897216796875, "logps/ref_chosen": -86.20362854003906, "logps/ref_rejected": -114.54719543457031, "logps/rejected": -394.8013916015625, "loss": 1.1243, "margin_dpo/margin_mean": 88.19882202148438, "margin_dpo/margin_std": 135.3408203125, "step": 555 }, { "KL/chosen_KL_mean": -172.41041564941406, "KL/mean": -219.8575439453125, "KL/rejected_KL_mean": -267.3046875, "KL/std": 104.11418151855469, "epoch": 0.8164464023494861, "fcm_dpo/beta": 0.0042260740883648396, "fcm_dpo/delta": -0.001250673085451126, "fcm_dpo/margin": 94.89425659179688, "fcm_dpo/q_t": 0.40639132261276245, "grad_norm": 10.330057144165039, "learning_rate": 5.049569317994012e-08, "logits/chosen": -1.6202504634857178, "logits/rejected": -1.3187134265899658, "logps/chosen": -260.81390380859375, "logps/ref_chosen": -88.40348815917969, "logps/ref_rejected": -106.63911437988281, "logps/rejected": -373.94378662109375, "loss": 1.085, "margin_dpo/margin_mean": 94.89425659179688, "margin_dpo/margin_std": 118.68606567382812, "step": 556 }, { "KL/chosen_KL_mean": -185.38319396972656, "KL/mean": -240.234619140625, "KL/rejected_KL_mean": -295.0860595703125, "KL/std": 120.69813537597656, "epoch": 0.8179148311306902, "fcm_dpo/beta": 0.004180575720965862, "fcm_dpo/delta": -0.06211596727371216, "fcm_dpo/margin": 109.70284271240234, "fcm_dpo/q_t": 0.39451897144317627, "grad_norm": 7.899259567260742, "learning_rate": 4.9724945830310144e-08, "logits/chosen": -1.9757657051086426, "logits/rejected": -1.8022551536560059, "logps/chosen": -284.21844482421875, "logps/ref_chosen": -98.83523559570312, "logps/ref_rejected": -115.64888763427734, "logps/rejected": -410.73492431640625, "loss": 1.0504, "margin_dpo/margin_mean": 109.70284271240234, "margin_dpo/margin_std": 130.8518524169922, "step": 557 }, { "KL/chosen_KL_mean": -183.36279296875, "KL/mean": -241.62454223632812, "KL/rejected_KL_mean": -299.88629150390625, "KL/std": 104.25341796875, "epoch": 0.8193832599118943, "fcm_dpo/beta": 0.0041241636499762535, "fcm_dpo/delta": -0.08491595089435577, "fcm_dpo/margin": 116.52346801757812, "fcm_dpo/q_t": 0.38678282499313354, "grad_norm": 10.671262741088867, "learning_rate": 4.8959475914614554e-08, "logits/chosen": -2.3789567947387695, "logits/rejected": -1.7708866596221924, "logps/chosen": -289.3746337890625, "logps/ref_chosen": -106.01183319091797, "logps/ref_rejected": -108.39131927490234, "logps/rejected": -408.277587890625, "loss": 1.009, "margin_dpo/margin_mean": 116.52346801757812, "margin_dpo/margin_std": 106.39876556396484, "step": 558 }, { "KL/chosen_KL_mean": -198.6157989501953, "KL/mean": -244.93408203125, "KL/rejected_KL_mean": -291.2523193359375, "KL/std": 106.57640075683594, "epoch": 0.8208516886930984, "fcm_dpo/beta": 0.004130173474550247, "fcm_dpo/delta": 0.018053701147437096, "fcm_dpo/margin": 92.63652038574219, "fcm_dpo/q_t": 0.40991270542144775, "grad_norm": 10.63416862487793, "learning_rate": 4.8199303603697614e-08, "logits/chosen": -2.2807705402374268, "logits/rejected": -1.7953400611877441, "logps/chosen": -299.04052734375, "logps/ref_chosen": -100.42474365234375, "logps/ref_rejected": -100.41270446777344, "logps/rejected": -391.6650390625, "loss": 1.0993, "margin_dpo/margin_mean": 92.63652038574219, "margin_dpo/margin_std": 120.48992919921875, "step": 559 }, { "KL/chosen_KL_mean": -189.1143035888672, "KL/mean": -234.44798278808594, "KL/rejected_KL_mean": -279.7816467285156, "KL/std": 104.3011474609375, "epoch": 0.8223201174743024, "fcm_dpo/beta": 0.0041424487717449665, "fcm_dpo/delta": 0.025375008583068848, "fcm_dpo/margin": 90.66734313964844, "fcm_dpo/q_t": 0.4123731851577759, "grad_norm": 11.172224998474121, "learning_rate": 4.7444448928806615e-08, "logits/chosen": -1.439769983291626, "logits/rejected": -0.9974866509437561, "logps/chosen": -276.74981689453125, "logps/ref_chosen": -87.63551330566406, "logps/ref_rejected": -87.37985229492188, "logps/rejected": -367.1614990234375, "loss": 1.1047, "margin_dpo/margin_mean": 90.66734313964844, "margin_dpo/margin_std": 120.47526550292969, "step": 560 }, { "KL/chosen_KL_mean": -198.3482208251953, "KL/mean": -240.470458984375, "KL/rejected_KL_mean": -282.59271240234375, "KL/std": 102.92167663574219, "epoch": 0.8237885462555066, "fcm_dpo/beta": 0.004189247265458107, "fcm_dpo/delta": 0.04863971099257469, "fcm_dpo/margin": 84.24447631835938, "fcm_dpo/q_t": 0.4168863892555237, "grad_norm": 8.635525703430176, "learning_rate": 4.669493178106432e-08, "logits/chosen": -1.856222152709961, "logits/rejected": -1.8283874988555908, "logps/chosen": -273.2296142578125, "logps/ref_chosen": -74.88140106201172, "logps/ref_rejected": -106.04796600341797, "logps/rejected": -388.64068603515625, "loss": 1.1055, "margin_dpo/margin_mean": 84.24449157714844, "margin_dpo/margin_std": 99.34193420410156, "step": 561 }, { "KL/chosen_KL_mean": -180.08209228515625, "KL/mean": -223.04071044921875, "KL/rejected_KL_mean": -265.99932861328125, "KL/std": 103.99583435058594, "epoch": 0.8252569750367107, "fcm_dpo/beta": 0.004203141201287508, "fcm_dpo/delta": 0.04024399816989899, "fcm_dpo/margin": 85.917236328125, "fcm_dpo/q_t": 0.41662245988845825, "grad_norm": 11.453095436096191, "learning_rate": 4.5950771910944596e-08, "logits/chosen": -2.207244396209717, "logits/rejected": -1.758447289466858, "logps/chosen": -280.74530029296875, "logps/ref_chosen": -100.66322326660156, "logps/ref_rejected": -102.24087524414062, "logps/rejected": -368.2402038574219, "loss": 1.1269, "margin_dpo/margin_mean": 85.91722869873047, "margin_dpo/margin_std": 128.87120056152344, "step": 562 }, { "KL/chosen_KL_mean": -189.82872009277344, "KL/mean": -225.5064697265625, "KL/rejected_KL_mean": -261.1841735839844, "KL/std": 104.61654663085938, "epoch": 0.8267254038179148, "fcm_dpo/beta": 0.004300978966057301, "fcm_dpo/delta": 0.09548667073249817, "fcm_dpo/margin": 71.35546112060547, "fcm_dpo/q_t": 0.42831218242645264, "grad_norm": 10.252458572387695, "learning_rate": 4.521198892775202e-08, "logits/chosen": -1.6837294101715088, "logits/rejected": -1.3441778421401978, "logps/chosen": -286.08795166015625, "logps/ref_chosen": -96.25921630859375, "logps/ref_rejected": -101.20503997802734, "logps/rejected": -362.38922119140625, "loss": 1.159, "margin_dpo/margin_mean": 71.35546112060547, "margin_dpo/margin_std": 111.72962951660156, "step": 563 }, { "KL/chosen_KL_mean": -186.2872314453125, "KL/mean": -226.03663635253906, "KL/rejected_KL_mean": -265.7860412597656, "KL/std": 105.17993927001953, "epoch": 0.8281938325991189, "fcm_dpo/beta": 0.0043524159118533134, "fcm_dpo/delta": 0.055709775537252426, "fcm_dpo/margin": 79.49880981445312, "fcm_dpo/q_t": 0.4181980490684509, "grad_norm": 12.237345695495605, "learning_rate": 4.447860229910544e-08, "logits/chosen": -2.404914379119873, "logits/rejected": -1.8411701917648315, "logps/chosen": -299.24761962890625, "logps/ref_chosen": -112.96040344238281, "logps/ref_rejected": -99.52803802490234, "logps/rejected": -365.3140869140625, "loss": 1.1128, "margin_dpo/margin_mean": 79.49880981445312, "margin_dpo/margin_std": 95.95743560791016, "step": 564 }, { "KL/chosen_KL_mean": -182.14346313476562, "KL/mean": -231.9015350341797, "KL/rejected_KL_mean": -281.65966796875, "KL/std": 114.21833801269531, "epoch": 0.8296622613803231, "fcm_dpo/beta": 0.004329100716859102, "fcm_dpo/delta": -0.032275453209877014, "fcm_dpo/margin": 99.51617431640625, "fcm_dpo/q_t": 0.4003847539424896, "grad_norm": 8.961211204528809, "learning_rate": 4.375063135042445e-08, "logits/chosen": -1.7828240394592285, "logits/rejected": -1.2836227416992188, "logps/chosen": -277.509033203125, "logps/ref_chosen": -95.36558532714844, "logps/ref_rejected": -92.21737670898438, "logps/rejected": -373.87701416015625, "loss": 1.0697, "margin_dpo/margin_mean": 99.51617431640625, "margin_dpo/margin_std": 124.67202758789062, "step": 565 }, { "KL/chosen_KL_mean": -190.8489532470703, "KL/mean": -237.7290496826172, "KL/rejected_KL_mean": -284.609130859375, "KL/std": 116.31834411621094, "epoch": 0.8311306901615272, "fcm_dpo/beta": 0.004333840683102608, "fcm_dpo/delta": -0.006719989702105522, "fcm_dpo/margin": 93.76020812988281, "fcm_dpo/q_t": 0.40456926822662354, "grad_norm": 8.611566543579102, "learning_rate": 4.3028095264420525e-08, "logits/chosen": -1.9020146131515503, "logits/rejected": -1.4681779146194458, "logps/chosen": -289.8221435546875, "logps/ref_chosen": -98.97320556640625, "logps/ref_rejected": -110.82090759277344, "logps/rejected": -395.4300537109375, "loss": 1.081, "margin_dpo/margin_mean": 93.76020812988281, "margin_dpo/margin_std": 115.12712097167969, "step": 566 }, { "KL/chosen_KL_mean": -184.54513549804688, "KL/mean": -226.3482666015625, "KL/rejected_KL_mean": -268.15142822265625, "KL/std": 104.99258422851562, "epoch": 0.8325991189427313, "fcm_dpo/beta": 0.004327400121837854, "fcm_dpo/delta": 0.03939405083656311, "fcm_dpo/margin": 83.60626220703125, "fcm_dpo/q_t": 0.4145927131175995, "grad_norm": 10.30115795135498, "learning_rate": 4.231101308059165e-08, "logits/chosen": -2.2086124420166016, "logits/rejected": -1.5272175073623657, "logps/chosen": -277.24871826171875, "logps/ref_chosen": -92.7035903930664, "logps/ref_rejected": -91.22431945800781, "logps/rejected": -359.375732421875, "loss": 1.1148, "margin_dpo/margin_mean": 83.60626220703125, "margin_dpo/margin_std": 110.74401092529297, "step": 567 }, { "KL/chosen_KL_mean": -175.39373779296875, "KL/mean": -230.40463256835938, "KL/rejected_KL_mean": -285.41552734375, "KL/std": 110.84016418457031, "epoch": 0.8340675477239354, "fcm_dpo/beta": 0.004291988909244537, "fcm_dpo/delta": -0.07628700882196426, "fcm_dpo/margin": 110.02178955078125, "fcm_dpo/q_t": 0.3895794749259949, "grad_norm": 8.607927322387695, "learning_rate": 4.1599403694720145e-08, "logits/chosen": -1.2725071907043457, "logits/rejected": -1.1388936042785645, "logps/chosen": -248.5260009765625, "logps/ref_chosen": -73.13226318359375, "logps/ref_rejected": -94.95265197753906, "logps/rejected": -380.3681945800781, "loss": 1.0234, "margin_dpo/margin_mean": 110.02178955078125, "margin_dpo/margin_std": 111.92556762695312, "step": 568 }, { "KL/chosen_KL_mean": -186.41482543945312, "KL/mean": -231.21548461914062, "KL/rejected_KL_mean": -276.01611328125, "KL/std": 111.94309997558594, "epoch": 0.8355359765051396, "fcm_dpo/beta": 0.004306395072489977, "fcm_dpo/delta": 0.014626980759203434, "fcm_dpo/margin": 89.60127258300781, "fcm_dpo/q_t": 0.4102838635444641, "grad_norm": 14.316227912902832, "learning_rate": 4.089328585837512e-08, "logits/chosen": -1.7964123487472534, "logits/rejected": -1.4078896045684814, "logps/chosen": -275.63189697265625, "logps/ref_chosen": -89.21708679199219, "logps/ref_rejected": -86.41105651855469, "logps/rejected": -362.42718505859375, "loss": 1.1182, "margin_dpo/margin_mean": 89.60127258300781, "margin_dpo/margin_std": 134.617919921875, "step": 569 }, { "KL/chosen_KL_mean": -166.097412109375, "KL/mean": -224.27291870117188, "KL/rejected_KL_mean": -282.44842529296875, "KL/std": 107.03939819335938, "epoch": 0.8370044052863436, "fcm_dpo/beta": 0.004237134475260973, "fcm_dpo/delta": -0.09789938479661942, "fcm_dpo/margin": 116.35102844238281, "fcm_dpo/q_t": 0.38357582688331604, "grad_norm": 9.164166450500488, "learning_rate": 4.019267817841834e-08, "logits/chosen": -2.2431373596191406, "logits/rejected": -1.551689624786377, "logps/chosen": -260.86016845703125, "logps/ref_chosen": -94.76277160644531, "logps/ref_rejected": -88.846923828125, "logps/rejected": -371.29534912109375, "loss": 1.0057, "margin_dpo/margin_mean": 116.35102844238281, "margin_dpo/margin_std": 111.74870300292969, "step": 570 }, { "KL/chosen_KL_mean": -173.78616333007812, "KL/mean": -225.00808715820312, "KL/rejected_KL_mean": -276.22998046875, "KL/std": 105.24479675292969, "epoch": 0.8384728340675477, "fcm_dpo/beta": 0.004208741709589958, "fcm_dpo/delta": -0.03260467201471329, "fcm_dpo/margin": 102.44383239746094, "fcm_dpo/q_t": 0.400029718875885, "grad_norm": 11.672890663146973, "learning_rate": 3.9497599116513705e-08, "logits/chosen": -1.712490439414978, "logits/rejected": -1.5556182861328125, "logps/chosen": -254.60745239257812, "logps/ref_chosen": -80.8212890625, "logps/ref_rejected": -97.01528930664062, "logps/rejected": -373.2452697753906, "loss": 1.0687, "margin_dpo/margin_mean": 102.44383239746094, "margin_dpo/margin_std": 126.82234191894531, "step": 571 }, { "KL/chosen_KL_mean": -177.79869079589844, "KL/mean": -229.5006561279297, "KL/rejected_KL_mean": -281.20263671875, "KL/std": 111.58793640136719, "epoch": 0.8399412628487518, "fcm_dpo/beta": 0.004166761413216591, "fcm_dpo/delta": -0.03231048956513405, "fcm_dpo/margin": 103.4039306640625, "fcm_dpo/q_t": 0.4004812240600586, "grad_norm": 9.372556686401367, "learning_rate": 3.880806698864086e-08, "logits/chosen": -1.8603551387786865, "logits/rejected": -1.6936845779418945, "logps/chosen": -255.17861938476562, "logps/ref_chosen": -77.37992858886719, "logps/ref_rejected": -90.56597900390625, "logps/rejected": -371.76861572265625, "loss": 1.0599, "margin_dpo/margin_mean": 103.4039306640625, "margin_dpo/margin_std": 120.28791046142578, "step": 572 }, { "KL/chosen_KL_mean": -173.040771484375, "KL/mean": -221.3245849609375, "KL/rejected_KL_mean": -269.6083984375, "KL/std": 111.16986083984375, "epoch": 0.8414096916299559, "fcm_dpo/beta": 0.004186637699604034, "fcm_dpo/delta": -0.005190014839172363, "fcm_dpo/margin": 96.56765747070312, "fcm_dpo/q_t": 0.40500408411026, "grad_norm": 9.036714553833008, "learning_rate": 3.812409996461275e-08, "logits/chosen": -2.0737667083740234, "logits/rejected": -1.8406906127929688, "logps/chosen": -256.7468566894531, "logps/ref_chosen": -83.7060775756836, "logps/ref_rejected": -90.69746398925781, "logps/rejected": -360.3058776855469, "loss": 1.0834, "margin_dpo/margin_mean": 96.5676498413086, "margin_dpo/margin_std": 117.7054672241211, "step": 573 }, { "KL/chosen_KL_mean": -181.8966064453125, "KL/mean": -233.9127197265625, "KL/rejected_KL_mean": -285.9288330078125, "KL/std": 110.39947509765625, "epoch": 0.8428781204111601, "fcm_dpo/beta": 0.004152457229793072, "fcm_dpo/delta": -0.03352772444486618, "fcm_dpo/margin": 104.03224182128906, "fcm_dpo/q_t": 0.3992612659931183, "grad_norm": 9.427274703979492, "learning_rate": 3.74457160675965e-08, "logits/chosen": -2.186664581298828, "logits/rejected": -1.8622626066207886, "logps/chosen": -267.0260925292969, "logps/ref_chosen": -85.12948608398438, "logps/ref_rejected": -99.05403137207031, "logps/rejected": -384.9828796386719, "loss": 1.0649, "margin_dpo/margin_mean": 104.0322494506836, "margin_dpo/margin_std": 125.00936889648438, "step": 574 }, { "KL/chosen_KL_mean": -188.0989227294922, "KL/mean": -236.54022216796875, "KL/rejected_KL_mean": -284.98150634765625, "KL/std": 113.03234100341797, "epoch": 0.8443465491923642, "fcm_dpo/beta": 0.004131317604333162, "fcm_dpo/delta": -0.00026449491269886494, "fcm_dpo/margin": 96.88261413574219, "fcm_dpo/q_t": 0.40731048583984375, "grad_norm": 11.019231796264648, "learning_rate": 3.677293317363864e-08, "logits/chosen": -1.4887568950653076, "logits/rejected": -1.0291008949279785, "logps/chosen": -293.8697509765625, "logps/ref_chosen": -105.77084350585938, "logps/ref_rejected": -100.94944763183594, "logps/rejected": -385.93096923828125, "loss": 1.0949, "margin_dpo/margin_mean": 96.88261413574219, "margin_dpo/margin_std": 131.52003479003906, "step": 575 }, { "KL/chosen_KL_mean": -187.9970703125, "KL/mean": -234.42626953125, "KL/rejected_KL_mean": -280.8554382324219, "KL/std": 104.52543640136719, "epoch": 0.8458149779735683, "fcm_dpo/beta": 0.004147705622017384, "fcm_dpo/delta": 0.015380189754068851, "fcm_dpo/margin": 92.85834503173828, "fcm_dpo/q_t": 0.41005727648735046, "grad_norm": 10.624781608581543, "learning_rate": 3.6105769011194224e-08, "logits/chosen": -2.010153293609619, "logits/rejected": -1.9495136737823486, "logps/chosen": -269.20159912109375, "logps/ref_chosen": -81.20452880859375, "logps/ref_rejected": -105.40940856933594, "logps/rejected": -386.26483154296875, "loss": 1.1028, "margin_dpo/margin_mean": 92.85835266113281, "margin_dpo/margin_std": 125.59454345703125, "step": 576 }, { "KL/chosen_KL_mean": -167.01742553710938, "KL/mean": -225.3931121826172, "KL/rejected_KL_mean": -283.768798828125, "KL/std": 113.59016418457031, "epoch": 0.8472834067547724, "fcm_dpo/beta": 0.0040807900950312614, "fcm_dpo/delta": -0.08081139624118805, "fcm_dpo/margin": 116.75135040283203, "fcm_dpo/q_t": 0.3888378441333771, "grad_norm": 8.150922775268555, "learning_rate": 3.5444241160659304e-08, "logits/chosen": -2.108572244644165, "logits/rejected": -1.420820951461792, "logps/chosen": -268.48162841796875, "logps/ref_chosen": -101.46419525146484, "logps/ref_rejected": -89.32637023925781, "logps/rejected": -373.09515380859375, "loss": 1.0319, "margin_dpo/margin_mean": 116.7513427734375, "margin_dpo/margin_std": 129.18267822265625, "step": 577 }, { "KL/chosen_KL_mean": -199.24859619140625, "KL/mean": -241.75115966796875, "KL/rejected_KL_mean": -284.25372314453125, "KL/std": 108.54114532470703, "epoch": 0.8487518355359766, "fcm_dpo/beta": 0.004108277149498463, "fcm_dpo/delta": 0.0526362881064415, "fcm_dpo/margin": 85.00509643554688, "fcm_dpo/q_t": 0.41863149404525757, "grad_norm": 13.33284854888916, "learning_rate": 3.478836705390808e-08, "logits/chosen": -1.2520579099655151, "logits/rejected": -1.0077526569366455, "logps/chosen": -276.541015625, "logps/ref_chosen": -77.29241943359375, "logps/ref_rejected": -88.24701690673828, "logps/rejected": -372.500732421875, "loss": 1.1289, "margin_dpo/margin_mean": 85.00509643554688, "margin_dpo/margin_std": 123.86869812011719, "step": 578 }, { "KL/chosen_KL_mean": -179.24444580078125, "KL/mean": -222.0959930419922, "KL/rejected_KL_mean": -264.9475402832031, "KL/std": 108.1878433227539, "epoch": 0.8502202643171806, "fcm_dpo/beta": 0.004134485963732004, "fcm_dpo/delta": 0.047208476811647415, "fcm_dpo/margin": 85.70309448242188, "fcm_dpo/q_t": 0.417565256357193, "grad_norm": 8.171529769897461, "learning_rate": 3.41381639738331e-08, "logits/chosen": -1.5103442668914795, "logits/rejected": -1.2028511762619019, "logps/chosen": -266.5874938964844, "logps/ref_chosen": -87.34305572509766, "logps/ref_rejected": -102.12100982666016, "logps/rejected": -367.06854248046875, "loss": 1.1174, "margin_dpo/margin_mean": 85.70309448242188, "margin_dpo/margin_std": 110.83352661132812, "step": 579 }, { "KL/chosen_KL_mean": -170.42135620117188, "KL/mean": -223.21624755859375, "KL/rejected_KL_mean": -276.01116943359375, "KL/std": 107.60038757324219, "epoch": 0.8516886930983847, "fcm_dpo/beta": 0.004122233483940363, "fcm_dpo/delta": -0.03729693964123726, "fcm_dpo/margin": 105.58979797363281, "fcm_dpo/q_t": 0.3997495770454407, "grad_norm": 9.752662658691406, "learning_rate": 3.349364905389032e-08, "logits/chosen": -1.3725589513778687, "logits/rejected": -1.0510061979293823, "logps/chosen": -259.35015869140625, "logps/ref_chosen": -88.92879486083984, "logps/ref_rejected": -88.69661712646484, "logps/rejected": -364.707763671875, "loss": 1.0649, "margin_dpo/margin_mean": 105.58979797363281, "margin_dpo/margin_std": 129.3506622314453, "step": 580 }, { "KL/chosen_KL_mean": -181.15432739257812, "KL/mean": -229.07485961914062, "KL/rejected_KL_mean": -276.995361328125, "KL/std": 108.88592529296875, "epoch": 0.8531571218795888, "fcm_dpo/beta": 0.0041353581473231316, "fcm_dpo/delta": 0.003798818215727806, "fcm_dpo/margin": 95.8410415649414, "fcm_dpo/q_t": 0.4079226851463318, "grad_norm": 10.837934494018555, "learning_rate": 3.285483927764726e-08, "logits/chosen": -1.8750321865081787, "logits/rejected": -1.519486904144287, "logps/chosen": -273.3775634765625, "logps/ref_chosen": -92.22323608398438, "logps/ref_rejected": -99.38943481445312, "logps/rejected": -376.38482666015625, "loss": 1.1046, "margin_dpo/margin_mean": 95.8410415649414, "margin_dpo/margin_std": 136.5115966796875, "step": 581 }, { "KL/chosen_KL_mean": -160.0307159423828, "KL/mean": -203.48455810546875, "KL/rejected_KL_mean": -246.93838500976562, "KL/std": 112.89453125, "epoch": 0.8546255506607929, "fcm_dpo/beta": 0.004191345535218716, "fcm_dpo/delta": 0.03521985560655594, "fcm_dpo/margin": 86.90767669677734, "fcm_dpo/q_t": 0.41680288314819336, "grad_norm": 9.827420234680176, "learning_rate": 3.222175147833556e-08, "logits/chosen": -2.0919480323791504, "logits/rejected": -2.08280611038208, "logps/chosen": -247.3663330078125, "logps/ref_chosen": -87.33561706542969, "logps/ref_rejected": -115.1544189453125, "logps/rejected": -362.0928039550781, "loss": 1.1248, "margin_dpo/margin_mean": 86.90767669677734, "margin_dpo/margin_std": 124.35946655273438, "step": 582 }, { "KL/chosen_KL_mean": -192.71041870117188, "KL/mean": -241.04473876953125, "KL/rejected_KL_mean": -289.37908935546875, "KL/std": 106.74190521240234, "epoch": 0.856093979441997, "fcm_dpo/beta": 0.004145767539739609, "fcm_dpo/delta": -0.0010952763259410858, "fcm_dpo/margin": 96.66867065429688, "fcm_dpo/q_t": 0.4074910283088684, "grad_norm": 10.918373107910156, "learning_rate": 3.159440233840763e-08, "logits/chosen": -1.5438284873962402, "logits/rejected": -1.3111217021942139, "logps/chosen": -276.427490234375, "logps/ref_chosen": -83.71707153320312, "logps/ref_rejected": -94.58465576171875, "logps/rejected": -383.9637451171875, "loss": 1.0903, "margin_dpo/margin_mean": 96.66867065429688, "margin_dpo/margin_std": 126.04806518554688, "step": 583 }, { "KL/chosen_KL_mean": -171.73846435546875, "KL/mean": -222.67364501953125, "KL/rejected_KL_mean": -273.6088562011719, "KL/std": 110.94793701171875, "epoch": 0.8575624082232012, "fcm_dpo/beta": 0.004173481836915016, "fcm_dpo/delta": -0.027065642178058624, "fcm_dpo/margin": 101.87039947509766, "fcm_dpo/q_t": 0.40064719319343567, "grad_norm": 7.989595890045166, "learning_rate": 3.0972808389096635e-08, "logits/chosen": -1.972607135772705, "logits/rejected": -1.5166950225830078, "logps/chosen": -265.9980163574219, "logps/ref_chosen": -94.25955963134766, "logps/ref_rejected": -104.48269653320312, "logps/rejected": -378.091552734375, "loss": 1.0761, "margin_dpo/margin_mean": 101.87040710449219, "margin_dpo/margin_std": 125.9791259765625, "step": 584 }, { "KL/chosen_KL_mean": -183.015869140625, "KL/mean": -239.93743896484375, "KL/rejected_KL_mean": -296.8590393066406, "KL/std": 126.43869018554688, "epoch": 0.8590308370044053, "fcm_dpo/beta": 0.004108230583369732, "fcm_dpo/delta": -0.07099392265081406, "fcm_dpo/margin": 113.84317016601562, "fcm_dpo/q_t": 0.3932206630706787, "grad_norm": 9.196124076843262, "learning_rate": 3.035698600998121e-08, "logits/chosen": -1.8066558837890625, "logits/rejected": -1.5242770910263062, "logps/chosen": -268.6526184082031, "logps/ref_chosen": -85.63676452636719, "logps/ref_rejected": -90.28547668457031, "logps/rejected": -387.1445007324219, "loss": 1.058, "margin_dpo/margin_mean": 113.84317779541016, "margin_dpo/margin_std": 147.35458374023438, "step": 585 }, { "KL/chosen_KL_mean": -203.01451110839844, "KL/mean": -240.9199676513672, "KL/rejected_KL_mean": -278.825439453125, "KL/std": 110.70780944824219, "epoch": 0.8604992657856094, "fcm_dpo/beta": 0.004134843125939369, "fcm_dpo/delta": 0.08934246003627777, "fcm_dpo/margin": 75.81089782714844, "fcm_dpo/q_t": 0.42629706859588623, "grad_norm": 10.492860794067383, "learning_rate": 2.974695142855388e-08, "logits/chosen": -1.9477040767669678, "logits/rejected": -1.688276767730713, "logps/chosen": -286.6956481933594, "logps/ref_chosen": -83.68114471435547, "logps/ref_rejected": -98.35916137695312, "logps/rejected": -377.1845703125, "loss": 1.1434, "margin_dpo/margin_mean": 75.81089782714844, "margin_dpo/margin_std": 105.11416625976562, "step": 586 }, { "KL/chosen_KL_mean": -165.01052856445312, "KL/mean": -206.79428100585938, "KL/rejected_KL_mean": -248.57803344726562, "KL/std": 107.04502868652344, "epoch": 0.8619676945668135, "fcm_dpo/beta": 0.0041906749829649925, "fcm_dpo/delta": 0.051520936191082, "fcm_dpo/margin": 83.5675048828125, "fcm_dpo/q_t": 0.41815000772476196, "grad_norm": 10.594428062438965, "learning_rate": 2.9142720719793122e-08, "logits/chosen": -2.332167625427246, "logits/rejected": -2.0463733673095703, "logps/chosen": -242.58802795410156, "logps/ref_chosen": -77.57750701904297, "logps/ref_rejected": -89.32032775878906, "logps/rejected": -337.89837646484375, "loss": 1.1181, "margin_dpo/margin_mean": 83.56751251220703, "margin_dpo/margin_std": 111.17457580566406, "step": 587 }, { "KL/chosen_KL_mean": -178.5463409423828, "KL/mean": -217.53875732421875, "KL/rejected_KL_mean": -256.53118896484375, "KL/std": 108.92408752441406, "epoch": 0.8634361233480177, "fcm_dpo/beta": 0.004234693944454193, "fcm_dpo/delta": 0.07220453768968582, "fcm_dpo/margin": 77.98480224609375, "fcm_dpo/q_t": 0.42472565174102783, "grad_norm": 13.654078483581543, "learning_rate": 2.8544309805740018e-08, "logits/chosen": -1.8392393589019775, "logits/rejected": -1.8003513813018799, "logps/chosen": -262.3505859375, "logps/ref_chosen": -83.80426025390625, "logps/ref_rejected": -113.11943054199219, "logps/rejected": -369.6506042480469, "loss": 1.1565, "margin_dpo/margin_mean": 77.98480224609375, "margin_dpo/margin_std": 131.14608764648438, "step": 588 }, { "KL/chosen_KL_mean": -190.62347412109375, "KL/mean": -242.119384765625, "KL/rejected_KL_mean": -293.61529541015625, "KL/std": 107.27945709228516, "epoch": 0.8649045521292217, "fcm_dpo/beta": 0.004235325381159782, "fcm_dpo/delta": -0.03784612566232681, "fcm_dpo/margin": 102.99179077148438, "fcm_dpo/q_t": 0.39690929651260376, "grad_norm": 8.783534049987793, "learning_rate": 2.7951734455078786e-08, "logits/chosen": -1.7675971984863281, "logits/rejected": -1.378097414970398, "logps/chosen": -287.0885009765625, "logps/ref_chosen": -96.46501922607422, "logps/ref_rejected": -116.03719329833984, "logps/rejected": -409.6524658203125, "loss": 1.0418, "margin_dpo/margin_mean": 102.99179077148438, "margin_dpo/margin_std": 102.50908660888672, "step": 589 }, { "KL/chosen_KL_mean": -176.222900390625, "KL/mean": -227.35452270507812, "KL/rejected_KL_mean": -278.48614501953125, "KL/std": 100.26134490966797, "epoch": 0.8663729809104258, "fcm_dpo/beta": 0.004221814684569836, "fcm_dpo/delta": -0.03335803002119064, "fcm_dpo/margin": 102.26322937011719, "fcm_dpo/q_t": 0.39879706501960754, "grad_norm": 11.606023788452148, "learning_rate": 2.736501028272095e-08, "logits/chosen": -1.9758179187774658, "logits/rejected": -1.8300034999847412, "logps/chosen": -264.6507568359375, "logps/ref_chosen": -88.4278564453125, "logps/ref_rejected": -110.19876861572266, "logps/rejected": -388.6849060058594, "loss": 1.0475, "margin_dpo/margin_mean": 102.26322937011719, "margin_dpo/margin_std": 104.72445678710938, "step": 590 }, { "KL/chosen_KL_mean": -167.87924194335938, "KL/mean": -222.17239379882812, "KL/rejected_KL_mean": -276.465576171875, "KL/std": 107.96830749511719, "epoch": 0.8678414096916299, "fcm_dpo/beta": 0.0041551245376467705, "fcm_dpo/delta": -0.05382894352078438, "fcm_dpo/margin": 108.58628845214844, "fcm_dpo/q_t": 0.39366090297698975, "grad_norm": 7.283422946929932, "learning_rate": 2.678415274939408e-08, "logits/chosen": -1.901911973953247, "logits/rejected": -1.2828800678253174, "logps/chosen": -255.72894287109375, "logps/ref_chosen": -87.84968566894531, "logps/ref_rejected": -89.57919311523438, "logps/rejected": -366.04473876953125, "loss": 1.0305, "margin_dpo/margin_mean": 108.5863037109375, "margin_dpo/margin_std": 104.78366088867188, "step": 591 }, { "KL/chosen_KL_mean": -198.79989624023438, "KL/mean": -240.76718139648438, "KL/rejected_KL_mean": -282.73443603515625, "KL/std": 104.67167663574219, "epoch": 0.869309838472834, "fcm_dpo/beta": 0.004186016507446766, "fcm_dpo/delta": 0.050395239144563675, "fcm_dpo/margin": 83.93453979492188, "fcm_dpo/q_t": 0.4181559085845947, "grad_norm": 11.353158950805664, "learning_rate": 2.6209177161234442e-08, "logits/chosen": -1.8997392654418945, "logits/rejected": -1.57634437084198, "logps/chosen": -273.775634765625, "logps/ref_chosen": -74.97573852539062, "logps/ref_rejected": -81.55262756347656, "logps/rejected": -364.2870788574219, "loss": 1.1373, "margin_dpo/margin_mean": 83.93453979492188, "margin_dpo/margin_std": 131.14930725097656, "step": 592 }, { "KL/chosen_KL_mean": -181.54666137695312, "KL/mean": -217.84310913085938, "KL/rejected_KL_mean": -254.13955688476562, "KL/std": 109.79293823242188, "epoch": 0.8707782672540382, "fcm_dpo/beta": 0.004241696558892727, "fcm_dpo/delta": 0.09510651230812073, "fcm_dpo/margin": 72.59286499023438, "fcm_dpo/q_t": 0.4291771352291107, "grad_norm": 14.327717781066895, "learning_rate": 2.564009866938349e-08, "logits/chosen": -0.5918596386909485, "logits/rejected": -0.27619442343711853, "logps/chosen": -253.78570556640625, "logps/ref_chosen": -72.23902893066406, "logps/ref_rejected": -70.09652709960938, "logps/rejected": -324.236083984375, "loss": 1.167, "margin_dpo/margin_mean": 72.59286499023438, "margin_dpo/margin_std": 122.03491973876953, "step": 593 }, { "KL/chosen_KL_mean": -162.92274475097656, "KL/mean": -218.3072967529297, "KL/rejected_KL_mean": -273.69183349609375, "KL/std": 106.78059387207031, "epoch": 0.8722466960352423, "fcm_dpo/beta": 0.004243612289428711, "fcm_dpo/delta": -0.07350478321313858, "fcm_dpo/margin": 110.76908874511719, "fcm_dpo/q_t": 0.38940542936325073, "grad_norm": 9.135778427124023, "learning_rate": 2.5076932269588708e-08, "logits/chosen": -1.895824909210205, "logits/rejected": -1.3656337261199951, "logps/chosen": -253.36045837402344, "logps/ref_chosen": -90.43771362304688, "logps/ref_rejected": -92.267578125, "logps/rejected": -365.95941162109375, "loss": 1.0174, "margin_dpo/margin_mean": 110.76908874511719, "margin_dpo/margin_std": 103.56658935546875, "step": 594 }, { "KL/chosen_KL_mean": -175.01531982421875, "KL/mean": -220.34744262695312, "KL/rejected_KL_mean": -265.6795654296875, "KL/std": 106.79657745361328, "epoch": 0.8737151248164464, "fcm_dpo/beta": 0.0042344555258750916, "fcm_dpo/delta": 0.0165504552423954, "fcm_dpo/margin": 90.66421508789062, "fcm_dpo/q_t": 0.41153764724731445, "grad_norm": 9.732483863830566, "learning_rate": 2.451969280180849e-08, "logits/chosen": -1.9349422454833984, "logits/rejected": -1.5335960388183594, "logps/chosen": -256.5301513671875, "logps/ref_chosen": -81.51480102539062, "logps/ref_rejected": -86.52047729492188, "logps/rejected": -352.20001220703125, "loss": 1.1031, "margin_dpo/margin_mean": 90.66422271728516, "margin_dpo/margin_std": 122.79666137695312, "step": 595 }, { "KL/chosen_KL_mean": -182.39028930664062, "KL/mean": -222.77413940429688, "KL/rejected_KL_mean": -263.1579895019531, "KL/std": 105.03718566894531, "epoch": 0.8751835535976505, "fcm_dpo/beta": 0.004266445990651846, "fcm_dpo/delta": 0.05733548849821091, "fcm_dpo/margin": 80.7677230834961, "fcm_dpo/q_t": 0.42100077867507935, "grad_norm": 12.99443531036377, "learning_rate": 2.396839494982103e-08, "logits/chosen": -1.4386746883392334, "logits/rejected": -0.8381572961807251, "logps/chosen": -270.59832763671875, "logps/ref_chosen": -88.20805358886719, "logps/ref_rejected": -86.09375, "logps/rejected": -349.25177001953125, "loss": 1.1482, "margin_dpo/margin_mean": 80.76771545410156, "margin_dpo/margin_std": 133.25094604492188, "step": 596 }, { "KL/chosen_KL_mean": -176.9459228515625, "KL/mean": -231.17718505859375, "KL/rejected_KL_mean": -285.408447265625, "KL/std": 113.64598083496094, "epoch": 0.8766519823788547, "fcm_dpo/beta": 0.004227799363434315, "fcm_dpo/delta": -0.061597324907779694, "fcm_dpo/margin": 108.4625015258789, "fcm_dpo/q_t": 0.3937196135520935, "grad_norm": 9.01308822631836, "learning_rate": 2.3423053240837514e-08, "logits/chosen": -1.4413721561431885, "logits/rejected": -1.3257654905319214, "logps/chosen": -260.535888671875, "logps/ref_chosen": -83.58998107910156, "logps/ref_rejected": -106.94439697265625, "logps/rejected": -392.35284423828125, "loss": 1.0432, "margin_dpo/margin_mean": 108.46250915527344, "margin_dpo/margin_std": 123.50918579101562, "step": 597 }, { "KL/chosen_KL_mean": -168.52398681640625, "KL/mean": -214.24098205566406, "KL/rejected_KL_mean": -259.9580078125, "KL/std": 105.9662094116211, "epoch": 0.8781204111600588, "fcm_dpo/beta": 0.004216291941702366, "fcm_dpo/delta": 0.014811497181653976, "fcm_dpo/margin": 91.43402862548828, "fcm_dpo/q_t": 0.4093155264854431, "grad_norm": 9.610456466674805, "learning_rate": 2.2883682045119062e-08, "logits/chosen": -1.853559136390686, "logits/rejected": -1.5343331098556519, "logps/chosen": -258.70660400390625, "logps/ref_chosen": -90.18262481689453, "logps/ref_rejected": -94.79856872558594, "logps/rejected": -354.7565612792969, "loss": 1.1001, "margin_dpo/margin_mean": 91.43402099609375, "margin_dpo/margin_std": 119.05941009521484, "step": 598 }, { "KL/chosen_KL_mean": -171.80758666992188, "KL/mean": -212.28414916992188, "KL/rejected_KL_mean": -252.76071166992188, "KL/std": 110.80136108398438, "epoch": 0.8795888399412628, "fcm_dpo/beta": 0.004252440761774778, "fcm_dpo/delta": 0.05755344033241272, "fcm_dpo/margin": 80.95314025878906, "fcm_dpo/q_t": 0.4197501242160797, "grad_norm": 9.275680541992188, "learning_rate": 2.2350295575598367e-08, "logits/chosen": -1.6830497980117798, "logits/rejected": -1.5600866079330444, "logps/chosen": -249.6498565673828, "logps/ref_chosen": -77.84226989746094, "logps/ref_rejected": -89.1976318359375, "logps/rejected": -341.9583435058594, "loss": 1.1383, "margin_dpo/margin_mean": 80.95314025878906, "margin_dpo/margin_std": 121.799560546875, "step": 599 }, { "KL/chosen_KL_mean": -191.43255615234375, "KL/mean": -236.75234985351562, "KL/rejected_KL_mean": -282.0721435546875, "KL/std": 104.69931030273438, "epoch": 0.8810572687224669, "fcm_dpo/beta": 0.004294979386031628, "fcm_dpo/delta": 0.011122666299343109, "fcm_dpo/margin": 90.63957214355469, "fcm_dpo/q_t": 0.40960273146629333, "grad_norm": 9.510931968688965, "learning_rate": 2.1822907887504932e-08, "logits/chosen": -2.3711938858032227, "logits/rejected": -2.024874687194824, "logps/chosen": -283.620361328125, "logps/ref_chosen": -92.18781280517578, "logps/ref_rejected": -92.95429992675781, "logps/rejected": -375.02642822265625, "loss": 1.0926, "margin_dpo/margin_mean": 90.63957214355469, "margin_dpo/margin_std": 114.36380004882812, "step": 600 }, { "KL/chosen_KL_mean": -192.6490478515625, "KL/mean": -241.32542419433594, "KL/rejected_KL_mean": -290.0018310546875, "KL/std": 106.32952117919922, "epoch": 0.882525697503671, "fcm_dpo/beta": 0.004271063953638077, "fcm_dpo/delta": -0.016721447929739952, "fcm_dpo/margin": 97.352783203125, "fcm_dpo/q_t": 0.4022361934185028, "grad_norm": 9.680984497070312, "learning_rate": 2.1301532877994742e-08, "logits/chosen": -1.4304683208465576, "logits/rejected": -1.0883485078811646, "logps/chosen": -281.86517333984375, "logps/ref_chosen": -89.21614074707031, "logps/ref_rejected": -100.17054748535156, "logps/rejected": -390.17236328125, "loss": 1.0589, "margin_dpo/margin_mean": 97.352783203125, "margin_dpo/margin_std": 102.13568115234375, "step": 601 }, { "KL/chosen_KL_mean": -152.53134155273438, "KL/mean": -215.50894165039062, "KL/rejected_KL_mean": -278.48651123046875, "KL/std": 116.09762573242188, "epoch": 0.8839941262848752, "fcm_dpo/beta": 0.004195379093289375, "fcm_dpo/delta": -0.1355825960636139, "fcm_dpo/margin": 125.9551773071289, "fcm_dpo/q_t": 0.37794002890586853, "grad_norm": 9.339836120605469, "learning_rate": 2.0786184285784298e-08, "logits/chosen": -1.9167630672454834, "logits/rejected": -1.646684169769287, "logps/chosen": -232.5889434814453, "logps/ref_chosen": -80.05760192871094, "logps/ref_rejected": -93.197509765625, "logps/rejected": -371.68402099609375, "loss": 0.9972, "margin_dpo/margin_mean": 125.9551773071289, "margin_dpo/margin_std": 132.66012573242188, "step": 602 }, { "KL/chosen_KL_mean": -163.23681640625, "KL/mean": -215.28732299804688, "KL/rejected_KL_mean": -267.33782958984375, "KL/std": 105.0837631225586, "epoch": 0.8854625550660793, "fcm_dpo/beta": 0.004148066975176334, "fcm_dpo/delta": -0.03324076160788536, "fcm_dpo/margin": 104.10101318359375, "fcm_dpo/q_t": 0.3994070589542389, "grad_norm": 8.081631660461426, "learning_rate": 2.0276875690788204e-08, "logits/chosen": -2.098011016845703, "logits/rejected": -1.5117430686950684, "logps/chosen": -265.54638671875, "logps/ref_chosen": -102.30958557128906, "logps/ref_rejected": -108.06884765625, "logps/rejected": -375.40667724609375, "loss": 1.0587, "margin_dpo/margin_mean": 104.10101318359375, "margin_dpo/margin_std": 120.96929931640625, "step": 603 }, { "KL/chosen_KL_mean": -171.24774169921875, "KL/mean": -224.9176025390625, "KL/rejected_KL_mean": -278.5875244140625, "KL/std": 110.2015380859375, "epoch": 0.8869309838472834, "fcm_dpo/beta": 0.004099584650248289, "fcm_dpo/delta": -0.042216863483190536, "fcm_dpo/margin": 107.33973693847656, "fcm_dpo/q_t": 0.3954828381538391, "grad_norm": 8.513572692871094, "learning_rate": 1.977362051376158e-08, "logits/chosen": -1.4836384057998657, "logits/rejected": -1.3265621662139893, "logps/chosen": -249.4218292236328, "logps/ref_chosen": -78.17408752441406, "logps/ref_rejected": -99.49610900878906, "logps/rejected": -378.0836181640625, "loss": 1.0456, "margin_dpo/margin_mean": 107.3397445678711, "margin_dpo/margin_std": 112.72280883789062, "step": 604 }, { "KL/chosen_KL_mean": -186.53225708007812, "KL/mean": -230.69920349121094, "KL/rejected_KL_mean": -274.86614990234375, "KL/std": 98.5725326538086, "epoch": 0.8883994126284875, "fcm_dpo/beta": 0.004123699851334095, "fcm_dpo/delta": 0.037103451788425446, "fcm_dpo/margin": 88.33387756347656, "fcm_dpo/q_t": 0.4148663580417633, "grad_norm": 9.174269676208496, "learning_rate": 1.9276432015946446e-08, "logits/chosen": -1.8333067893981934, "logits/rejected": -1.3369500637054443, "logps/chosen": -281.30560302734375, "logps/ref_chosen": -94.77332305908203, "logps/ref_rejected": -107.30490112304688, "logps/rejected": -382.1710510253906, "loss": 1.1049, "margin_dpo/margin_mean": 88.33387756347656, "margin_dpo/margin_std": 111.20333862304688, "step": 605 }, { "KL/chosen_KL_mean": -179.97634887695312, "KL/mean": -225.9715576171875, "KL/rejected_KL_mean": -271.966796875, "KL/std": 108.05996704101562, "epoch": 0.8898678414096917, "fcm_dpo/beta": 0.004165642894804478, "fcm_dpo/delta": 0.016849756240844727, "fcm_dpo/margin": 91.99044036865234, "fcm_dpo/q_t": 0.4097779095172882, "grad_norm": 8.532909393310547, "learning_rate": 1.8785323298722093e-08, "logits/chosen": -1.7642053365707397, "logits/rejected": -1.3710261583328247, "logps/chosen": -267.729736328125, "logps/ref_chosen": -87.7533950805664, "logps/ref_rejected": -104.2422103881836, "logps/rejected": -376.208984375, "loss": 1.0967, "margin_dpo/margin_mean": 91.99044036865234, "margin_dpo/margin_std": 115.39258575439453, "step": 606 }, { "KL/chosen_KL_mean": -174.13687133789062, "KL/mean": -226.08477783203125, "KL/rejected_KL_mean": -278.03271484375, "KL/std": 114.94685363769531, "epoch": 0.8913362701908958, "fcm_dpo/beta": 0.004132311325520277, "fcm_dpo/delta": -0.030634162947535515, "fcm_dpo/margin": 103.89582061767578, "fcm_dpo/q_t": 0.4010041356086731, "grad_norm": 11.600451469421387, "learning_rate": 1.8300307303259904e-08, "logits/chosen": -1.6594361066818237, "logits/rejected": -1.1199301481246948, "logps/chosen": -262.4659118652344, "logps/ref_chosen": -88.32904052734375, "logps/ref_rejected": -86.76810455322266, "logps/rejected": -364.80078125, "loss": 1.0832, "margin_dpo/margin_mean": 103.89582061767578, "margin_dpo/margin_std": 141.76866149902344, "step": 607 }, { "KL/chosen_KL_mean": -158.42884826660156, "KL/mean": -212.6431884765625, "KL/rejected_KL_mean": -266.8575439453125, "KL/std": 113.4287109375, "epoch": 0.8928046989720999, "fcm_dpo/beta": 0.004068960435688496, "fcm_dpo/delta": -0.04424865543842316, "fcm_dpo/margin": 108.42870330810547, "fcm_dpo/q_t": 0.39609676599502563, "grad_norm": 8.687485694885254, "learning_rate": 1.7821396810182437e-08, "logits/chosen": -2.0134570598602295, "logits/rejected": -1.7219434976577759, "logps/chosen": -244.19821166992188, "logps/ref_chosen": -85.76937866210938, "logps/ref_rejected": -100.23281860351562, "logps/rejected": -367.0903625488281, "loss": 1.058, "margin_dpo/margin_mean": 108.42870330810547, "margin_dpo/margin_std": 123.67848205566406, "step": 608 }, { "KL/chosen_KL_mean": -177.2882537841797, "KL/mean": -228.3763427734375, "KL/rejected_KL_mean": -279.46441650390625, "KL/std": 114.50660705566406, "epoch": 0.8942731277533039, "fcm_dpo/beta": 0.004059626720845699, "fcm_dpo/delta": -0.015860531479120255, "fcm_dpo/margin": 102.17616271972656, "fcm_dpo/q_t": 0.4036392271518707, "grad_norm": 10.571024894714355, "learning_rate": 1.7348604439226617e-08, "logits/chosen": -2.290658712387085, "logits/rejected": -1.871590495109558, "logps/chosen": -270.25482177734375, "logps/ref_chosen": -92.9665756225586, "logps/ref_rejected": -95.91818237304688, "logps/rejected": -375.382568359375, "loss": 1.0765, "margin_dpo/margin_mean": 102.17616271972656, "margin_dpo/margin_std": 121.29409790039062, "step": 609 }, { "KL/chosen_KL_mean": -162.18838500976562, "KL/mean": -216.93170166015625, "KL/rejected_KL_mean": -271.67498779296875, "KL/std": 108.85682678222656, "epoch": 0.895741556534508, "fcm_dpo/beta": 0.0040379539132118225, "fcm_dpo/delta": -0.04423649236559868, "fcm_dpo/margin": 109.48660278320312, "fcm_dpo/q_t": 0.39528539776802063, "grad_norm": 8.469439506530762, "learning_rate": 1.6881942648911074e-08, "logits/chosen": -1.6258387565612793, "logits/rejected": -1.0699334144592285, "logps/chosen": -256.888671875, "logps/ref_chosen": -94.70028686523438, "logps/ref_rejected": -89.68739318847656, "logps/rejected": -361.3623962402344, "loss": 1.044, "margin_dpo/margin_mean": 109.48660278320312, "margin_dpo/margin_std": 115.1426773071289, "step": 610 }, { "KL/chosen_KL_mean": -171.3548583984375, "KL/mean": -224.23443603515625, "KL/rejected_KL_mean": -277.11407470703125, "KL/std": 112.71803283691406, "epoch": 0.8972099853157122, "fcm_dpo/beta": 0.004038108512759209, "fcm_dpo/delta": -0.02854507975280285, "fcm_dpo/margin": 105.75917053222656, "fcm_dpo/q_t": 0.4003354609012604, "grad_norm": 10.060083389282227, "learning_rate": 1.6421423736208e-08, "logits/chosen": -2.1642470359802246, "logits/rejected": -1.8243929147720337, "logps/chosen": -258.13818359375, "logps/ref_chosen": -86.78334045410156, "logps/ref_rejected": -89.84307861328125, "logps/rejected": -366.9571228027344, "loss": 1.0562, "margin_dpo/margin_mean": 105.75917053222656, "margin_dpo/margin_std": 114.37024688720703, "step": 611 }, { "KL/chosen_KL_mean": -188.76541137695312, "KL/mean": -243.06631469726562, "KL/rejected_KL_mean": -297.36724853515625, "KL/std": 106.55699920654297, "epoch": 0.8986784140969163, "fcm_dpo/beta": 0.003989209420979023, "fcm_dpo/delta": -0.03480519354343414, "fcm_dpo/margin": 108.60176849365234, "fcm_dpo/q_t": 0.39796483516693115, "grad_norm": 9.093473434448242, "learning_rate": 1.5967059836219042e-08, "logits/chosen": -1.9941918849945068, "logits/rejected": -1.1363801956176758, "logps/chosen": -289.78558349609375, "logps/ref_chosen": -101.02015686035156, "logps/ref_rejected": -93.78302764892578, "logps/rejected": -391.1502685546875, "loss": 1.0456, "margin_dpo/margin_mean": 108.60177612304688, "margin_dpo/margin_std": 111.40904998779297, "step": 612 }, { "KL/chosen_KL_mean": -150.7563018798828, "KL/mean": -212.39462280273438, "KL/rejected_KL_mean": -274.032958984375, "KL/std": 109.2419662475586, "epoch": 0.9001468428781204, "fcm_dpo/beta": 0.003927746322005987, "fcm_dpo/delta": -0.08869104087352753, "fcm_dpo/margin": 123.27667236328125, "fcm_dpo/q_t": 0.38514143228530884, "grad_norm": 8.120635986328125, "learning_rate": 1.551886292185553e-08, "logits/chosen": -1.9999380111694336, "logits/rejected": -1.8038638830184937, "logps/chosen": -239.7449188232422, "logps/ref_chosen": -88.98861694335938, "logps/ref_rejected": -109.99551391601562, "logps/rejected": -384.0284729003906, "loss": 1.0008, "margin_dpo/margin_mean": 123.27666473388672, "margin_dpo/margin_std": 106.64390563964844, "step": 613 }, { "KL/chosen_KL_mean": -189.6427001953125, "KL/mean": -239.39923095703125, "KL/rejected_KL_mean": -289.15576171875, "KL/std": 103.69692993164062, "epoch": 0.9016152716593245, "fcm_dpo/beta": 0.003918469417840242, "fcm_dpo/delta": 0.010457588359713554, "fcm_dpo/margin": 99.5130615234375, "fcm_dpo/q_t": 0.40834999084472656, "grad_norm": 10.459037780761719, "learning_rate": 1.507684480352292e-08, "logits/chosen": -1.2895487546920776, "logits/rejected": -1.281791090965271, "logps/chosen": -269.8427734375, "logps/ref_chosen": -80.20005798339844, "logps/ref_rejected": -109.86239624023438, "logps/rejected": -399.01812744140625, "loss": 1.094, "margin_dpo/margin_mean": 99.51304626464844, "margin_dpo/margin_std": 127.11903381347656, "step": 614 }, { "KL/chosen_KL_mean": -177.69140625, "KL/mean": -218.6724853515625, "KL/rejected_KL_mean": -259.65350341796875, "KL/std": 104.69528198242188, "epoch": 0.9030837004405287, "fcm_dpo/beta": 0.003992682322859764, "fcm_dpo/delta": 0.07399415969848633, "fcm_dpo/margin": 81.96208953857422, "fcm_dpo/q_t": 0.4222257435321808, "grad_norm": 8.807668685913086, "learning_rate": 1.4641017128809801e-08, "logits/chosen": -1.9038972854614258, "logits/rejected": -1.5114322900772095, "logps/chosen": -278.1266784667969, "logps/ref_chosen": -100.43525695800781, "logps/ref_rejected": -101.1800537109375, "logps/rejected": -360.83355712890625, "loss": 1.1279, "margin_dpo/margin_mean": 81.96208953857422, "margin_dpo/margin_std": 100.40618133544922, "step": 615 }, { "KL/chosen_KL_mean": -179.99668884277344, "KL/mean": -235.75399780273438, "KL/rejected_KL_mean": -291.51129150390625, "KL/std": 104.38822937011719, "epoch": 0.9045521292217328, "fcm_dpo/beta": 0.003971125464886427, "fcm_dpo/delta": -0.04492996260523796, "fcm_dpo/margin": 111.51458740234375, "fcm_dpo/q_t": 0.39619868993759155, "grad_norm": 8.678057670593262, "learning_rate": 1.4211391382180637e-08, "logits/chosen": -1.9386075735092163, "logits/rejected": -1.4512126445770264, "logps/chosen": -272.4896240234375, "logps/ref_chosen": -92.49292755126953, "logps/ref_rejected": -82.06065368652344, "logps/rejected": -373.57196044921875, "loss": 1.0479, "margin_dpo/margin_mean": 111.51458740234375, "margin_dpo/margin_std": 122.03363037109375, "step": 616 }, { "KL/chosen_KL_mean": -186.71258544921875, "KL/mean": -231.34072875976562, "KL/rejected_KL_mean": -275.9688720703125, "KL/std": 102.0836181640625, "epoch": 0.9060205580029369, "fcm_dpo/beta": 0.0039738137274980545, "fcm_dpo/delta": 0.0469934418797493, "fcm_dpo/margin": 89.25631713867188, "fcm_dpo/q_t": 0.4163208305835724, "grad_norm": 9.150815963745117, "learning_rate": 1.378797888467345e-08, "logits/chosen": -1.5016717910766602, "logits/rejected": -0.8497835397720337, "logps/chosen": -277.8095703125, "logps/ref_chosen": -91.09699249267578, "logps/ref_rejected": -70.41004943847656, "logps/rejected": -346.3789367675781, "loss": 1.1135, "margin_dpo/margin_mean": 89.25631713867188, "margin_dpo/margin_std": 114.75865173339844, "step": 617 }, { "KL/chosen_KL_mean": -189.54953002929688, "KL/mean": -239.1862335205078, "KL/rejected_KL_mean": -288.82293701171875, "KL/std": 112.74347686767578, "epoch": 0.9074889867841409, "fcm_dpo/beta": 0.003982514142990112, "fcm_dpo/delta": 0.004804821684956551, "fcm_dpo/margin": 99.27340698242188, "fcm_dpo/q_t": 0.40650007128715515, "grad_norm": 8.44237995147705, "learning_rate": 1.3370790793601371e-08, "logits/chosen": -2.0582468509674072, "logits/rejected": -1.6970760822296143, "logps/chosen": -291.57012939453125, "logps/ref_chosen": -102.02059936523438, "logps/ref_rejected": -99.80119323730469, "logps/rejected": -388.6241455078125, "loss": 1.0893, "margin_dpo/margin_mean": 99.27340698242188, "margin_dpo/margin_std": 124.66029357910156, "step": 618 }, { "KL/chosen_KL_mean": -183.12069702148438, "KL/mean": -227.24288940429688, "KL/rejected_KL_mean": -271.3651123046875, "KL/std": 111.331787109375, "epoch": 0.908957415565345, "fcm_dpo/beta": 0.004015929065644741, "fcm_dpo/delta": 0.04730312526226044, "fcm_dpo/margin": 88.24444580078125, "fcm_dpo/q_t": 0.4189585745334625, "grad_norm": 8.45223617553711, "learning_rate": 1.2959838102258535e-08, "logits/chosen": -1.4905574321746826, "logits/rejected": -1.123539924621582, "logps/chosen": -272.862060546875, "logps/ref_chosen": -89.74136352539062, "logps/ref_rejected": -99.90138244628906, "logps/rejected": -371.2665100097656, "loss": 1.1304, "margin_dpo/margin_mean": 88.24443817138672, "margin_dpo/margin_std": 133.6453094482422, "step": 619 }, { "KL/chosen_KL_mean": -174.9085693359375, "KL/mean": -223.426025390625, "KL/rejected_KL_mean": -271.9435119628906, "KL/std": 117.03223419189453, "epoch": 0.9104258443465492, "fcm_dpo/beta": 0.003997547551989555, "fcm_dpo/delta": 0.01105613261461258, "fcm_dpo/margin": 97.03494262695312, "fcm_dpo/q_t": 0.4096406102180481, "grad_norm": 9.341211318969727, "learning_rate": 1.2555131639630567e-08, "logits/chosen": -2.0588772296905518, "logits/rejected": -1.5201891660690308, "logps/chosen": -260.0328674316406, "logps/ref_chosen": -85.12431335449219, "logps/ref_rejected": -85.41253662109375, "logps/rejected": -357.3560485839844, "loss": 1.1043, "margin_dpo/margin_mean": 97.03494262695312, "margin_dpo/margin_std": 125.9666748046875, "step": 620 }, { "KL/chosen_KL_mean": -178.04391479492188, "KL/mean": -225.4053192138672, "KL/rejected_KL_mean": -272.7667541503906, "KL/std": 112.88932800292969, "epoch": 0.9118942731277533, "fcm_dpo/beta": 0.004078746773302555, "fcm_dpo/delta": 0.012524455785751343, "fcm_dpo/margin": 94.72285461425781, "fcm_dpo/q_t": 0.41036227345466614, "grad_norm": 9.231380462646484, "learning_rate": 1.2156682070109086e-08, "logits/chosen": -2.026763916015625, "logits/rejected": -1.7662785053253174, "logps/chosen": -267.2923278808594, "logps/ref_chosen": -89.24842071533203, "logps/ref_rejected": -95.46463775634766, "logps/rejected": -368.23138427734375, "loss": 1.1129, "margin_dpo/margin_mean": 94.72285461425781, "margin_dpo/margin_std": 131.51112365722656, "step": 621 }, { "KL/chosen_KL_mean": -175.9618377685547, "KL/mean": -226.94540405273438, "KL/rejected_KL_mean": -277.928955078125, "KL/std": 101.05702209472656, "epoch": 0.9133627019089574, "fcm_dpo/beta": 0.004030452109873295, "fcm_dpo/delta": -0.011552580632269382, "fcm_dpo/margin": 101.96710205078125, "fcm_dpo/q_t": 0.40298715233802795, "grad_norm": 10.233354568481445, "learning_rate": 1.1764499893210878e-08, "logits/chosen": -1.3777191638946533, "logits/rejected": -0.7063345909118652, "logps/chosen": -275.7559814453125, "logps/ref_chosen": -99.79412841796875, "logps/ref_rejected": -90.82821655273438, "logps/rejected": -368.7571716308594, "loss": 1.064, "margin_dpo/margin_mean": 101.96710205078125, "margin_dpo/margin_std": 109.577392578125, "step": 622 }, { "KL/chosen_KL_mean": -184.06317138671875, "KL/mean": -219.6446990966797, "KL/rejected_KL_mean": -255.22625732421875, "KL/std": 108.89497375488281, "epoch": 0.9148311306901615, "fcm_dpo/beta": 0.004092751070857048, "fcm_dpo/delta": 0.11216405779123306, "fcm_dpo/margin": 71.1630859375, "fcm_dpo/q_t": 0.4333891272544861, "grad_norm": 9.860190391540527, "learning_rate": 1.1378595443300998e-08, "logits/chosen": -2.2024056911468506, "logits/rejected": -1.7604541778564453, "logps/chosen": -274.51873779296875, "logps/ref_chosen": -90.45555114746094, "logps/ref_rejected": -91.32276153564453, "logps/rejected": -346.54901123046875, "loss": 1.1888, "margin_dpo/margin_mean": 71.1630859375, "margin_dpo/margin_std": 133.92840576171875, "step": 623 }, { "KL/chosen_KL_mean": -189.0701904296875, "KL/mean": -246.57859802246094, "KL/rejected_KL_mean": -304.0870361328125, "KL/std": 108.13809204101562, "epoch": 0.9162995594713657, "fcm_dpo/beta": 0.004082166124135256, "fcm_dpo/delta": -0.0729648768901825, "fcm_dpo/margin": 115.0168228149414, "fcm_dpo/q_t": 0.39053380489349365, "grad_norm": 9.783458709716797, "learning_rate": 1.0998978889320582e-08, "logits/chosen": -2.462695837020874, "logits/rejected": -1.5314414501190186, "logps/chosen": -298.9454345703125, "logps/ref_chosen": -109.87522888183594, "logps/ref_rejected": -104.77320861816406, "logps/rejected": -408.8602294921875, "loss": 1.0312, "margin_dpo/margin_mean": 115.01681518554688, "margin_dpo/margin_std": 124.2376708984375, "step": 624 }, { "KL/chosen_KL_mean": -196.15560913085938, "KL/mean": -237.61073303222656, "KL/rejected_KL_mean": -279.06585693359375, "KL/std": 107.20172882080078, "epoch": 0.9177679882525698, "fcm_dpo/beta": 0.004113738890737295, "fcm_dpo/delta": 0.06079289689660072, "fcm_dpo/margin": 82.91023254394531, "fcm_dpo/q_t": 0.42043623328208923, "grad_norm": 8.7804594039917, "learning_rate": 1.0625660234518913e-08, "logits/chosen": -1.7285656929016113, "logits/rejected": -1.354683518409729, "logps/chosen": -283.32379150390625, "logps/ref_chosen": -87.16815948486328, "logps/ref_rejected": -91.86148071289062, "logps/rejected": -370.9273376464844, "loss": 1.1267, "margin_dpo/margin_mean": 82.91023254394531, "margin_dpo/margin_std": 114.33120727539062, "step": 625 }, { "KL/chosen_KL_mean": -185.7237548828125, "KL/mean": -227.11334228515625, "KL/rejected_KL_mean": -268.5029296875, "KL/std": 118.6675796508789, "epoch": 0.9192364170337739, "fcm_dpo/beta": 0.004162913653999567, "fcm_dpo/delta": 0.05712512508034706, "fcm_dpo/margin": 82.77915954589844, "fcm_dpo/q_t": 0.42091184854507446, "grad_norm": 11.050243377685547, "learning_rate": 1.0258649316189721e-08, "logits/chosen": -1.161201000213623, "logits/rejected": -0.7074933052062988, "logps/chosen": -289.9479675292969, "logps/ref_chosen": -104.22421264648438, "logps/ref_rejected": -104.1774673461914, "logps/rejected": -372.6803894042969, "loss": 1.1407, "margin_dpo/margin_mean": 82.77915954589844, "margin_dpo/margin_std": 130.26788330078125, "step": 626 }, { "KL/chosen_KL_mean": -164.6634063720703, "KL/mean": -216.6180877685547, "KL/rejected_KL_mean": -268.57281494140625, "KL/std": 121.40113830566406, "epoch": 0.920704845814978, "fcm_dpo/beta": 0.004145722836256027, "fcm_dpo/delta": -0.03216177970170975, "fcm_dpo/margin": 103.90938568115234, "fcm_dpo/q_t": 0.40146297216415405, "grad_norm": 7.568896770477295, "learning_rate": 9.897955805412e-09, "logits/chosen": -1.570462942123413, "logits/rejected": -1.7182931900024414, "logps/chosen": -239.5980224609375, "logps/ref_chosen": -74.93461608886719, "logps/ref_rejected": -112.57289123535156, "logps/rejected": -381.14569091796875, "loss": 1.0723, "margin_dpo/margin_mean": 103.90938568115234, "margin_dpo/margin_std": 134.21583557128906, "step": 627 }, { "KL/chosen_KL_mean": -174.17642211914062, "KL/mean": -228.18255615234375, "KL/rejected_KL_mean": -282.188720703125, "KL/std": 110.02069091796875, "epoch": 0.922173274596182, "fcm_dpo/beta": 0.0041107600554823875, "fcm_dpo/delta": -0.046051569283008575, "fcm_dpo/margin": 108.01229858398438, "fcm_dpo/q_t": 0.39653655886650085, "grad_norm": 10.314324378967285, "learning_rate": 9.543589206795238e-09, "logits/chosen": -2.147139310836792, "logits/rejected": -1.8886983394622803, "logps/chosen": -267.86749267578125, "logps/ref_chosen": -93.69107818603516, "logps/ref_rejected": -107.34395599365234, "logps/rejected": -389.53265380859375, "loss": 1.0508, "margin_dpo/margin_mean": 108.01229858398438, "margin_dpo/margin_std": 122.15483093261719, "step": 628 }, { "KL/chosen_KL_mean": -187.43673706054688, "KL/mean": -236.20724487304688, "KL/rejected_KL_mean": -284.9776916503906, "KL/std": 110.47042846679688, "epoch": 0.9236417033773862, "fcm_dpo/beta": 0.004088602960109711, "fcm_dpo/delta": 0.0010542776435613632, "fcm_dpo/margin": 97.54096221923828, "fcm_dpo/q_t": 0.4059574007987976, "grad_norm": 9.603934288024902, "learning_rate": 9.19555885822887e-09, "logits/chosen": -1.9504039287567139, "logits/rejected": -1.507066249847412, "logps/chosen": -290.6671142578125, "logps/ref_chosen": -103.23037719726562, "logps/ref_rejected": -97.16841888427734, "logps/rejected": -382.1461181640625, "loss": 1.0837, "margin_dpo/margin_mean": 97.54096221923828, "margin_dpo/margin_std": 118.29389953613281, "step": 629 }, { "KL/chosen_KL_mean": -170.95223999023438, "KL/mean": -202.36854553222656, "KL/rejected_KL_mean": -233.7848358154297, "KL/std": 102.48808288574219, "epoch": 0.9251101321585903, "fcm_dpo/beta": 0.004201427102088928, "fcm_dpo/delta": 0.13901160657405853, "fcm_dpo/margin": 62.83258056640625, "fcm_dpo/q_t": 0.4382888078689575, "grad_norm": 12.086967468261719, "learning_rate": 8.85387393063622e-09, "logits/chosen": -1.9129951000213623, "logits/rejected": -1.376857042312622, "logps/chosen": -264.84979248046875, "logps/ref_chosen": -93.89755249023438, "logps/ref_rejected": -89.3743896484375, "logps/rejected": -323.15924072265625, "loss": 1.1979, "margin_dpo/margin_mean": 62.832584381103516, "margin_dpo/margin_std": 115.70326232910156, "step": 630 }, { "KL/chosen_KL_mean": -168.54290771484375, "KL/mean": -215.57589721679688, "KL/rejected_KL_mean": -262.60888671875, "KL/std": 101.86968994140625, "epoch": 0.9265785609397944, "fcm_dpo/beta": 0.004215255379676819, "fcm_dpo/delta": 0.003616894129663706, "fcm_dpo/margin": 94.06595611572266, "fcm_dpo/q_t": 0.40688514709472656, "grad_norm": 8.507015228271484, "learning_rate": 8.518543427732949e-09, "logits/chosen": -2.5255701541900635, "logits/rejected": -2.100771427154541, "logps/chosen": -256.313720703125, "logps/ref_chosen": -87.77082061767578, "logps/ref_rejected": -88.68241882324219, "logps/rejected": -351.29132080078125, "loss": 1.0865, "margin_dpo/margin_mean": 94.06596374511719, "margin_dpo/margin_std": 116.93255615234375, "step": 631 }, { "KL/chosen_KL_mean": -164.38775634765625, "KL/mean": -216.46266174316406, "KL/rejected_KL_mean": -268.53759765625, "KL/std": 107.63964080810547, "epoch": 0.9280469897209985, "fcm_dpo/beta": 0.004176911432296038, "fcm_dpo/delta": -0.03725190460681915, "fcm_dpo/margin": 104.14982604980469, "fcm_dpo/q_t": 0.39939671754837036, "grad_norm": 11.150821685791016, "learning_rate": 8.189576185789637e-09, "logits/chosen": -2.108255624771118, "logits/rejected": -1.6864230632781982, "logps/chosen": -253.0142822265625, "logps/ref_chosen": -88.62652587890625, "logps/ref_rejected": -91.45091247558594, "logps/rejected": -359.9884948730469, "loss": 1.072, "margin_dpo/margin_mean": 104.14982604980469, "margin_dpo/margin_std": 132.97486877441406, "step": 632 }, { "KL/chosen_KL_mean": -205.92520141601562, "KL/mean": -236.29946899414062, "KL/rejected_KL_mean": -266.6737365722656, "KL/std": 108.80809020996094, "epoch": 0.9295154185022027, "fcm_dpo/beta": 0.004300587810575962, "fcm_dpo/delta": 0.1411598026752472, "fcm_dpo/margin": 60.74851608276367, "fcm_dpo/q_t": 0.44169795513153076, "grad_norm": 11.446995735168457, "learning_rate": 7.866980873399015e-09, "logits/chosen": -2.398768901824951, "logits/rejected": -2.2140583992004395, "logps/chosen": -287.29962158203125, "logps/ref_chosen": -81.37442016601562, "logps/ref_rejected": -98.62571716308594, "logps/rejected": -365.2994384765625, "loss": 1.2368, "margin_dpo/margin_mean": 60.74851989746094, "margin_dpo/margin_std": 146.28341674804688, "step": 633 }, { "KL/chosen_KL_mean": -189.24517822265625, "KL/mean": -229.2075653076172, "KL/rejected_KL_mean": -269.16998291015625, "KL/std": 98.8389892578125, "epoch": 0.9309838472834068, "fcm_dpo/beta": 0.00437537394464016, "fcm_dpo/delta": 0.050480917096138, "fcm_dpo/margin": 79.9247817993164, "fcm_dpo/q_t": 0.41794538497924805, "grad_norm": 10.96252727508545, "learning_rate": 7.550765991247654e-09, "logits/chosen": -1.38177490234375, "logits/rejected": -1.2072207927703857, "logps/chosen": -285.3680419921875, "logps/ref_chosen": -96.12284851074219, "logps/ref_rejected": -112.84780883789062, "logps/rejected": -382.01776123046875, "loss": 1.1186, "margin_dpo/margin_mean": 79.92477416992188, "margin_dpo/margin_std": 100.46638488769531, "step": 634 }, { "KL/chosen_KL_mean": -198.55641174316406, "KL/mean": -229.16107177734375, "KL/rejected_KL_mean": -259.7657775878906, "KL/std": 107.78771209716797, "epoch": 0.9324522760646109, "fcm_dpo/beta": 0.00445893220603466, "fcm_dpo/delta": 0.12953412532806396, "fcm_dpo/margin": 61.2093505859375, "fcm_dpo/q_t": 0.43787121772766113, "grad_norm": 13.274846076965332, "learning_rate": 7.240939871891699e-09, "logits/chosen": -1.7207732200622559, "logits/rejected": -1.1769323348999023, "logps/chosen": -297.24053955078125, "logps/ref_chosen": -98.68411254882812, "logps/ref_rejected": -89.89920043945312, "logps/rejected": -349.66497802734375, "loss": 1.212, "margin_dpo/margin_mean": 61.2093505859375, "margin_dpo/margin_std": 129.17730712890625, "step": 635 }, { "KL/chosen_KL_mean": -177.15414428710938, "KL/mean": -229.08460998535156, "KL/rejected_KL_mean": -281.0150451660156, "KL/std": 110.58282470703125, "epoch": 0.933920704845815, "fcm_dpo/beta": 0.004430863540619612, "fcm_dpo/delta": -0.06304943561553955, "fcm_dpo/margin": 103.86089324951172, "fcm_dpo/q_t": 0.3928070366382599, "grad_norm": 9.716262817382812, "learning_rate": 6.937510679537628e-09, "logits/chosen": -1.4924399852752686, "logits/rejected": -1.0093767642974854, "logps/chosen": -267.5721130371094, "logps/ref_chosen": -90.41796112060547, "logps/ref_rejected": -87.70687866210938, "logps/rejected": -368.721923828125, "loss": 1.035, "margin_dpo/margin_mean": 103.86089324951172, "margin_dpo/margin_std": 110.84523010253906, "step": 636 }, { "KL/chosen_KL_mean": -176.53689575195312, "KL/mean": -232.363037109375, "KL/rejected_KL_mean": -288.1891784667969, "KL/std": 122.73499298095703, "epoch": 0.9353891336270191, "fcm_dpo/beta": 0.004357256926596165, "fcm_dpo/delta": -0.09083692729473114, "fcm_dpo/margin": 111.65229034423828, "fcm_dpo/q_t": 0.38627082109451294, "grad_norm": 7.959272384643555, "learning_rate": 6.640486409826785e-09, "logits/chosen": -2.105187177658081, "logits/rejected": -2.003416061401367, "logps/chosen": -258.98663330078125, "logps/ref_chosen": -82.44971466064453, "logps/ref_rejected": -104.02860260009766, "logps/rejected": -392.2177734375, "loss": 1.0192, "margin_dpo/margin_mean": 111.65229797363281, "margin_dpo/margin_std": 116.10621643066406, "step": 637 }, { "KL/chosen_KL_mean": -166.69107055664062, "KL/mean": -220.6617431640625, "KL/rejected_KL_mean": -274.6324462890625, "KL/std": 108.78456115722656, "epoch": 0.9368575624082232, "fcm_dpo/beta": 0.004309596493840218, "fcm_dpo/delta": -0.06855485588312149, "fcm_dpo/margin": 107.94137573242188, "fcm_dpo/q_t": 0.3903169631958008, "grad_norm": 8.880730628967285, "learning_rate": 6.349874889624962e-09, "logits/chosen": -1.541294813156128, "logits/rejected": -0.8539109826087952, "logps/chosen": -258.6160583496094, "logps/ref_chosen": -91.92498779296875, "logps/ref_rejected": -86.28703308105469, "logps/rejected": -360.91949462890625, "loss": 1.0224, "margin_dpo/margin_mean": 107.94137573242188, "margin_dpo/margin_std": 100.90206146240234, "step": 638 }, { "KL/chosen_KL_mean": -220.72616577148438, "KL/mean": -246.5460662841797, "KL/rejected_KL_mean": -272.36590576171875, "KL/std": 101.26468658447266, "epoch": 0.9383259911894273, "fcm_dpo/beta": 0.00440070778131485, "fcm_dpo/delta": 0.17596019804477692, "fcm_dpo/margin": 51.639747619628906, "fcm_dpo/q_t": 0.4471595287322998, "grad_norm": 9.977910995483398, "learning_rate": 6.065683776815933e-09, "logits/chosen": -1.4857425689697266, "logits/rejected": -0.7344560027122498, "logps/chosen": -325.25372314453125, "logps/ref_chosen": -104.52755737304688, "logps/ref_rejected": -81.4803466796875, "logps/rejected": -353.84625244140625, "loss": 1.2465, "margin_dpo/margin_mean": 51.639739990234375, "margin_dpo/margin_std": 126.24737548828125, "step": 639 }, { "KL/chosen_KL_mean": -187.2372589111328, "KL/mean": -236.16000366210938, "KL/rejected_KL_mean": -285.0827331542969, "KL/std": 109.60596466064453, "epoch": 0.9397944199706314, "fcm_dpo/beta": 0.004440665245056152, "fcm_dpo/delta": -0.037377938628196716, "fcm_dpo/margin": 97.84546661376953, "fcm_dpo/q_t": 0.3983241319656372, "grad_norm": 12.679488182067871, "learning_rate": 5.7879205600998296e-09, "logits/chosen": -1.5973081588745117, "logits/rejected": -1.0602827072143555, "logps/chosen": -285.1225280761719, "logps/ref_chosen": -97.88526916503906, "logps/ref_rejected": -112.70502471923828, "logps/rejected": -397.7877502441406, "loss": 1.0635, "margin_dpo/margin_mean": 97.84547424316406, "margin_dpo/margin_std": 109.55726623535156, "step": 640 }, { "KL/chosen_KL_mean": -171.5295867919922, "KL/mean": -222.37042236328125, "KL/rejected_KL_mean": -273.2112731933594, "KL/std": 127.00564575195312, "epoch": 0.9412628487518355, "fcm_dpo/beta": 0.004365661181509495, "fcm_dpo/delta": -0.04592466354370117, "fcm_dpo/margin": 101.68169403076172, "fcm_dpo/q_t": 0.4008108377456665, "grad_norm": 8.9500150680542, "learning_rate": 5.516592558795746e-09, "logits/chosen": -2.2062900066375732, "logits/rejected": -1.6539539098739624, "logps/chosen": -267.97528076171875, "logps/ref_chosen": -96.4456787109375, "logps/ref_rejected": -95.13568878173828, "logps/rejected": -368.34698486328125, "loss": 1.0741, "margin_dpo/margin_mean": 101.68169403076172, "margin_dpo/margin_std": 140.5269775390625, "step": 641 }, { "KL/chosen_KL_mean": -191.478759765625, "KL/mean": -239.13446044921875, "KL/rejected_KL_mean": -286.7901611328125, "KL/std": 109.11629486083984, "epoch": 0.9427312775330396, "fcm_dpo/beta": 0.004353306721895933, "fcm_dpo/delta": -0.01566200703382492, "fcm_dpo/margin": 95.31141662597656, "fcm_dpo/q_t": 0.40626126527786255, "grad_norm": 11.626405715942383, "learning_rate": 5.251706922648868e-09, "logits/chosen": -1.35894775390625, "logits/rejected": -1.0146660804748535, "logps/chosen": -292.2386169433594, "logps/ref_chosen": -100.75984954833984, "logps/ref_rejected": -114.7076416015625, "logps/rejected": -401.497802734375, "loss": 1.0934, "margin_dpo/margin_mean": 95.31141662597656, "margin_dpo/margin_std": 135.86036682128906, "step": 642 }, { "KL/chosen_KL_mean": -179.059326171875, "KL/mean": -224.10733032226562, "KL/rejected_KL_mean": -269.1552734375, "KL/std": 101.28324890136719, "epoch": 0.9441997063142438, "fcm_dpo/beta": 0.004315345548093319, "fcm_dpo/delta": 0.01073531061410904, "fcm_dpo/margin": 90.09595489501953, "fcm_dpo/q_t": 0.40898841619491577, "grad_norm": 11.943429946899414, "learning_rate": 4.993270631642038e-09, "logits/chosen": -2.0791831016540527, "logits/rejected": -1.696735143661499, "logps/chosen": -263.802978515625, "logps/ref_chosen": -84.74365997314453, "logps/ref_rejected": -94.31842041015625, "logps/rejected": -363.47369384765625, "loss": 1.1016, "margin_dpo/margin_mean": 90.09595489501953, "margin_dpo/margin_std": 117.8892822265625, "step": 643 }, { "KL/chosen_KL_mean": -184.73651123046875, "KL/mean": -222.509765625, "KL/rejected_KL_mean": -260.28302001953125, "KL/std": 102.1156005859375, "epoch": 0.9456681350954479, "fcm_dpo/beta": 0.004393700044602156, "fcm_dpo/delta": 0.07041022926568985, "fcm_dpo/margin": 75.54650115966797, "fcm_dpo/q_t": 0.4225154519081116, "grad_norm": 11.27793025970459, "learning_rate": 4.741290495811873e-09, "logits/chosen": -1.8944928646087646, "logits/rejected": -1.6283773183822632, "logps/chosen": -270.05926513671875, "logps/ref_chosen": -85.32274627685547, "logps/ref_rejected": -94.60861206054688, "logps/rejected": -354.8916320800781, "loss": 1.1375, "margin_dpo/margin_mean": 75.54650115966797, "margin_dpo/margin_std": 110.111328125, "step": 644 }, { "KL/chosen_KL_mean": -196.74502563476562, "KL/mean": -221.00958251953125, "KL/rejected_KL_mean": -245.2741241455078, "KL/std": 111.3653564453125, "epoch": 0.947136563876652, "fcm_dpo/beta": 0.004542603623121977, "fcm_dpo/delta": 0.18305739760398865, "fcm_dpo/margin": 48.529090881347656, "fcm_dpo/q_t": 0.44977158308029175, "grad_norm": 11.038433074951172, "learning_rate": 4.495773155069299e-09, "logits/chosen": -1.6790955066680908, "logits/rejected": -1.515622854232788, "logps/chosen": -279.33526611328125, "logps/ref_chosen": -82.59024047851562, "logps/ref_rejected": -103.00375366210938, "logps/rejected": -348.27789306640625, "loss": 1.2576, "margin_dpo/margin_mean": 48.529090881347656, "margin_dpo/margin_std": 126.15511322021484, "step": 645 }, { "KL/chosen_KL_mean": -173.53436279296875, "KL/mean": -221.92477416992188, "KL/rejected_KL_mean": -270.3151550292969, "KL/std": 97.35064697265625, "epoch": 0.9486049926578561, "fcm_dpo/beta": 0.004526478238403797, "fcm_dpo/delta": -0.04040028154850006, "fcm_dpo/margin": 96.78079223632812, "fcm_dpo/q_t": 0.39727216958999634, "grad_norm": 9.560938835144043, "learning_rate": 4.256725079024553e-09, "logits/chosen": -2.005054473876953, "logits/rejected": -1.4161031246185303, "logps/chosen": -264.7264099121094, "logps/ref_chosen": -91.1920394897461, "logps/ref_rejected": -83.77833557128906, "logps/rejected": -354.093505859375, "loss": 1.0513, "margin_dpo/margin_mean": 96.78079223632812, "margin_dpo/margin_std": 105.70700073242188, "step": 646 }, { "KL/chosen_KL_mean": -188.26365661621094, "KL/mean": -230.80426025390625, "KL/rejected_KL_mean": -273.3448486328125, "KL/std": 116.93202209472656, "epoch": 0.9500734214390602, "fcm_dpo/beta": 0.004546056967228651, "fcm_dpo/delta": 0.013748741708695889, "fcm_dpo/margin": 85.08122253417969, "fcm_dpo/q_t": 0.41216498613357544, "grad_norm": 10.63406753540039, "learning_rate": 4.024152566816791e-09, "logits/chosen": -1.4981589317321777, "logits/rejected": -1.3597617149353027, "logps/chosen": -277.10809326171875, "logps/ref_chosen": -88.8444595336914, "logps/ref_rejected": -99.49833679199219, "logps/rejected": -372.84320068359375, "loss": 1.1185, "margin_dpo/margin_mean": 85.08122253417969, "margin_dpo/margin_std": 130.28533935546875, "step": 647 }, { "KL/chosen_KL_mean": -175.35934448242188, "KL/mean": -234.02639770507812, "KL/rejected_KL_mean": -292.69342041015625, "KL/std": 119.82608032226562, "epoch": 0.9515418502202643, "fcm_dpo/beta": 0.00447250297293067, "fcm_dpo/delta": -0.13149042427539825, "fcm_dpo/margin": 117.33409118652344, "fcm_dpo/q_t": 0.37865111231803894, "grad_norm": 10.731439590454102, "learning_rate": 3.798061746947995e-09, "logits/chosen": -1.715616226196289, "logits/rejected": -1.6419222354888916, "logps/chosen": -263.20745849609375, "logps/ref_chosen": -87.84810638427734, "logps/ref_rejected": -104.67005920410156, "logps/rejected": -397.3634948730469, "loss": 1.0044, "margin_dpo/margin_mean": 117.33409118652344, "margin_dpo/margin_std": 127.09063720703125, "step": 648 }, { "KL/chosen_KL_mean": -161.1766815185547, "KL/mean": -210.62973022460938, "KL/rejected_KL_mean": -260.0827941894531, "KL/std": 105.01527404785156, "epoch": 0.9530102790014684, "fcm_dpo/beta": 0.004386160057038069, "fcm_dpo/delta": -0.03602888435125351, "fcm_dpo/margin": 98.9061050415039, "fcm_dpo/q_t": 0.3980611562728882, "grad_norm": 11.376351356506348, "learning_rate": 3.5784585771215235e-09, "logits/chosen": -2.277283191680908, "logits/rejected": -1.8637611865997314, "logps/chosen": -250.8691864013672, "logps/ref_chosen": -89.6925048828125, "logps/ref_rejected": -88.70658111572266, "logps/rejected": -348.78936767578125, "loss": 1.049, "margin_dpo/margin_mean": 98.90609741210938, "margin_dpo/margin_std": 102.10493469238281, "step": 649 }, { "KL/chosen_KL_mean": -190.6533660888672, "KL/mean": -236.83970642089844, "KL/rejected_KL_mean": -283.02606201171875, "KL/std": 113.3672866821289, "epoch": 0.9544787077826725, "fcm_dpo/beta": 0.004398618824779987, "fcm_dpo/delta": -0.006585496477782726, "fcm_dpo/margin": 92.37268829345703, "fcm_dpo/q_t": 0.4064382314682007, "grad_norm": 10.460981369018555, "learning_rate": 3.3653488440851253e-09, "logits/chosen": -1.5607702732086182, "logits/rejected": -1.3692331314086914, "logps/chosen": -280.583984375, "logps/ref_chosen": -89.93060302734375, "logps/ref_rejected": -102.61282348632812, "logps/rejected": -385.6388854980469, "loss": 1.1015, "margin_dpo/margin_mean": 92.37268829345703, "margin_dpo/margin_std": 134.46815490722656, "step": 650 }, { "KL/chosen_KL_mean": -158.3472442626953, "KL/mean": -215.02801513671875, "KL/rejected_KL_mean": -271.70880126953125, "KL/std": 109.23676300048828, "epoch": 0.9559471365638766, "fcm_dpo/beta": 0.004328923299908638, "fcm_dpo/delta": -0.09549497067928314, "fcm_dpo/margin": 113.36157989501953, "fcm_dpo/q_t": 0.3841865658760071, "grad_norm": 9.252904891967773, "learning_rate": 3.158738163478475e-09, "logits/chosen": -2.155019760131836, "logits/rejected": -1.9802674055099487, "logps/chosen": -237.53456115722656, "logps/ref_chosen": -79.18731689453125, "logps/ref_rejected": -105.93333435058594, "logps/rejected": -377.64215087890625, "loss": 1.0066, "margin_dpo/margin_mean": 113.36157989501953, "margin_dpo/margin_std": 107.9755630493164, "step": 651 }, { "KL/chosen_KL_mean": -165.15280151367188, "KL/mean": -212.763916015625, "KL/rejected_KL_mean": -260.37506103515625, "KL/std": 101.59744262695312, "epoch": 0.9574155653450808, "fcm_dpo/beta": 0.004313184879720211, "fcm_dpo/delta": -0.011245248839259148, "fcm_dpo/margin": 95.22225189208984, "fcm_dpo/q_t": 0.4031199812889099, "grad_norm": 9.584260940551758, "learning_rate": 2.9586319796851555e-09, "logits/chosen": -2.4843358993530273, "logits/rejected": -2.1884689331054688, "logps/chosen": -266.9430236816406, "logps/ref_chosen": -101.79022216796875, "logps/ref_rejected": -116.3245849609375, "logps/rejected": -376.69964599609375, "loss": 1.0586, "margin_dpo/margin_mean": 95.22225189208984, "margin_dpo/margin_std": 96.86058044433594, "step": 652 }, { "KL/chosen_KL_mean": -189.51956176757812, "KL/mean": -235.5393524169922, "KL/rejected_KL_mean": -281.55914306640625, "KL/std": 111.22758483886719, "epoch": 0.9588839941262849, "fcm_dpo/beta": 0.004309739917516708, "fcm_dpo/delta": 0.003421064466238022, "fcm_dpo/margin": 92.03958129882812, "fcm_dpo/q_t": 0.4057605564594269, "grad_norm": 9.846468925476074, "learning_rate": 2.7650355656892166e-09, "logits/chosen": -2.1832971572875977, "logits/rejected": -1.9320926666259766, "logps/chosen": -282.8731689453125, "logps/ref_chosen": -93.35359191894531, "logps/ref_rejected": -109.12324523925781, "logps/rejected": -390.682373046875, "loss": 1.0823, "margin_dpo/margin_mean": 92.03958129882812, "margin_dpo/margin_std": 109.22705078125, "step": 653 }, { "KL/chosen_KL_mean": -189.8115234375, "KL/mean": -229.39462280273438, "KL/rejected_KL_mean": -268.9777526855469, "KL/std": 106.67190551757812, "epoch": 0.960352422907489, "fcm_dpo/beta": 0.0043411231599748135, "fcm_dpo/delta": 0.058342210948467255, "fcm_dpo/margin": 79.16622924804688, "fcm_dpo/q_t": 0.4225510358810425, "grad_norm": 12.769107818603516, "learning_rate": 2.577954022936174e-09, "logits/chosen": -2.406515598297119, "logits/rejected": -2.229417324066162, "logps/chosen": -278.92706298828125, "logps/ref_chosen": -89.11553955078125, "logps/ref_rejected": -104.91995239257812, "logps/rejected": -373.897705078125, "loss": 1.1518, "margin_dpo/margin_mean": 79.16622924804688, "margin_dpo/margin_std": 135.0435791015625, "step": 654 }, { "KL/chosen_KL_mean": -172.78921508789062, "KL/mean": -220.78318786621094, "KL/rejected_KL_mean": -268.77716064453125, "KL/std": 116.16748809814453, "epoch": 0.9618208516886931, "fcm_dpo/beta": 0.0043481276370584965, "fcm_dpo/delta": -0.01813214272260666, "fcm_dpo/margin": 95.9879379272461, "fcm_dpo/q_t": 0.4026939868927002, "grad_norm": 8.977787971496582, "learning_rate": 2.397392281198729e-09, "logits/chosen": -2.124497890472412, "logits/rejected": -2.1254940032958984, "logps/chosen": -253.8253173828125, "logps/ref_chosen": -81.03610229492188, "logps/ref_rejected": -102.80233764648438, "logps/rejected": -371.5794677734375, "loss": 1.0825, "margin_dpo/margin_mean": 95.98794555664062, "margin_dpo/margin_std": 124.80957794189453, "step": 655 }, { "KL/chosen_KL_mean": -174.1330108642578, "KL/mean": -235.03611755371094, "KL/rejected_KL_mean": -295.9392395019531, "KL/std": 116.13006591796875, "epoch": 0.9632892804698973, "fcm_dpo/beta": 0.004253302700817585, "fcm_dpo/delta": -0.12474177777767181, "fcm_dpo/margin": 121.80622100830078, "fcm_dpo/q_t": 0.37971699237823486, "grad_norm": 9.894128799438477, "learning_rate": 2.223355098446622e-09, "logits/chosen": -1.398581862449646, "logits/rejected": -1.4688255786895752, "logps/chosen": -259.4583740234375, "logps/ref_chosen": -85.32534790039062, "logps/ref_rejected": -118.33866119384766, "logps/rejected": -414.27789306640625, "loss": 0.9927, "margin_dpo/margin_mean": 121.80622863769531, "margin_dpo/margin_std": 119.20272827148438, "step": 656 }, { "KL/chosen_KL_mean": -190.07293701171875, "KL/mean": -234.95394897460938, "KL/rejected_KL_mean": -279.8349609375, "KL/std": 107.40159606933594, "epoch": 0.9647577092511013, "fcm_dpo/beta": 0.004225913435220718, "fcm_dpo/delta": 0.021155185997486115, "fcm_dpo/margin": 89.76201629638672, "fcm_dpo/q_t": 0.4129956364631653, "grad_norm": 11.025970458984375, "learning_rate": 2.055847060721566e-09, "logits/chosen": -2.004633903503418, "logits/rejected": -1.8398029804229736, "logps/chosen": -270.2706604003906, "logps/ref_chosen": -80.19772338867188, "logps/ref_rejected": -102.581298828125, "logps/rejected": -382.416259765625, "loss": 1.1156, "margin_dpo/margin_mean": 89.76201629638672, "margin_dpo/margin_std": 129.81704711914062, "step": 657 }, { "KL/chosen_KL_mean": -187.68370056152344, "KL/mean": -229.50814819335938, "KL/rejected_KL_mean": -271.33258056640625, "KL/std": 101.30854034423828, "epoch": 0.9662261380323054, "fcm_dpo/beta": 0.004289310425519943, "fcm_dpo/delta": 0.04244206100702286, "fcm_dpo/margin": 83.64889526367188, "fcm_dpo/q_t": 0.41597241163253784, "grad_norm": 9.153238296508789, "learning_rate": 1.8948725820160662e-09, "logits/chosen": -2.1516153812408447, "logits/rejected": -1.692945957183838, "logps/chosen": -282.3182373046875, "logps/ref_chosen": -94.634521484375, "logps/ref_rejected": -101.6316146850586, "logps/rejected": -372.9642028808594, "loss": 1.1124, "margin_dpo/margin_mean": 83.64889526367188, "margin_dpo/margin_std": 108.68551635742188, "step": 658 }, { "KL/chosen_KL_mean": -170.25009155273438, "KL/mean": -220.640625, "KL/rejected_KL_mean": -271.0311279296875, "KL/std": 109.05287170410156, "epoch": 0.9676945668135095, "fcm_dpo/beta": 0.004249989986419678, "fcm_dpo/delta": -0.030032040551304817, "fcm_dpo/margin": 100.78103637695312, "fcm_dpo/q_t": 0.40189552307128906, "grad_norm": 8.81241226196289, "learning_rate": 1.7404359041573723e-09, "logits/chosen": -1.9971462488174438, "logits/rejected": -1.009421467781067, "logps/chosen": -282.80596923828125, "logps/ref_chosen": -112.55587005615234, "logps/ref_rejected": -93.9216079711914, "logps/rejected": -364.9527587890625, "loss": 1.092, "margin_dpo/margin_mean": 100.78104400634766, "margin_dpo/margin_std": 141.86572265625, "step": 659 }, { "KL/chosen_KL_mean": -178.76365661621094, "KL/mean": -230.75421142578125, "KL/rejected_KL_mean": -282.74481201171875, "KL/std": 106.23770904541016, "epoch": 0.9691629955947136, "fcm_dpo/beta": 0.004233683459460735, "fcm_dpo/delta": -0.0420772060751915, "fcm_dpo/margin": 103.98114013671875, "fcm_dpo/q_t": 0.39728349447250366, "grad_norm": 11.592752456665039, "learning_rate": 1.592541096695571e-09, "logits/chosen": -1.8468358516693115, "logits/rejected": -1.2184410095214844, "logps/chosen": -272.1410827636719, "logps/ref_chosen": -93.37742614746094, "logps/ref_rejected": -81.39482116699219, "logps/rejected": -364.1396179199219, "loss": 1.0508, "margin_dpo/margin_mean": 103.98114013671875, "margin_dpo/margin_std": 116.861083984375, "step": 660 }, { "KL/chosen_KL_mean": -176.52395629882812, "KL/mean": -234.44757080078125, "KL/rejected_KL_mean": -292.3711853027344, "KL/std": 118.56643676757812, "epoch": 0.9706314243759178, "fcm_dpo/beta": 0.004177877213805914, "fcm_dpo/delta": -0.08817372471094131, "fcm_dpo/margin": 115.84723663330078, "fcm_dpo/q_t": 0.38853415846824646, "grad_norm": 9.423646926879883, "learning_rate": 1.4511920567963908e-09, "logits/chosen": -2.059849977493286, "logits/rejected": -1.5649185180664062, "logps/chosen": -264.37908935546875, "logps/ref_chosen": -87.85516357421875, "logps/ref_rejected": -92.40330505371094, "logps/rejected": -384.77447509765625, "loss": 1.032, "margin_dpo/margin_mean": 115.84723663330078, "margin_dpo/margin_std": 132.23533630371094, "step": 661 }, { "KL/chosen_KL_mean": -172.51837158203125, "KL/mean": -225.8707275390625, "KL/rejected_KL_mean": -279.22308349609375, "KL/std": 103.93362426757812, "epoch": 0.9720998531571219, "fcm_dpo/beta": 0.004131439607590437, "fcm_dpo/delta": -0.04271296411752701, "fcm_dpo/margin": 106.7047119140625, "fcm_dpo/q_t": 0.3964042067527771, "grad_norm": 10.14755916595459, "learning_rate": 1.3163925091384532e-09, "logits/chosen": -1.5226922035217285, "logits/rejected": -0.931685209274292, "logps/chosen": -275.29815673828125, "logps/ref_chosen": -102.77980041503906, "logps/ref_rejected": -95.22531127929688, "logps/rejected": -374.44842529296875, "loss": 1.0433, "margin_dpo/margin_mean": 106.7047119140625, "margin_dpo/margin_std": 110.0166015625, "step": 662 }, { "KL/chosen_KL_mean": -174.3593292236328, "KL/mean": -222.62863159179688, "KL/rejected_KL_mean": -270.89794921875, "KL/std": 107.32845306396484, "epoch": 0.973568281938326, "fcm_dpo/beta": 0.004124980419874191, "fcm_dpo/delta": 0.0017675179988145828, "fcm_dpo/margin": 96.53861999511719, "fcm_dpo/q_t": 0.4073456823825836, "grad_norm": 10.843335151672363, "learning_rate": 1.1881460058152382e-09, "logits/chosen": -2.0939552783966064, "logits/rejected": -1.8712527751922607, "logps/chosen": -270.7059326171875, "logps/ref_chosen": -96.34658813476562, "logps/ref_rejected": -120.52645874023438, "logps/rejected": -391.4244079589844, "loss": 1.0906, "margin_dpo/margin_mean": 96.53861999511719, "margin_dpo/margin_std": 124.6082763671875, "step": 663 }, { "KL/chosen_KL_mean": -193.64511108398438, "KL/mean": -237.367919921875, "KL/rejected_KL_mean": -281.0906982421875, "KL/std": 110.62553405761719, "epoch": 0.9750367107195301, "fcm_dpo/beta": 0.004149041138589382, "fcm_dpo/delta": 0.038424424827098846, "fcm_dpo/margin": 87.44561767578125, "fcm_dpo/q_t": 0.4147990942001343, "grad_norm": 8.507647514343262, "learning_rate": 1.066455926241383e-09, "logits/chosen": -1.7284271717071533, "logits/rejected": -1.556849479675293, "logps/chosen": -285.487548828125, "logps/ref_chosen": -91.84242248535156, "logps/ref_rejected": -111.8366928100586, "logps/rejected": -392.92742919921875, "loss": 1.1068, "margin_dpo/margin_mean": 87.44561004638672, "margin_dpo/margin_std": 109.2135009765625, "step": 664 }, { "KL/chosen_KL_mean": -165.79922485351562, "KL/mean": -213.50680541992188, "KL/rejected_KL_mean": -261.21441650390625, "KL/std": 106.64515686035156, "epoch": 0.9765051395007343, "fcm_dpo/beta": 0.004159211181104183, "fcm_dpo/delta": 0.0031878389418125153, "fcm_dpo/margin": 95.41516876220703, "fcm_dpo/q_t": 0.40613293647766113, "grad_norm": 7.824140548706055, "learning_rate": 9.513254770636137e-10, "logits/chosen": -2.5570077896118164, "logits/rejected": -2.1397852897644043, "logps/chosen": -253.98541259765625, "logps/ref_chosen": -88.18618774414062, "logps/ref_rejected": -91.9120101928711, "logps/rejected": -353.12640380859375, "loss": 1.0787, "margin_dpo/margin_mean": 95.41517639160156, "margin_dpo/margin_std": 110.35000610351562, "step": 665 }, { "KL/chosen_KL_mean": -182.7670440673828, "KL/mean": -231.52920532226562, "KL/rejected_KL_mean": -280.2913513183594, "KL/std": 104.12007904052734, "epoch": 0.9779735682819384, "fcm_dpo/beta": 0.004138185642659664, "fcm_dpo/delta": -0.003874862566590309, "fcm_dpo/margin": 97.52427673339844, "fcm_dpo/q_t": 0.4064302146434784, "grad_norm": 10.861021995544434, "learning_rate": 8.427576920763956e-10, "logits/chosen": -1.6349728107452393, "logits/rejected": -1.2554893493652344, "logps/chosen": -283.74163818359375, "logps/ref_chosen": -100.97459411621094, "logps/ref_rejected": -101.24992370605469, "logps/rejected": -381.541259765625, "loss": 1.0863, "margin_dpo/margin_mean": 97.52427673339844, "margin_dpo/margin_std": 123.077392578125, "step": 666 }, { "KL/chosen_KL_mean": -198.56985473632812, "KL/mean": -248.4011688232422, "KL/rejected_KL_mean": -298.23248291015625, "KL/std": 108.48700714111328, "epoch": 0.9794419970631424, "fcm_dpo/beta": 0.004135057330131531, "fcm_dpo/delta": -0.012681795284152031, "fcm_dpo/margin": 99.66262817382812, "fcm_dpo/q_t": 0.4025360345840454, "grad_norm": 9.738076210021973, "learning_rate": 7.407554321417764e-10, "logits/chosen": -1.538848638534546, "logits/rejected": -0.8880414962768555, "logps/chosen": -296.1410217285156, "logps/ref_chosen": -97.5711669921875, "logps/ref_rejected": -93.58476257324219, "logps/rejected": -391.8172607421875, "loss": 1.0612, "margin_dpo/margin_mean": 99.66262817382812, "margin_dpo/margin_std": 105.06712341308594, "step": 667 }, { "KL/chosen_KL_mean": -175.859130859375, "KL/mean": -233.8070068359375, "KL/rejected_KL_mean": -291.75494384765625, "KL/std": 116.04315948486328, "epoch": 0.9809104258443465, "fcm_dpo/beta": 0.00409449078142643, "fcm_dpo/delta": -0.07818560302257538, "fcm_dpo/margin": 115.89578247070312, "fcm_dpo/q_t": 0.39062923192977905, "grad_norm": 10.113677978515625, "learning_rate": 6.453213851142225e-10, "logits/chosen": -2.4140195846557617, "logits/rejected": -1.9029655456542969, "logps/chosen": -278.4341735839844, "logps/ref_chosen": -102.5750503540039, "logps/ref_rejected": -108.81768798828125, "logps/rejected": -400.5726318359375, "loss": 1.0292, "margin_dpo/margin_mean": 115.89578247070312, "margin_dpo/margin_std": 126.80303955078125, "step": 668 }, { "KL/chosen_KL_mean": -160.46368408203125, "KL/mean": -216.34658813476562, "KL/rejected_KL_mean": -272.2294921875, "KL/std": 113.12890625, "epoch": 0.9823788546255506, "fcm_dpo/beta": 0.004042389802634716, "fcm_dpo/delta": -0.05423373728990555, "fcm_dpo/margin": 111.76577758789062, "fcm_dpo/q_t": 0.3943328261375427, "grad_norm": 9.42497730255127, "learning_rate": 5.564580657695939e-10, "logits/chosen": -1.6934189796447754, "logits/rejected": -1.2958654165267944, "logps/chosen": -249.95846557617188, "logps/ref_chosen": -89.49478149414062, "logps/ref_rejected": -82.51950073242188, "logps/rejected": -354.74896240234375, "loss": 1.0406, "margin_dpo/margin_mean": 111.76578521728516, "margin_dpo/margin_std": 120.68971252441406, "step": 669 }, { "KL/chosen_KL_mean": -160.22274780273438, "KL/mean": -222.0571746826172, "KL/rejected_KL_mean": -283.8916015625, "KL/std": 115.48844909667969, "epoch": 0.9838472834067548, "fcm_dpo/beta": 0.0039776707999408245, "fcm_dpo/delta": -0.09657521545886993, "fcm_dpo/margin": 123.66888427734375, "fcm_dpo/q_t": 0.3835490942001343, "grad_norm": 7.44149112701416, "learning_rate": 4.741678157389739e-10, "logits/chosen": -1.6388193368911743, "logits/rejected": -1.0927828550338745, "logps/chosen": -255.67733764648438, "logps/ref_chosen": -95.45459747314453, "logps/ref_rejected": -101.53292846679688, "logps/rejected": -385.424560546875, "loss": 1.0087, "margin_dpo/margin_mean": 123.66887664794922, "margin_dpo/margin_std": 120.63509368896484, "step": 670 }, { "KL/chosen_KL_mean": -195.82919311523438, "KL/mean": -237.9037322998047, "KL/rejected_KL_mean": -279.978271484375, "KL/std": 106.79075622558594, "epoch": 0.9853157121879589, "fcm_dpo/beta": 0.00401078537106514, "fcm_dpo/delta": 0.0638190358877182, "fcm_dpo/margin": 84.14907836914062, "fcm_dpo/q_t": 0.42157915234565735, "grad_norm": 9.48272705078125, "learning_rate": 3.9845280344705245e-10, "logits/chosen": -1.6397138833999634, "logits/rejected": -1.4144715070724487, "logps/chosen": -277.95233154296875, "logps/ref_chosen": -82.12312316894531, "logps/ref_rejected": -90.21969604492188, "logps/rejected": -370.1979675292969, "loss": 1.1336, "margin_dpo/margin_mean": 84.14908599853516, "margin_dpo/margin_std": 118.96661376953125, "step": 671 }, { "KL/chosen_KL_mean": -195.37799072265625, "KL/mean": -231.71615600585938, "KL/rejected_KL_mean": -268.0543212890625, "KL/std": 102.59355163574219, "epoch": 0.986784140969163, "fcm_dpo/beta": 0.004063246306031942, "fcm_dpo/delta": 0.10798829048871994, "fcm_dpo/margin": 72.67633819580078, "fcm_dpo/q_t": 0.4309203624725342, "grad_norm": 12.774223327636719, "learning_rate": 3.293150240547549e-10, "logits/chosen": -2.069598436355591, "logits/rejected": -1.7431080341339111, "logps/chosen": -285.4399108886719, "logps/ref_chosen": -90.0619125366211, "logps/ref_rejected": -100.45323181152344, "logps/rejected": -368.507568359375, "loss": 1.1624, "margin_dpo/margin_mean": 72.67633819580078, "margin_dpo/margin_std": 108.65247344970703, "step": 672 }, { "KL/chosen_KL_mean": -175.35751342773438, "KL/mean": -216.31100463867188, "KL/rejected_KL_mean": -257.26446533203125, "KL/std": 105.82122802734375, "epoch": 0.9882525697503671, "fcm_dpo/beta": 0.004139385186135769, "fcm_dpo/delta": 0.06281546503305435, "fcm_dpo/margin": 81.90696716308594, "fcm_dpo/q_t": 0.4205656051635742, "grad_norm": 11.100406646728516, "learning_rate": 2.6675629940689504e-10, "logits/chosen": -1.9961262941360474, "logits/rejected": -1.6332600116729736, "logps/chosen": -254.62066650390625, "logps/ref_chosen": -79.26315307617188, "logps/ref_rejected": -91.34925079345703, "logps/rejected": -348.61370849609375, "loss": 1.1244, "margin_dpo/margin_mean": 81.90696716308594, "margin_dpo/margin_std": 108.67463684082031, "step": 673 }, { "KL/chosen_KL_mean": -163.64492797851562, "KL/mean": -222.18167114257812, "KL/rejected_KL_mean": -280.7183837890625, "KL/std": 105.79374694824219, "epoch": 0.9897209985315712, "fcm_dpo/beta": 0.004088378045707941, "fcm_dpo/delta": -0.08267657458782196, "fcm_dpo/margin": 117.07347106933594, "fcm_dpo/q_t": 0.38835328817367554, "grad_norm": 7.834228038787842, "learning_rate": 2.1077827798404725e-10, "logits/chosen": -1.666238784790039, "logits/rejected": -1.2661975622177124, "logps/chosen": -239.10324096679688, "logps/ref_chosen": -75.45831298828125, "logps/ref_rejected": -76.20362854003906, "logps/rejected": -356.9220275878906, "loss": 1.0152, "margin_dpo/margin_mean": 117.07347106933594, "margin_dpo/margin_std": 115.59072875976562, "step": 674 }, { "KL/chosen_KL_mean": -180.50228881835938, "KL/mean": -232.09658813476562, "KL/rejected_KL_mean": -283.69085693359375, "KL/std": 114.59227752685547, "epoch": 0.9911894273127754, "fcm_dpo/beta": 0.004038950428366661, "fcm_dpo/delta": -0.018356915563344955, "fcm_dpo/margin": 103.18855285644531, "fcm_dpo/q_t": 0.4029632806777954, "grad_norm": 10.151354789733887, "learning_rate": 1.6138243485910863e-10, "logits/chosen": -1.4934881925582886, "logits/rejected": -1.1527193784713745, "logps/chosen": -260.41180419921875, "logps/ref_chosen": -79.90953063964844, "logps/ref_rejected": -81.21824645996094, "logps/rejected": -364.90911865234375, "loss": 1.0915, "margin_dpo/margin_mean": 103.18855285644531, "margin_dpo/margin_std": 140.4902801513672, "step": 675 }, { "KL/chosen_KL_mean": -194.29684448242188, "KL/mean": -244.83590698242188, "KL/rejected_KL_mean": -295.37493896484375, "KL/std": 109.94676208496094, "epoch": 0.9926578560939795, "fcm_dpo/beta": 0.004045488312840462, "fcm_dpo/delta": -0.009396681562066078, "fcm_dpo/margin": 101.0781021118164, "fcm_dpo/q_t": 0.40403151512145996, "grad_norm": 8.622685432434082, "learning_rate": 1.1857007165852472e-10, "logits/chosen": -1.9971487522125244, "logits/rejected": -1.4139882326126099, "logps/chosen": -292.46795654296875, "logps/ref_chosen": -98.17111206054688, "logps/ref_rejected": -95.024658203125, "logps/rejected": -390.39959716796875, "loss": 1.0797, "margin_dpo/margin_mean": 101.07810974121094, "margin_dpo/margin_std": 123.6147232055664, "step": 676 }, { "KL/chosen_KL_mean": -173.26318359375, "KL/mean": -224.42764282226562, "KL/rejected_KL_mean": -275.59210205078125, "KL/std": 108.45816040039062, "epoch": 0.9941262848751835, "fcm_dpo/beta": 0.004034699406474829, "fcm_dpo/delta": -0.01355313416570425, "fcm_dpo/margin": 102.32893371582031, "fcm_dpo/q_t": 0.40264731645584106, "grad_norm": 8.279085159301758, "learning_rate": 8.23423165278725e-11, "logits/chosen": -1.7284295558929443, "logits/rejected": -1.054389476776123, "logps/chosen": -264.6424560546875, "logps/ref_chosen": -91.37928771972656, "logps/ref_rejected": -82.87776947021484, "logps/rejected": -358.4698791503906, "loss": 1.0611, "margin_dpo/margin_mean": 102.32893371582031, "margin_dpo/margin_std": 106.91799926757812, "step": 677 }, { "KL/chosen_KL_mean": -164.06060791015625, "KL/mean": -224.62562561035156, "KL/rejected_KL_mean": -285.190673828125, "KL/std": 115.8128662109375, "epoch": 0.9955947136563876, "fcm_dpo/beta": 0.004004926420748234, "fcm_dpo/delta": -0.08945269882678986, "fcm_dpo/margin": 121.13008117675781, "fcm_dpo/q_t": 0.38600683212280273, "grad_norm": 9.828141212463379, "learning_rate": 5.270012410216185e-11, "logits/chosen": -1.7069041728973389, "logits/rejected": -1.3808274269104004, "logps/chosen": -239.7064666748047, "logps/ref_chosen": -75.6458740234375, "logps/ref_rejected": -86.96611022949219, "logps/rejected": -372.1567687988281, "loss": 1.0129, "margin_dpo/margin_mean": 121.13008117675781, "margin_dpo/margin_std": 117.40084838867188, "step": 678 }, { "KL/chosen_KL_mean": -189.58999633789062, "KL/mean": -239.7615966796875, "KL/rejected_KL_mean": -289.93316650390625, "KL/std": 96.58641052246094, "epoch": 0.9970631424375918, "fcm_dpo/beta": 0.003967978060245514, "fcm_dpo/delta": 0.00191023712977767, "fcm_dpo/margin": 100.34319305419922, "fcm_dpo/q_t": 0.4062592387199402, "grad_norm": 8.582398414611816, "learning_rate": 2.9644275480772416e-11, "logits/chosen": -2.2233400344848633, "logits/rejected": -1.706078052520752, "logps/chosen": -270.36346435546875, "logps/ref_chosen": -80.77344512939453, "logps/ref_rejected": -82.87850189208984, "logps/rejected": -372.81170654296875, "loss": 1.0663, "margin_dpo/margin_mean": 100.34319305419922, "margin_dpo/margin_std": 103.00176239013672, "step": 679 }, { "KL/chosen_KL_mean": -190.04348754882812, "KL/mean": -244.38632202148438, "KL/rejected_KL_mean": -298.72918701171875, "KL/std": 114.8513412475586, "epoch": 0.9985315712187959, "fcm_dpo/beta": 0.00397725822404027, "fcm_dpo/delta": -0.03441997990012169, "fcm_dpo/margin": 108.68569946289062, "fcm_dpo/q_t": 0.39964038133621216, "grad_norm": 7.0583391189575195, "learning_rate": 1.31753782067201e-11, "logits/chosen": -1.9306485652923584, "logits/rejected": -1.5985172986984253, "logps/chosen": -297.7264099121094, "logps/ref_chosen": -107.68292999267578, "logps/ref_rejected": -116.0948715209961, "logps/rejected": -414.82403564453125, "loss": 1.0619, "margin_dpo/margin_mean": 108.68571472167969, "margin_dpo/margin_std": 124.371337890625, "step": 680 }, { "KL/chosen_KL_mean": -173.37803649902344, "KL/mean": -220.93002319335938, "KL/rejected_KL_mean": -268.48199462890625, "KL/std": 108.56916809082031, "epoch": 1.0, "fcm_dpo/beta": 0.003963897004723549, "fcm_dpo/delta": 0.023829761892557144, "fcm_dpo/margin": 95.10395812988281, "fcm_dpo/q_t": 0.41195446252822876, "grad_norm": 9.78291130065918, "learning_rate": 3.2938662507808745e-12, "logits/chosen": -2.3133158683776855, "logits/rejected": -2.003218173980713, "logps/chosen": -266.38909912109375, "logps/ref_chosen": -93.01106262207031, "logps/ref_rejected": -94.8221664428711, "logps/rejected": -363.3041687011719, "loss": 1.0974, "margin_dpo/margin_mean": 95.10395812988281, "margin_dpo/margin_std": 118.562255859375, "step": 681 }, { "epoch": 1.0, "step": 681, "total_flos": 0.0, "train_loss": 1.0967794093091408, "train_runtime": 1994.2396, "train_samples_per_second": 21.862, "train_steps_per_second": 0.341 } ], "logging_steps": 1, "max_steps": 681, "num_input_tokens_seen": 0, "num_train_epochs": 1, "save_steps": 50, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": false, "should_training_stop": false }, "attributes": {} } }, "total_flos": 0.0, "train_batch_size": 8, "trial_name": null, "trial_params": null }