{ "best_global_step": null, "best_metric": null, "best_model_checkpoint": null, "epoch": 1.0, "eval_steps": 100, "global_step": 681, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "KL/chosen_KL_mean": 0.05419158935546875, "KL/mean": 0.023573115468025208, "KL/rejected_KL_mean": -0.007045745849609375, "KL/std": 0.33097243309020996, "epoch": 0.0014684287812041115, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.06123492121696472, "fcm_dpo/q_t": 0.4984707534313202, "grad_norm": 38.820674896240234, "learning_rate": 0.0, "logits/chosen": 0.2677389979362488, "logits/rejected": 0.6191319227218628, "logps/chosen": -85.37664031982422, "logps/ref_chosen": -85.43083190917969, "logps/ref_rejected": -79.90458679199219, "logps/rejected": -79.91163635253906, "loss": 1.3807, "margin_dpo/margin_mean": 0.061235010623931885, "margin_dpo/margin_std": 0.4424324035644531, "step": 1 }, { "KL/chosen_KL_mean": -0.044902801513671875, "KL/mean": 0.017303764820098877, "KL/rejected_KL_mean": 0.07951736450195312, "KL/std": 0.2933448553085327, "epoch": 0.002936857562408223, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.12442219257354736, "fcm_dpo/q_t": 0.5031089186668396, "grad_norm": 29.758089065551758, "learning_rate": 7.246376811594203e-09, "logits/chosen": 0.47771453857421875, "logits/rejected": 0.5908545851707458, "logps/chosen": -82.11383056640625, "logps/ref_chosen": -82.06893157958984, "logps/ref_rejected": -81.65457916259766, "logps/rejected": -81.57505798339844, "loss": 1.3992, "margin_dpo/margin_mean": -0.12442246079444885, "margin_dpo/margin_std": 0.41358208656311035, "step": 2 }, { "KL/chosen_KL_mean": 0.01465606689453125, "KL/mean": 0.01496800035238266, "KL/rejected_KL_mean": 0.015285491943359375, "KL/std": 0.3061980605125427, "epoch": 0.004405286343612335, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.0006299763917922974, "fcm_dpo/q_t": 0.5000157356262207, "grad_norm": 26.069414138793945, "learning_rate": 1.4492753623188406e-08, "logits/chosen": 0.4271789789199829, "logits/rejected": 0.6502995491027832, "logps/chosen": -93.79632568359375, "logps/ref_chosen": -93.81098937988281, "logps/ref_rejected": -74.22950744628906, "logps/rejected": -74.21421813964844, "loss": 1.3869, "margin_dpo/margin_mean": -0.0006296485662460327, "margin_dpo/margin_std": 0.4710845947265625, "step": 3 }, { "KL/chosen_KL_mean": -0.015625, "KL/mean": -0.028287947177886963, "KL/rejected_KL_mean": -0.040950775146484375, "KL/std": 0.34648823738098145, "epoch": 0.005873715124816446, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.025322318077087402, "fcm_dpo/q_t": 0.499367892742157, "grad_norm": 32.61881637573242, "learning_rate": 2.1739130434782606e-08, "logits/chosen": 0.6476162672042847, "logits/rejected": 0.7791898250579834, "logps/chosen": -87.30809020996094, "logps/ref_chosen": -87.29246520996094, "logps/ref_rejected": -93.82425689697266, "logps/rejected": -93.86520385742188, "loss": 1.3843, "margin_dpo/margin_mean": 0.02532219886779785, "margin_dpo/margin_std": 0.47982242703437805, "step": 4 }, { "KL/chosen_KL_mean": 0.023525238037109375, "KL/mean": -0.0032288432121276855, "KL/rejected_KL_mean": -0.0299835205078125, "KL/std": 0.28698718547821045, "epoch": 0.007342143906020558, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.05350852012634277, "fcm_dpo/q_t": 0.49866265058517456, "grad_norm": 35.11767578125, "learning_rate": 2.898550724637681e-08, "logits/chosen": 0.4449205696582794, "logits/rejected": 0.656495213508606, "logps/chosen": -89.313232421875, "logps/ref_chosen": -89.33676147460938, "logps/ref_rejected": -88.74783325195312, "logps/rejected": -88.77781677246094, "loss": 1.3814, "margin_dpo/margin_mean": 0.05350884795188904, "margin_dpo/margin_std": 0.4278198480606079, "step": 5 }, { "KL/chosen_KL_mean": 0.039897918701171875, "KL/mean": 0.008364655077457428, "KL/rejected_KL_mean": -0.02317047119140625, "KL/std": 0.2923298478126526, "epoch": 0.00881057268722467, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.06306080520153046, "fcm_dpo/q_t": 0.4984240233898163, "grad_norm": 35.921546936035156, "learning_rate": 3.6231884057971014e-08, "logits/chosen": 0.4856651723384857, "logits/rejected": 0.8012295961380005, "logps/chosen": -97.28157806396484, "logps/ref_chosen": -97.32147216796875, "logps/ref_rejected": -97.88345336914062, "logps/rejected": -97.9066162109375, "loss": 1.3804, "margin_dpo/margin_mean": 0.06306031346321106, "margin_dpo/margin_std": 0.38511377573013306, "step": 6 }, { "KL/chosen_KL_mean": 0.0255126953125, "KL/mean": -0.0052457451820373535, "KL/rejected_KL_mean": -0.0360107421875, "KL/std": 0.34397822618484497, "epoch": 0.010279001468428781, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.061517417430877686, "fcm_dpo/q_t": 0.4984646439552307, "grad_norm": 37.94108581542969, "learning_rate": 4.347826086956521e-08, "logits/chosen": 0.4881408214569092, "logits/rejected": 0.5354626774787903, "logps/chosen": -86.62301635742188, "logps/ref_chosen": -86.64852905273438, "logps/ref_rejected": -109.61618041992188, "logps/rejected": -109.65219116210938, "loss": 1.3806, "margin_dpo/margin_mean": 0.061517298221588135, "margin_dpo/margin_std": 0.43569451570510864, "step": 7 }, { "KL/chosen_KL_mean": -0.06678009033203125, "KL/mean": -0.00782717764377594, "KL/rejected_KL_mean": 0.051128387451171875, "KL/std": 0.336764931678772, "epoch": 0.011747430249632892, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.11790210008621216, "fcm_dpo/q_t": 0.502944827079773, "grad_norm": 32.93342971801758, "learning_rate": 5.0724637681159424e-08, "logits/chosen": 0.4219074845314026, "logits/rejected": 0.7341209053993225, "logps/chosen": -89.99046325683594, "logps/ref_chosen": -89.9236831665039, "logps/ref_rejected": -86.22804260253906, "logps/rejected": -86.17691040039062, "loss": 1.3988, "margin_dpo/margin_mean": -0.11790190637111664, "margin_dpo/margin_std": 0.49930205941200256, "step": 8 }, { "KL/chosen_KL_mean": 0.05693817138671875, "KL/mean": 0.03145362436771393, "KL/rejected_KL_mean": 0.005970001220703125, "KL/std": 0.3200056552886963, "epoch": 0.013215859030837005, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.050966084003448486, "fcm_dpo/q_t": 0.49872636795043945, "grad_norm": 36.001705169677734, "learning_rate": 5.797101449275362e-08, "logits/chosen": 0.7285146117210388, "logits/rejected": 0.9130157232284546, "logps/chosen": -103.8001937866211, "logps/ref_chosen": -103.85713195800781, "logps/ref_rejected": -104.31932067871094, "logps/rejected": -104.3133544921875, "loss": 1.3817, "margin_dpo/margin_mean": 0.050965458154678345, "margin_dpo/margin_std": 0.4331083297729492, "step": 9 }, { "KL/chosen_KL_mean": 0.018100738525390625, "KL/mean": 0.04509608447551727, "KL/rejected_KL_mean": 0.07208633422851562, "KL/std": 0.25571948289871216, "epoch": 0.014684287812041116, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.05399295687675476, "fcm_dpo/q_t": 0.501349925994873, "grad_norm": 33.59144973754883, "learning_rate": 6.521739130434782e-08, "logits/chosen": 0.5794335603713989, "logits/rejected": 0.7947685122489929, "logps/chosen": -76.18626403808594, "logps/ref_chosen": -76.20436096191406, "logps/ref_rejected": -87.152099609375, "logps/rejected": -87.08001708984375, "loss": 1.392, "margin_dpo/margin_mean": -0.05399322509765625, "margin_dpo/margin_std": 0.3645928204059601, "step": 10 }, { "KL/chosen_KL_mean": 0.012393951416015625, "KL/mean": -0.0013011395931243896, "KL/rejected_KL_mean": -0.014995574951171875, "KL/std": 0.32870516180992126, "epoch": 0.016152716593245228, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.02738875150680542, "fcm_dpo/q_t": 0.49931570887565613, "grad_norm": 36.94203567504883, "learning_rate": 7.246376811594203e-08, "logits/chosen": 0.2282310128211975, "logits/rejected": 0.44162872433662415, "logps/chosen": -82.3541030883789, "logps/ref_chosen": -82.36650085449219, "logps/ref_rejected": -94.26461791992188, "logps/rejected": -94.27961730957031, "loss": 1.3841, "margin_dpo/margin_mean": 0.027389109134674072, "margin_dpo/margin_std": 0.44703277945518494, "step": 11 }, { "KL/chosen_KL_mean": 0.009220123291015625, "KL/mean": -0.006621226668357849, "KL/rejected_KL_mean": -0.022464752197265625, "KL/std": 0.3435856103897095, "epoch": 0.01762114537444934, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.031681835651397705, "fcm_dpo/q_t": 0.49920886754989624, "grad_norm": 40.490726470947266, "learning_rate": 7.971014492753623e-08, "logits/chosen": 0.4049307107925415, "logits/rejected": 0.5326098203659058, "logps/chosen": -99.09628295898438, "logps/ref_chosen": -99.10549926757812, "logps/ref_rejected": -110.27140808105469, "logps/rejected": -110.29387664794922, "loss": 1.3838, "margin_dpo/margin_mean": 0.03168192505836487, "margin_dpo/margin_std": 0.49787259101867676, "step": 12 }, { "KL/chosen_KL_mean": 0.021770477294921875, "KL/mean": 0.04439286142587662, "KL/rejected_KL_mean": 0.0670166015625, "KL/std": 0.32605552673339844, "epoch": 0.01908957415565345, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.04524601995944977, "fcm_dpo/q_t": 0.5011304020881653, "grad_norm": 41.879478454589844, "learning_rate": 8.695652173913042e-08, "logits/chosen": 0.5337778329849243, "logits/rejected": 0.7933570146560669, "logps/chosen": -90.53795623779297, "logps/ref_chosen": -90.55972290039062, "logps/ref_rejected": -93.69110107421875, "logps/rejected": -93.62408447265625, "loss": 1.3914, "margin_dpo/margin_mean": -0.0452461838722229, "margin_dpo/margin_std": 0.4874368906021118, "step": 13 }, { "KL/chosen_KL_mean": -0.08868789672851562, "KL/mean": -0.03641383349895477, "KL/rejected_KL_mean": 0.01586151123046875, "KL/std": 0.33402109146118164, "epoch": 0.020558002936857563, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.10455295443534851, "fcm_dpo/q_t": 0.5026113986968994, "grad_norm": 36.025638580322266, "learning_rate": 9.420289855072464e-08, "logits/chosen": 0.7584762573242188, "logits/rejected": 0.8417384624481201, "logps/chosen": -99.91586303710938, "logps/ref_chosen": -99.82717895507812, "logps/ref_rejected": -108.94200134277344, "logps/rejected": -108.92613983154297, "loss": 1.3973, "margin_dpo/margin_mean": -0.10455304384231567, "margin_dpo/margin_std": 0.4466242492198944, "step": 14 }, { "KL/chosen_KL_mean": -0.021053314208984375, "KL/mean": -0.0028785765171051025, "KL/rejected_KL_mean": 0.015293121337890625, "KL/std": 0.29913845658302307, "epoch": 0.022026431718061675, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.03634488582611084, "fcm_dpo/q_t": 0.5009075999259949, "grad_norm": 31.936511993408203, "learning_rate": 1.0144927536231885e-07, "logits/chosen": 0.40792274475097656, "logits/rejected": 0.5764085054397583, "logps/chosen": -78.93102264404297, "logps/ref_chosen": -78.90997314453125, "logps/ref_rejected": -90.06234741210938, "logps/rejected": -90.04705047607422, "loss": 1.3904, "margin_dpo/margin_mean": -0.036345332860946655, "margin_dpo/margin_std": 0.42946162819862366, "step": 15 }, { "KL/chosen_KL_mean": 0.02458953857421875, "KL/mean": 0.018733836710453033, "KL/rejected_KL_mean": 0.01287078857421875, "KL/std": 0.34945398569107056, "epoch": 0.023494860499265784, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.011714145541191101, "fcm_dpo/q_t": 0.4997081160545349, "grad_norm": 34.51431655883789, "learning_rate": 1.0869565217391303e-07, "logits/chosen": 0.48071449995040894, "logits/rejected": 0.7300193309783936, "logps/chosen": -97.39868927001953, "logps/ref_chosen": -97.42327880859375, "logps/ref_rejected": -90.59945678710938, "logps/rejected": -90.58659362792969, "loss": 1.3857, "margin_dpo/margin_mean": 0.01171422004699707, "margin_dpo/margin_std": 0.4923936724662781, "step": 16 }, { "KL/chosen_KL_mean": 0.00018310546875, "KL/mean": -0.001642562448978424, "KL/rejected_KL_mean": -0.003475189208984375, "KL/std": 0.3160494565963745, "epoch": 0.024963289280469897, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.003662601113319397, "fcm_dpo/q_t": 0.4999082684516907, "grad_norm": 35.67642593383789, "learning_rate": 1.1594202898550725e-07, "logits/chosen": 0.5002927780151367, "logits/rejected": 0.7326413989067078, "logps/chosen": -104.36412048339844, "logps/ref_chosen": -104.36430358886719, "logps/ref_rejected": -90.46772766113281, "logps/rejected": -90.47119903564453, "loss": 1.3865, "margin_dpo/margin_mean": 0.0036616921424865723, "margin_dpo/margin_std": 0.4501694142818451, "step": 17 }, { "KL/chosen_KL_mean": 0.02106475830078125, "KL/mean": 0.02845171093940735, "KL/rejected_KL_mean": 0.03585052490234375, "KL/std": 0.27513402700424194, "epoch": 0.02643171806167401, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.014791727066040039, "fcm_dpo/q_t": 0.5003697872161865, "grad_norm": 41.84176254272461, "learning_rate": 1.2318840579710146e-07, "logits/chosen": 0.48476219177246094, "logits/rejected": 0.8407496213912964, "logps/chosen": -87.07089233398438, "logps/ref_chosen": -87.09195709228516, "logps/ref_rejected": -81.85072326660156, "logps/rejected": -81.81486511230469, "loss": 1.3881, "margin_dpo/margin_mean": -0.014791086316108704, "margin_dpo/margin_std": 0.37340396642684937, "step": 18 }, { "KL/chosen_KL_mean": 0.03704071044921875, "KL/mean": 0.00791950523853302, "KL/rejected_KL_mean": -0.021205902099609375, "KL/std": 0.32189714908599854, "epoch": 0.027900146842878122, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.05824217200279236, "fcm_dpo/q_t": 0.49854540824890137, "grad_norm": 31.105785369873047, "learning_rate": 1.3043478260869563e-07, "logits/chosen": 0.5374953150749207, "logits/rejected": 0.808600902557373, "logps/chosen": -105.83650207519531, "logps/ref_chosen": -105.87354278564453, "logps/ref_rejected": -96.93023681640625, "logps/rejected": -96.9514389038086, "loss": 1.3809, "margin_dpo/margin_mean": 0.05824229121208191, "margin_dpo/margin_std": 0.43151164054870605, "step": 19 }, { "KL/chosen_KL_mean": 0.06446075439453125, "KL/mean": 0.03968258202075958, "KL/rejected_KL_mean": 0.014904022216796875, "KL/std": 0.3084867596626282, "epoch": 0.02936857562408223, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.04955431818962097, "fcm_dpo/q_t": 0.498762309551239, "grad_norm": 32.53604507446289, "learning_rate": 1.3768115942028986e-07, "logits/chosen": 0.47317224740982056, "logits/rejected": 0.7738533020019531, "logps/chosen": -90.69366455078125, "logps/ref_chosen": -90.75811767578125, "logps/ref_rejected": -85.91232299804688, "logps/rejected": -85.89741516113281, "loss": 1.3818, "margin_dpo/margin_mean": 0.04955416917800903, "margin_dpo/margin_std": 0.4128846526145935, "step": 20 }, { "KL/chosen_KL_mean": 0.033283233642578125, "KL/mean": 0.0017265155911445618, "KL/rejected_KL_mean": -0.029827117919921875, "KL/std": 0.2668415307998657, "epoch": 0.030837004405286344, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.06311012804508209, "fcm_dpo/q_t": 0.4984225630760193, "grad_norm": 31.62078857421875, "learning_rate": 1.4492753623188405e-07, "logits/chosen": 0.6600567102432251, "logits/rejected": 0.8490397930145264, "logps/chosen": -80.3001708984375, "logps/ref_chosen": -80.33345794677734, "logps/ref_rejected": -83.93373107910156, "logps/rejected": -83.96356201171875, "loss": 1.3803, "margin_dpo/margin_mean": 0.06311002373695374, "margin_dpo/margin_std": 0.37183040380477905, "step": 21 }, { "KL/chosen_KL_mean": 0.022003173828125, "KL/mean": 0.010841697454452515, "KL/rejected_KL_mean": -0.000316619873046875, "KL/std": 0.36299848556518555, "epoch": 0.032305433186490456, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.02230769395828247, "fcm_dpo/q_t": 0.49944305419921875, "grad_norm": 42.550689697265625, "learning_rate": 1.5217391304347825e-07, "logits/chosen": 0.7412316799163818, "logits/rejected": 0.8664436936378479, "logps/chosen": -95.37329864501953, "logps/ref_chosen": -95.39530181884766, "logps/ref_rejected": -103.47351837158203, "logps/rejected": -103.47383117675781, "loss": 1.3846, "margin_dpo/margin_mean": 0.02230784296989441, "margin_dpo/margin_std": 0.4530148506164551, "step": 22 }, { "KL/chosen_KL_mean": 0.03369331359863281, "KL/mean": 0.06140320748090744, "KL/rejected_KL_mean": 0.08911895751953125, "KL/std": 0.2708756923675537, "epoch": 0.033773861967694566, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.055423811078071594, "fcm_dpo/q_t": 0.5013852119445801, "grad_norm": 31.583332061767578, "learning_rate": 1.5942028985507245e-07, "logits/chosen": 0.769604504108429, "logits/rejected": 0.9496768712997437, "logps/chosen": -90.60382080078125, "logps/ref_chosen": -90.63751220703125, "logps/ref_rejected": -86.59425354003906, "logps/rejected": -86.50514221191406, "loss": 1.3922, "margin_dpo/margin_mean": -0.05542415380477905, "margin_dpo/margin_std": 0.36274829506874084, "step": 23 }, { "KL/chosen_KL_mean": 0.07908439636230469, "KL/mean": 0.032233938574790955, "KL/rejected_KL_mean": -0.014617919921875, "KL/std": 0.31843966245651245, "epoch": 0.03524229074889868, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.09370556473731995, "fcm_dpo/q_t": 0.49766063690185547, "grad_norm": 44.20138931274414, "learning_rate": 1.6666666666666665e-07, "logits/chosen": 0.5037790536880493, "logits/rejected": 0.5362599492073059, "logps/chosen": -69.83821105957031, "logps/ref_chosen": -69.91728973388672, "logps/ref_rejected": -106.60990142822266, "logps/rejected": -106.62451934814453, "loss": 1.3775, "margin_dpo/margin_mean": 0.09370668232440948, "margin_dpo/margin_std": 0.47933924198150635, "step": 24 }, { "KL/chosen_KL_mean": -0.06927871704101562, "KL/mean": -0.07646503299474716, "KL/rejected_KL_mean": -0.08365249633789062, "KL/std": 0.312234103679657, "epoch": 0.03671071953010279, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.014364555478096008, "fcm_dpo/q_t": 0.49964118003845215, "grad_norm": 37.118465423583984, "learning_rate": 1.7391304347826085e-07, "logits/chosen": 0.4768267869949341, "logits/rejected": 0.6433182954788208, "logps/chosen": -80.89476013183594, "logps/ref_chosen": -80.82548522949219, "logps/ref_rejected": -95.95710754394531, "logps/rejected": -96.04075622558594, "loss": 1.3853, "margin_dpo/margin_mean": 0.014364391565322876, "margin_dpo/margin_std": 0.41830477118492126, "step": 25 }, { "KL/chosen_KL_mean": -0.0384368896484375, "KL/mean": -0.027527883648872375, "KL/rejected_KL_mean": -0.016620635986328125, "KL/std": 0.32468181848526, "epoch": 0.0381791483113069, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.021814316511154175, "fcm_dpo/q_t": 0.5005432367324829, "grad_norm": 40.95606231689453, "learning_rate": 1.8115942028985507e-07, "logits/chosen": 0.431133508682251, "logits/rejected": 0.6434487104415894, "logps/chosen": -88.93960571289062, "logps/ref_chosen": -88.90116882324219, "logps/ref_rejected": -109.82819366455078, "logps/rejected": -109.84481811523438, "loss": 1.3891, "margin_dpo/margin_mean": -0.021814018487930298, "margin_dpo/margin_std": 0.4856882691383362, "step": 26 }, { "KL/chosen_KL_mean": -0.01734161376953125, "KL/mean": -0.058997683227062225, "KL/rejected_KL_mean": -0.10065841674804688, "KL/std": 0.33262455463409424, "epoch": 0.039647577092511016, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.08332325518131256, "fcm_dpo/q_t": 0.49791765213012695, "grad_norm": 44.292057037353516, "learning_rate": 1.8840579710144927e-07, "logits/chosen": 0.1446508765220642, "logits/rejected": 0.33246272802352905, "logps/chosen": -77.61335754394531, "logps/ref_chosen": -77.59600830078125, "logps/ref_rejected": -103.93850708007812, "logps/rejected": -104.03916931152344, "loss": 1.3785, "margin_dpo/margin_mean": 0.08332285284996033, "margin_dpo/margin_std": 0.4558706283569336, "step": 27 }, { "KL/chosen_KL_mean": 0.0346221923828125, "KL/mean": -0.004966810345649719, "KL/rejected_KL_mean": -0.0445556640625, "KL/std": 0.34315457940101624, "epoch": 0.041116005873715125, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.07918068766593933, "fcm_dpo/q_t": 0.49801960587501526, "grad_norm": 36.65188980102539, "learning_rate": 1.9565217391304347e-07, "logits/chosen": 0.6428591012954712, "logits/rejected": 0.8668360710144043, "logps/chosen": -102.1939468383789, "logps/ref_chosen": -102.22856903076172, "logps/ref_rejected": -96.9594955444336, "logps/rejected": -97.0040512084961, "loss": 1.3789, "margin_dpo/margin_mean": 0.07918080687522888, "margin_dpo/margin_std": 0.4384033679962158, "step": 28 }, { "KL/chosen_KL_mean": 0.0294036865234375, "KL/mean": -0.053695522248744965, "KL/rejected_KL_mean": -0.13679122924804688, "KL/std": 0.3311346471309662, "epoch": 0.042584434654919234, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.1661987155675888, "fcm_dpo/q_t": 0.49584776163101196, "grad_norm": 42.23774719238281, "learning_rate": 2.028985507246377e-07, "logits/chosen": 0.3736269474029541, "logits/rejected": 0.6361100077629089, "logps/chosen": -88.61764526367188, "logps/ref_chosen": -88.64704895019531, "logps/ref_rejected": -102.96011352539062, "logps/rejected": -103.09690856933594, "loss": 1.3702, "margin_dpo/margin_mean": 0.16619959473609924, "margin_dpo/margin_std": 0.4080168604850769, "step": 29 }, { "KL/chosen_KL_mean": 0.03119659423828125, "KL/mean": -0.043515101075172424, "KL/rejected_KL_mean": -0.11823654174804688, "KL/std": 0.30171334743499756, "epoch": 0.04405286343612335, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.14942720532417297, "fcm_dpo/q_t": 0.4962664842605591, "grad_norm": 39.048545837402344, "learning_rate": 2.1014492753623187e-07, "logits/chosen": 0.8060142397880554, "logits/rejected": 0.9297041893005371, "logps/chosen": -88.35719299316406, "logps/ref_chosen": -88.38838195800781, "logps/ref_rejected": -102.31889343261719, "logps/rejected": -102.4371337890625, "loss": 1.3718, "margin_dpo/margin_mean": 0.14942720532417297, "margin_dpo/margin_std": 0.3693804144859314, "step": 30 }, { "KL/chosen_KL_mean": -0.030895233154296875, "KL/mean": -0.09597525745630264, "KL/rejected_KL_mean": -0.16105270385742188, "KL/std": 0.3533359169960022, "epoch": 0.04552129221732746, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.13015665113925934, "fcm_dpo/q_t": 0.4967484176158905, "grad_norm": 31.51624298095703, "learning_rate": 2.1739130434782607e-07, "logits/chosen": 0.9528893232345581, "logits/rejected": 1.2278332710266113, "logps/chosen": -101.15655517578125, "logps/ref_chosen": -101.12565612792969, "logps/ref_rejected": -79.8584213256836, "logps/rejected": -80.01947021484375, "loss": 1.3738, "margin_dpo/margin_mean": 0.13015678524971008, "margin_dpo/margin_std": 0.45469146966934204, "step": 31 }, { "KL/chosen_KL_mean": -0.01691436767578125, "KL/mean": -0.06670171022415161, "KL/rejected_KL_mean": -0.11648941040039062, "KL/std": 0.3256070613861084, "epoch": 0.04698972099853157, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.09957391023635864, "fcm_dpo/q_t": 0.4975109100341797, "grad_norm": 35.107818603515625, "learning_rate": 2.2463768115942027e-07, "logits/chosen": 0.38375186920166016, "logits/rejected": 0.522384524345398, "logps/chosen": -96.63394165039062, "logps/ref_chosen": -96.61702728271484, "logps/ref_rejected": -96.224365234375, "logps/rejected": -96.34085083007812, "loss": 1.3769, "margin_dpo/margin_mean": 0.09957461059093475, "margin_dpo/margin_std": 0.46503159403800964, "step": 32 }, { "KL/chosen_KL_mean": 0.003997802734375, "KL/mean": -0.011204198002815247, "KL/rejected_KL_mean": -0.026409149169921875, "KL/std": 0.32523515820503235, "epoch": 0.048458149779735685, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.03040716052055359, "fcm_dpo/q_t": 0.49924027919769287, "grad_norm": 33.73084259033203, "learning_rate": 2.318840579710145e-07, "logits/chosen": 0.5409725904464722, "logits/rejected": 0.7198845148086548, "logps/chosen": -81.51701354980469, "logps/ref_chosen": -81.52101135253906, "logps/ref_rejected": -93.80595397949219, "logps/rejected": -93.83236694335938, "loss": 1.3838, "margin_dpo/margin_mean": 0.03040754795074463, "margin_dpo/margin_std": 0.4421288073062897, "step": 33 }, { "KL/chosen_KL_mean": 0.06410980224609375, "KL/mean": -0.02580152451992035, "KL/rejected_KL_mean": -0.1157073974609375, "KL/std": 0.35330280661582947, "epoch": 0.049926578560939794, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.17982324957847595, "fcm_dpo/q_t": 0.4955122768878937, "grad_norm": 40.87218475341797, "learning_rate": 2.391304347826087e-07, "logits/chosen": 0.5539125800132751, "logits/rejected": 0.7267269492149353, "logps/chosen": -77.13792419433594, "logps/ref_chosen": -77.20204162597656, "logps/ref_rejected": -106.71876525878906, "logps/rejected": -106.83447265625, "loss": 1.369, "margin_dpo/margin_mean": 0.17982378602027893, "margin_dpo/margin_std": 0.4962542653083801, "step": 34 }, { "KL/chosen_KL_mean": -0.005828857421875, "KL/mean": -0.09057293832302094, "KL/rejected_KL_mean": -0.17531585693359375, "KL/std": 0.3251180052757263, "epoch": 0.0513950073421439, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.169493168592453, "fcm_dpo/q_t": 0.49576517939567566, "grad_norm": 42.73767852783203, "learning_rate": 2.463768115942029e-07, "logits/chosen": 0.42941057682037354, "logits/rejected": 0.5840994715690613, "logps/chosen": -77.57617950439453, "logps/ref_chosen": -77.57035064697266, "logps/ref_rejected": -112.18855285644531, "logps/rejected": -112.3638687133789, "loss": 1.3699, "margin_dpo/margin_mean": 0.16949301958084106, "margin_dpo/margin_std": 0.4189368486404419, "step": 35 }, { "KL/chosen_KL_mean": -0.01593017578125, "KL/mean": -0.1278805285692215, "KL/rejected_KL_mean": -0.23983001708984375, "KL/std": 0.38666093349456787, "epoch": 0.05286343612334802, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.2238941490650177, "fcm_dpo/q_t": 0.494412362575531, "grad_norm": 32.33586502075195, "learning_rate": 2.536231884057971e-07, "logits/chosen": 0.4637696444988251, "logits/rejected": 0.6560626029968262, "logps/chosen": -83.30416870117188, "logps/ref_chosen": -83.28823852539062, "logps/ref_rejected": -91.75741577148438, "logps/rejected": -91.99724578857422, "loss": 1.3648, "margin_dpo/margin_mean": 0.2238946259021759, "margin_dpo/margin_std": 0.5497259497642517, "step": 36 }, { "KL/chosen_KL_mean": -0.07549667358398438, "KL/mean": -0.18632854521274567, "KL/rejected_KL_mean": -0.29715728759765625, "KL/std": 0.4600166082382202, "epoch": 0.05433186490455213, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.22165650129318237, "fcm_dpo/q_t": 0.4944719076156616, "grad_norm": 35.99590301513672, "learning_rate": 2.6086956521739126e-07, "logits/chosen": 0.7685495615005493, "logits/rejected": 0.936451256275177, "logps/chosen": -94.84658813476562, "logps/ref_chosen": -94.77108764648438, "logps/ref_rejected": -85.71726989746094, "logps/rejected": -86.01441955566406, "loss": 1.3655, "margin_dpo/margin_mean": 0.2216568887233734, "margin_dpo/margin_std": 0.695243239402771, "step": 37 }, { "KL/chosen_KL_mean": 0.055355072021484375, "KL/mean": -0.06861411780118942, "KL/rejected_KL_mean": -0.1925811767578125, "KL/std": 0.40513813495635986, "epoch": 0.055800293685756244, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.24793694913387299, "fcm_dpo/q_t": 0.49381375312805176, "grad_norm": 36.653480529785156, "learning_rate": 2.681159420289855e-07, "logits/chosen": 0.5632282495498657, "logits/rejected": 0.7024773359298706, "logps/chosen": -75.92314910888672, "logps/ref_chosen": -75.97850036621094, "logps/ref_rejected": -104.10401916503906, "logps/rejected": -104.29660034179688, "loss": 1.3626, "margin_dpo/margin_mean": 0.2479374259710312, "margin_dpo/margin_std": 0.5943114757537842, "step": 38 }, { "KL/chosen_KL_mean": -0.016330718994140625, "KL/mean": -0.1703965663909912, "KL/rejected_KL_mean": -0.324462890625, "KL/std": 0.3898355960845947, "epoch": 0.05726872246696035, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.3081251382827759, "fcm_dpo/q_t": 0.4923045337200165, "grad_norm": 36.08936309814453, "learning_rate": 2.753623188405797e-07, "logits/chosen": 0.5016517639160156, "logits/rejected": 0.6918469667434692, "logps/chosen": -81.20211029052734, "logps/ref_chosen": -81.18577575683594, "logps/ref_rejected": -84.1959228515625, "logps/rejected": -84.5203857421875, "loss": 1.3564, "margin_dpo/margin_mean": 0.3081253170967102, "margin_dpo/margin_std": 0.5036002397537231, "step": 39 }, { "KL/chosen_KL_mean": -0.02194976806640625, "KL/mean": -0.11939875781536102, "KL/rejected_KL_mean": -0.21683883666992188, "KL/std": 0.3163018226623535, "epoch": 0.05873715124816446, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.1948917806148529, "fcm_dpo/q_t": 0.4951309561729431, "grad_norm": 29.204524993896484, "learning_rate": 2.8260869565217386e-07, "logits/chosen": 0.5341147184371948, "logits/rejected": 0.724126935005188, "logps/chosen": -83.35452270507812, "logps/ref_chosen": -83.33257293701172, "logps/ref_rejected": -80.25591278076172, "logps/rejected": -80.47274780273438, "loss": 1.3674, "margin_dpo/margin_mean": 0.19489142298698425, "margin_dpo/margin_std": 0.4238033890724182, "step": 40 }, { "KL/chosen_KL_mean": -0.050930023193359375, "KL/mean": -0.17703256011009216, "KL/rejected_KL_mean": -0.3031349182128906, "KL/std": 0.4483546018600464, "epoch": 0.06020558002936858, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.2522096037864685, "fcm_dpo/q_t": 0.4937064051628113, "grad_norm": 33.518550872802734, "learning_rate": 2.898550724637681e-07, "logits/chosen": 0.5285093784332275, "logits/rejected": 0.7094418406486511, "logps/chosen": -93.19959259033203, "logps/ref_chosen": -93.14866638183594, "logps/ref_rejected": -102.07920837402344, "logps/rejected": -102.38233947753906, "loss": 1.3623, "margin_dpo/margin_mean": 0.25220993161201477, "margin_dpo/margin_std": 0.6251505017280579, "step": 41 }, { "KL/chosen_KL_mean": 0.06290435791015625, "KL/mean": -0.15234878659248352, "KL/rejected_KL_mean": -0.3675994873046875, "KL/std": 0.469764769077301, "epoch": 0.06167400881057269, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.4305035471916199, "fcm_dpo/q_t": 0.48924633860588074, "grad_norm": 40.858097076416016, "learning_rate": 2.971014492753623e-07, "logits/chosen": 0.40404701232910156, "logits/rejected": 0.5445290207862854, "logps/chosen": -90.63871765136719, "logps/ref_chosen": -90.70162200927734, "logps/ref_rejected": -113.8646469116211, "logps/rejected": -114.23224639892578, "loss": 1.3444, "margin_dpo/margin_mean": 0.4305039644241333, "margin_dpo/margin_std": 0.5120759010314941, "step": 42 }, { "KL/chosen_KL_mean": 0.033443450927734375, "KL/mean": -0.17150002717971802, "KL/rejected_KL_mean": -0.3764495849609375, "KL/std": 0.5307700037956238, "epoch": 0.0631424375917768, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.4098924398422241, "fcm_dpo/q_t": 0.48977774381637573, "grad_norm": 37.972496032714844, "learning_rate": 3.043478260869565e-07, "logits/chosen": 0.8372114300727844, "logits/rejected": 1.0393562316894531, "logps/chosen": -89.61058807373047, "logps/ref_chosen": -89.64402770996094, "logps/ref_rejected": -103.95185852050781, "logps/rejected": -104.32830810546875, "loss": 1.3472, "margin_dpo/margin_mean": 0.40989261865615845, "margin_dpo/margin_std": 0.7539166808128357, "step": 43 }, { "KL/chosen_KL_mean": -0.029296875, "KL/mean": -0.25405848026275635, "KL/rejected_KL_mean": -0.47882080078125, "KL/std": 0.5066498517990112, "epoch": 0.06461086637298091, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.44952452182769775, "fcm_dpo/q_t": 0.48877984285354614, "grad_norm": 34.2753791809082, "learning_rate": 3.115942028985507e-07, "logits/chosen": 0.8688725233078003, "logits/rejected": 0.9891389608383179, "logps/chosen": -81.9076156616211, "logps/ref_chosen": -81.8783187866211, "logps/ref_rejected": -113.421630859375, "logps/rejected": -113.90045166015625, "loss": 1.3429, "margin_dpo/margin_mean": 0.4495249092578888, "margin_dpo/margin_std": 0.6458636522293091, "step": 44 }, { "KL/chosen_KL_mean": -0.13150405883789062, "KL/mean": -0.22282178699970245, "KL/rejected_KL_mean": -0.31414031982421875, "KL/std": 0.4229486286640167, "epoch": 0.06607929515418502, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.1826362907886505, "fcm_dpo/q_t": 0.49544259905815125, "grad_norm": 24.359228134155273, "learning_rate": 3.188405797101449e-07, "logits/chosen": 0.6244760751724243, "logits/rejected": 0.7353973388671875, "logps/chosen": -77.47610473632812, "logps/ref_chosen": -77.34459686279297, "logps/ref_rejected": -84.24774169921875, "logps/rejected": -84.5618896484375, "loss": 1.3689, "margin_dpo/margin_mean": 0.18263721466064453, "margin_dpo/margin_std": 0.5607451796531677, "step": 45 }, { "KL/chosen_KL_mean": 0.00995635986328125, "KL/mean": -0.2442992627620697, "KL/rejected_KL_mean": -0.4985542297363281, "KL/std": 0.6066877841949463, "epoch": 0.06754772393538913, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.5085097551345825, "fcm_dpo/q_t": 0.4873088300228119, "grad_norm": 33.07052993774414, "learning_rate": 3.260869565217391e-07, "logits/chosen": 0.4818155765533447, "logits/rejected": 0.7236320972442627, "logps/chosen": -90.33087158203125, "logps/ref_chosen": -90.3408203125, "logps/ref_rejected": -100.676513671875, "logps/rejected": -101.1750717163086, "loss": 1.3372, "margin_dpo/margin_mean": 0.5085091590881348, "margin_dpo/margin_std": 0.6580162048339844, "step": 46 }, { "KL/chosen_KL_mean": -0.014850616455078125, "KL/mean": -0.22118790447711945, "KL/rejected_KL_mean": -0.42752838134765625, "KL/std": 0.5509482622146606, "epoch": 0.06901615271659324, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.41267693042755127, "fcm_dpo/q_t": 0.48970478773117065, "grad_norm": 33.9089241027832, "learning_rate": 3.333333333333333e-07, "logits/chosen": 0.5086853504180908, "logits/rejected": 0.6941236257553101, "logps/chosen": -104.4261474609375, "logps/ref_chosen": -104.41130065917969, "logps/ref_rejected": -101.25489807128906, "logps/rejected": -101.68242645263672, "loss": 1.3469, "margin_dpo/margin_mean": 0.41267627477645874, "margin_dpo/margin_std": 0.7492132186889648, "step": 47 }, { "KL/chosen_KL_mean": 0.053989410400390625, "KL/mean": -0.26624077558517456, "KL/rejected_KL_mean": -0.5864715576171875, "KL/std": 0.6990648508071899, "epoch": 0.07048458149779736, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.6404613256454468, "fcm_dpo/q_t": 0.4840337336063385, "grad_norm": 38.71017837524414, "learning_rate": 3.4057971014492755e-07, "logits/chosen": 0.18323175609111786, "logits/rejected": 0.3808895945549011, "logps/chosen": -91.04627990722656, "logps/ref_chosen": -91.10027313232422, "logps/ref_rejected": -95.08058166503906, "logps/rejected": -95.66705322265625, "loss": 1.3252, "margin_dpo/margin_mean": 0.6404622197151184, "margin_dpo/margin_std": 0.8738809823989868, "step": 48 }, { "KL/chosen_KL_mean": 0.06303024291992188, "KL/mean": -0.33870628476142883, "KL/rejected_KL_mean": -0.7404403686523438, "KL/std": 0.8242291212081909, "epoch": 0.07195301027900147, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.803463876247406, "fcm_dpo/q_t": 0.4799991548061371, "grad_norm": 39.60509490966797, "learning_rate": 3.478260869565217e-07, "logits/chosen": 0.1192077174782753, "logits/rejected": 0.5923252105712891, "logps/chosen": -92.9406509399414, "logps/ref_chosen": -93.00367736816406, "logps/ref_rejected": -91.74899291992188, "logps/rejected": -92.48944091796875, "loss": 1.3101, "margin_dpo/margin_mean": 0.8034640550613403, "margin_dpo/margin_std": 1.0122640132904053, "step": 49 }, { "KL/chosen_KL_mean": -0.08350372314453125, "KL/mean": -0.41640323400497437, "KL/rejected_KL_mean": -0.749298095703125, "KL/std": 0.731054425239563, "epoch": 0.07342143906020558, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.6657922863960266, "fcm_dpo/q_t": 0.4833972454071045, "grad_norm": 33.44935607910156, "learning_rate": 3.5507246376811595e-07, "logits/chosen": 0.5877135396003723, "logits/rejected": 0.7597104907035828, "logps/chosen": -94.71031188964844, "logps/ref_chosen": -94.62681579589844, "logps/ref_rejected": -103.57435607910156, "logps/rejected": -104.32365417480469, "loss": 1.3225, "margin_dpo/margin_mean": 0.6657923460006714, "margin_dpo/margin_std": 0.8076422214508057, "step": 50 }, { "KL/chosen_KL_mean": -0.07152175903320312, "KL/mean": -0.4052048623561859, "KL/rejected_KL_mean": -0.7388877868652344, "KL/std": 0.8468261957168579, "epoch": 0.07488986784140969, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.6673593521118164, "fcm_dpo/q_t": 0.48339325189590454, "grad_norm": 29.8863582611084, "learning_rate": 3.6231884057971015e-07, "logits/chosen": 0.5101909637451172, "logits/rejected": 0.6459024548530579, "logps/chosen": -87.57879638671875, "logps/ref_chosen": -87.50727844238281, "logps/ref_rejected": -83.47235107421875, "logps/rejected": -84.21124267578125, "loss": 1.3236, "margin_dpo/margin_mean": 0.6673594117164612, "margin_dpo/margin_std": 1.0721827745437622, "step": 51 }, { "KL/chosen_KL_mean": -0.052768707275390625, "KL/mean": -0.4799800217151642, "KL/rejected_KL_mean": -0.9071998596191406, "KL/std": 0.7781846523284912, "epoch": 0.0763582966226138, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.8544255495071411, "fcm_dpo/q_t": 0.4787142872810364, "grad_norm": 32.08740997314453, "learning_rate": 3.695652173913043e-07, "logits/chosen": 0.3984390199184418, "logits/rejected": 0.6092727184295654, "logps/chosen": -90.68302917480469, "logps/ref_chosen": -90.63026428222656, "logps/ref_rejected": -87.0390625, "logps/rejected": -87.94625854492188, "loss": 1.3046, "margin_dpo/margin_mean": 0.8544254302978516, "margin_dpo/margin_std": 0.8648147583007812, "step": 52 }, { "KL/chosen_KL_mean": -0.042057037353515625, "KL/mean": -0.5769430994987488, "KL/rejected_KL_mean": -1.1118316650390625, "KL/std": 1.0826855897903442, "epoch": 0.07782672540381791, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.0697749853134155, "fcm_dpo/q_t": 0.47352221608161926, "grad_norm": 39.484519958496094, "learning_rate": 3.7681159420289855e-07, "logits/chosen": -0.07025596499443054, "logits/rejected": 0.038687705993652344, "logps/chosen": -81.62513732910156, "logps/ref_chosen": -81.58307647705078, "logps/ref_rejected": -95.66152954101562, "logps/rejected": -96.77336120605469, "loss": 1.2871, "margin_dpo/margin_mean": 1.0697749853134155, "margin_dpo/margin_std": 1.3265681266784668, "step": 53 }, { "KL/chosen_KL_mean": -0.069305419921875, "KL/mean": -0.6200127005577087, "KL/rejected_KL_mean": -1.1707229614257812, "KL/std": 1.2855360507965088, "epoch": 0.07929515418502203, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.1014137268066406, "fcm_dpo/q_t": 0.47267693281173706, "grad_norm": 39.66839599609375, "learning_rate": 3.8405797101449274e-07, "logits/chosen": 0.3582454323768616, "logits/rejected": 0.536725640296936, "logps/chosen": -88.97947692871094, "logps/ref_chosen": -88.91017150878906, "logps/ref_rejected": -99.1175537109375, "logps/rejected": -100.28827667236328, "loss": 1.2836, "margin_dpo/margin_mean": 1.101413369178772, "margin_dpo/margin_std": 1.3248322010040283, "step": 54 }, { "KL/chosen_KL_mean": 0.15243911743164062, "KL/mean": -0.4502677917480469, "KL/rejected_KL_mean": -1.0529708862304688, "KL/std": 1.2290518283843994, "epoch": 0.08076358296622614, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.2054040431976318, "fcm_dpo/q_t": 0.4701889157295227, "grad_norm": 35.24320983886719, "learning_rate": 3.9130434782608694e-07, "logits/chosen": 0.19396163523197174, "logits/rejected": 0.5096533298492432, "logps/chosen": -92.30348205566406, "logps/ref_chosen": -92.45591735839844, "logps/ref_rejected": -92.97093200683594, "logps/rejected": -94.02391052246094, "loss": 1.2758, "margin_dpo/margin_mean": 1.2054039239883423, "margin_dpo/margin_std": 1.568965196609497, "step": 55 }, { "KL/chosen_KL_mean": -0.05075836181640625, "KL/mean": -0.5847801566123962, "KL/rejected_KL_mean": -1.1188011169433594, "KL/std": 1.2063649892807007, "epoch": 0.08223201174743025, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.0680392980575562, "fcm_dpo/q_t": 0.4736279249191284, "grad_norm": 33.231956481933594, "learning_rate": 3.9855072463768114e-07, "logits/chosen": 0.009950734674930573, "logits/rejected": 0.340020090341568, "logps/chosen": -87.28741455078125, "logps/ref_chosen": -87.23665618896484, "logps/ref_rejected": -100.64553833007812, "logps/rejected": -101.76434326171875, "loss": 1.289, "margin_dpo/margin_mean": 1.0680384635925293, "margin_dpo/margin_std": 1.6126031875610352, "step": 56 }, { "KL/chosen_KL_mean": -0.06392669677734375, "KL/mean": -0.612309455871582, "KL/rejected_KL_mean": -1.1606979370117188, "KL/std": 1.1600854396820068, "epoch": 0.08370044052863436, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.0967847108840942, "fcm_dpo/q_t": 0.4727925658226013, "grad_norm": 33.92351531982422, "learning_rate": 4.057971014492754e-07, "logits/chosen": 0.28535905480384827, "logits/rejected": 0.5003440976142883, "logps/chosen": -98.21466827392578, "logps/ref_chosen": -98.15074157714844, "logps/ref_rejected": -101.05284118652344, "logps/rejected": -102.21353149414062, "loss": 1.2848, "margin_dpo/margin_mean": 1.096785068511963, "margin_dpo/margin_std": 1.4413888454437256, "step": 57 }, { "KL/chosen_KL_mean": 0.2508087158203125, "KL/mean": -0.5253406763076782, "KL/rejected_KL_mean": -1.301483154296875, "KL/std": 1.470930814743042, "epoch": 0.08516886930983847, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.5522925853729248, "fcm_dpo/q_t": 0.46189388632774353, "grad_norm": 37.65719223022461, "learning_rate": 4.1304347826086954e-07, "logits/chosen": 0.3724132776260376, "logits/rejected": 0.7173783779144287, "logps/chosen": -99.33016204833984, "logps/ref_chosen": -99.58097076416016, "logps/ref_rejected": -91.22227478027344, "logps/rejected": -92.52375793457031, "loss": 1.2471, "margin_dpo/margin_mean": 1.5522925853729248, "margin_dpo/margin_std": 1.9381825923919678, "step": 58 }, { "KL/chosen_KL_mean": -0.010650634765625, "KL/mean": -0.6070826053619385, "KL/rejected_KL_mean": -1.2035102844238281, "KL/std": 1.1752407550811768, "epoch": 0.08663729809104258, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.192854404449463, "fcm_dpo/q_t": 0.47046053409576416, "grad_norm": 34.20338821411133, "learning_rate": 4.2028985507246374e-07, "logits/chosen": -0.03345263749361038, "logits/rejected": 0.14757108688354492, "logps/chosen": -89.81297302246094, "logps/ref_chosen": -89.80232238769531, "logps/ref_rejected": -94.27667236328125, "logps/rejected": -95.48018646240234, "loss": 1.2764, "margin_dpo/margin_mean": 1.1928536891937256, "margin_dpo/margin_std": 1.5170061588287354, "step": 59 }, { "KL/chosen_KL_mean": 0.07042312622070312, "KL/mean": -0.5525708794593811, "KL/rejected_KL_mean": -1.175567626953125, "KL/std": 1.2916783094406128, "epoch": 0.0881057268722467, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.245985746383667, "fcm_dpo/q_t": 0.469086229801178, "grad_norm": 28.587860107421875, "learning_rate": 4.2753623188405794e-07, "logits/chosen": 0.21156209707260132, "logits/rejected": 0.4694107472896576, "logps/chosen": -95.08528900146484, "logps/ref_chosen": -95.15571594238281, "logps/ref_rejected": -91.3724365234375, "logps/rejected": -92.54800415039062, "loss": 1.2706, "margin_dpo/margin_mean": 1.245985984802246, "margin_dpo/margin_std": 1.4123971462249756, "step": 60 }, { "KL/chosen_KL_mean": -0.194976806640625, "KL/mean": -0.7813370227813721, "KL/rejected_KL_mean": -1.3676986694335938, "KL/std": 1.4176148176193237, "epoch": 0.08957415565345081, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.1727262735366821, "fcm_dpo/q_t": 0.4712342917919159, "grad_norm": 28.317663192749023, "learning_rate": 4.3478260869565214e-07, "logits/chosen": 0.09041240066289902, "logits/rejected": 0.22927650809288025, "logps/chosen": -85.92728424072266, "logps/ref_chosen": -85.73230743408203, "logps/ref_rejected": -97.96575927734375, "logps/rejected": -99.33345794677734, "loss": 1.2813, "margin_dpo/margin_mean": 1.1727261543273926, "margin_dpo/margin_std": 1.838467001914978, "step": 61 }, { "KL/chosen_KL_mean": -0.1353302001953125, "KL/mean": -0.771751344203949, "KL/rejected_KL_mean": -1.4081687927246094, "KL/std": 1.4471325874328613, "epoch": 0.09104258443465492, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.2728437185287476, "fcm_dpo/q_t": 0.4685220718383789, "grad_norm": 28.620708465576172, "learning_rate": 4.420289855072464e-07, "logits/chosen": 0.23102353513240814, "logits/rejected": 0.4310774803161621, "logps/chosen": -81.77071380615234, "logps/ref_chosen": -81.63538360595703, "logps/ref_rejected": -84.03831481933594, "logps/rejected": -85.44647979736328, "loss": 1.2724, "margin_dpo/margin_mean": 1.272843360900879, "margin_dpo/margin_std": 1.8278840780258179, "step": 62 }, { "KL/chosen_KL_mean": -0.016002655029296875, "KL/mean": -0.8360881805419922, "KL/rejected_KL_mean": -1.6561737060546875, "KL/std": 1.404371976852417, "epoch": 0.09251101321585903, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.6401724815368652, "fcm_dpo/q_t": 0.4593884348869324, "grad_norm": 34.704429626464844, "learning_rate": 4.4927536231884053e-07, "logits/chosen": 0.5016651749610901, "logits/rejected": 0.7214891910552979, "logps/chosen": -103.64006042480469, "logps/ref_chosen": -103.62405395507812, "logps/ref_rejected": -103.40303039550781, "logps/rejected": -105.0592041015625, "loss": 1.2353, "margin_dpo/margin_mean": 1.6401718854904175, "margin_dpo/margin_std": 1.563086986541748, "step": 63 }, { "KL/chosen_KL_mean": -0.14995956420898438, "KL/mean": -1.0391714572906494, "KL/rejected_KL_mean": -1.9283905029296875, "KL/std": 1.9013859033584595, "epoch": 0.09397944199706314, "fcm_dpo/beta": 0.10091481357812881, "fcm_dpo/delta": 0.09106548875570297, "fcm_dpo/margin": 1.7784286737442017, "fcm_dpo/q_t": 0.45633113384246826, "grad_norm": 35.040748596191406, "learning_rate": 4.5652173913043473e-07, "logits/chosen": 0.6507229804992676, "logits/rejected": 0.8773005604743958, "logps/chosen": -87.15155029296875, "logps/ref_chosen": -87.0015869140625, "logps/ref_rejected": -100.5854721069336, "logps/rejected": -102.51386260986328, "loss": 1.2275, "margin_dpo/margin_mean": 1.778428554534912, "margin_dpo/margin_std": 2.1595983505249023, "step": 64 }, { "KL/chosen_KL_mean": -0.10332107543945312, "KL/mean": -0.9983298778533936, "KL/rejected_KL_mean": -1.8933448791503906, "KL/std": 1.6365978717803955, "epoch": 0.09544787077826726, "fcm_dpo/beta": 0.10091481357812881, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.7900145053863525, "fcm_dpo/q_t": 0.45561760663986206, "grad_norm": 38.56663513183594, "learning_rate": 4.63768115942029e-07, "logits/chosen": 0.10252522677183151, "logits/rejected": 0.24797284603118896, "logps/chosen": -91.32524108886719, "logps/ref_chosen": -91.22191619873047, "logps/ref_rejected": -115.33553314208984, "logps/rejected": -117.2288818359375, "loss": 1.2244, "margin_dpo/margin_mean": 1.7900141477584839, "margin_dpo/margin_std": 2.0485870838165283, "step": 65 }, { "KL/chosen_KL_mean": -0.2866096496582031, "KL/mean": -0.9195466041564941, "KL/rejected_KL_mean": -1.5524826049804688, "KL/std": 1.5795402526855469, "epoch": 0.09691629955947137, "fcm_dpo/beta": 0.10091481357812881, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.265866994857788, "fcm_dpo/q_t": 0.46858981251716614, "grad_norm": 25.785856246948242, "learning_rate": 4.7101449275362313e-07, "logits/chosen": 0.21335312724113464, "logits/rejected": 0.420886754989624, "logps/chosen": -84.07083129882812, "logps/ref_chosen": -83.78422546386719, "logps/ref_rejected": -82.7520980834961, "logps/rejected": -84.30458068847656, "loss": 1.2726, "margin_dpo/margin_mean": 1.2658671140670776, "margin_dpo/margin_std": 1.9339399337768555, "step": 66 }, { "KL/chosen_KL_mean": -0.8789558410644531, "KL/mean": -1.4693105220794678, "KL/rejected_KL_mean": -2.059661865234375, "KL/std": 2.0459136962890625, "epoch": 0.09838472834067548, "fcm_dpo/beta": 0.10091481357812881, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.1807036399841309, "fcm_dpo/q_t": 0.47112709283828735, "grad_norm": 24.621028900146484, "learning_rate": 4.782608695652174e-07, "logits/chosen": 0.3478488326072693, "logits/rejected": 0.6293653249740601, "logps/chosen": -88.55191802978516, "logps/ref_chosen": -87.67295837402344, "logps/ref_rejected": -79.08674621582031, "logps/rejected": -81.14640808105469, "loss": 1.2894, "margin_dpo/margin_mean": 1.1807037591934204, "margin_dpo/margin_std": 2.723046064376831, "step": 67 }, { "KL/chosen_KL_mean": -0.2891349792480469, "KL/mean": -1.2964884042739868, "KL/rejected_KL_mean": -2.3038368225097656, "KL/std": 2.195040702819824, "epoch": 0.09985315712187959, "fcm_dpo/beta": 0.10091481357812881, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 2.0147013664245605, "fcm_dpo/q_t": 0.45103344321250916, "grad_norm": 28.788509368896484, "learning_rate": 4.855072463768116e-07, "logits/chosen": -0.002613719552755356, "logits/rejected": 0.29605117440223694, "logps/chosen": -97.36798095703125, "logps/ref_chosen": -97.07884216308594, "logps/ref_rejected": -84.11872863769531, "logps/rejected": -86.42256164550781, "loss": 1.2123, "margin_dpo/margin_mean": 2.0147013664245605, "margin_dpo/margin_std": 2.7739837169647217, "step": 68 }, { "KL/chosen_KL_mean": -0.33010101318359375, "KL/mean": -1.4739819765090942, "KL/rejected_KL_mean": -2.6178627014160156, "KL/std": 2.517026662826538, "epoch": 0.1013215859030837, "fcm_dpo/beta": 0.1022500991821289, "fcm_dpo/delta": 0.16793254017829895, "fcm_dpo/margin": 2.2877681255340576, "fcm_dpo/q_t": 0.4444032311439514, "grad_norm": 30.719749450683594, "learning_rate": 4.927536231884058e-07, "logits/chosen": 0.3961479663848877, "logits/rejected": 0.5032382607460022, "logps/chosen": -86.04981231689453, "logps/ref_chosen": -85.71971130371094, "logps/ref_rejected": -109.48020935058594, "logps/rejected": -112.09806823730469, "loss": 1.1911, "margin_dpo/margin_mean": 2.2877681255340576, "margin_dpo/margin_std": 3.1567816734313965, "step": 69 }, { "KL/chosen_KL_mean": -0.5609512329101562, "KL/mean": -1.8873111009597778, "KL/rejected_KL_mean": -3.213672637939453, "KL/std": 2.824026584625244, "epoch": 0.1027900146842878, "fcm_dpo/beta": 0.10360650718212128, "fcm_dpo/delta": 0.1269008070230484, "fcm_dpo/margin": 2.6527225971221924, "fcm_dpo/q_t": 0.43533632159233093, "grad_norm": 29.7752685546875, "learning_rate": 5e-07, "logits/chosen": 0.20913109183311462, "logits/rejected": 0.38281166553497314, "logps/chosen": -95.5708999633789, "logps/ref_chosen": -95.00994873046875, "logps/ref_rejected": -96.21272277832031, "logps/rejected": -99.4263916015625, "loss": 1.1615, "margin_dpo/margin_mean": 2.6527228355407715, "margin_dpo/margin_std": 3.4347667694091797, "step": 70 }, { "KL/chosen_KL_mean": -0.10221099853515625, "KL/mean": -1.9368947744369507, "KL/rejected_KL_mean": -3.7715797424316406, "KL/std": 3.5194363594055176, "epoch": 0.10425844346549193, "fcm_dpo/beta": 0.10395423322916031, "fcm_dpo/delta": 0.01887689344584942, "fcm_dpo/margin": 3.669370651245117, "fcm_dpo/q_t": 0.41130685806274414, "grad_norm": 35.17735290527344, "learning_rate": 4.999967061337492e-07, "logits/chosen": -0.19604744017124176, "logits/rejected": 0.16736896336078644, "logps/chosen": -91.07955932617188, "logps/ref_chosen": -90.97735595703125, "logps/ref_rejected": -102.59103393554688, "logps/rejected": -106.36260986328125, "loss": 1.0806, "margin_dpo/margin_mean": 3.66936993598938, "margin_dpo/margin_std": 4.033270359039307, "step": 71 }, { "KL/chosen_KL_mean": -0.5927886962890625, "KL/mean": -2.0484261512756348, "KL/rejected_KL_mean": -3.5040664672851562, "KL/std": 2.9484262466430664, "epoch": 0.10572687224669604, "fcm_dpo/beta": 0.10483211278915405, "fcm_dpo/delta": 0.09631133079528809, "fcm_dpo/margin": 2.911280632019043, "fcm_dpo/q_t": 0.4275033473968506, "grad_norm": 30.39789390563965, "learning_rate": 4.999868246217933e-07, "logits/chosen": -0.17072129249572754, "logits/rejected": 0.153425931930542, "logps/chosen": -98.48431396484375, "logps/ref_chosen": -97.89152526855469, "logps/ref_rejected": -100.19171142578125, "logps/rejected": -103.69578552246094, "loss": 1.1339, "margin_dpo/margin_mean": 2.911280632019043, "margin_dpo/margin_std": 3.363623857498169, "step": 72 }, { "KL/chosen_KL_mean": -0.774932861328125, "KL/mean": -2.313994884490967, "KL/rejected_KL_mean": -3.8530616760253906, "KL/std": 3.796140670776367, "epoch": 0.10719530102790015, "fcm_dpo/beta": 0.10574303567409515, "fcm_dpo/delta": 0.07573653757572174, "fcm_dpo/margin": 3.0781307220458984, "fcm_dpo/q_t": 0.4269424080848694, "grad_norm": 28.92380142211914, "learning_rate": 4.999703557245192e-07, "logits/chosen": -0.32564064860343933, "logits/rejected": -0.019538961350917816, "logps/chosen": -96.54395294189453, "logps/ref_chosen": -95.7690200805664, "logps/ref_rejected": -95.93243408203125, "logps/rejected": -99.7854995727539, "loss": 1.148, "margin_dpo/margin_mean": 3.0781302452087402, "margin_dpo/margin_std": 4.89613151550293, "step": 73 }, { "KL/chosen_KL_mean": -1.0813827514648438, "KL/mean": -2.8240084648132324, "KL/rejected_KL_mean": -4.566627502441406, "KL/std": 3.7865257263183594, "epoch": 0.10866372980910426, "fcm_dpo/beta": 0.10610372573137283, "fcm_dpo/delta": 0.03077266737818718, "fcm_dpo/margin": 3.485240936279297, "fcm_dpo/q_t": 0.41483768820762634, "grad_norm": 32.126708984375, "learning_rate": 4.999472998758977e-07, "logits/chosen": -0.08175422996282578, "logits/rejected": 0.06209857016801834, "logps/chosen": -79.88977813720703, "logps/ref_chosen": -78.80839538574219, "logps/ref_rejected": -101.64677429199219, "logps/rejected": -106.21339416503906, "loss": 1.1028, "margin_dpo/margin_mean": 3.485240936279297, "margin_dpo/margin_std": 4.4861273765563965, "step": 74 }, { "KL/chosen_KL_mean": -1.0846595764160156, "KL/mean": -3.354384422302246, "KL/rejected_KL_mean": -5.624114990234375, "KL/std": 4.560100078582764, "epoch": 0.11013215859030837, "fcm_dpo/beta": 0.10583330690860748, "fcm_dpo/delta": -0.08240952342748642, "fcm_dpo/margin": 4.539459705352783, "fcm_dpo/q_t": 0.3928443193435669, "grad_norm": 33.178955078125, "learning_rate": 4.999176576834721e-07, "logits/chosen": -0.26280421018600464, "logits/rejected": -0.06077182665467262, "logps/chosen": -79.36651611328125, "logps/ref_chosen": -78.28185272216797, "logps/ref_rejected": -115.40311431884766, "logps/rejected": -121.02722930908203, "loss": 1.0345, "margin_dpo/margin_mean": 4.539460182189941, "margin_dpo/margin_std": 5.392035007476807, "step": 75 }, { "KL/chosen_KL_mean": -1.5043144226074219, "KL/mean": -3.0296833515167236, "KL/rejected_KL_mean": -4.555049896240234, "KL/std": 3.8164029121398926, "epoch": 0.11160058737151249, "fcm_dpo/beta": 0.10599271208047867, "fcm_dpo/delta": 0.07791441679000854, "fcm_dpo/margin": 3.05073618888855, "fcm_dpo/q_t": 0.42637109756469727, "grad_norm": 23.96317481994629, "learning_rate": 4.998814299283415e-07, "logits/chosen": -0.4992431402206421, "logits/rejected": -0.11019206047058105, "logps/chosen": -89.38145446777344, "logps/ref_chosen": -87.87713623046875, "logps/ref_rejected": -85.71968078613281, "logps/rejected": -90.27473449707031, "loss": 1.1435, "margin_dpo/margin_mean": 3.0507359504699707, "margin_dpo/margin_std": 4.603917121887207, "step": 76 }, { "KL/chosen_KL_mean": -1.5969352722167969, "KL/mean": -3.3078389167785645, "KL/rejected_KL_mean": -5.018741607666016, "KL/std": 4.51671028137207, "epoch": 0.1130690161527166, "fcm_dpo/beta": 0.10639140754938126, "fcm_dpo/delta": 0.0366068109869957, "fcm_dpo/margin": 3.421811819076538, "fcm_dpo/q_t": 0.4167407155036926, "grad_norm": 28.361101150512695, "learning_rate": 4.998386175651409e-07, "logits/chosen": -0.24063727259635925, "logits/rejected": 0.12350298464298248, "logps/chosen": -101.29728698730469, "logps/ref_chosen": -99.70034790039062, "logps/ref_rejected": -98.20576477050781, "logps/rejected": -103.22450256347656, "loss": 1.1246, "margin_dpo/margin_mean": 3.42181134223938, "margin_dpo/margin_std": 5.1160993576049805, "step": 77 }, { "KL/chosen_KL_mean": -0.6455001831054688, "KL/mean": -2.507826566696167, "KL/rejected_KL_mean": -4.370151519775391, "KL/std": 3.7427563667297363, "epoch": 0.1145374449339207, "fcm_dpo/beta": 0.10662194341421127, "fcm_dpo/delta": 0.002928401343524456, "fcm_dpo/margin": 3.7246530055999756, "fcm_dpo/q_t": 0.4083814322948456, "grad_norm": 26.000186920166016, "learning_rate": 4.997892217220159e-07, "logits/chosen": 0.061221569776535034, "logits/rejected": 0.2950375974178314, "logps/chosen": -90.94221496582031, "logps/ref_chosen": -90.29670715332031, "logps/ref_rejected": -91.13772583007812, "logps/rejected": -95.50787353515625, "loss": 1.0785, "margin_dpo/margin_mean": 3.7246532440185547, "margin_dpo/margin_std": 4.426640510559082, "step": 78 }, { "KL/chosen_KL_mean": -0.8708000183105469, "KL/mean": -3.269526720046997, "KL/rejected_KL_mean": -5.6682586669921875, "KL/std": 5.387819290161133, "epoch": 0.11600587371512482, "fcm_dpo/beta": 0.10585636645555496, "fcm_dpo/delta": -0.11058422923088074, "fcm_dpo/margin": 4.797456741333008, "fcm_dpo/q_t": 0.38731497526168823, "grad_norm": 27.461631774902344, "learning_rate": 4.997332437005931e-07, "logits/chosen": -0.4942951798439026, "logits/rejected": -0.10180403292179108, "logps/chosen": -87.24913024902344, "logps/ref_chosen": -86.37832641601562, "logps/ref_rejected": -94.10777282714844, "logps/rejected": -99.77603149414062, "loss": 1.0262, "margin_dpo/margin_mean": 4.79745626449585, "margin_dpo/margin_std": 5.924061298370361, "step": 79 }, { "KL/chosen_KL_mean": -1.5459213256835938, "KL/mean": -3.4492573738098145, "KL/rejected_KL_mean": -5.35260009765625, "KL/std": 4.738081932067871, "epoch": 0.11747430249632893, "fcm_dpo/beta": 0.10541613399982452, "fcm_dpo/delta": -0.0013172058388590813, "fcm_dpo/margin": 3.8066768646240234, "fcm_dpo/q_t": 0.41029495000839233, "grad_norm": 26.267602920532227, "learning_rate": 4.996706849759452e-07, "logits/chosen": -0.591657280921936, "logits/rejected": -0.2247426062822342, "logps/chosen": -95.5162353515625, "logps/ref_chosen": -93.97032165527344, "logps/ref_rejected": -92.57441711425781, "logps/rejected": -97.92701721191406, "loss": 1.0923, "margin_dpo/margin_mean": 3.8066771030426025, "margin_dpo/margin_std": 5.206352233886719, "step": 80 }, { "KL/chosen_KL_mean": -1.0616722106933594, "KL/mean": -3.7419490814208984, "KL/rejected_KL_mean": -6.422214508056641, "KL/std": 5.828679084777832, "epoch": 0.11894273127753303, "fcm_dpo/beta": 0.1038893461227417, "fcm_dpo/delta": -0.16217456758022308, "fcm_dpo/margin": 5.360544681549072, "fcm_dpo/q_t": 0.38168466091156006, "grad_norm": 28.045499801635742, "learning_rate": 4.996015471965529e-07, "logits/chosen": -0.6947442293167114, "logits/rejected": -0.2753222584724426, "logps/chosen": -100.89179992675781, "logps/ref_chosen": -99.83012390136719, "logps/ref_rejected": -133.67247009277344, "logps/rejected": -140.0946807861328, "loss": 1.0107, "margin_dpo/margin_mean": 5.360545635223389, "margin_dpo/margin_std": 6.830893516540527, "step": 81 }, { "KL/chosen_KL_mean": -1.967254638671875, "KL/mean": -3.8327736854553223, "KL/rejected_KL_mean": -5.698291778564453, "KL/std": 4.645172595977783, "epoch": 0.12041116005873716, "fcm_dpo/beta": 0.10416808724403381, "fcm_dpo/delta": 0.0111357681453228, "fcm_dpo/margin": 3.7310373783111572, "fcm_dpo/q_t": 0.41196805238723755, "grad_norm": 26.52179718017578, "learning_rate": 4.995258321842611e-07, "logits/chosen": -0.09177055209875107, "logits/rejected": 0.1289501190185547, "logps/chosen": -85.01323699951172, "logps/ref_chosen": -83.04598236083984, "logps/ref_rejected": -94.52595520019531, "logps/rejected": -100.2242431640625, "loss": 1.1039, "margin_dpo/margin_mean": 3.731037139892578, "margin_dpo/margin_std": 5.140506744384766, "step": 82 }, { "KL/chosen_KL_mean": -2.4234390258789062, "KL/mean": -4.465487480163574, "KL/rejected_KL_mean": -6.507537841796875, "KL/std": 5.202995300292969, "epoch": 0.12187958883994127, "fcm_dpo/beta": 0.10365696251392365, "fcm_dpo/delta": -0.0238596573472023, "fcm_dpo/margin": 4.084100246429443, "fcm_dpo/q_t": 0.4082214832305908, "grad_norm": 28.356550216674805, "learning_rate": 4.994435419342304e-07, "logits/chosen": -0.5093056559562683, "logits/rejected": -0.0835685282945633, "logps/chosen": -94.59965515136719, "logps/ref_chosen": -92.17621612548828, "logps/ref_rejected": -107.74464416503906, "logps/rejected": -114.25218200683594, "loss": 1.114, "margin_dpo/margin_mean": 4.084100723266602, "margin_dpo/margin_std": 6.6873674392700195, "step": 83 }, { "KL/chosen_KL_mean": -2.9660682678222656, "KL/mean": -4.26534366607666, "KL/rejected_KL_mean": -5.564628601074219, "KL/std": 4.639596939086914, "epoch": 0.12334801762114538, "fcm_dpo/beta": 0.10344299674034119, "fcm_dpo/delta": -0.017332036048173904, "fcm_dpo/margin": 2.598557949066162, "fcm_dpo/q_t": 0.43926888704299927, "grad_norm": 29.824649810791016, "learning_rate": 4.993546786148857e-07, "logits/chosen": -0.4992828965187073, "logits/rejected": -0.11458448320627213, "logps/chosen": -104.49256134033203, "logps/ref_chosen": -101.5264892578125, "logps/ref_rejected": -92.42608642578125, "logps/rejected": -97.99071502685547, "loss": 1.2035, "margin_dpo/margin_mean": 2.5985586643218994, "margin_dpo/margin_std": 4.967041969299316, "step": 84 }, { "KL/chosen_KL_mean": -2.628711700439453, "KL/mean": -4.383569717407227, "KL/rejected_KL_mean": -6.138431549072266, "KL/std": 5.105294227600098, "epoch": 0.12481644640234948, "fcm_dpo/beta": 0.10396669805049896, "fcm_dpo/delta": 0.03544096648693085, "fcm_dpo/margin": 3.509721040725708, "fcm_dpo/q_t": 0.418870210647583, "grad_norm": 27.975391387939453, "learning_rate": 4.992592445678582e-07, "logits/chosen": -0.33947592973709106, "logits/rejected": -0.005286738276481628, "logps/chosen": -98.75608825683594, "logps/ref_chosen": -96.12738037109375, "logps/ref_rejected": -85.05519104003906, "logps/rejected": -91.19361877441406, "loss": 1.1498, "margin_dpo/margin_mean": 3.509720802307129, "margin_dpo/margin_std": 6.147680759429932, "step": 85 }, { "KL/chosen_KL_mean": -2.0420303344726562, "KL/mean": -4.219537734985352, "KL/rejected_KL_mean": -6.3970489501953125, "KL/std": 5.82932186126709, "epoch": 0.1262848751835536, "fcm_dpo/beta": 0.10334603488445282, "fcm_dpo/delta": -0.05127780884504318, "fcm_dpo/margin": 4.355024337768555, "fcm_dpo/q_t": 0.40677374601364136, "grad_norm": 22.87744140625, "learning_rate": 4.991572423079235e-07, "logits/chosen": -0.3594822287559509, "logits/rejected": -0.10969389975070953, "logps/chosen": -83.74629211425781, "logps/ref_chosen": -81.70426177978516, "logps/ref_rejected": -93.65550231933594, "logps/rejected": -100.05255126953125, "loss": 1.1017, "margin_dpo/margin_mean": 4.355024337768555, "margin_dpo/margin_std": 7.25318717956543, "step": 86 }, { "KL/chosen_KL_mean": -2.446044921875, "KL/mean": -4.826696872711182, "KL/rejected_KL_mean": -7.207344055175781, "KL/std": 6.524242401123047, "epoch": 0.1277533039647577, "fcm_dpo/beta": 0.10246125608682632, "fcm_dpo/delta": -0.09021423757076263, "fcm_dpo/margin": 4.761297225952148, "fcm_dpo/q_t": 0.3927533030509949, "grad_norm": 27.681407928466797, "learning_rate": 4.990486745229364e-07, "logits/chosen": -0.9670562744140625, "logits/rejected": -0.600102961063385, "logps/chosen": -95.13201141357422, "logps/ref_chosen": -92.68596649169922, "logps/ref_rejected": -102.91818237304688, "logps/rejected": -110.12551879882812, "loss": 1.092, "margin_dpo/margin_mean": 4.761298179626465, "margin_dpo/margin_std": 7.676843643188477, "step": 87 }, { "KL/chosen_KL_mean": -2.8869285583496094, "KL/mean": -4.812328338623047, "KL/rejected_KL_mean": -6.7377166748046875, "KL/std": 6.324530601501465, "epoch": 0.12922173274596183, "fcm_dpo/beta": 0.1020563393831253, "fcm_dpo/delta": 0.006699658930301666, "fcm_dpo/margin": 3.850785255432129, "fcm_dpo/q_t": 0.4147062301635742, "grad_norm": 25.180274963378906, "learning_rate": 4.989335440737586e-07, "logits/chosen": -0.20077624917030334, "logits/rejected": -0.010758977383375168, "logps/chosen": -103.64990997314453, "logps/ref_chosen": -100.76298522949219, "logps/ref_rejected": -113.15037536621094, "logps/rejected": -119.88809204101562, "loss": 1.1368, "margin_dpo/margin_mean": 3.850785255432129, "margin_dpo/margin_std": 6.611970901489258, "step": 88 }, { "KL/chosen_KL_mean": -2.3459930419921875, "KL/mean": -4.444387435913086, "KL/rejected_KL_mean": -6.542781829833984, "KL/std": 5.318692207336426, "epoch": 0.13069016152716592, "fcm_dpo/beta": 0.10206518322229385, "fcm_dpo/delta": -0.02907172590494156, "fcm_dpo/margin": 4.196779251098633, "fcm_dpo/q_t": 0.4096394181251526, "grad_norm": 24.131677627563477, "learning_rate": 4.988118539941847e-07, "logits/chosen": -0.5229376554489136, "logits/rejected": -0.17269718647003174, "logps/chosen": -92.03707885742188, "logps/ref_chosen": -89.69108581542969, "logps/ref_rejected": -88.56832885742188, "logps/rejected": -95.11111450195312, "loss": 1.106, "margin_dpo/margin_mean": 4.196779251098633, "margin_dpo/margin_std": 6.996875286102295, "step": 89 }, { "KL/chosen_KL_mean": -2.174266815185547, "KL/mean": -4.874780654907227, "KL/rejected_KL_mean": -7.575290679931641, "KL/std": 6.236407279968262, "epoch": 0.13215859030837004, "fcm_dpo/beta": 0.10118772089481354, "fcm_dpo/delta": -0.1504947543144226, "fcm_dpo/margin": 5.401022434234619, "fcm_dpo/q_t": 0.38438451290130615, "grad_norm": 26.3984432220459, "learning_rate": 4.986836074908615e-07, "logits/chosen": -0.5396679639816284, "logits/rejected": -0.41308701038360596, "logps/chosen": -83.55682373046875, "logps/ref_chosen": -81.38255310058594, "logps/ref_rejected": -117.77714538574219, "logps/rejected": -125.35243225097656, "loss": 1.0342, "margin_dpo/margin_mean": 5.401022434234619, "margin_dpo/margin_std": 7.518261909484863, "step": 90 }, { "KL/chosen_KL_mean": -2.4635009765625, "KL/mean": -4.7295145988464355, "KL/rejected_KL_mean": -6.9955291748046875, "KL/std": 6.072656631469727, "epoch": 0.13362701908957417, "fcm_dpo/beta": 0.10020003467798233, "fcm_dpo/delta": -0.05535896122455597, "fcm_dpo/margin": 4.532036781311035, "fcm_dpo/q_t": 0.40370216965675354, "grad_norm": 26.665502548217773, "learning_rate": 4.985488079432037e-07, "logits/chosen": -0.51636803150177, "logits/rejected": -0.11353670805692673, "logps/chosen": -99.68537902832031, "logps/ref_chosen": -97.22187805175781, "logps/ref_rejected": -92.97674560546875, "logps/rejected": -99.97227478027344, "loss": 1.089, "margin_dpo/margin_mean": 4.532036781311035, "margin_dpo/margin_std": 7.05333948135376, "step": 91 }, { "KL/chosen_KL_mean": -2.6842422485351562, "KL/mean": -4.686981201171875, "KL/rejected_KL_mean": -6.689720153808594, "KL/std": 6.0587615966796875, "epoch": 0.13509544787077826, "fcm_dpo/beta": 0.10010424256324768, "fcm_dpo/delta": -0.0010038092732429504, "fcm_dpo/margin": 4.005477428436279, "fcm_dpo/q_t": 0.4130065143108368, "grad_norm": 23.575939178466797, "learning_rate": 4.984074589033043e-07, "logits/chosen": -0.6186103820800781, "logits/rejected": -0.2355697751045227, "logps/chosen": -87.21452331542969, "logps/ref_chosen": -84.5302734375, "logps/ref_rejected": -84.5013198852539, "logps/rejected": -91.1910400390625, "loss": 1.1182, "margin_dpo/margin_mean": 4.005477428436279, "margin_dpo/margin_std": 6.5184855461120605, "step": 92 }, { "KL/chosen_KL_mean": -2.9597320556640625, "KL/mean": -4.883044719696045, "KL/rejected_KL_mean": -6.806362152099609, "KL/std": 5.744448661804199, "epoch": 0.13656387665198239, "fcm_dpo/beta": 0.10023744404315948, "fcm_dpo/delta": 0.014656535349786282, "fcm_dpo/margin": 3.8466296195983887, "fcm_dpo/q_t": 0.4162563383579254, "grad_norm": 27.524620056152344, "learning_rate": 4.982595640958425e-07, "logits/chosen": -0.8400024771690369, "logits/rejected": -0.2551325857639313, "logps/chosen": -93.21016693115234, "logps/ref_chosen": -90.25043487548828, "logps/ref_rejected": -84.0942153930664, "logps/rejected": -90.90057373046875, "loss": 1.1504, "margin_dpo/margin_mean": 3.8466293811798096, "margin_dpo/margin_std": 7.025146007537842, "step": 93 }, { "KL/chosen_KL_mean": -3.051441192626953, "KL/mean": -5.944498538970947, "KL/rejected_KL_mean": -8.837554931640625, "KL/std": 6.747977256774902, "epoch": 0.13803230543318648, "fcm_dpo/beta": 0.09844072163105011, "fcm_dpo/delta": -0.17609989643096924, "fcm_dpo/margin": 5.786115646362305, "fcm_dpo/q_t": 0.3771990239620209, "grad_norm": 26.635517120361328, "learning_rate": 4.98105127417984e-07, "logits/chosen": -0.7531778812408447, "logits/rejected": -0.45261555910110474, "logps/chosen": -95.50566101074219, "logps/ref_chosen": -92.4542236328125, "logps/ref_rejected": -105.2472915649414, "logps/rejected": -114.08484649658203, "loss": 1.0277, "margin_dpo/margin_mean": 5.7861151695251465, "margin_dpo/margin_std": 7.817741394042969, "step": 94 }, { "KL/chosen_KL_mean": -2.452472686767578, "KL/mean": -4.749147415161133, "KL/rejected_KL_mean": -7.045818328857422, "KL/std": 5.657369613647461, "epoch": 0.1395007342143906, "fcm_dpo/beta": 0.09824827313423157, "fcm_dpo/delta": -0.052536968141794205, "fcm_dpo/margin": 4.593348026275635, "fcm_dpo/q_t": 0.40398532152175903, "grad_norm": 22.408796310424805, "learning_rate": 4.979441529392784e-07, "logits/chosen": -0.8851953744888306, "logits/rejected": -0.5490449666976929, "logps/chosen": -81.32617950439453, "logps/ref_chosen": -78.87370300292969, "logps/ref_rejected": -83.59121704101562, "logps/rejected": -90.63703155517578, "loss": 1.0906, "margin_dpo/margin_mean": 4.593347549438477, "margin_dpo/margin_std": 7.032950401306152, "step": 95 }, { "KL/chosen_KL_mean": -2.179168701171875, "KL/mean": -4.9832258224487305, "KL/rejected_KL_mean": -7.787284851074219, "KL/std": 6.742409706115723, "epoch": 0.14096916299559473, "fcm_dpo/beta": 0.09673945605754852, "fcm_dpo/delta": -0.146717369556427, "fcm_dpo/margin": 5.608129024505615, "fcm_dpo/q_t": 0.3837338984012604, "grad_norm": 23.041353225708008, "learning_rate": 4.977766449015534e-07, "logits/chosen": -0.7213720679283142, "logits/rejected": -0.27590179443359375, "logps/chosen": -108.77133178710938, "logps/ref_chosen": -106.5921630859375, "logps/ref_rejected": -101.76802825927734, "logps/rejected": -109.55531311035156, "loss": 1.0295, "margin_dpo/margin_mean": 5.608128547668457, "margin_dpo/margin_std": 7.449648857116699, "step": 96 }, { "KL/chosen_KL_mean": -3.1914710998535156, "KL/mean": -5.713850975036621, "KL/rejected_KL_mean": -8.236225128173828, "KL/std": 5.867135047912598, "epoch": 0.14243759177679882, "fcm_dpo/beta": 0.09624665975570679, "fcm_dpo/delta": -0.08797576278448105, "fcm_dpo/margin": 5.044759273529053, "fcm_dpo/q_t": 0.3921036422252655, "grad_norm": 26.164541244506836, "learning_rate": 4.976026077188012e-07, "logits/chosen": -0.6073750257492065, "logits/rejected": -0.21226121485233307, "logps/chosen": -91.87135314941406, "logps/ref_chosen": -88.67988586425781, "logps/ref_rejected": -84.81229400634766, "logps/rejected": -93.04852294921875, "loss": 1.0451, "margin_dpo/margin_mean": 5.044759750366211, "margin_dpo/margin_std": 6.2583441734313965, "step": 97 }, { "KL/chosen_KL_mean": -2.384429931640625, "KL/mean": -5.292548179626465, "KL/rejected_KL_mean": -8.200664520263672, "KL/std": 5.67898416519165, "epoch": 0.14390602055800295, "fcm_dpo/beta": 0.09430339932441711, "fcm_dpo/delta": -0.15337209403514862, "fcm_dpo/margin": 5.816230773925781, "fcm_dpo/q_t": 0.3783724009990692, "grad_norm": 23.957752227783203, "learning_rate": 4.974220459770639e-07, "logits/chosen": -0.6470411419868469, "logits/rejected": -0.38252896070480347, "logps/chosen": -94.62692260742188, "logps/ref_chosen": -92.24249267578125, "logps/ref_rejected": -101.51948547363281, "logps/rejected": -109.72015380859375, "loss": 0.9928, "margin_dpo/margin_mean": 5.816230773925781, "margin_dpo/margin_std": 6.394502639770508, "step": 98 }, { "KL/chosen_KL_mean": -1.8036651611328125, "KL/mean": -5.0089640617370605, "KL/rejected_KL_mean": -8.214271545410156, "KL/std": 6.808899879455566, "epoch": 0.14537444933920704, "fcm_dpo/beta": 0.0927506685256958, "fcm_dpo/delta": -0.20063996315002441, "fcm_dpo/margin": 6.410608291625977, "fcm_dpo/q_t": 0.3782620429992676, "grad_norm": 23.110204696655273, "learning_rate": 4.972349644343108e-07, "logits/chosen": -0.5918841361999512, "logits/rejected": -0.37998056411743164, "logps/chosen": -73.98831176757812, "logps/ref_chosen": -72.18464660644531, "logps/ref_rejected": -91.88131713867188, "logps/rejected": -100.09559631347656, "loss": 1.001, "margin_dpo/margin_mean": 6.410608291625977, "margin_dpo/margin_std": 8.656423568725586, "step": 99 }, { "KL/chosen_KL_mean": -2.1305465698242188, "KL/mean": -4.4404144287109375, "KL/rejected_KL_mean": -6.750282287597656, "KL/std": 5.1163201332092285, "epoch": 0.14684287812041116, "fcm_dpo/beta": 0.09197448194026947, "fcm_dpo/delta": -0.025614582002162933, "fcm_dpo/margin": 4.6197285652160645, "fcm_dpo/q_t": 0.40307557582855225, "grad_norm": 20.810955047607422, "learning_rate": 4.970413680203148e-07, "logits/chosen": -0.5232473611831665, "logits/rejected": -0.024309255182743073, "logps/chosen": -91.64437866210938, "logps/ref_chosen": -89.51382446289062, "logps/ref_rejected": -81.21713256835938, "logps/rejected": -87.9674072265625, "loss": 1.0647, "margin_dpo/margin_mean": 4.6197285652160645, "margin_dpo/margin_std": 5.532007217407227, "step": 100 }, { "KL/chosen_KL_mean": -3.430400848388672, "KL/mean": -5.466066360473633, "KL/rejected_KL_mean": -7.501731872558594, "KL/std": 6.601434707641602, "epoch": 0.14831130690161526, "fcm_dpo/beta": 0.09205891937017441, "fcm_dpo/delta": 0.025547703728079796, "fcm_dpo/margin": 4.071328163146973, "fcm_dpo/q_t": 0.41568803787231445, "grad_norm": 25.63942527770996, "learning_rate": 4.968412618365215e-07, "logits/chosen": -0.9160046577453613, "logits/rejected": -0.65730881690979, "logps/chosen": -97.8335952758789, "logps/ref_chosen": -94.4031982421875, "logps/ref_rejected": -89.91001892089844, "logps/rejected": -97.4117431640625, "loss": 1.145, "margin_dpo/margin_mean": 4.071328639984131, "margin_dpo/margin_std": 7.069766998291016, "step": 101 }, { "KL/chosen_KL_mean": -3.951282501220703, "KL/mean": -5.729787826538086, "KL/rejected_KL_mean": -7.508289337158203, "KL/std": 6.338902473449707, "epoch": 0.14977973568281938, "fcm_dpo/beta": 0.09206026792526245, "fcm_dpo/delta": -0.0555599108338356, "fcm_dpo/margin": 3.557008743286133, "fcm_dpo/q_t": 0.42690375447273254, "grad_norm": 31.55438995361328, "learning_rate": 4.966346511559149e-07, "logits/chosen": -0.8539567589759827, "logits/rejected": -0.4901812672615051, "logps/chosen": -95.56070709228516, "logps/ref_chosen": -91.60942077636719, "logps/ref_rejected": -75.7689208984375, "logps/rejected": -83.27720642089844, "loss": 1.1896, "margin_dpo/margin_mean": 3.557008981704712, "margin_dpo/margin_std": 7.183685779571533, "step": 102 }, { "KL/chosen_KL_mean": -3.019145965576172, "KL/mean": -6.016874313354492, "KL/rejected_KL_mean": -9.014602661132812, "KL/std": 6.7016448974609375, "epoch": 0.1512481644640235, "fcm_dpo/beta": 0.09072312712669373, "fcm_dpo/delta": -0.14809420704841614, "fcm_dpo/margin": 5.995456695556641, "fcm_dpo/q_t": 0.37926697731018066, "grad_norm": 24.419452667236328, "learning_rate": 4.964215414228785e-07, "logits/chosen": -0.8054560422897339, "logits/rejected": -0.41973745822906494, "logps/chosen": -100.4215316772461, "logps/ref_chosen": -97.40238952636719, "logps/ref_rejected": -98.981689453125, "logps/rejected": -107.99629211425781, "loss": 1.0202, "margin_dpo/margin_mean": 5.995457649230957, "margin_dpo/margin_std": 7.514443397521973, "step": 103 }, { "KL/chosen_KL_mean": -2.2058448791503906, "KL/mean": -5.464951038360596, "KL/rejected_KL_mean": -8.724052429199219, "KL/std": 7.273146629333496, "epoch": 0.1527165932452276, "fcm_dpo/beta": 0.0891357883810997, "fcm_dpo/delta": -0.18659019470214844, "fcm_dpo/margin": 6.51821231842041, "fcm_dpo/q_t": 0.3779059648513794, "grad_norm": 20.173734664916992, "learning_rate": 4.96201938253052e-07, "logits/chosen": -1.0304596424102783, "logits/rejected": -0.6911474466323853, "logps/chosen": -97.06221008300781, "logps/ref_chosen": -94.85636138916016, "logps/ref_rejected": -97.02669525146484, "logps/rejected": -105.75074768066406, "loss": 1.0009, "margin_dpo/margin_mean": 6.518212795257568, "margin_dpo/margin_std": 8.57598876953125, "step": 104 }, { "KL/chosen_KL_mean": -3.0236968994140625, "KL/mean": -6.25477409362793, "KL/rejected_KL_mean": -9.485855102539062, "KL/std": 6.6038665771484375, "epoch": 0.15418502202643172, "fcm_dpo/beta": 0.08787080645561218, "fcm_dpo/delta": -0.17256246507167816, "fcm_dpo/margin": 6.462156772613525, "fcm_dpo/q_t": 0.37594395875930786, "grad_norm": 23.62158966064453, "learning_rate": 4.959758474331832e-07, "logits/chosen": -0.9158233404159546, "logits/rejected": -0.44189178943634033, "logps/chosen": -99.78160095214844, "logps/ref_chosen": -96.75790405273438, "logps/ref_rejected": -101.5774917602539, "logps/rejected": -111.06334686279297, "loss": 1.0109, "margin_dpo/margin_mean": 6.462157249450684, "margin_dpo/margin_std": 8.205889701843262, "step": 105 }, { "KL/chosen_KL_mean": -2.996379852294922, "KL/mean": -6.073016166687012, "KL/rejected_KL_mean": -9.149650573730469, "KL/std": 6.815079689025879, "epoch": 0.15565345080763582, "fcm_dpo/beta": 0.08653544634580612, "fcm_dpo/delta": -0.1359855979681015, "fcm_dpo/margin": 6.153269290924072, "fcm_dpo/q_t": 0.38602960109710693, "grad_norm": 21.566967010498047, "learning_rate": 4.957432749209755e-07, "logits/chosen": -0.7674145102500916, "logits/rejected": -0.3388713598251343, "logps/chosen": -95.43428802490234, "logps/ref_chosen": -92.43791198730469, "logps/ref_rejected": -91.00970458984375, "logps/rejected": -100.15934753417969, "loss": 1.0312, "margin_dpo/margin_mean": 6.153268814086914, "margin_dpo/margin_std": 8.230663299560547, "step": 106 }, { "KL/chosen_KL_mean": -2.1653709411621094, "KL/mean": -5.71470832824707, "KL/rejected_KL_mean": -9.264049530029297, "KL/std": 8.664938926696777, "epoch": 0.15712187958883994, "fcm_dpo/beta": 0.08475795388221741, "fcm_dpo/delta": -0.20793211460113525, "fcm_dpo/margin": 7.098676681518555, "fcm_dpo/q_t": 0.3809083104133606, "grad_norm": 21.78976058959961, "learning_rate": 4.955042268449307e-07, "logits/chosen": -0.6691031455993652, "logits/rejected": -0.14687854051589966, "logps/chosen": -107.29037475585938, "logps/ref_chosen": -105.12500762939453, "logps/ref_rejected": -101.1824951171875, "logps/rejected": -110.44654846191406, "loss": 1.0201, "margin_dpo/margin_mean": 7.098676681518555, "margin_dpo/margin_std": 10.808680534362793, "step": 107 }, { "KL/chosen_KL_mean": -3.2004547119140625, "KL/mean": -6.492118835449219, "KL/rejected_KL_mean": -9.78378677368164, "KL/std": 7.806471824645996, "epoch": 0.15859030837004406, "fcm_dpo/beta": 0.08315321803092957, "fcm_dpo/delta": -0.15220709145069122, "fcm_dpo/margin": 6.583327293395996, "fcm_dpo/q_t": 0.38251468539237976, "grad_norm": 21.383575439453125, "learning_rate": 4.952587095041881e-07, "logits/chosen": -0.9045896530151367, "logits/rejected": -0.39962753653526306, "logps/chosen": -92.40982055664062, "logps/ref_chosen": -89.20936584472656, "logps/ref_rejected": -101.83784484863281, "logps/rejected": -111.62163543701172, "loss": 1.0227, "margin_dpo/margin_mean": 6.583327770233154, "margin_dpo/margin_std": 8.523290634155273, "step": 108 }, { "KL/chosen_KL_mean": -2.5288658142089844, "KL/mean": -6.615765571594238, "KL/rejected_KL_mean": -10.70267105102539, "KL/std": 7.9392781257629395, "epoch": 0.16005873715124816, "fcm_dpo/beta": 0.08126156032085419, "fcm_dpo/delta": -0.2736997902393341, "fcm_dpo/margin": 8.17380142211914, "fcm_dpo/q_t": 0.35937485098838806, "grad_norm": 20.563512802124023, "learning_rate": 4.95006729368358e-07, "logits/chosen": -0.5691713094711304, "logits/rejected": -0.3416387438774109, "logps/chosen": -102.71595764160156, "logps/ref_chosen": -100.18708801269531, "logps/ref_rejected": -103.53739929199219, "logps/rejected": -114.24006652832031, "loss": 0.9554, "margin_dpo/margin_mean": 8.17380142211914, "margin_dpo/margin_std": 9.741275787353516, "step": 109 }, { "KL/chosen_KL_mean": -3.2194061279296875, "KL/mean": -5.741009712219238, "KL/rejected_KL_mean": -8.262622833251953, "KL/std": 6.528242111206055, "epoch": 0.16152716593245228, "fcm_dpo/beta": 0.08052003383636475, "fcm_dpo/delta": -0.006579872220754623, "fcm_dpo/margin": 5.043218612670898, "fcm_dpo/q_t": 0.40900078415870667, "grad_norm": 22.38878059387207, "learning_rate": 4.947482930773511e-07, "logits/chosen": -0.3172037899494171, "logits/rejected": 0.0823855847120285, "logps/chosen": -110.11512756347656, "logps/ref_chosen": -106.89572143554688, "logps/ref_rejected": -85.81120300292969, "logps/rejected": -94.07382202148438, "loss": 1.1029, "margin_dpo/margin_mean": 5.043219089508057, "margin_dpo/margin_std": 7.362325668334961, "step": 110 }, { "KL/chosen_KL_mean": -2.8408241271972656, "KL/mean": -7.634045600891113, "KL/rejected_KL_mean": -12.42727279663086, "KL/std": 9.210634231567383, "epoch": 0.16299559471365638, "fcm_dpo/beta": 0.07832729816436768, "fcm_dpo/delta": -0.3661164939403534, "fcm_dpo/margin": 9.586448669433594, "fcm_dpo/q_t": 0.34120920300483704, "grad_norm": 18.927616119384766, "learning_rate": 4.944834074412042e-07, "logits/chosen": -0.8964242339134216, "logits/rejected": -0.6079606413841248, "logps/chosen": -96.17019653320312, "logps/ref_chosen": -93.3293685913086, "logps/ref_rejected": -104.177490234375, "logps/rejected": -116.6047592163086, "loss": 0.9037, "margin_dpo/margin_mean": 9.586448669433594, "margin_dpo/margin_std": 10.123699188232422, "step": 111 }, { "KL/chosen_KL_mean": -3.3148574829101562, "KL/mean": -5.744173049926758, "KL/rejected_KL_mean": -8.17348861694336, "KL/std": 5.294613361358643, "epoch": 0.1644640234948605, "fcm_dpo/beta": 0.07792125642299652, "fcm_dpo/delta": 0.02182052843272686, "fcm_dpo/margin": 4.858631134033203, "fcm_dpo/q_t": 0.41413456201553345, "grad_norm": 19.789918899536133, "learning_rate": 4.942120794399002e-07, "logits/chosen": -1.259549617767334, "logits/rejected": -0.7757831811904907, "logps/chosen": -86.06190490722656, "logps/ref_chosen": -82.74703979492188, "logps/ref_rejected": -71.5584487915039, "logps/rejected": -79.73193359375, "loss": 1.1156, "margin_dpo/margin_mean": 4.8586320877075195, "margin_dpo/margin_std": 7.166678428649902, "step": 112 }, { "KL/chosen_KL_mean": -4.149452209472656, "KL/mean": -6.810981750488281, "KL/rejected_KL_mean": -9.472515106201172, "KL/std": 6.994219779968262, "epoch": 0.16593245227606462, "fcm_dpo/beta": 0.07794740796089172, "fcm_dpo/delta": -0.015234194695949554, "fcm_dpo/margin": 5.323063850402832, "fcm_dpo/q_t": 0.4060831069946289, "grad_norm": 22.869123458862305, "learning_rate": 4.939343162231841e-07, "logits/chosen": -0.932394802570343, "logits/rejected": -0.412416011095047, "logps/chosen": -102.0489501953125, "logps/ref_chosen": -97.89949035644531, "logps/ref_rejected": -84.41960144042969, "logps/rejected": -93.8921127319336, "loss": 1.0836, "margin_dpo/margin_mean": 5.323063850402832, "margin_dpo/margin_std": 6.816515922546387, "step": 113 }, { "KL/chosen_KL_mean": -3.8686180114746094, "KL/mean": -7.531517505645752, "KL/rejected_KL_mean": -11.194408416748047, "KL/std": 8.236284255981445, "epoch": 0.16740088105726872, "fcm_dpo/beta": 0.07679440081119537, "fcm_dpo/delta": -0.1676870882511139, "fcm_dpo/margin": 7.325790882110596, "fcm_dpo/q_t": 0.3829652667045593, "grad_norm": 23.045766830444336, "learning_rate": 4.936501251103751e-07, "logits/chosen": -1.1101850271224976, "logits/rejected": -0.545001745223999, "logps/chosen": -100.36373901367188, "logps/ref_chosen": -96.4951171875, "logps/ref_rejected": -92.01536560058594, "logps/rejected": -103.20977020263672, "loss": 1.0432, "margin_dpo/margin_mean": 7.325790882110596, "margin_dpo/margin_std": 10.658713340759277, "step": 114 }, { "KL/chosen_KL_mean": -3.942584991455078, "KL/mean": -7.6357102394104, "KL/rejected_KL_mean": -11.328826904296875, "KL/std": 8.60621452331543, "epoch": 0.16886930983847284, "fcm_dpo/beta": 0.0756983682513237, "fcm_dpo/delta": -0.16364595293998718, "fcm_dpo/margin": 7.386244773864746, "fcm_dpo/q_t": 0.3789907693862915, "grad_norm": 19.960857391357422, "learning_rate": 4.933595135901732e-07, "logits/chosen": -0.941020667552948, "logits/rejected": -0.5389047861099243, "logps/chosen": -107.12271118164062, "logps/ref_chosen": -103.18013000488281, "logps/ref_rejected": -105.71595764160156, "logps/rejected": -117.04478454589844, "loss": 1.0126, "margin_dpo/margin_mean": 7.386245250701904, "margin_dpo/margin_std": 9.282920837402344, "step": 115 }, { "KL/chosen_KL_mean": -3.50628662109375, "KL/mean": -6.5818634033203125, "KL/rejected_KL_mean": -9.657432556152344, "KL/std": 6.736128807067871, "epoch": 0.17033773861967694, "fcm_dpo/beta": 0.07512612640857697, "fcm_dpo/delta": -0.06358997523784637, "fcm_dpo/margin": 6.151145935058594, "fcm_dpo/q_t": 0.3964797258377075, "grad_norm": 18.697132110595703, "learning_rate": 4.930624893204624e-07, "logits/chosen": -1.1671193838119507, "logits/rejected": -0.7518138885498047, "logps/chosen": -84.66465759277344, "logps/ref_chosen": -81.15837097167969, "logps/ref_rejected": -86.30586242675781, "logps/rejected": -95.96328735351562, "loss": 1.0543, "margin_dpo/margin_mean": 6.15114688873291, "margin_dpo/margin_std": 7.7508649826049805, "step": 116 }, { "KL/chosen_KL_mean": -4.286472320556641, "KL/mean": -6.812287330627441, "KL/rejected_KL_mean": -9.33810043334961, "KL/std": 6.816826820373535, "epoch": 0.17180616740088106, "fcm_dpo/beta": 0.07474479079246521, "fcm_dpo/delta": 0.02244337648153305, "fcm_dpo/margin": 5.051626205444336, "fcm_dpo/q_t": 0.4147650897502899, "grad_norm": 26.997217178344727, "learning_rate": 4.927590601281083e-07, "logits/chosen": -0.7938641309738159, "logits/rejected": -0.3972617983818054, "logps/chosen": -101.367431640625, "logps/ref_chosen": -97.0809555053711, "logps/ref_rejected": -75.1399154663086, "logps/rejected": -84.47801208496094, "loss": 1.134, "margin_dpo/margin_mean": 5.051626205444336, "margin_dpo/margin_std": 8.219548225402832, "step": 117 }, { "KL/chosen_KL_mean": -2.9889373779296875, "KL/mean": -6.541358947753906, "KL/rejected_KL_mean": -10.09377670288086, "KL/std": 7.373363494873047, "epoch": 0.17327459618208516, "fcm_dpo/beta": 0.07425746321678162, "fcm_dpo/delta": -0.13116824626922607, "fcm_dpo/margin": 7.10483455657959, "fcm_dpo/q_t": 0.3799889087677002, "grad_norm": 18.047367095947266, "learning_rate": 4.924492340087524e-07, "logits/chosen": -1.0602984428405762, "logits/rejected": -0.7288836240768433, "logps/chosen": -89.52517700195312, "logps/ref_chosen": -86.53623962402344, "logps/ref_rejected": -82.93563842773438, "logps/rejected": -93.0294189453125, "loss": 1.0, "margin_dpo/margin_mean": 7.10483455657959, "margin_dpo/margin_std": 7.688782691955566, "step": 118 }, { "KL/chosen_KL_mean": -4.538562774658203, "KL/mean": -8.278428077697754, "KL/rejected_KL_mean": -12.018291473388672, "KL/std": 8.79874038696289, "epoch": 0.17474302496328928, "fcm_dpo/beta": 0.0731423944234848, "fcm_dpo/delta": -0.15147875249385834, "fcm_dpo/margin": 7.479732513427734, "fcm_dpo/q_t": 0.38009971380233765, "grad_norm": 21.215991973876953, "learning_rate": 4.92133019126601e-07, "logits/chosen": -1.1346720457077026, "logits/rejected": -0.8178671598434448, "logps/chosen": -109.14764404296875, "logps/ref_chosen": -104.60908508300781, "logps/ref_rejected": -109.15780639648438, "logps/rejected": -121.17610168457031, "loss": 1.0516, "margin_dpo/margin_mean": 7.479732513427734, "margin_dpo/margin_std": 10.982650756835938, "step": 119 }, { "KL/chosen_KL_mean": -3.5213851928710938, "KL/mean": -8.43623161315918, "KL/rejected_KL_mean": -13.35107421875, "KL/std": 10.65475082397461, "epoch": 0.1762114537444934, "fcm_dpo/beta": 0.0712634027004242, "fcm_dpo/delta": -0.3114854097366333, "fcm_dpo/margin": 9.82968521118164, "fcm_dpo/q_t": 0.3578925132751465, "grad_norm": 24.70150375366211, "learning_rate": 4.918104238142103e-07, "logits/chosen": -1.0647270679473877, "logits/rejected": -0.5055477619171143, "logps/chosen": -118.20881652832031, "logps/ref_chosen": -114.68742370605469, "logps/ref_rejected": -111.18586730957031, "logps/rejected": -124.53694152832031, "loss": 0.9866, "margin_dpo/margin_mean": 9.82968521118164, "margin_dpo/margin_std": 13.401922225952148, "step": 120 }, { "KL/chosen_KL_mean": -3.261371612548828, "KL/mean": -7.696924209594727, "KL/rejected_KL_mean": -12.132484436035156, "KL/std": 8.299503326416016, "epoch": 0.1776798825256975, "fcm_dpo/beta": 0.06942278146743774, "fcm_dpo/delta": -0.22325626015663147, "fcm_dpo/margin": 8.871124267578125, "fcm_dpo/q_t": 0.36388280987739563, "grad_norm": 20.17177391052246, "learning_rate": 4.91481456572267e-07, "logits/chosen": -1.051828145980835, "logits/rejected": -0.7274699211120605, "logps/chosen": -95.37803649902344, "logps/ref_chosen": -92.11666107177734, "logps/ref_rejected": -114.972900390625, "logps/rejected": -127.10538482666016, "loss": 0.9505, "margin_dpo/margin_mean": 8.871125221252441, "margin_dpo/margin_std": 9.24101448059082, "step": 121 }, { "KL/chosen_KL_mean": -3.4649505615234375, "KL/mean": -8.507323265075684, "KL/rejected_KL_mean": -13.549697875976562, "KL/std": 8.976645469665527, "epoch": 0.17914831130690162, "fcm_dpo/beta": 0.0678216814994812, "fcm_dpo/delta": -0.2935524582862854, "fcm_dpo/margin": 10.084745407104492, "fcm_dpo/q_t": 0.35014358162879944, "grad_norm": 20.3961124420166, "learning_rate": 4.911461260693638e-07, "logits/chosen": -0.9080691337585449, "logits/rejected": -0.8025715947151184, "logps/chosen": -88.34252166748047, "logps/ref_chosen": -84.87757110595703, "logps/ref_rejected": -111.0829849243164, "logps/rejected": -124.63268280029297, "loss": 0.9244, "margin_dpo/margin_mean": 10.084745407104492, "margin_dpo/margin_std": 10.37466812133789, "step": 122 }, { "KL/chosen_KL_mean": -3.9832916259765625, "KL/mean": -7.614042282104492, "KL/rejected_KL_mean": -11.244792938232422, "KL/std": 8.180074691772461, "epoch": 0.18061674008810572, "fcm_dpo/beta": 0.06648887693881989, "fcm_dpo/delta": -0.08567877858877182, "fcm_dpo/margin": 7.261500358581543, "fcm_dpo/q_t": 0.392147958278656, "grad_norm": 18.00908851623535, "learning_rate": 4.908044411417711e-07, "logits/chosen": -1.065259337425232, "logits/rejected": -0.7510160207748413, "logps/chosen": -102.4620361328125, "logps/ref_chosen": -98.47874450683594, "logps/ref_rejected": -93.85417175292969, "logps/rejected": -105.09896850585938, "loss": 1.051, "margin_dpo/margin_mean": 7.261499404907227, "margin_dpo/margin_std": 9.348323822021484, "step": 123 }, { "KL/chosen_KL_mean": -3.6701316833496094, "KL/mean": -8.171102523803711, "KL/rejected_KL_mean": -12.672077178955078, "KL/std": 10.353570938110352, "epoch": 0.18208516886930984, "fcm_dpo/beta": 0.06562966108322144, "fcm_dpo/delta": -0.19655519723892212, "fcm_dpo/margin": 9.001935958862305, "fcm_dpo/q_t": 0.38174426555633545, "grad_norm": 18.429401397705078, "learning_rate": 4.904564107932048e-07, "logits/chosen": -0.8598624467849731, "logits/rejected": -0.5949076414108276, "logps/chosen": -106.43302917480469, "logps/ref_chosen": -102.76290130615234, "logps/ref_rejected": -123.005615234375, "logps/rejected": -135.6776885986328, "loss": 1.0331, "margin_dpo/margin_mean": 9.001935958862305, "margin_dpo/margin_std": 13.422502517700195, "step": 124 }, { "KL/chosen_KL_mean": -5.103397369384766, "KL/mean": -8.399026870727539, "KL/rejected_KL_mean": -11.694656372070312, "KL/std": 8.292282104492188, "epoch": 0.18355359765051396, "fcm_dpo/beta": 0.06512059271335602, "fcm_dpo/delta": -0.02988032065331936, "fcm_dpo/margin": 6.591255187988281, "fcm_dpo/q_t": 0.4051329493522644, "grad_norm": 19.402973175048828, "learning_rate": 4.90102044194588e-07, "logits/chosen": -0.7145662903785706, "logits/rejected": -0.498498797416687, "logps/chosen": -95.52967834472656, "logps/ref_chosen": -90.42627716064453, "logps/ref_rejected": -89.24901580810547, "logps/rejected": -100.94367218017578, "loss": 1.1017, "margin_dpo/margin_mean": 6.591256141662598, "margin_dpo/margin_std": 10.130295753479004, "step": 125 }, { "KL/chosen_KL_mean": -3.9728050231933594, "KL/mean": -8.766082763671875, "KL/rejected_KL_mean": -13.55936050415039, "KL/std": 8.964065551757812, "epoch": 0.18502202643171806, "fcm_dpo/beta": 0.06405161321163177, "fcm_dpo/delta": -0.22074109315872192, "fcm_dpo/margin": 9.586554527282715, "fcm_dpo/q_t": 0.36139988899230957, "grad_norm": 18.165380477905273, "learning_rate": 4.897413506838102e-07, "logits/chosen": -1.0754961967468262, "logits/rejected": -0.808986246585846, "logps/chosen": -98.39569091796875, "logps/ref_chosen": -94.42288208007812, "logps/ref_rejected": -103.88156127929688, "logps/rejected": -117.44091796875, "loss": 0.9434, "margin_dpo/margin_mean": 9.586553573608398, "margin_dpo/margin_std": 9.355989456176758, "step": 126 }, { "KL/chosen_KL_mean": -4.0654144287109375, "KL/mean": -7.662539482116699, "KL/rejected_KL_mean": -11.259658813476562, "KL/std": 8.346498489379883, "epoch": 0.18649045521292218, "fcm_dpo/beta": 0.06349340826272964, "fcm_dpo/delta": -0.05812225490808487, "fcm_dpo/margin": 7.194247722625732, "fcm_dpo/q_t": 0.39998042583465576, "grad_norm": 18.715473175048828, "learning_rate": 4.89374339765481e-07, "logits/chosen": -0.6462635397911072, "logits/rejected": -0.29867732524871826, "logps/chosen": -88.81646728515625, "logps/ref_chosen": -84.75105285644531, "logps/ref_rejected": -83.54523468017578, "logps/rejected": -94.80489349365234, "loss": 1.0788, "margin_dpo/margin_mean": 7.194248199462891, "margin_dpo/margin_std": 10.462494850158691, "step": 127 }, { "KL/chosen_KL_mean": -4.039558410644531, "KL/mean": -7.756995677947998, "KL/rejected_KL_mean": -11.474431991577148, "KL/std": 10.233675003051758, "epoch": 0.18795888399412627, "fcm_dpo/beta": 0.06312422454357147, "fcm_dpo/delta": -0.07106203585863113, "fcm_dpo/margin": 7.434878826141357, "fcm_dpo/q_t": 0.3959639370441437, "grad_norm": 20.942453384399414, "learning_rate": 4.890010211106795e-07, "logits/chosen": -0.7935303449630737, "logits/rejected": -0.3440946340560913, "logps/chosen": -94.6592025756836, "logps/ref_chosen": -90.61964416503906, "logps/ref_rejected": -83.10347747802734, "logps/rejected": -94.57791137695312, "loss": 1.0983, "margin_dpo/margin_mean": 7.434878349304199, "margin_dpo/margin_std": 12.458648681640625, "step": 128 }, { "KL/chosen_KL_mean": -4.548439025878906, "KL/mean": -8.365934371948242, "KL/rejected_KL_mean": -12.18343734741211, "KL/std": 8.30251693725586, "epoch": 0.1894273127753304, "fcm_dpo/beta": 0.06259674578905106, "fcm_dpo/delta": -0.07982572168111801, "fcm_dpo/margin": 7.634991645812988, "fcm_dpo/q_t": 0.39814749360084534, "grad_norm": 16.94500160217285, "learning_rate": 4.88621404556699e-07, "logits/chosen": -0.8383136987686157, "logits/rejected": -0.5419360399246216, "logps/chosen": -103.46714782714844, "logps/ref_chosen": -98.91870880126953, "logps/ref_rejected": -103.28419494628906, "logps/rejected": -115.46763610839844, "loss": 1.0741, "margin_dpo/margin_mean": 7.634992599487305, "margin_dpo/margin_std": 11.058738708496094, "step": 129 }, { "KL/chosen_KL_mean": -3.546092987060547, "KL/mean": -8.515026092529297, "KL/rejected_KL_mean": -13.483959197998047, "KL/std": 9.850883483886719, "epoch": 0.19089574155653452, "fcm_dpo/beta": 0.06161799281835556, "fcm_dpo/delta": -0.21888461709022522, "fcm_dpo/margin": 9.937867164611816, "fcm_dpo/q_t": 0.369597464799881, "grad_norm": 15.980816841125488, "learning_rate": 4.882355001067891e-07, "logits/chosen": -1.1546411514282227, "logits/rejected": -0.9106923937797546, "logps/chosen": -76.72978210449219, "logps/ref_chosen": -73.18368530273438, "logps/ref_rejected": -88.02154541015625, "logps/rejected": -101.50550079345703, "loss": 0.9805, "margin_dpo/margin_mean": 9.937867164611816, "margin_dpo/margin_std": 11.562911987304688, "step": 130 }, { "KL/chosen_KL_mean": -2.9912147521972656, "KL/mean": -7.364982604980469, "KL/rejected_KL_mean": -11.738754272460938, "KL/std": 9.631525039672852, "epoch": 0.19236417033773862, "fcm_dpo/beta": 0.06060262769460678, "fcm_dpo/delta": -0.1336764544248581, "fcm_dpo/margin": 8.74753475189209, "fcm_dpo/q_t": 0.3834349811077118, "grad_norm": 17.951488494873047, "learning_rate": 4.878433179298909e-07, "logits/chosen": -0.9362294673919678, "logits/rejected": -0.7212625741958618, "logps/chosen": -76.94766235351562, "logps/ref_chosen": -73.9564437866211, "logps/ref_rejected": -94.45582580566406, "logps/rejected": -106.194580078125, "loss": 1.0259, "margin_dpo/margin_mean": 8.747533798217773, "margin_dpo/margin_std": 10.92578125, "step": 131 }, { "KL/chosen_KL_mean": -4.258876800537109, "KL/mean": -8.843465805053711, "KL/rejected_KL_mean": -13.428050994873047, "KL/std": 10.096443176269531, "epoch": 0.19383259911894274, "fcm_dpo/beta": 0.05940578132867813, "fcm_dpo/delta": -0.1492229551076889, "fcm_dpo/margin": 9.169175148010254, "fcm_dpo/q_t": 0.3784756064414978, "grad_norm": 18.906333923339844, "learning_rate": 4.874448683603694e-07, "logits/chosen": -1.1890652179718018, "logits/rejected": -0.7812725305557251, "logps/chosen": -96.6983642578125, "logps/ref_chosen": -92.43948364257812, "logps/ref_rejected": -94.13252258300781, "logps/rejected": -107.5605697631836, "loss": 1.0119, "margin_dpo/margin_mean": 9.169175148010254, "margin_dpo/margin_std": 11.212705612182617, "step": 132 }, { "KL/chosen_KL_mean": -3.5074806213378906, "KL/mean": -8.207819938659668, "KL/rejected_KL_mean": -12.908153533935547, "KL/std": 10.135307312011719, "epoch": 0.19530102790014683, "fcm_dpo/beta": 0.05866105109453201, "fcm_dpo/delta": -0.1556541472673416, "fcm_dpo/margin": 9.400674819946289, "fcm_dpo/q_t": 0.3776993751525879, "grad_norm": 17.024778366088867, "learning_rate": 4.870401618977415e-07, "logits/chosen": -1.5914839506149292, "logits/rejected": -1.2172819375991821, "logps/chosen": -96.6241455078125, "logps/ref_chosen": -93.11666107177734, "logps/ref_rejected": -102.09490966796875, "logps/rejected": -115.00306701660156, "loss": 0.9918, "margin_dpo/margin_mean": 9.400675773620605, "margin_dpo/margin_std": 10.504207611083984, "step": 133 }, { "KL/chosen_KL_mean": -3.8177337646484375, "KL/mean": -8.840692520141602, "KL/rejected_KL_mean": -13.863651275634766, "KL/std": 9.723281860351562, "epoch": 0.19676945668135096, "fcm_dpo/beta": 0.0577988401055336, "fcm_dpo/delta": -0.18591542541980743, "fcm_dpo/margin": 10.045913696289062, "fcm_dpo/q_t": 0.3702012300491333, "grad_norm": 16.099592208862305, "learning_rate": 4.866292092063986e-07, "logits/chosen": -1.5010484457015991, "logits/rejected": -1.1738357543945312, "logps/chosen": -87.36785888671875, "logps/ref_chosen": -83.55012512207031, "logps/ref_rejected": -93.8815689086914, "logps/rejected": -107.74522399902344, "loss": 0.9735, "margin_dpo/margin_mean": 10.045912742614746, "margin_dpo/margin_std": 10.403275489807129, "step": 134 }, { "KL/chosen_KL_mean": -3.6349849700927734, "KL/mean": -8.507829666137695, "KL/rejected_KL_mean": -13.380680084228516, "KL/std": 10.651508331298828, "epoch": 0.19823788546255505, "fcm_dpo/beta": 0.05672912672162056, "fcm_dpo/delta": -0.15706497430801392, "fcm_dpo/margin": 9.745683670043945, "fcm_dpo/q_t": 0.38048338890075684, "grad_norm": 21.376148223876953, "learning_rate": 4.862120211153265e-07, "logits/chosen": -1.1055209636688232, "logits/rejected": -0.9738554954528809, "logps/chosen": -81.21761322021484, "logps/ref_chosen": -77.58262634277344, "logps/ref_rejected": -121.05648040771484, "logps/rejected": -134.43716430664062, "loss": 1.0071, "margin_dpo/margin_mean": 9.745684623718262, "margin_dpo/margin_std": 11.885605812072754, "step": 135 }, { "KL/chosen_KL_mean": -3.9594955444335938, "KL/mean": -8.905628204345703, "KL/rejected_KL_mean": -13.851760864257812, "KL/std": 10.700824737548828, "epoch": 0.19970631424375918, "fcm_dpo/beta": 0.05573910474777222, "fcm_dpo/delta": -0.15575990080833435, "fcm_dpo/margin": 9.892263412475586, "fcm_dpo/q_t": 0.37610870599746704, "grad_norm": 16.077150344848633, "learning_rate": 4.857886086178193e-07, "logits/chosen": -1.350375771522522, "logits/rejected": -0.9199120998382568, "logps/chosen": -98.13223266601562, "logps/ref_chosen": -94.1727294921875, "logps/ref_rejected": -103.26428985595703, "logps/rejected": -117.11605072021484, "loss": 0.9829, "margin_dpo/margin_mean": 9.892263412475586, "margin_dpo/margin_std": 10.458353042602539, "step": 136 }, { "KL/chosen_KL_mean": -3.131031036376953, "KL/mean": -10.73232650756836, "KL/rejected_KL_mean": -18.3336181640625, "KL/std": 13.573356628417969, "epoch": 0.2011747430249633, "fcm_dpo/beta": 0.05359583720564842, "fcm_dpo/delta": -0.43436291813850403, "fcm_dpo/margin": 15.202593803405762, "fcm_dpo/q_t": 0.3282920718193054, "grad_norm": 18.279951095581055, "learning_rate": 4.853589828711902e-07, "logits/chosen": -1.4103870391845703, "logits/rejected": -1.1845248937606812, "logps/chosen": -81.24977111816406, "logps/ref_chosen": -78.11874389648438, "logps/ref_rejected": -117.92237091064453, "logps/rejected": -136.2559814453125, "loss": 0.849, "margin_dpo/margin_mean": 15.202594757080078, "margin_dpo/margin_std": 14.650882720947266, "step": 137 }, { "KL/chosen_KL_mean": -5.621013641357422, "KL/mean": -10.408590316772461, "KL/rejected_KL_mean": -15.196170806884766, "KL/std": 10.220415115356445, "epoch": 0.2026431718061674, "fcm_dpo/beta": 0.052701517939567566, "fcm_dpo/delta": -0.10749885439872742, "fcm_dpo/margin": 9.575159072875977, "fcm_dpo/q_t": 0.3867577314376831, "grad_norm": 14.945446968078613, "learning_rate": 4.849231551964771e-07, "logits/chosen": -0.8759540319442749, "logits/rejected": -0.6245163083076477, "logps/chosen": -98.45095825195312, "logps/ref_chosen": -92.82994079589844, "logps/ref_rejected": -98.64840698242188, "logps/rejected": -113.84457397460938, "loss": 1.0208, "margin_dpo/margin_mean": 9.575159072875977, "margin_dpo/margin_std": 11.127583503723145, "step": 138 }, { "KL/chosen_KL_mean": -3.721515655517578, "KL/mean": -9.296195030212402, "KL/rejected_KL_mean": -14.870872497558594, "KL/std": 11.25031566619873, "epoch": 0.20411160058737152, "fcm_dpo/beta": 0.05184052884578705, "fcm_dpo/delta": -0.18354931473731995, "fcm_dpo/margin": 11.149362564086914, "fcm_dpo/q_t": 0.36877477169036865, "grad_norm": 15.274247169494629, "learning_rate": 4.844811370781446e-07, "logits/chosen": -1.1952546834945679, "logits/rejected": -0.8718482851982117, "logps/chosen": -82.72453308105469, "logps/ref_chosen": -79.00301361083984, "logps/ref_rejected": -85.82733154296875, "logps/rejected": -100.69820404052734, "loss": 0.9901, "margin_dpo/margin_mean": 11.149362564086914, "margin_dpo/margin_std": 13.138825416564941, "step": 139 }, { "KL/chosen_KL_mean": -4.0208740234375, "KL/mean": -9.122602462768555, "KL/rejected_KL_mean": -14.22433090209961, "KL/std": 10.645638465881348, "epoch": 0.2055800293685756, "fcm_dpo/beta": 0.05118384212255478, "fcm_dpo/delta": -0.12551403045654297, "fcm_dpo/margin": 10.203458786010742, "fcm_dpo/q_t": 0.3816570043563843, "grad_norm": 14.812681198120117, "learning_rate": 4.840329401637809e-07, "logits/chosen": -1.5316461324691772, "logits/rejected": -1.1344819068908691, "logps/chosen": -92.37332153320312, "logps/ref_chosen": -88.35244750976562, "logps/ref_rejected": -90.57476806640625, "logps/rejected": -104.79910278320312, "loss": 1.0042, "margin_dpo/margin_mean": 10.203459739685059, "margin_dpo/margin_std": 11.142803192138672, "step": 140 }, { "KL/chosen_KL_mean": -5.387966156005859, "KL/mean": -11.237958908081055, "KL/rejected_KL_mean": -17.087955474853516, "KL/std": 13.490775108337402, "epoch": 0.20704845814977973, "fcm_dpo/beta": 0.05030140280723572, "fcm_dpo/delta": -0.1941697895526886, "fcm_dpo/margin": 11.699989318847656, "fcm_dpo/q_t": 0.36932051181793213, "grad_norm": 17.78990936279297, "learning_rate": 4.83578576263792e-07, "logits/chosen": -1.0591198205947876, "logits/rejected": -0.6729257106781006, "logps/chosen": -112.60812377929688, "logps/ref_chosen": -107.22016143798828, "logps/ref_rejected": -104.71223449707031, "logps/rejected": -121.80018615722656, "loss": 0.9833, "margin_dpo/margin_mean": 11.699989318847656, "margin_dpo/margin_std": 13.58017349243164, "step": 141 }, { "KL/chosen_KL_mean": -5.916538238525391, "KL/mean": -11.583646774291992, "KL/rejected_KL_mean": -17.250747680664062, "KL/std": 12.79791259765625, "epoch": 0.20851688693098386, "fcm_dpo/beta": 0.04936206340789795, "fcm_dpo/delta": -0.16433660686016083, "fcm_dpo/margin": 11.334210395812988, "fcm_dpo/q_t": 0.377848744392395, "grad_norm": 14.731356620788574, "learning_rate": 4.83118057351089e-07, "logits/chosen": -1.1909927129745483, "logits/rejected": -0.9384269714355469, "logps/chosen": -94.6454849243164, "logps/ref_chosen": -88.72894287109375, "logps/ref_rejected": -99.73038482666016, "logps/rejected": -116.98113250732422, "loss": 0.9918, "margin_dpo/margin_mean": 11.334209442138672, "margin_dpo/margin_std": 13.271568298339844, "step": 142 }, { "KL/chosen_KL_mean": -4.093166351318359, "KL/mean": -8.823657989501953, "KL/rejected_KL_mean": -13.554141998291016, "KL/std": 10.805572509765625, "epoch": 0.20998531571218795, "fcm_dpo/beta": 0.04887082800269127, "fcm_dpo/delta": -0.06419782340526581, "fcm_dpo/margin": 9.460975646972656, "fcm_dpo/q_t": 0.3951960802078247, "grad_norm": 14.635172843933105, "learning_rate": 4.826513955607734e-07, "logits/chosen": -1.4836841821670532, "logits/rejected": -1.0334839820861816, "logps/chosen": -89.59774780273438, "logps/ref_chosen": -85.50457763671875, "logps/ref_rejected": -85.8056640625, "logps/rejected": -99.35980224609375, "loss": 1.0542, "margin_dpo/margin_mean": 9.460975646972656, "margin_dpo/margin_std": 11.875649452209473, "step": 143 }, { "KL/chosen_KL_mean": -4.831794738769531, "KL/mean": -9.687114715576172, "KL/rejected_KL_mean": -14.542434692382812, "KL/std": 11.324934005737305, "epoch": 0.21145374449339208, "fcm_dpo/beta": 0.04879078269004822, "fcm_dpo/delta": -0.07578147947788239, "fcm_dpo/margin": 9.710639953613281, "fcm_dpo/q_t": 0.3955634832382202, "grad_norm": 18.605396270751953, "learning_rate": 4.821786031898176e-07, "logits/chosen": -1.3822683095932007, "logits/rejected": -0.860374927520752, "logps/chosen": -109.51889038085938, "logps/ref_chosen": -104.68708801269531, "logps/ref_rejected": -87.71908569335938, "logps/rejected": -102.26152038574219, "loss": 1.0726, "margin_dpo/margin_mean": 9.710639953613281, "margin_dpo/margin_std": 13.93049430847168, "step": 144 }, { "KL/chosen_KL_mean": -4.181056976318359, "KL/mean": -9.877605438232422, "KL/rejected_KL_mean": -15.574150085449219, "KL/std": 11.09048080444336, "epoch": 0.21292217327459617, "fcm_dpo/beta": 0.04799002408981323, "fcm_dpo/delta": -0.15088051557540894, "fcm_dpo/margin": 11.393094062805176, "fcm_dpo/q_t": 0.37679004669189453, "grad_norm": 14.918173789978027, "learning_rate": 4.816996926967401e-07, "logits/chosen": -0.7419586777687073, "logits/rejected": -0.1369239240884781, "logps/chosen": -92.70094299316406, "logps/ref_chosen": -88.51988983154297, "logps/ref_rejected": -83.76513671875, "logps/rejected": -99.33929443359375, "loss": 0.9927, "margin_dpo/margin_mean": 11.393095016479492, "margin_dpo/margin_std": 12.529277801513672, "step": 145 }, { "KL/chosen_KL_mean": -4.472137451171875, "KL/mean": -10.820611000061035, "KL/rejected_KL_mean": -17.169078826904297, "KL/std": 11.977924346923828, "epoch": 0.2143906020558003, "fcm_dpo/beta": 0.046952325850725174, "fcm_dpo/delta": -0.20309990644454956, "fcm_dpo/margin": 12.696939468383789, "fcm_dpo/q_t": 0.3654606342315674, "grad_norm": 14.66407299041748, "learning_rate": 4.812146767012779e-07, "logits/chosen": -1.5521221160888672, "logits/rejected": -0.9385132789611816, "logps/chosen": -107.37800598144531, "logps/ref_chosen": -102.90586853027344, "logps/ref_rejected": -88.9493637084961, "logps/rejected": -106.11843872070312, "loss": 0.9592, "margin_dpo/margin_mean": 12.696939468383789, "margin_dpo/margin_std": 12.848381042480469, "step": 146 }, { "KL/chosen_KL_mean": -4.505889892578125, "KL/mean": -11.035873413085938, "KL/rejected_KL_mean": -17.565860748291016, "KL/std": 11.696565628051758, "epoch": 0.21585903083700442, "fcm_dpo/beta": 0.046130623668432236, "fcm_dpo/delta": -0.2087171971797943, "fcm_dpo/margin": 13.059974670410156, "fcm_dpo/q_t": 0.3635420799255371, "grad_norm": 13.682374000549316, "learning_rate": 4.807235679840536e-07, "logits/chosen": -1.2152023315429688, "logits/rejected": -0.6197528839111328, "logps/chosen": -93.76808166503906, "logps/ref_chosen": -89.26219177246094, "logps/ref_rejected": -76.50011444091797, "logps/rejected": -94.06597900390625, "loss": 0.9452, "margin_dpo/margin_mean": 13.05997371673584, "margin_dpo/margin_std": 12.494707107543945, "step": 147 }, { "KL/chosen_KL_mean": -2.899089813232422, "KL/mean": -8.756926536560059, "KL/rejected_KL_mean": -14.614761352539062, "KL/std": 11.9217529296875, "epoch": 0.2173274596182085, "fcm_dpo/beta": 0.04560375213623047, "fcm_dpo/delta": -0.13811685144901276, "fcm_dpo/margin": 11.715672492980957, "fcm_dpo/q_t": 0.38347914814949036, "grad_norm": 14.807011604309082, "learning_rate": 4.802263794862384e-07, "logits/chosen": -1.4776225090026855, "logits/rejected": -1.0828218460083008, "logps/chosen": -94.66197204589844, "logps/ref_chosen": -91.76287841796875, "logps/ref_rejected": -110.38552856445312, "logps/rejected": -125.00028991699219, "loss": 1.0147, "margin_dpo/margin_mean": 11.71567153930664, "margin_dpo/margin_std": 13.522621154785156, "step": 148 }, { "KL/chosen_KL_mean": -5.884937286376953, "KL/mean": -11.543502807617188, "KL/rejected_KL_mean": -17.20205307006836, "KL/std": 12.635601997375488, "epoch": 0.21879588839941264, "fcm_dpo/beta": 0.04478982090950012, "fcm_dpo/delta": -0.11004670709371567, "fcm_dpo/margin": 11.317113876342773, "fcm_dpo/q_t": 0.3919112980365753, "grad_norm": 14.414762496948242, "learning_rate": 4.797231243092118e-07, "logits/chosen": -1.1135563850402832, "logits/rejected": -0.7892755270004272, "logps/chosen": -99.36895751953125, "logps/ref_chosen": -93.48401641845703, "logps/ref_rejected": -105.6436996459961, "logps/rejected": -122.84574890136719, "loss": 1.0461, "margin_dpo/margin_mean": 11.317113876342773, "margin_dpo/margin_std": 15.54418659210205, "step": 149 }, { "KL/chosen_KL_mean": -4.5919647216796875, "KL/mean": -10.652318954467773, "KL/rejected_KL_mean": -16.71268081665039, "KL/std": 13.615423202514648, "epoch": 0.22026431718061673, "fcm_dpo/beta": 0.04422898590564728, "fcm_dpo/delta": -0.1400938630104065, "fcm_dpo/margin": 12.120716094970703, "fcm_dpo/q_t": 0.38221991062164307, "grad_norm": 12.894203186035156, "learning_rate": 4.792138157142157e-07, "logits/chosen": -1.1924772262573242, "logits/rejected": -0.9238916635513306, "logps/chosen": -80.37065124511719, "logps/ref_chosen": -75.7786865234375, "logps/ref_rejected": -90.02676391601562, "logps/rejected": -106.73944091796875, "loss": 1.0143, "margin_dpo/margin_mean": 12.120716094970703, "margin_dpo/margin_std": 15.066704750061035, "step": 150 }, { "KL/chosen_KL_mean": -6.401336669921875, "KL/mean": -12.14889144897461, "KL/rejected_KL_mean": -17.896438598632812, "KL/std": 13.28571891784668, "epoch": 0.22173274596182085, "fcm_dpo/beta": 0.04378341883420944, "fcm_dpo/delta": -0.10596097260713577, "fcm_dpo/margin": 11.49510383605957, "fcm_dpo/q_t": 0.3886474668979645, "grad_norm": 37.3143310546875, "learning_rate": 4.786984671220053e-07, "logits/chosen": -1.9626964330673218, "logits/rejected": -1.3813412189483643, "logps/chosen": -109.13555908203125, "logps/ref_chosen": -102.73422241210938, "logps/ref_rejected": -106.03781127929688, "logps/rejected": -123.93424987792969, "loss": 1.0359, "margin_dpo/margin_mean": 11.49510383605957, "margin_dpo/margin_std": 14.829866409301758, "step": 151 }, { "KL/chosen_KL_mean": -3.565887451171875, "KL/mean": -10.432666778564453, "KL/rejected_KL_mean": -17.29944610595703, "KL/std": 12.927406311035156, "epoch": 0.22320117474302498, "fcm_dpo/beta": 0.04299573600292206, "fcm_dpo/delta": -0.19646520912647247, "fcm_dpo/margin": 13.733552932739258, "fcm_dpo/q_t": 0.3675119876861572, "grad_norm": 14.107396125793457, "learning_rate": 4.78177092112495e-07, "logits/chosen": -1.2471699714660645, "logits/rejected": -0.8568304181098938, "logps/chosen": -101.15663146972656, "logps/ref_chosen": -97.59074401855469, "logps/ref_rejected": -110.49913024902344, "logps/rejected": -127.79857635498047, "loss": 0.9685, "margin_dpo/margin_mean": 13.733552932739258, "margin_dpo/margin_std": 14.831411361694336, "step": 152 }, { "KL/chosen_KL_mean": -6.0180206298828125, "KL/mean": -10.982166290283203, "KL/rejected_KL_mean": -15.946319580078125, "KL/std": 12.665582656860352, "epoch": 0.22466960352422907, "fcm_dpo/beta": 0.042730726301670074, "fcm_dpo/delta": -0.024773703888058662, "fcm_dpo/margin": 9.928305625915527, "fcm_dpo/q_t": 0.4047371745109558, "grad_norm": 14.003578186035156, "learning_rate": 4.776497044244016e-07, "logits/chosen": -1.8810559511184692, "logits/rejected": -1.5746769905090332, "logps/chosen": -93.04362487792969, "logps/ref_chosen": -87.02560424804688, "logps/ref_rejected": -91.06854248046875, "logps/rejected": -107.01486206054688, "loss": 1.0942, "margin_dpo/margin_mean": 9.928304672241211, "margin_dpo/margin_std": 14.343067169189453, "step": 153 }, { "KL/chosen_KL_mean": -4.3714447021484375, "KL/mean": -12.421117782592773, "KL/rejected_KL_mean": -20.470794677734375, "KL/std": 13.896186828613281, "epoch": 0.2261380323054332, "fcm_dpo/beta": 0.04198497161269188, "fcm_dpo/delta": -0.285095751285553, "fcm_dpo/margin": 16.099349975585938, "fcm_dpo/q_t": 0.3494638204574585, "grad_norm": 13.727378845214844, "learning_rate": 4.771163179548808e-07, "logits/chosen": -1.3958674669265747, "logits/rejected": -1.0705538988113403, "logps/chosen": -95.47711181640625, "logps/ref_chosen": -91.10566711425781, "logps/ref_rejected": -111.37238311767578, "logps/rejected": -131.84317016601562, "loss": 0.9048, "margin_dpo/margin_mean": 16.099349975585938, "margin_dpo/margin_std": 14.315711975097656, "step": 154 }, { "KL/chosen_KL_mean": -6.303314208984375, "KL/mean": -12.42654800415039, "KL/rejected_KL_mean": -18.54977035522461, "KL/std": 14.319586753845215, "epoch": 0.2276064610866373, "fcm_dpo/beta": 0.041066911071538925, "fcm_dpo/delta": -0.10619339346885681, "fcm_dpo/margin": 12.2464599609375, "fcm_dpo/q_t": 0.3852418065071106, "grad_norm": 17.043603897094727, "learning_rate": 4.7657694675916247e-07, "logits/chosen": -1.2808641195297241, "logits/rejected": -0.9110965728759766, "logps/chosen": -100.86428833007812, "logps/ref_chosen": -94.56097412109375, "logps/ref_rejected": -84.77359008789062, "logps/rejected": -103.3233642578125, "loss": 1.0596, "margin_dpo/margin_mean": 12.2464599609375, "margin_dpo/margin_std": 17.368968963623047, "step": 155 }, { "KL/chosen_KL_mean": -7.784755706787109, "KL/mean": -13.246843338012695, "KL/rejected_KL_mean": -18.70893096923828, "KL/std": 12.75959587097168, "epoch": 0.2290748898678414, "fcm_dpo/beta": 0.04083424434065819, "fcm_dpo/delta": -0.047442518174648285, "fcm_dpo/margin": 10.924179077148438, "fcm_dpo/q_t": 0.40129154920578003, "grad_norm": 14.939629554748535, "learning_rate": 4.7603160505017893e-07, "logits/chosen": -1.513357162475586, "logits/rejected": -1.016481637954712, "logps/chosen": -102.44013977050781, "logps/ref_chosen": -94.65538787841797, "logps/ref_rejected": -84.41934204101562, "logps/rejected": -103.1282730102539, "loss": 1.0795, "margin_dpo/margin_mean": 10.924178123474121, "margin_dpo/margin_std": 15.263725280761719, "step": 156 }, { "KL/chosen_KL_mean": -7.3359527587890625, "KL/mean": -14.783267974853516, "KL/rejected_KL_mean": -22.230579376220703, "KL/std": 13.633243560791016, "epoch": 0.2305433186490455, "fcm_dpo/beta": 0.040285103023052216, "fcm_dpo/delta": -0.20608282089233398, "fcm_dpo/margin": 14.894618034362793, "fcm_dpo/q_t": 0.3640595078468323, "grad_norm": 14.807397842407227, "learning_rate": 4.7548030719819154e-07, "logits/chosen": -1.7941582202911377, "logits/rejected": -1.3317137956619263, "logps/chosen": -104.34834289550781, "logps/ref_chosen": -97.01239013671875, "logps/ref_rejected": -111.70133972167969, "logps/rejected": -133.93191528320312, "loss": 0.9485, "margin_dpo/margin_mean": 14.894618034362793, "margin_dpo/margin_std": 14.313398361206055, "step": 157 }, { "KL/chosen_KL_mean": -6.633209228515625, "KL/mean": -14.574542045593262, "KL/rejected_KL_mean": -22.51587677001953, "KL/std": 14.739847183227539, "epoch": 0.23201174743024963, "fcm_dpo/beta": 0.039386287331581116, "fcm_dpo/delta": -0.2326827347278595, "fcm_dpo/margin": 15.882665634155273, "fcm_dpo/q_t": 0.360154926776886, "grad_norm": 13.769851684570312, "learning_rate": 4.7492306773041136e-07, "logits/chosen": -1.3303098678588867, "logits/rejected": -1.053748369216919, "logps/chosen": -101.34617614746094, "logps/ref_chosen": -94.71296691894531, "logps/ref_rejected": -120.93733978271484, "logps/rejected": -143.45321655273438, "loss": 0.9366, "margin_dpo/margin_mean": 15.882665634155273, "margin_dpo/margin_std": 15.536252975463867, "step": 158 }, { "KL/chosen_KL_mean": -6.028942108154297, "KL/mean": -13.174751281738281, "KL/rejected_KL_mean": -20.320568084716797, "KL/std": 14.50330924987793, "epoch": 0.23348017621145375, "fcm_dpo/beta": 0.038659144192934036, "fcm_dpo/delta": -0.15686848759651184, "fcm_dpo/margin": 14.291629791259766, "fcm_dpo/q_t": 0.37667417526245117, "grad_norm": 12.453608512878418, "learning_rate": 4.743599013306165e-07, "logits/chosen": -1.9605467319488525, "logits/rejected": -1.4825139045715332, "logps/chosen": -114.24717712402344, "logps/ref_chosen": -108.2182388305664, "logps/ref_rejected": -94.83901977539062, "logps/rejected": -115.15959167480469, "loss": 0.9976, "margin_dpo/margin_mean": 14.291629791259766, "margin_dpo/margin_std": 16.47472381591797, "step": 159 }, { "KL/chosen_KL_mean": -5.065681457519531, "KL/mean": -12.013028144836426, "KL/rejected_KL_mean": -18.960372924804688, "KL/std": 13.426647186279297, "epoch": 0.23494860499265785, "fcm_dpo/beta": 0.03809487819671631, "fcm_dpo/delta": -0.13300462067127228, "fcm_dpo/margin": 13.89468765258789, "fcm_dpo/q_t": 0.3783968687057495, "grad_norm": 13.004213333129883, "learning_rate": 4.737908228387656e-07, "logits/chosen": -1.5780268907546997, "logits/rejected": -1.0309470891952515, "logps/chosen": -103.22909545898438, "logps/ref_chosen": -98.16340637207031, "logps/ref_rejected": -102.28097534179688, "logps/rejected": -121.24134826660156, "loss": 0.9928, "margin_dpo/margin_mean": 13.89468765258789, "margin_dpo/margin_std": 14.312393188476562, "step": 160 }, { "KL/chosen_KL_mean": -7.168087005615234, "KL/mean": -13.505789756774902, "KL/rejected_KL_mean": -19.843490600585938, "KL/std": 12.472017288208008, "epoch": 0.23641703377386197, "fcm_dpo/beta": 0.037692271173000336, "fcm_dpo/delta": -0.0800580084323883, "fcm_dpo/margin": 12.675409317016602, "fcm_dpo/q_t": 0.3917338252067566, "grad_norm": 12.99565315246582, "learning_rate": 4.7321584725060594e-07, "logits/chosen": -1.7988574504852295, "logits/rejected": -1.3133783340454102, "logps/chosen": -93.75995635986328, "logps/ref_chosen": -86.59187316894531, "logps/ref_rejected": -89.01643371582031, "logps/rejected": -108.85992431640625, "loss": 1.0484, "margin_dpo/margin_mean": 12.675409317016602, "margin_dpo/margin_std": 15.851043701171875, "step": 161 }, { "KL/chosen_KL_mean": -6.2622222900390625, "KL/mean": -13.219793319702148, "KL/rejected_KL_mean": -20.17736053466797, "KL/std": 14.358290672302246, "epoch": 0.23788546255506607, "fcm_dpo/beta": 0.037312839180231094, "fcm_dpo/delta": -0.12259788811206818, "fcm_dpo/margin": 13.915141105651855, "fcm_dpo/q_t": 0.38064777851104736, "grad_norm": 12.318243026733398, "learning_rate": 4.7263498971727905e-07, "logits/chosen": -1.2412667274475098, "logits/rejected": -0.8445541262626648, "logps/chosen": -103.66510772705078, "logps/ref_chosen": -97.40288543701172, "logps/ref_rejected": -95.86923217773438, "logps/rejected": -116.04659271240234, "loss": 1.0053, "margin_dpo/margin_mean": 13.915142059326172, "margin_dpo/margin_std": 15.116000175476074, "step": 162 }, { "KL/chosen_KL_mean": -6.106884002685547, "KL/mean": -14.508097648620605, "KL/rejected_KL_mean": -22.909305572509766, "KL/std": 16.542448043823242, "epoch": 0.2393538913362702, "fcm_dpo/beta": 0.03658631816506386, "fcm_dpo/delta": -0.22187933325767517, "fcm_dpo/margin": 16.802425384521484, "fcm_dpo/q_t": 0.3645501136779785, "grad_norm": 11.049089431762695, "learning_rate": 4.720482655449212e-07, "logits/chosen": -1.6136537790298462, "logits/rejected": -1.221135139465332, "logps/chosen": -88.28227233886719, "logps/ref_chosen": -82.1753921508789, "logps/ref_rejected": -82.86473846435547, "logps/rejected": -105.7740478515625, "loss": 0.951, "margin_dpo/margin_mean": 16.802425384521484, "margin_dpo/margin_std": 17.695819854736328, "step": 163 }, { "KL/chosen_KL_mean": -9.724273681640625, "KL/mean": -15.369344711303711, "KL/rejected_KL_mean": -21.01441192626953, "KL/std": 12.990394592285156, "epoch": 0.24082232011747431, "fcm_dpo/beta": 0.03625613451004028, "fcm_dpo/delta": -0.010182276368141174, "fcm_dpo/margin": 11.290143966674805, "fcm_dpo/q_t": 0.405105859041214, "grad_norm": 12.987998962402344, "learning_rate": 4.714556901942599e-07, "logits/chosen": -1.510499119758606, "logits/rejected": -1.0614402294158936, "logps/chosen": -100.19854736328125, "logps/ref_chosen": -90.47427368164062, "logps/ref_rejected": -87.02809143066406, "logps/rejected": -108.0425033569336, "loss": 1.0837, "margin_dpo/margin_mean": 11.290144920349121, "margin_dpo/margin_std": 13.968591690063477, "step": 164 }, { "KL/chosen_KL_mean": -6.497180938720703, "KL/mean": -14.752361297607422, "KL/rejected_KL_mean": -23.007537841796875, "KL/std": 15.71397590637207, "epoch": 0.2422907488986784, "fcm_dpo/beta": 0.035905808210372925, "fcm_dpo/delta": -0.19850437343120575, "fcm_dpo/margin": 16.510358810424805, "fcm_dpo/q_t": 0.3695904016494751, "grad_norm": 10.840771675109863, "learning_rate": 4.708572792802069e-07, "logits/chosen": -1.5680269002914429, "logits/rejected": -1.1433031558990479, "logps/chosen": -93.81040954589844, "logps/ref_chosen": -87.313232421875, "logps/ref_rejected": -81.902587890625, "logps/rejected": -104.91012573242188, "loss": 0.9611, "margin_dpo/margin_mean": 16.510356903076172, "margin_dpo/margin_std": 17.693790435791016, "step": 165 }, { "KL/chosen_KL_mean": -10.9293212890625, "KL/mean": -18.99458885192871, "KL/rejected_KL_mean": -27.059856414794922, "KL/std": 16.688522338867188, "epoch": 0.24375917767988253, "fcm_dpo/beta": 0.035214848816394806, "fcm_dpo/delta": -0.1729484498500824, "fcm_dpo/margin": 16.13052749633789, "fcm_dpo/q_t": 0.37816840410232544, "grad_norm": 12.214245796203613, "learning_rate": 4.702530485714461e-07, "logits/chosen": -1.3524874448776245, "logits/rejected": -1.1449264287948608, "logps/chosen": -92.51538848876953, "logps/ref_chosen": -81.58606719970703, "logps/ref_rejected": -103.16517639160156, "logps/rejected": -130.22503662109375, "loss": 0.9948, "margin_dpo/margin_mean": 16.13052749633789, "margin_dpo/margin_std": 19.863107681274414, "step": 166 }, { "KL/chosen_KL_mean": -9.674354553222656, "KL/mean": -18.437549591064453, "KL/rejected_KL_mean": -27.20074462890625, "KL/std": 15.704032897949219, "epoch": 0.24522760646108663, "fcm_dpo/beta": 0.03441347926855087, "fcm_dpo/delta": -0.21042993664741516, "fcm_dpo/margin": 17.526382446289062, "fcm_dpo/q_t": 0.3631550073623657, "grad_norm": 11.399792671203613, "learning_rate": 4.6964301399001877e-07, "logits/chosen": -1.013858437538147, "logits/rejected": -0.8334169983863831, "logps/chosen": -93.23602294921875, "logps/ref_chosen": -83.56167602539062, "logps/ref_rejected": -99.05305480957031, "logps/rejected": -126.25379943847656, "loss": 0.9384, "margin_dpo/margin_mean": 17.526382446289062, "margin_dpo/margin_std": 15.840325355529785, "step": 167 }, { "KL/chosen_KL_mean": -9.375076293945312, "KL/mean": -16.787124633789062, "KL/rejected_KL_mean": -24.199176788330078, "KL/std": 15.416728973388672, "epoch": 0.24669603524229075, "fcm_dpo/beta": 0.03408646583557129, "fcm_dpo/delta": -0.10796555876731873, "fcm_dpo/margin": 14.824094772338867, "fcm_dpo/q_t": 0.3842093348503113, "grad_norm": 12.589140892028809, "learning_rate": 4.690271916109034e-07, "logits/chosen": -1.4754741191864014, "logits/rejected": -1.1326795816421509, "logps/chosen": -89.30631256103516, "logps/ref_chosen": -79.93123626708984, "logps/ref_rejected": -80.10216522216797, "logps/rejected": -104.30134582519531, "loss": 1.0077, "margin_dpo/margin_mean": 14.824095726013184, "margin_dpo/margin_std": 15.493619918823242, "step": 168 }, { "KL/chosen_KL_mean": -7.942226409912109, "KL/mean": -16.428653717041016, "KL/rejected_KL_mean": -24.915081024169922, "KL/std": 16.50726318359375, "epoch": 0.24816446402349487, "fcm_dpo/beta": 0.03352634236216545, "fcm_dpo/delta": -0.17397752404212952, "fcm_dpo/margin": 16.972850799560547, "fcm_dpo/q_t": 0.372464656829834, "grad_norm": 12.184320449829102, "learning_rate": 4.6840559766159235e-07, "logits/chosen": -1.4945714473724365, "logits/rejected": -1.2713801860809326, "logps/chosen": -90.84593200683594, "logps/ref_chosen": -82.90370178222656, "logps/ref_rejected": -90.12191009521484, "logps/rejected": -115.0369873046875, "loss": 0.9722, "margin_dpo/margin_mean": 16.972850799560547, "margin_dpo/margin_std": 17.618022918701172, "step": 169 }, { "KL/chosen_KL_mean": -9.581035614013672, "KL/mean": -17.108070373535156, "KL/rejected_KL_mean": -24.635108947753906, "KL/std": 14.05484390258789, "epoch": 0.24963289280469897, "fcm_dpo/beta": 0.03304228186607361, "fcm_dpo/delta": -0.10042858868837357, "fcm_dpo/margin": 15.0540771484375, "fcm_dpo/q_t": 0.38384944200515747, "grad_norm": 12.375666618347168, "learning_rate": 4.6777824852166437e-07, "logits/chosen": -1.1229568719863892, "logits/rejected": -0.8939340114593506, "logps/chosen": -84.56300354003906, "logps/ref_chosen": -74.98197174072266, "logps/ref_rejected": -82.16560363769531, "logps/rejected": -106.80071258544922, "loss": 1.0014, "margin_dpo/margin_mean": 15.0540771484375, "margin_dpo/margin_std": 14.039011001586914, "step": 170 }, { "KL/chosen_KL_mean": -12.106010437011719, "KL/mean": -20.578533172607422, "KL/rejected_KL_mean": -29.051067352294922, "KL/std": 19.843276977539062, "epoch": 0.2511013215859031, "fcm_dpo/beta": 0.032671112567186356, "fcm_dpo/delta": -0.15792220830917358, "fcm_dpo/margin": 16.94504737854004, "fcm_dpo/q_t": 0.3833596706390381, "grad_norm": 13.793436050415039, "learning_rate": 4.6714516072235273e-07, "logits/chosen": -1.3876206874847412, "logits/rejected": -0.823296070098877, "logps/chosen": -131.08453369140625, "logps/ref_chosen": -118.97853088378906, "logps/ref_rejected": -115.76930236816406, "logps/rejected": -144.82037353515625, "loss": 1.0246, "margin_dpo/margin_mean": 16.945049285888672, "margin_dpo/margin_std": 23.416908264160156, "step": 171 }, { "KL/chosen_KL_mean": -13.023738861083984, "KL/mean": -20.552703857421875, "KL/rejected_KL_mean": -28.0816650390625, "KL/std": 14.397984504699707, "epoch": 0.2525697503671072, "fcm_dpo/beta": 0.03226371109485626, "fcm_dpo/delta": -0.08818166702985764, "fcm_dpo/margin": 15.05793285369873, "fcm_dpo/q_t": 0.38461050391197205, "grad_norm": 11.24765682220459, "learning_rate": 4.6650635094610966e-07, "logits/chosen": -2.1441149711608887, "logits/rejected": -1.3875584602355957, "logps/chosen": -114.17752838134766, "logps/ref_chosen": -101.15379333496094, "logps/ref_rejected": -91.00568389892578, "logps/rejected": -119.08734893798828, "loss": 0.9936, "margin_dpo/margin_mean": 15.05793285369873, "margin_dpo/margin_std": 11.797309875488281, "step": 172 }, { "KL/chosen_KL_mean": -11.86862564086914, "KL/mean": -18.42430877685547, "KL/rejected_KL_mean": -24.97998809814453, "KL/std": 13.644248008728027, "epoch": 0.2540381791483113, "fcm_dpo/beta": 0.03220270574092865, "fcm_dpo/delta": -0.022700361907482147, "fcm_dpo/margin": 13.111363410949707, "fcm_dpo/q_t": 0.40119650959968567, "grad_norm": 10.96784782409668, "learning_rate": 4.6586183602616687e-07, "logits/chosen": -1.4542639255523682, "logits/rejected": -0.9804751873016357, "logps/chosen": -110.92530822753906, "logps/ref_chosen": -99.05668640136719, "logps/ref_rejected": -83.8507308959961, "logps/rejected": -108.83071899414062, "loss": 1.058, "margin_dpo/margin_mean": 13.111363410949707, "margin_dpo/margin_std": 14.308538436889648, "step": 173 }, { "KL/chosen_KL_mean": -11.715520858764648, "KL/mean": -20.533138275146484, "KL/rejected_KL_mean": -29.35076141357422, "KL/std": 18.424793243408203, "epoch": 0.2555066079295154, "fcm_dpo/beta": 0.031793683767318726, "fcm_dpo/delta": -0.16523197293281555, "fcm_dpo/margin": 17.63523292541504, "fcm_dpo/q_t": 0.37261199951171875, "grad_norm": 12.907157897949219, "learning_rate": 4.652116329460919e-07, "logits/chosen": -1.7240040302276611, "logits/rejected": -1.6728994846343994, "logps/chosen": -92.12442779541016, "logps/ref_chosen": -80.40890502929688, "logps/ref_rejected": -108.20171356201172, "logps/rejected": -137.55247497558594, "loss": 0.9768, "margin_dpo/margin_mean": 17.635231018066406, "margin_dpo/margin_std": 18.236366271972656, "step": 174 }, { "KL/chosen_KL_mean": -14.907695770263672, "KL/mean": -23.549697875976562, "KL/rejected_KL_mean": -32.19169616699219, "KL/std": 16.576953887939453, "epoch": 0.25697503671071953, "fcm_dpo/beta": 0.031342871487140656, "fcm_dpo/delta": -0.1455506533384323, "fcm_dpo/margin": 17.284000396728516, "fcm_dpo/q_t": 0.3745259642601013, "grad_norm": 12.524797439575195, "learning_rate": 4.645557588393406e-07, "logits/chosen": -1.1668583154678345, "logits/rejected": -0.9017741680145264, "logps/chosen": -97.62419128417969, "logps/ref_chosen": -82.71649169921875, "logps/ref_rejected": -96.66712951660156, "logps/rejected": -128.85882568359375, "loss": 0.9736, "margin_dpo/margin_mean": 17.284000396728516, "margin_dpo/margin_std": 15.672529220581055, "step": 175 }, { "KL/chosen_KL_mean": -12.217338562011719, "KL/mean": -21.009323120117188, "KL/rejected_KL_mean": -29.80130386352539, "KL/std": 18.087621688842773, "epoch": 0.25844346549192365, "fcm_dpo/beta": 0.030842404812574387, "fcm_dpo/delta": -0.14630496501922607, "fcm_dpo/margin": 17.583965301513672, "fcm_dpo/q_t": 0.379184365272522, "grad_norm": 11.971929550170898, "learning_rate": 4.638942309888058e-07, "logits/chosen": -1.5903464555740356, "logits/rejected": -1.423057198524475, "logps/chosen": -91.84664154052734, "logps/ref_chosen": -79.62930297851562, "logps/ref_rejected": -102.6885986328125, "logps/rejected": -132.48989868164062, "loss": 1.0057, "margin_dpo/margin_mean": 17.583965301513672, "margin_dpo/margin_std": 20.579524993896484, "step": 176 }, { "KL/chosen_KL_mean": -13.681251525878906, "KL/mean": -23.94095230102539, "KL/rejected_KL_mean": -34.20064926147461, "KL/std": 20.58495330810547, "epoch": 0.2599118942731278, "fcm_dpo/beta": 0.030286850407719612, "fcm_dpo/delta": -0.22825388610363007, "fcm_dpo/margin": 20.519397735595703, "fcm_dpo/q_t": 0.3649108409881592, "grad_norm": 10.868305206298828, "learning_rate": 4.6322706682636137e-07, "logits/chosen": -1.5100200176239014, "logits/rejected": -1.165813684463501, "logps/chosen": -106.24235534667969, "logps/ref_chosen": -92.56109619140625, "logps/ref_rejected": -103.33839416503906, "logps/rejected": -137.53903198242188, "loss": 0.947, "margin_dpo/margin_mean": 20.519397735595703, "margin_dpo/margin_std": 21.47791290283203, "step": 177 }, { "KL/chosen_KL_mean": -12.522510528564453, "KL/mean": -24.78708839416504, "KL/rejected_KL_mean": -37.051658630371094, "KL/std": 20.10458755493164, "epoch": 0.26138032305433184, "fcm_dpo/beta": 0.029294239357113838, "fcm_dpo/delta": -0.3305700123310089, "fcm_dpo/margin": 24.52914810180664, "fcm_dpo/q_t": 0.34108418226242065, "grad_norm": 12.256343841552734, "learning_rate": 4.6255428393240354e-07, "logits/chosen": -1.7404441833496094, "logits/rejected": -1.512343168258667, "logps/chosen": -96.7883071899414, "logps/ref_chosen": -84.26579284667969, "logps/ref_rejected": -110.79730987548828, "logps/rejected": -147.84896850585938, "loss": 0.8748, "margin_dpo/margin_mean": 24.52914810180664, "margin_dpo/margin_std": 21.7408447265625, "step": 178 }, { "KL/chosen_KL_mean": -16.122901916503906, "KL/mean": -24.08049774169922, "KL/rejected_KL_mean": -32.038089752197266, "KL/std": 19.43193817138672, "epoch": 0.26284875183553597, "fcm_dpo/beta": 0.02887153998017311, "fcm_dpo/delta": -0.06115927919745445, "fcm_dpo/margin": 15.91518783569336, "fcm_dpo/q_t": 0.39799466729164124, "grad_norm": 11.934700965881348, "learning_rate": 4.6187590003538724e-07, "logits/chosen": -1.3663103580474854, "logits/rejected": -0.9279817938804626, "logps/chosen": -109.57087707519531, "logps/ref_chosen": -93.4479751586914, "logps/ref_rejected": -94.25541687011719, "logps/rejected": -126.29350280761719, "loss": 1.0752, "margin_dpo/margin_mean": 15.91518783569336, "margin_dpo/margin_std": 22.444398880004883, "step": 179 }, { "KL/chosen_KL_mean": -13.435199737548828, "KL/mean": -23.536970138549805, "KL/rejected_KL_mean": -33.63874053955078, "KL/std": 18.837366104125977, "epoch": 0.2643171806167401, "fcm_dpo/beta": 0.028447844088077545, "fcm_dpo/delta": -0.18033847212791443, "fcm_dpo/margin": 20.203540802001953, "fcm_dpo/q_t": 0.3693804442882538, "grad_norm": 14.07954216003418, "learning_rate": 4.611919330113591e-07, "logits/chosen": -1.31528639793396, "logits/rejected": -1.0739433765411377, "logps/chosen": -102.27073669433594, "logps/ref_chosen": -88.83554077148438, "logps/ref_rejected": -103.90696716308594, "logps/rejected": -137.54571533203125, "loss": 0.9565, "margin_dpo/margin_mean": 20.203540802001953, "margin_dpo/margin_std": 19.297584533691406, "step": 180 }, { "KL/chosen_KL_mean": -13.542984008789062, "KL/mean": -21.364585876464844, "KL/rejected_KL_mean": -29.18618392944336, "KL/std": 18.008453369140625, "epoch": 0.2657856093979442, "fcm_dpo/beta": 0.02828541398048401, "fcm_dpo/delta": -0.04342437535524368, "fcm_dpo/margin": 15.643199920654297, "fcm_dpo/q_t": 0.3998418152332306, "grad_norm": 10.888174057006836, "learning_rate": 4.605024008834863e-07, "logits/chosen": -1.6260757446289062, "logits/rejected": -1.17562997341156, "logps/chosen": -92.77694702148438, "logps/ref_chosen": -79.23396301269531, "logps/ref_rejected": -69.17442321777344, "logps/rejected": -98.36061096191406, "loss": 1.0574, "margin_dpo/margin_mean": 15.643199920654297, "margin_dpo/margin_std": 19.189762115478516, "step": 181 }, { "KL/chosen_KL_mean": -17.108013153076172, "KL/mean": -27.242496490478516, "KL/rejected_KL_mean": -37.376976013183594, "KL/std": 20.71778106689453, "epoch": 0.26725403817914833, "fcm_dpo/beta": 0.0278313010931015, "fcm_dpo/delta": -0.16938933730125427, "fcm_dpo/margin": 20.26896858215332, "fcm_dpo/q_t": 0.37418854236602783, "grad_norm": 11.24946117401123, "learning_rate": 4.598073218215817e-07, "logits/chosen": -1.1280932426452637, "logits/rejected": -0.8971935510635376, "logps/chosen": -93.19020080566406, "logps/ref_chosen": -76.08218383789062, "logps/ref_rejected": -94.9686050415039, "logps/rejected": -132.3455810546875, "loss": 0.9877, "margin_dpo/margin_mean": 20.26896858215332, "margin_dpo/margin_std": 22.696243286132812, "step": 182 }, { "KL/chosen_KL_mean": -17.526500701904297, "KL/mean": -27.564620971679688, "KL/rejected_KL_mean": -37.60274887084961, "KL/std": 19.01822853088379, "epoch": 0.2687224669603524, "fcm_dpo/beta": 0.027393944561481476, "fcm_dpo/delta": -0.15469077229499817, "fcm_dpo/margin": 20.07624053955078, "fcm_dpo/q_t": 0.374215304851532, "grad_norm": 11.697032928466797, "learning_rate": 4.5910671414162484e-07, "logits/chosen": -1.566564917564392, "logits/rejected": -0.965617299079895, "logps/chosen": -111.43440246582031, "logps/ref_chosen": -93.90790557861328, "logps/ref_rejected": -83.56061553955078, "logps/rejected": -121.16336059570312, "loss": 0.9763, "margin_dpo/margin_mean": 20.076242446899414, "margin_dpo/margin_std": 19.54137420654297, "step": 183 }, { "KL/chosen_KL_mean": -15.65963363647461, "KL/mean": -22.424062728881836, "KL/rejected_KL_mean": -29.188491821289062, "KL/std": 19.558467864990234, "epoch": 0.2701908957415565, "fcm_dpo/beta": 0.027409566566348076, "fcm_dpo/delta": 0.02972198650240898, "fcm_dpo/margin": 13.528854370117188, "fcm_dpo/q_t": 0.4141917824745178, "grad_norm": 11.068546295166016, "learning_rate": 4.5840059630527985e-07, "logits/chosen": -1.6560285091400146, "logits/rejected": -1.3880600929260254, "logps/chosen": -102.52754211425781, "logps/ref_chosen": -86.86790466308594, "logps/ref_rejected": -84.28456115722656, "logps/rejected": -113.47305297851562, "loss": 1.1288, "margin_dpo/margin_mean": 13.528854370117188, "margin_dpo/margin_std": 21.266395568847656, "step": 184 }, { "KL/chosen_KL_mean": -16.67963409423828, "KL/mean": -24.917774200439453, "KL/rejected_KL_mean": -33.155914306640625, "KL/std": 19.262619018554688, "epoch": 0.27165932452276065, "fcm_dpo/beta": 0.02736147865653038, "fcm_dpo/delta": -0.052008435130119324, "fcm_dpo/margin": 16.476280212402344, "fcm_dpo/q_t": 0.3988819718360901, "grad_norm": 12.26574420928955, "learning_rate": 4.5768898691940836e-07, "logits/chosen": -1.9388419389724731, "logits/rejected": -1.3091202974319458, "logps/chosen": -110.70105743408203, "logps/ref_chosen": -94.02142333984375, "logps/ref_rejected": -81.45668029785156, "logps/rejected": -114.61259460449219, "loss": 1.0696, "margin_dpo/margin_mean": 16.476280212402344, "margin_dpo/margin_std": 22.24062728881836, "step": 185 }, { "KL/chosen_KL_mean": -20.566402435302734, "KL/mean": -29.49085807800293, "KL/rejected_KL_mean": -38.41530990600586, "KL/std": 18.53443145751953, "epoch": 0.27312775330396477, "fcm_dpo/beta": 0.02705829031765461, "fcm_dpo/delta": -0.08530790358781815, "fcm_dpo/margin": 17.84890365600586, "fcm_dpo/q_t": 0.38912177085876465, "grad_norm": 12.649065017700195, "learning_rate": 4.5697190473557947e-07, "logits/chosen": -2.0586376190185547, "logits/rejected": -1.1304185390472412, "logps/chosen": -134.75808715820312, "logps/ref_chosen": -114.19168090820312, "logps/ref_rejected": -93.33912658691406, "logps/rejected": -131.7544403076172, "loss": 1.0289, "margin_dpo/margin_mean": 17.84890365600586, "margin_dpo/margin_std": 20.041854858398438, "step": 186 }, { "KL/chosen_KL_mean": -17.577930450439453, "KL/mean": -26.029033660888672, "KL/rejected_KL_mean": -34.48012924194336, "KL/std": 20.709232330322266, "epoch": 0.2745961820851689, "fcm_dpo/beta": 0.02697415091097355, "fcm_dpo/delta": -0.05723072960972786, "fcm_dpo/margin": 16.90220069885254, "fcm_dpo/q_t": 0.3974665403366089, "grad_norm": 11.207185745239258, "learning_rate": 4.5624936864957555e-07, "logits/chosen": -1.668868064880371, "logits/rejected": -1.4298734664916992, "logps/chosen": -94.44747924804688, "logps/ref_chosen": -76.86955261230469, "logps/ref_rejected": -88.16494750976562, "logps/rejected": -122.64507293701172, "loss": 1.0664, "margin_dpo/margin_mean": 16.90220069885254, "margin_dpo/margin_std": 22.61894416809082, "step": 187 }, { "KL/chosen_KL_mean": -13.562904357910156, "KL/mean": -23.228605270385742, "KL/rejected_KL_mean": -32.894309997558594, "KL/std": 20.53191375732422, "epoch": 0.27606461086637296, "fcm_dpo/beta": 0.026604700833559036, "fcm_dpo/delta": -0.1177414134144783, "fcm_dpo/margin": 19.331411361694336, "fcm_dpo/q_t": 0.3803586959838867, "grad_norm": 11.503058433532715, "learning_rate": 4.5552139770089454e-07, "logits/chosen": -2.0378894805908203, "logits/rejected": -1.7957950830459595, "logps/chosen": -92.75770568847656, "logps/ref_chosen": -79.1948013305664, "logps/ref_rejected": -96.82889556884766, "logps/rejected": -129.72320556640625, "loss": 1.0017, "margin_dpo/margin_mean": 19.331411361694336, "margin_dpo/margin_std": 19.365528106689453, "step": 188 }, { "KL/chosen_KL_mean": -17.918109893798828, "KL/mean": -27.877155303955078, "KL/rejected_KL_mean": -37.83620071411133, "KL/std": 20.739830017089844, "epoch": 0.2775330396475771, "fcm_dpo/beta": 0.02640739642083645, "fcm_dpo/delta": -0.1293770670890808, "fcm_dpo/margin": 19.9180965423584, "fcm_dpo/q_t": 0.3835380971431732, "grad_norm": 10.30174446105957, "learning_rate": 4.5478801107224794e-07, "logits/chosen": -1.5843603610992432, "logits/rejected": -0.9261908531188965, "logps/chosen": -101.8109130859375, "logps/ref_chosen": -83.8927993774414, "logps/ref_rejected": -79.46589660644531, "logps/rejected": -117.30209350585938, "loss": 1.0112, "margin_dpo/margin_mean": 19.9180965423584, "margin_dpo/margin_std": 23.127300262451172, "step": 189 }, { "KL/chosen_KL_mean": -14.227371215820312, "KL/mean": -26.166576385498047, "KL/rejected_KL_mean": -38.10578918457031, "KL/std": 22.842117309570312, "epoch": 0.2790014684287812, "fcm_dpo/beta": 0.02579706907272339, "fcm_dpo/delta": -0.22294239699840546, "fcm_dpo/margin": 23.8784122467041, "fcm_dpo/q_t": 0.3596351146697998, "grad_norm": 11.203171730041504, "learning_rate": 4.5404922808905543e-07, "logits/chosen": -1.945347547531128, "logits/rejected": -1.3640046119689941, "logps/chosen": -117.66883850097656, "logps/ref_chosen": -103.44146728515625, "logps/ref_rejected": -109.21502685546875, "logps/rejected": -147.32081604003906, "loss": 0.9311, "margin_dpo/margin_mean": 23.8784122467041, "margin_dpo/margin_std": 21.502368927001953, "step": 190 }, { "KL/chosen_KL_mean": -13.442211151123047, "KL/mean": -26.74632453918457, "KL/rejected_KL_mean": -40.050437927246094, "KL/std": 24.585084915161133, "epoch": 0.28046989720998533, "fcm_dpo/beta": 0.025156119838356972, "fcm_dpo/delta": -0.278470903635025, "fcm_dpo/margin": 26.60822296142578, "fcm_dpo/q_t": 0.35160866379737854, "grad_norm": 11.979606628417969, "learning_rate": 4.5330506821893565e-07, "logits/chosen": -1.5345382690429688, "logits/rejected": -1.2213177680969238, "logps/chosen": -113.22772216796875, "logps/ref_chosen": -99.78551483154297, "logps/ref_rejected": -116.00028228759766, "logps/rejected": -156.05072021484375, "loss": 0.91, "margin_dpo/margin_mean": 26.60822296142578, "margin_dpo/margin_std": 25.055011749267578, "step": 191 }, { "KL/chosen_KL_mean": -22.374202728271484, "KL/mean": -31.703136444091797, "KL/rejected_KL_mean": -41.03207015991211, "KL/std": 19.758882522583008, "epoch": 0.28193832599118945, "fcm_dpo/beta": 0.024819843471050262, "fcm_dpo/delta": -0.06463722139596939, "fcm_dpo/margin": 18.65787124633789, "fcm_dpo/q_t": 0.39531421661376953, "grad_norm": 12.722247123718262, "learning_rate": 4.5255555107119336e-07, "logits/chosen": -1.6833852529525757, "logits/rejected": -1.4359793663024902, "logps/chosen": -123.23452758789062, "logps/ref_chosen": -100.86032104492188, "logps/ref_rejected": -109.44500732421875, "logps/rejected": -150.47708129882812, "loss": 1.051, "margin_dpo/margin_mean": 18.65787124633789, "margin_dpo/margin_std": 23.250160217285156, "step": 192 }, { "KL/chosen_KL_mean": -18.400806427001953, "KL/mean": -26.098926544189453, "KL/rejected_KL_mean": -33.79704666137695, "KL/std": 20.717302322387695, "epoch": 0.2834067547723935, "fcm_dpo/beta": 0.024865545332431793, "fcm_dpo/delta": 0.01737535372376442, "fcm_dpo/margin": 15.396240234375, "fcm_dpo/q_t": 0.41288137435913086, "grad_norm": 12.00924015045166, "learning_rate": 4.5180069639630236e-07, "logits/chosen": -2.10103702545166, "logits/rejected": -1.528651475906372, "logps/chosen": -111.33912658691406, "logps/ref_chosen": -92.93831634521484, "logps/ref_rejected": -83.10917663574219, "logps/rejected": -116.9062271118164, "loss": 1.1323, "margin_dpo/margin_mean": 15.396240234375, "margin_dpo/margin_std": 25.021080017089844, "step": 193 }, { "KL/chosen_KL_mean": -18.98476791381836, "KL/mean": -27.93246078491211, "KL/rejected_KL_mean": -36.88015365600586, "KL/std": 21.56507110595703, "epoch": 0.28487518355359764, "fcm_dpo/beta": 0.024799171835184097, "fcm_dpo/delta": -0.04483334720134735, "fcm_dpo/margin": 17.8953914642334, "fcm_dpo/q_t": 0.39921244978904724, "grad_norm": 10.03604507446289, "learning_rate": 4.510405240853854e-07, "logits/chosen": -2.3598215579986572, "logits/rejected": -1.9853980541229248, "logps/chosen": -82.77633666992188, "logps/ref_chosen": -63.79157257080078, "logps/ref_rejected": -68.72665405273438, "logps/rejected": -105.6068115234375, "loss": 1.065, "margin_dpo/margin_mean": 17.8953914642334, "margin_dpo/margin_std": 22.79195213317871, "step": 194 }, { "KL/chosen_KL_mean": -18.17306137084961, "KL/mean": -29.984188079833984, "KL/rejected_KL_mean": -41.79531478881836, "KL/std": 21.90328598022461, "epoch": 0.28634361233480177, "fcm_dpo/beta": 0.02442985773086548, "fcm_dpo/delta": -0.18216848373413086, "fcm_dpo/margin": 23.62226104736328, "fcm_dpo/q_t": 0.3691832721233368, "grad_norm": 10.49765682220459, "learning_rate": 4.5027505416968985e-07, "logits/chosen": -1.4883873462677002, "logits/rejected": -1.3629332780838013, "logps/chosen": -102.05851745605469, "logps/ref_chosen": -83.88545989990234, "logps/ref_rejected": -104.00125122070312, "logps/rejected": -145.79656982421875, "loss": 0.9761, "margin_dpo/margin_mean": 23.62226104736328, "margin_dpo/margin_std": 25.13174819946289, "step": 195 }, { "KL/chosen_KL_mean": -21.14596939086914, "KL/mean": -32.32801818847656, "KL/rejected_KL_mean": -43.51006317138672, "KL/std": 23.47501564025879, "epoch": 0.2878120411160059, "fcm_dpo/beta": 0.024023931473493576, "fcm_dpo/delta": -0.14107677340507507, "fcm_dpo/margin": 22.364093780517578, "fcm_dpo/q_t": 0.37529122829437256, "grad_norm": 9.886722564697266, "learning_rate": 4.495043068200599e-07, "logits/chosen": -1.3276188373565674, "logits/rejected": -0.8795911073684692, "logps/chosen": -110.54733276367188, "logps/ref_chosen": -89.4013671875, "logps/ref_rejected": -83.34735107421875, "logps/rejected": -126.85740661621094, "loss": 0.9791, "margin_dpo/margin_mean": 22.36409568786621, "margin_dpo/margin_std": 21.09688949584961, "step": 196 }, { "KL/chosen_KL_mean": -21.491859436035156, "KL/mean": -28.39409828186035, "KL/rejected_KL_mean": -35.29633331298828, "KL/std": 18.966053009033203, "epoch": 0.28928046989721, "fcm_dpo/beta": 0.024061817675828934, "fcm_dpo/delta": 0.06899162381887436, "fcm_dpo/margin": 13.804469108581543, "fcm_dpo/q_t": 0.42184683680534363, "grad_norm": 10.425678253173828, "learning_rate": 4.4872830234640493e-07, "logits/chosen": -1.316481590270996, "logits/rejected": -0.985496461391449, "logps/chosen": -108.33256530761719, "logps/ref_chosen": -86.84070587158203, "logps/ref_rejected": -82.79289245605469, "logps/rejected": -118.0892333984375, "loss": 1.136, "margin_dpo/margin_mean": 13.80447006225586, "margin_dpo/margin_std": 20.02841567993164, "step": 197 }, { "KL/chosen_KL_mean": -24.829265594482422, "KL/mean": -36.009063720703125, "KL/rejected_KL_mean": -47.18885040283203, "KL/std": 22.302886962890625, "epoch": 0.2907488986784141, "fcm_dpo/beta": 0.02386770024895668, "fcm_dpo/delta": -0.1372983455657959, "fcm_dpo/margin": 22.359588623046875, "fcm_dpo/q_t": 0.3786056935787201, "grad_norm": 12.300978660583496, "learning_rate": 4.479470611971645e-07, "logits/chosen": -1.814868688583374, "logits/rejected": -1.6386330127716064, "logps/chosen": -109.87751007080078, "logps/ref_chosen": -85.04824829101562, "logps/ref_rejected": -104.10365295410156, "logps/rejected": -151.29251098632812, "loss": 0.9894, "margin_dpo/margin_mean": 22.359588623046875, "margin_dpo/margin_std": 23.068775177001953, "step": 198 }, { "KL/chosen_KL_mean": -24.310989379882812, "KL/mean": -33.53852462768555, "KL/rejected_KL_mean": -42.76606369018555, "KL/std": 21.856117248535156, "epoch": 0.2922173274596182, "fcm_dpo/beta": 0.023724917322397232, "fcm_dpo/delta": -0.0386846698820591, "fcm_dpo/margin": 18.455078125, "fcm_dpo/q_t": 0.39935144782066345, "grad_norm": 10.691583633422852, "learning_rate": 4.471606039587695e-07, "logits/chosen": -1.6142724752426147, "logits/rejected": -1.3537925481796265, "logps/chosen": -106.09768676757812, "logps/ref_chosen": -81.78669738769531, "logps/ref_rejected": -90.790771484375, "logps/rejected": -133.5568389892578, "loss": 1.0631, "margin_dpo/margin_mean": 18.455076217651367, "margin_dpo/margin_std": 22.664146423339844, "step": 199 }, { "KL/chosen_KL_mean": -23.895748138427734, "KL/mean": -34.87153625488281, "KL/rejected_KL_mean": -45.84732437133789, "KL/std": 23.32733154296875, "epoch": 0.2936857562408223, "fcm_dpo/beta": 0.02342352643609047, "fcm_dpo/delta": -0.11769238114356995, "fcm_dpo/margin": 21.951568603515625, "fcm_dpo/q_t": 0.38429421186447144, "grad_norm": 10.506084442138672, "learning_rate": 4.4636895135509966e-07, "logits/chosen": -2.019775152206421, "logits/rejected": -1.5217949151992798, "logps/chosen": -108.1638412475586, "logps/ref_chosen": -84.26809692382812, "logps/ref_rejected": -86.96786499023438, "logps/rejected": -132.815185546875, "loss": 1.0279, "margin_dpo/margin_mean": 21.951566696166992, "margin_dpo/margin_std": 26.699819564819336, "step": 200 }, { "KL/chosen_KL_mean": -25.67392349243164, "KL/mean": -36.26042938232422, "KL/rejected_KL_mean": -46.84693908691406, "KL/std": 25.164995193481445, "epoch": 0.29515418502202645, "fcm_dpo/beta": 0.023270417004823685, "fcm_dpo/delta": -0.09499061107635498, "fcm_dpo/margin": 21.173011779785156, "fcm_dpo/q_t": 0.3889583349227905, "grad_norm": 10.843786239624023, "learning_rate": 4.455721242469372e-07, "logits/chosen": -1.3048158884048462, "logits/rejected": -0.9126987457275391, "logps/chosen": -133.4464111328125, "logps/ref_chosen": -107.77249145507812, "logps/ref_rejected": -119.79248046875, "logps/rejected": -166.63941955566406, "loss": 1.0451, "margin_dpo/margin_mean": 21.173011779785156, "margin_dpo/margin_std": 27.416826248168945, "step": 201 }, { "KL/chosen_KL_mean": -24.164569854736328, "KL/mean": -33.96553039550781, "KL/rejected_KL_mean": -43.7664909362793, "KL/std": 21.009117126464844, "epoch": 0.2966226138032305, "fcm_dpo/beta": 0.023106824606657028, "fcm_dpo/delta": -0.054147012531757355, "fcm_dpo/margin": 19.60192108154297, "fcm_dpo/q_t": 0.39703041315078735, "grad_norm": 11.08411693572998, "learning_rate": 4.4477014363141755e-07, "logits/chosen": -1.60884690284729, "logits/rejected": -1.478827953338623, "logps/chosen": -100.13702392578125, "logps/ref_chosen": -75.97245788574219, "logps/ref_rejected": -94.4599838256836, "logps/rejected": -138.22647094726562, "loss": 1.0557, "margin_dpo/margin_mean": 19.60192108154297, "margin_dpo/margin_std": 23.767866134643555, "step": 202 }, { "KL/chosen_KL_mean": -21.290264129638672, "KL/mean": -32.43567657470703, "KL/rejected_KL_mean": -43.581092834472656, "KL/std": 21.94060516357422, "epoch": 0.29809104258443464, "fcm_dpo/beta": 0.022885257378220558, "fcm_dpo/delta": -0.11295238882303238, "fcm_dpo/margin": 22.290817260742188, "fcm_dpo/q_t": 0.38228052854537964, "grad_norm": 9.854025840759277, "learning_rate": 4.439630306414758e-07, "logits/chosen": -2.015516996383667, "logits/rejected": -1.5743533372879028, "logps/chosen": -116.25741577148438, "logps/ref_chosen": -94.96715545654297, "logps/ref_rejected": -92.8876724243164, "logps/rejected": -136.46876525878906, "loss": 1.0029, "margin_dpo/margin_mean": 22.290817260742188, "margin_dpo/margin_std": 22.838611602783203, "step": 203 }, { "KL/chosen_KL_mean": -27.039588928222656, "KL/mean": -36.678768157958984, "KL/rejected_KL_mean": -46.317935943603516, "KL/std": 21.865049362182617, "epoch": 0.29955947136563876, "fcm_dpo/beta": 0.022766832262277603, "fcm_dpo/delta": -0.03980274498462677, "fcm_dpo/margin": 19.27835464477539, "fcm_dpo/q_t": 0.40027329325675964, "grad_norm": 11.082603454589844, "learning_rate": 4.431508065452897e-07, "logits/chosen": -1.7771902084350586, "logits/rejected": -1.113682508468628, "logps/chosen": -143.39678955078125, "logps/ref_chosen": -116.35719299316406, "logps/ref_rejected": -93.39759063720703, "logps/rejected": -139.7155303955078, "loss": 1.0736, "margin_dpo/margin_mean": 19.27835464477539, "margin_dpo/margin_std": 25.28366470336914, "step": 204 }, { "KL/chosen_KL_mean": -25.126937866210938, "KL/mean": -37.59545135498047, "KL/rejected_KL_mean": -50.06396484375, "KL/std": 24.260665893554688, "epoch": 0.3010279001468429, "fcm_dpo/beta": 0.022413738071918488, "fcm_dpo/delta": -0.16370999813079834, "fcm_dpo/margin": 24.937026977539062, "fcm_dpo/q_t": 0.3693287968635559, "grad_norm": 9.562270164489746, "learning_rate": 4.4233349274571974e-07, "logits/chosen": -2.130115509033203, "logits/rejected": -1.6934013366699219, "logps/chosen": -113.98628234863281, "logps/ref_chosen": -88.85934448242188, "logps/ref_rejected": -91.8544921875, "logps/rejected": -141.91845703125, "loss": 0.9543, "margin_dpo/margin_mean": 24.937026977539062, "margin_dpo/margin_std": 20.895641326904297, "step": 205 }, { "KL/chosen_KL_mean": -25.727798461914062, "KL/mean": -37.07975769042969, "KL/rejected_KL_mean": -48.43171310424805, "KL/std": 23.603069305419922, "epoch": 0.302496328928047, "fcm_dpo/beta": 0.022127434611320496, "fcm_dpo/delta": -0.10535117238759995, "fcm_dpo/margin": 22.703920364379883, "fcm_dpo/q_t": 0.382033109664917, "grad_norm": 11.559245109558105, "learning_rate": 4.415111107797445e-07, "logits/chosen": -1.5030491352081299, "logits/rejected": -1.348142147064209, "logps/chosen": -102.2741470336914, "logps/ref_chosen": -76.54634857177734, "logps/ref_rejected": -102.95314025878906, "logps/rejected": -151.38485717773438, "loss": 0.993, "margin_dpo/margin_mean": 22.703922271728516, "margin_dpo/margin_std": 20.112821578979492, "step": 206 }, { "KL/chosen_KL_mean": -27.073078155517578, "KL/mean": -39.712303161621094, "KL/rejected_KL_mean": -52.35152816772461, "KL/std": 28.472858428955078, "epoch": 0.3039647577092511, "fcm_dpo/beta": 0.021858762949705124, "fcm_dpo/delta": -0.1568467617034912, "fcm_dpo/margin": 25.27845001220703, "fcm_dpo/q_t": 0.3798820972442627, "grad_norm": 11.571463584899902, "learning_rate": 4.4068368231789365e-07, "logits/chosen": -1.9767465591430664, "logits/rejected": -1.5483746528625488, "logps/chosen": -113.30472564697266, "logps/ref_chosen": -86.23164367675781, "logps/ref_rejected": -90.65512084960938, "logps/rejected": -143.00665283203125, "loss": 1.0188, "margin_dpo/margin_mean": 25.278446197509766, "margin_dpo/margin_std": 32.78599548339844, "step": 207 }, { "KL/chosen_KL_mean": -27.880897521972656, "KL/mean": -40.984561920166016, "KL/rejected_KL_mean": -54.088226318359375, "KL/std": 26.311359405517578, "epoch": 0.3054331864904552, "fcm_dpo/beta": 0.021541737020015717, "fcm_dpo/delta": -0.16915631294250488, "fcm_dpo/margin": 26.207324981689453, "fcm_dpo/q_t": 0.36920031905174255, "grad_norm": 11.050517082214355, "learning_rate": 4.398512291636768e-07, "logits/chosen": -1.604923963546753, "logits/rejected": -1.2396068572998047, "logps/chosen": -122.04044342041016, "logps/ref_chosen": -94.1595458984375, "logps/ref_rejected": -100.96233367919922, "logps/rejected": -155.05056762695312, "loss": 0.9631, "margin_dpo/margin_mean": 26.207324981689453, "margin_dpo/margin_std": 24.11130714416504, "step": 208 }, { "KL/chosen_KL_mean": -23.627521514892578, "KL/mean": -35.13088607788086, "KL/rejected_KL_mean": -46.634254455566406, "KL/std": 23.409576416015625, "epoch": 0.3069016152716593, "fcm_dpo/beta": 0.021210001781582832, "fcm_dpo/delta": -0.09055854380130768, "fcm_dpo/margin": 23.006729125976562, "fcm_dpo/q_t": 0.38881275057792664, "grad_norm": 9.982840538024902, "learning_rate": 4.3901377325300857e-07, "logits/chosen": -2.139176368713379, "logits/rejected": -1.6714458465576172, "logps/chosen": -107.79808807373047, "logps/ref_chosen": -84.17056274414062, "logps/ref_rejected": -87.61955261230469, "logps/rejected": -134.25379943847656, "loss": 1.031, "margin_dpo/margin_mean": 23.006729125976562, "margin_dpo/margin_std": 26.52461051940918, "step": 209 }, { "KL/chosen_KL_mean": -26.833881378173828, "KL/mean": -37.803192138671875, "KL/rejected_KL_mean": -48.77250289916992, "KL/std": 24.912778854370117, "epoch": 0.30837004405286345, "fcm_dpo/beta": 0.02108333259820938, "fcm_dpo/delta": -0.06413492560386658, "fcm_dpo/margin": 21.93861961364746, "fcm_dpo/q_t": 0.3942887783050537, "grad_norm": 9.301071166992188, "learning_rate": 4.381713366536311e-07, "logits/chosen": -1.9791654348373413, "logits/rejected": -1.5943191051483154, "logps/chosen": -108.00505065917969, "logps/ref_chosen": -81.17117309570312, "logps/ref_rejected": -84.17478942871094, "logps/rejected": -132.94729614257812, "loss": 1.0423, "margin_dpo/margin_mean": 21.938621520996094, "margin_dpo/margin_std": 25.38385009765625, "step": 210 }, { "KL/chosen_KL_mean": -29.871662139892578, "KL/mean": -40.694236755371094, "KL/rejected_KL_mean": -51.51681137084961, "KL/std": 26.111743927001953, "epoch": 0.30983847283406757, "fcm_dpo/beta": 0.02100434899330139, "fcm_dpo/delta": -0.05589741840958595, "fcm_dpo/margin": 21.645160675048828, "fcm_dpo/q_t": 0.39645177125930786, "grad_norm": 11.116700172424316, "learning_rate": 4.373239415645323e-07, "logits/chosen": -2.050020933151245, "logits/rejected": -1.431436538696289, "logps/chosen": -138.58438110351562, "logps/ref_chosen": -108.71270751953125, "logps/ref_rejected": -93.55564880371094, "logps/rejected": -145.0724639892578, "loss": 1.0536, "margin_dpo/margin_mean": 21.645156860351562, "margin_dpo/margin_std": 26.638874053955078, "step": 211 }, { "KL/chosen_KL_mean": -20.77536392211914, "KL/mean": -36.436119079589844, "KL/rejected_KL_mean": -52.09687042236328, "KL/std": 29.639421463012695, "epoch": 0.31130690161527164, "fcm_dpo/beta": 0.02053999900817871, "fcm_dpo/delta": -0.2517920732498169, "fcm_dpo/margin": 31.32150650024414, "fcm_dpo/q_t": 0.355773389339447, "grad_norm": 9.708909034729004, "learning_rate": 4.3647161031536086e-07, "logits/chosen": -1.9287019968032837, "logits/rejected": -1.5204100608825684, "logps/chosen": -119.13731384277344, "logps/ref_chosen": -98.36195373535156, "logps/ref_rejected": -109.88999938964844, "logps/rejected": -161.98687744140625, "loss": 0.9232, "margin_dpo/margin_mean": 31.32150650024414, "margin_dpo/margin_std": 29.584415435791016, "step": 212 }, { "KL/chosen_KL_mean": -24.694488525390625, "KL/mean": -36.65623474121094, "KL/rejected_KL_mean": -48.61798095703125, "KL/std": 27.293115615844727, "epoch": 0.31277533039647576, "fcm_dpo/beta": 0.020276721566915512, "fcm_dpo/delta": -0.0873616486787796, "fcm_dpo/margin": 23.92348861694336, "fcm_dpo/q_t": 0.3891180157661438, "grad_norm": 12.534401893615723, "learning_rate": 4.3561436536583774e-07, "logits/chosen": -1.8651275634765625, "logits/rejected": -1.0763912200927734, "logps/chosen": -132.7498016357422, "logps/ref_chosen": -108.05531311035156, "logps/ref_rejected": -100.14414978027344, "logps/rejected": -148.7621307373047, "loss": 1.0405, "margin_dpo/margin_mean": 23.923484802246094, "margin_dpo/margin_std": 29.411853790283203, "step": 213 }, { "KL/chosen_KL_mean": -26.1181640625, "KL/mean": -37.24357604980469, "KL/rejected_KL_mean": -48.36900329589844, "KL/std": 29.137386322021484, "epoch": 0.3142437591776799, "fcm_dpo/beta": 0.020191876217722893, "fcm_dpo/delta": -0.05040303245186806, "fcm_dpo/margin": 22.250835418701172, "fcm_dpo/q_t": 0.3999677896499634, "grad_norm": 9.73897647857666, "learning_rate": 4.3475222930516473e-07, "logits/chosen": -1.7345507144927979, "logits/rejected": -1.6115949153900146, "logps/chosen": -103.92289733886719, "logps/ref_chosen": -77.80473327636719, "logps/ref_rejected": -84.27578735351562, "logps/rejected": -132.64479064941406, "loss": 1.0817, "margin_dpo/margin_mean": 22.250835418701172, "margin_dpo/margin_std": 32.05826950073242, "step": 214 }, { "KL/chosen_KL_mean": -22.53055191040039, "KL/mean": -37.4747314453125, "KL/rejected_KL_mean": -52.41891860961914, "KL/std": 28.717050552368164, "epoch": 0.315712187958884, "fcm_dpo/beta": 0.019857052713632584, "fcm_dpo/delta": -0.1994040459394455, "fcm_dpo/margin": 29.888351440429688, "fcm_dpo/q_t": 0.3654733896255493, "grad_norm": 9.478631019592285, "learning_rate": 4.3388522485142885e-07, "logits/chosen": -1.8485474586486816, "logits/rejected": -1.3810014724731445, "logps/chosen": -107.64444732666016, "logps/ref_chosen": -85.1138916015625, "logps/ref_rejected": -96.86151123046875, "logps/rejected": -149.28042602539062, "loss": 0.9468, "margin_dpo/margin_mean": 29.888351440429688, "margin_dpo/margin_std": 28.164031982421875, "step": 215 }, { "KL/chosen_KL_mean": -24.637920379638672, "KL/mean": -37.65586853027344, "KL/rejected_KL_mean": -50.67382049560547, "KL/std": 25.739974975585938, "epoch": 0.31718061674008813, "fcm_dpo/beta": 0.019632035866379738, "fcm_dpo/delta": -0.11399906873703003, "fcm_dpo/margin": 26.035905838012695, "fcm_dpo/q_t": 0.3831835985183716, "grad_norm": 9.020357131958008, "learning_rate": 4.330133748510036e-07, "logits/chosen": -1.9177556037902832, "logits/rejected": -1.5238847732543945, "logps/chosen": -105.230224609375, "logps/ref_chosen": -80.59230041503906, "logps/ref_rejected": -81.41983795166016, "logps/rejected": -132.09365844726562, "loss": 1.0026, "margin_dpo/margin_mean": 26.035905838012695, "margin_dpo/margin_std": 26.6428279876709, "step": 216 }, { "KL/chosen_KL_mean": -29.788307189941406, "KL/mean": -40.5751953125, "KL/rejected_KL_mean": -51.36207962036133, "KL/std": 26.362041473388672, "epoch": 0.3186490455212922, "fcm_dpo/beta": 0.019519049674272537, "fcm_dpo/delta": -0.021569598466157913, "fcm_dpo/margin": 21.57377052307129, "fcm_dpo/q_t": 0.4050278663635254, "grad_norm": 11.093538284301758, "learning_rate": 4.3213670227794757e-07, "logits/chosen": -2.045012950897217, "logits/rejected": -1.7392696142196655, "logps/chosen": -123.26089477539062, "logps/ref_chosen": -93.47257995605469, "logps/ref_rejected": -103.488525390625, "logps/rejected": -154.85060119628906, "loss": 1.0863, "margin_dpo/margin_mean": 21.57377052307129, "margin_dpo/margin_std": 29.89163589477539, "step": 217 }, { "KL/chosen_KL_mean": -29.10724639892578, "KL/mean": -42.07308578491211, "KL/rejected_KL_mean": -55.03892517089844, "KL/std": 26.86640739440918, "epoch": 0.3201174743024963, "fcm_dpo/beta": 0.019312191754579544, "fcm_dpo/delta": -0.10361404716968536, "fcm_dpo/margin": 25.931678771972656, "fcm_dpo/q_t": 0.38676512241363525, "grad_norm": 9.545085906982422, "learning_rate": 4.3125523023339815e-07, "logits/chosen": -1.3147671222686768, "logits/rejected": -0.925396203994751, "logps/chosen": -118.16607666015625, "logps/ref_chosen": -89.05883026123047, "logps/ref_rejected": -94.30680847167969, "logps/rejected": -149.34573364257812, "loss": 1.0344, "margin_dpo/margin_mean": 25.931678771972656, "margin_dpo/margin_std": 31.981922149658203, "step": 218 }, { "KL/chosen_KL_mean": -26.121810913085938, "KL/mean": -37.84553527832031, "KL/rejected_KL_mean": -49.56926345825195, "KL/std": 31.050649642944336, "epoch": 0.32158590308370044, "fcm_dpo/beta": 0.019267480820417404, "fcm_dpo/delta": -0.05314759165048599, "fcm_dpo/margin": 23.447450637817383, "fcm_dpo/q_t": 0.39811718463897705, "grad_norm": 10.957161903381348, "learning_rate": 4.303689819449636e-07, "logits/chosen": -1.813549518585205, "logits/rejected": -1.3702311515808105, "logps/chosen": -127.129150390625, "logps/ref_chosen": -101.00733947753906, "logps/ref_rejected": -92.46794128417969, "logps/rejected": -142.03720092773438, "loss": 1.0779, "margin_dpo/margin_mean": 23.447450637817383, "margin_dpo/margin_std": 32.93583679199219, "step": 219 }, { "KL/chosen_KL_mean": -26.876251220703125, "KL/mean": -37.77546691894531, "KL/rejected_KL_mean": -48.6746826171875, "KL/std": 25.723241806030273, "epoch": 0.32305433186490456, "fcm_dpo/beta": 0.019164690747857094, "fcm_dpo/delta": -0.01813708432018757, "fcm_dpo/margin": 21.798431396484375, "fcm_dpo/q_t": 0.4029002785682678, "grad_norm": 11.019100189208984, "learning_rate": 4.2947798076611047e-07, "logits/chosen": -1.5448203086853027, "logits/rejected": -1.1973944902420044, "logps/chosen": -122.41346740722656, "logps/ref_chosen": -95.53721618652344, "logps/ref_rejected": -94.30703735351562, "logps/rejected": -142.98171997070312, "loss": 1.0687, "margin_dpo/margin_mean": 21.798431396484375, "margin_dpo/margin_std": 25.541053771972656, "step": 220 }, { "KL/chosen_KL_mean": -25.478347778320312, "KL/mean": -42.03856658935547, "KL/rejected_KL_mean": -58.59878158569336, "KL/std": 32.21840286254883, "epoch": 0.3245227606461087, "fcm_dpo/beta": 0.018888358026742935, "fcm_dpo/delta": -0.23255622386932373, "fcm_dpo/margin": 33.12043762207031, "fcm_dpo/q_t": 0.35842031240463257, "grad_norm": 10.607196807861328, "learning_rate": 4.285822501755485e-07, "logits/chosen": -1.2882272005081177, "logits/rejected": -1.147545337677002, "logps/chosen": -108.32321166992188, "logps/ref_chosen": -82.84486389160156, "logps/ref_rejected": -110.81179809570312, "logps/rejected": -169.41058349609375, "loss": 0.9264, "margin_dpo/margin_mean": 33.12043762207031, "margin_dpo/margin_std": 28.470827102661133, "step": 221 }, { "KL/chosen_KL_mean": -32.747859954833984, "KL/mean": -46.346282958984375, "KL/rejected_KL_mean": -59.94470977783203, "KL/std": 30.98981475830078, "epoch": 0.32599118942731276, "fcm_dpo/beta": 0.018602095544338226, "fcm_dpo/delta": -0.10862651467323303, "fcm_dpo/margin": 27.196847915649414, "fcm_dpo/q_t": 0.3865847587585449, "grad_norm": 11.217174530029297, "learning_rate": 4.276818137766118e-07, "logits/chosen": -1.979528784751892, "logits/rejected": -1.5519062280654907, "logps/chosen": -127.88983917236328, "logps/ref_chosen": -95.14198303222656, "logps/ref_rejected": -106.80441284179688, "logps/rejected": -166.74911499023438, "loss": 1.0494, "margin_dpo/margin_mean": 27.196847915649414, "margin_dpo/margin_std": 36.32723617553711, "step": 222 }, { "KL/chosen_KL_mean": -31.653461456298828, "KL/mean": -41.45649719238281, "KL/rejected_KL_mean": -51.25953674316406, "KL/std": 27.23687744140625, "epoch": 0.3274596182085169, "fcm_dpo/beta": 0.018547065556049347, "fcm_dpo/delta": 0.03704346343874931, "fcm_dpo/margin": 19.6060791015625, "fcm_dpo/q_t": 0.41654157638549805, "grad_norm": 10.666226387023926, "learning_rate": 4.2677669529663686e-07, "logits/chosen": -2.2680158615112305, "logits/rejected": -1.8826873302459717, "logps/chosen": -117.22857666015625, "logps/ref_chosen": -85.57511138916016, "logps/ref_rejected": -86.45237731933594, "logps/rejected": -137.7119140625, "loss": 1.1276, "margin_dpo/margin_mean": 19.6060791015625, "margin_dpo/margin_std": 29.853656768798828, "step": 223 }, { "KL/chosen_KL_mean": -29.190956115722656, "KL/mean": -42.92631149291992, "KL/rejected_KL_mean": -56.66166687011719, "KL/std": 30.213489532470703, "epoch": 0.328928046989721, "fcm_dpo/beta": 0.01842355541884899, "fcm_dpo/delta": -0.1088898628950119, "fcm_dpo/margin": 27.470718383789062, "fcm_dpo/q_t": 0.38506603240966797, "grad_norm": 10.07646369934082, "learning_rate": 4.2586691858633747e-07, "logits/chosen": -1.7676796913146973, "logits/rejected": -1.310835599899292, "logps/chosen": -111.91476440429688, "logps/ref_chosen": -82.72380065917969, "logps/ref_rejected": -82.59538269042969, "logps/rejected": -139.25704956054688, "loss": 1.02, "margin_dpo/margin_mean": 27.470718383789062, "margin_dpo/margin_std": 31.601285934448242, "step": 224 }, { "KL/chosen_KL_mean": -32.643470764160156, "KL/mean": -47.088714599609375, "KL/rejected_KL_mean": -61.53395080566406, "KL/std": 31.94729232788086, "epoch": 0.3303964757709251, "fcm_dpo/beta": 0.018235545605421066, "fcm_dpo/delta": -0.1301664113998413, "fcm_dpo/margin": 28.890485763549805, "fcm_dpo/q_t": 0.38260000944137573, "grad_norm": 10.93416976928711, "learning_rate": 4.249525076191759e-07, "logits/chosen": -2.0174734592437744, "logits/rejected": -1.5444540977478027, "logps/chosen": -128.32115173339844, "logps/ref_chosen": -95.67768859863281, "logps/ref_rejected": -105.09687805175781, "logps/rejected": -166.63082885742188, "loss": 1.021, "margin_dpo/margin_mean": 28.890485763549805, "margin_dpo/margin_std": 35.68040466308594, "step": 225 }, { "KL/chosen_KL_mean": -27.118175506591797, "KL/mean": -40.76454162597656, "KL/rejected_KL_mean": -54.41090393066406, "KL/std": 30.987037658691406, "epoch": 0.33186490455212925, "fcm_dpo/beta": 0.01796567440032959, "fcm_dpo/delta": -0.09313695132732391, "fcm_dpo/margin": 27.292728424072266, "fcm_dpo/q_t": 0.3869597315788269, "grad_norm": 9.017721176147461, "learning_rate": 4.2403348649073167e-07, "logits/chosen": -1.7463873624801636, "logits/rejected": -1.2063143253326416, "logps/chosen": -120.5791015625, "logps/ref_chosen": -93.46092224121094, "logps/ref_rejected": -86.7017593383789, "logps/rejected": -141.1126708984375, "loss": 1.0144, "margin_dpo/margin_mean": 27.292728424072266, "margin_dpo/margin_std": 27.82759666442871, "step": 226 }, { "KL/chosen_KL_mean": -37.06584548950195, "KL/mean": -51.057289123535156, "KL/rejected_KL_mean": -65.0487289428711, "KL/std": 29.772377014160156, "epoch": 0.3333333333333333, "fcm_dpo/beta": 0.017840225249528885, "fcm_dpo/delta": -0.1017354428768158, "fcm_dpo/margin": 27.982887268066406, "fcm_dpo/q_t": 0.3877747654914856, "grad_norm": 10.030227661132812, "learning_rate": 4.2310987941806615e-07, "logits/chosen": -1.449934482574463, "logits/rejected": -1.1016008853912354, "logps/chosen": -129.88011169433594, "logps/ref_chosen": -92.81427001953125, "logps/ref_rejected": -104.73693084716797, "logps/rejected": -169.78565979003906, "loss": 1.039, "margin_dpo/margin_mean": 27.982891082763672, "margin_dpo/margin_std": 35.22080993652344, "step": 227 }, { "KL/chosen_KL_mean": -34.84355545043945, "KL/mean": -45.62939453125, "KL/rejected_KL_mean": -56.41524124145508, "KL/std": 27.799240112304688, "epoch": 0.33480176211453744, "fcm_dpo/beta": 0.01780383661389351, "fcm_dpo/delta": 0.01625359058380127, "fcm_dpo/margin": 21.571680068969727, "fcm_dpo/q_t": 0.41096535325050354, "grad_norm": 10.279170989990234, "learning_rate": 4.2218171073908463e-07, "logits/chosen": -1.8968825340270996, "logits/rejected": -1.4655301570892334, "logps/chosen": -128.88067626953125, "logps/ref_chosen": -94.03712463378906, "logps/ref_rejected": -96.02151489257812, "logps/rejected": -152.436767578125, "loss": 1.1107, "margin_dpo/margin_mean": 21.571678161621094, "margin_dpo/margin_std": 30.920761108398438, "step": 228 }, { "KL/chosen_KL_mean": -30.579425811767578, "KL/mean": -48.401878356933594, "KL/rejected_KL_mean": -66.22433471679688, "KL/std": 34.21904754638672, "epoch": 0.33627019089574156, "fcm_dpo/beta": 0.017521101981401443, "fcm_dpo/delta": -0.23167923092842102, "fcm_dpo/margin": 35.64491271972656, "fcm_dpo/q_t": 0.36183321475982666, "grad_norm": 9.84212589263916, "learning_rate": 4.212490049118951e-07, "logits/chosen": -2.056971788406372, "logits/rejected": -1.4521591663360596, "logps/chosen": -126.15708923339844, "logps/ref_chosen": -95.57766723632812, "logps/ref_rejected": -89.17379760742188, "logps/rejected": -155.39813232421875, "loss": 0.9474, "margin_dpo/margin_mean": 35.64491271972656, "margin_dpo/margin_std": 37.09995651245117, "step": 229 }, { "KL/chosen_KL_mean": -32.383731842041016, "KL/mean": -50.360008239746094, "KL/rejected_KL_mean": -68.33627319335938, "KL/std": 33.618438720703125, "epoch": 0.3377386196769457, "fcm_dpo/beta": 0.017122924327850342, "fcm_dpo/delta": -0.22247090935707092, "fcm_dpo/margin": 35.95253372192383, "fcm_dpo/q_t": 0.3597046136856079, "grad_norm": 11.008466720581055, "learning_rate": 4.203117865141635e-07, "logits/chosen": -1.8617911338806152, "logits/rejected": -1.7950533628463745, "logps/chosen": -96.09735870361328, "logps/ref_chosen": -63.713626861572266, "logps/ref_rejected": -91.90870666503906, "logps/rejected": -160.2449951171875, "loss": 0.9357, "margin_dpo/margin_mean": 35.95253372192383, "margin_dpo/margin_std": 33.33916473388672, "step": 230 }, { "KL/chosen_KL_mean": -35.83055114746094, "KL/mean": -51.51530456542969, "KL/rejected_KL_mean": -67.20005798339844, "KL/std": 32.7164421081543, "epoch": 0.3392070484581498, "fcm_dpo/beta": 0.016819078475236893, "fcm_dpo/delta": -0.13163301348686218, "fcm_dpo/margin": 31.369503021240234, "fcm_dpo/q_t": 0.3774837851524353, "grad_norm": 8.95595932006836, "learning_rate": 4.1937008024246625e-07, "logits/chosen": -1.8387259244918823, "logits/rejected": -1.292878270149231, "logps/chosen": -131.28622436523438, "logps/ref_chosen": -95.45567321777344, "logps/ref_rejected": -80.95568084716797, "logps/rejected": -148.15573120117188, "loss": 0.9801, "margin_dpo/margin_mean": 31.369503021240234, "margin_dpo/margin_std": 28.148853302001953, "step": 231 }, { "KL/chosen_KL_mean": -37.87982940673828, "KL/mean": -50.96852111816406, "KL/rejected_KL_mean": -64.05720520019531, "KL/std": 29.987892150878906, "epoch": 0.3406754772393539, "fcm_dpo/beta": 0.01674450933933258, "fcm_dpo/delta": -0.03922766074538231, "fcm_dpo/margin": 26.177379608154297, "fcm_dpo/q_t": 0.3984566926956177, "grad_norm": 9.795026779174805, "learning_rate": 4.1842391091163933e-07, "logits/chosen": -1.301193356513977, "logits/rejected": -0.8046658039093018, "logps/chosen": -134.777099609375, "logps/ref_chosen": -96.89726257324219, "logps/ref_rejected": -89.76461791992188, "logps/rejected": -153.8218231201172, "loss": 1.0571, "margin_dpo/margin_mean": 26.177379608154297, "margin_dpo/margin_std": 30.455860137939453, "step": 232 }, { "KL/chosen_KL_mean": -39.56000518798828, "KL/mean": -57.731449127197266, "KL/rejected_KL_mean": -75.90288543701172, "KL/std": 37.8310546875, "epoch": 0.342143906020558, "fcm_dpo/beta": 0.01647075265645981, "fcm_dpo/delta": -0.20492511987686157, "fcm_dpo/margin": 36.342891693115234, "fcm_dpo/q_t": 0.3646617531776428, "grad_norm": 10.39029598236084, "learning_rate": 4.174733034541245e-07, "logits/chosen": -1.7545757293701172, "logits/rejected": -1.5448105335235596, "logps/chosen": -128.61032104492188, "logps/ref_chosen": -89.05032348632812, "logps/ref_rejected": -112.75917053222656, "logps/rejected": -188.66204833984375, "loss": 0.9562, "margin_dpo/margin_mean": 36.34288787841797, "margin_dpo/margin_std": 36.51426696777344, "step": 233 }, { "KL/chosen_KL_mean": -38.70917892456055, "KL/mean": -58.01055145263672, "KL/rejected_KL_mean": -77.31192016601562, "KL/std": 36.25314712524414, "epoch": 0.3436123348017621, "fcm_dpo/beta": 0.01605828106403351, "fcm_dpo/delta": -0.22846683859825134, "fcm_dpo/margin": 38.602745056152344, "fcm_dpo/q_t": 0.36210471391677856, "grad_norm": 9.296764373779297, "learning_rate": 4.165182829193126e-07, "logits/chosen": -1.3572031259536743, "logits/rejected": -1.2857710123062134, "logps/chosen": -113.02754211425781, "logps/ref_chosen": -74.318359375, "logps/ref_rejected": -106.38758850097656, "logps/rejected": -183.6995086669922, "loss": 0.952, "margin_dpo/margin_mean": 38.60274124145508, "margin_dpo/margin_std": 39.63816452026367, "step": 234 }, { "KL/chosen_KL_mean": -46.259681701660156, "KL/mean": -60.927589416503906, "KL/rejected_KL_mean": -75.59549713134766, "KL/std": 32.40122985839844, "epoch": 0.34508076358296624, "fcm_dpo/beta": 0.015955356881022453, "fcm_dpo/delta": -0.06966708600521088, "fcm_dpo/margin": 29.335826873779297, "fcm_dpo/q_t": 0.3913593590259552, "grad_norm": 9.633562088012695, "learning_rate": 4.1555887447288255e-07, "logits/chosen": -1.4421420097351074, "logits/rejected": -0.8724038004875183, "logps/chosen": -144.4767303466797, "logps/ref_chosen": -98.217041015625, "logps/ref_rejected": -97.24677276611328, "logps/rejected": -172.84226989746094, "loss": 1.0279, "margin_dpo/margin_mean": 29.33582878112793, "margin_dpo/margin_std": 30.706283569335938, "step": 235 }, { "KL/chosen_KL_mean": -39.13382339477539, "KL/mean": -57.956756591796875, "KL/rejected_KL_mean": -76.77969360351562, "KL/std": 40.56707000732422, "epoch": 0.3465491923641703, "fcm_dpo/beta": 0.015704363584518433, "fcm_dpo/delta": -0.1968509554862976, "fcm_dpo/margin": 37.645870208740234, "fcm_dpo/q_t": 0.37307411432266235, "grad_norm": 10.049556732177734, "learning_rate": 4.1459510339613946e-07, "logits/chosen": -1.4512578248977661, "logits/rejected": -1.4482207298278809, "logps/chosen": -117.9715576171875, "logps/ref_chosen": -78.83773040771484, "logps/ref_rejected": -109.06343078613281, "logps/rejected": -185.8431396484375, "loss": 0.9779, "margin_dpo/margin_mean": 37.645870208740234, "margin_dpo/margin_std": 44.13715744018555, "step": 236 }, { "KL/chosen_KL_mean": -46.144588470458984, "KL/mean": -60.08417892456055, "KL/rejected_KL_mean": -74.02376556396484, "KL/std": 35.94622039794922, "epoch": 0.34801762114537443, "fcm_dpo/beta": 0.015564993023872375, "fcm_dpo/delta": -0.034696053713560104, "fcm_dpo/margin": 27.87917137145996, "fcm_dpo/q_t": 0.40636691451072693, "grad_norm": 9.613080024719238, "learning_rate": 4.136269950853473e-07, "logits/chosen": -1.5294795036315918, "logits/rejected": -1.1701977252960205, "logps/chosen": -131.3558807373047, "logps/ref_chosen": -85.21128845214844, "logps/ref_rejected": -99.90999603271484, "logps/rejected": -173.9337615966797, "loss": 1.1159, "margin_dpo/margin_mean": 27.87917137145996, "margin_dpo/margin_std": 46.68815994262695, "step": 237 }, { "KL/chosen_KL_mean": -50.71162033081055, "KL/mean": -66.95100402832031, "KL/rejected_KL_mean": -83.19038391113281, "KL/std": 37.65531921386719, "epoch": 0.34948604992657856, "fcm_dpo/beta": 0.015422451309859753, "fcm_dpo/delta": -0.10358598828315735, "fcm_dpo/margin": 32.478759765625, "fcm_dpo/q_t": 0.3876338601112366, "grad_norm": 8.941457748413086, "learning_rate": 4.126545750510605e-07, "logits/chosen": -1.2918500900268555, "logits/rejected": -1.1402816772460938, "logps/chosen": -129.44285583496094, "logps/ref_chosen": -78.73123168945312, "logps/ref_rejected": -95.41840362548828, "logps/rejected": -178.60879516601562, "loss": 1.0285, "margin_dpo/margin_mean": 32.478759765625, "margin_dpo/margin_std": 39.18052673339844, "step": 238 }, { "KL/chosen_KL_mean": -45.85738754272461, "KL/mean": -61.02899169921875, "KL/rejected_KL_mean": -76.20060729980469, "KL/std": 33.35694122314453, "epoch": 0.3509544787077827, "fcm_dpo/beta": 0.01526646874845028, "fcm_dpo/delta": -0.0654076337814331, "fcm_dpo/margin": 30.343219757080078, "fcm_dpo/q_t": 0.39459776878356934, "grad_norm": 9.568467140197754, "learning_rate": 4.116778689174514e-07, "logits/chosen": -1.8963329792022705, "logits/rejected": -1.4235320091247559, "logps/chosen": -138.4583282470703, "logps/ref_chosen": -92.60093688964844, "logps/ref_rejected": -100.51769256591797, "logps/rejected": -176.71829223632812, "loss": 1.0539, "margin_dpo/margin_mean": 30.343219757080078, "margin_dpo/margin_std": 36.983619689941406, "step": 239 }, { "KL/chosen_KL_mean": -47.82500076293945, "KL/mean": -61.33987045288086, "KL/rejected_KL_mean": -74.854736328125, "KL/std": 33.456268310546875, "epoch": 0.3524229074889868, "fcm_dpo/beta": 0.01525870431214571, "fcm_dpo/delta": -0.012780064716935158, "fcm_dpo/margin": 27.029741287231445, "fcm_dpo/q_t": 0.4050754904747009, "grad_norm": 10.682613372802734, "learning_rate": 4.106969024216348e-07, "logits/chosen": -2.207782506942749, "logits/rejected": -1.8618403673171997, "logps/chosen": -133.98477172851562, "logps/ref_chosen": -86.15977478027344, "logps/ref_rejected": -80.45567321777344, "logps/rejected": -155.31040954589844, "loss": 1.0862, "margin_dpo/margin_mean": 27.029743194580078, "margin_dpo/margin_std": 35.72768783569336, "step": 240 }, { "KL/chosen_KL_mean": -43.549434661865234, "KL/mean": -67.24105072021484, "KL/rejected_KL_mean": -90.93266296386719, "KL/std": 38.74498748779297, "epoch": 0.35389133627019087, "fcm_dpo/beta": 0.014955822378396988, "fcm_dpo/delta": -0.3194202482700348, "fcm_dpo/margin": 47.38322448730469, "fcm_dpo/q_t": 0.34337469935417175, "grad_norm": 9.246628761291504, "learning_rate": 4.097117014129903e-07, "logits/chosen": -1.9965972900390625, "logits/rejected": -1.3547818660736084, "logps/chosen": -144.59536743164062, "logps/ref_chosen": -101.04594421386719, "logps/ref_rejected": -94.04934692382812, "logps/rejected": -184.9820098876953, "loss": 0.8907, "margin_dpo/margin_mean": 47.38322448730469, "margin_dpo/margin_std": 43.335205078125, "step": 241 }, { "KL/chosen_KL_mean": -51.869964599609375, "KL/mean": -68.90657043457031, "KL/rejected_KL_mean": -85.94316864013672, "KL/std": 34.14404296875, "epoch": 0.355359765051395, "fcm_dpo/beta": 0.014704184606671333, "fcm_dpo/delta": -0.10355594009160995, "fcm_dpo/margin": 34.07319641113281, "fcm_dpo/q_t": 0.3845828175544739, "grad_norm": 9.631617546081543, "learning_rate": 4.087222918524807e-07, "logits/chosen": -1.7768795490264893, "logits/rejected": -1.1070683002471924, "logps/chosen": -147.54263305664062, "logps/ref_chosen": -95.67266845703125, "logps/ref_rejected": -90.65454864501953, "logps/rejected": -176.59771728515625, "loss": 1.0193, "margin_dpo/margin_mean": 34.07319641113281, "margin_dpo/margin_std": 37.71101379394531, "step": 242 }, { "KL/chosen_KL_mean": -51.03168487548828, "KL/mean": -67.81936645507812, "KL/rejected_KL_mean": -84.6070556640625, "KL/std": 36.448333740234375, "epoch": 0.3568281938325991, "fcm_dpo/beta": 0.014600668102502823, "fcm_dpo/delta": -0.09281398355960846, "fcm_dpo/margin": 33.57537841796875, "fcm_dpo/q_t": 0.39008110761642456, "grad_norm": 9.396341323852539, "learning_rate": 4.07728699811968e-07, "logits/chosen": -1.964916467666626, "logits/rejected": -1.1543402671813965, "logps/chosen": -149.06307983398438, "logps/ref_chosen": -98.03140258789062, "logps/ref_rejected": -83.18806457519531, "logps/rejected": -167.7951202392578, "loss": 1.0516, "margin_dpo/margin_mean": 33.57537841796875, "margin_dpo/margin_std": 42.99260330200195, "step": 243 }, { "KL/chosen_KL_mean": -53.12769317626953, "KL/mean": -70.82644653320312, "KL/rejected_KL_mean": -88.52519226074219, "KL/std": 40.055110931396484, "epoch": 0.35829662261380324, "fcm_dpo/beta": 0.014370383694767952, "fcm_dpo/delta": -0.11166246235370636, "fcm_dpo/margin": 35.39748764038086, "fcm_dpo/q_t": 0.3844931125640869, "grad_norm": 9.389265060424805, "learning_rate": 4.067309514735267e-07, "logits/chosen": -2.054870367050171, "logits/rejected": -1.7165900468826294, "logps/chosen": -142.0216064453125, "logps/ref_chosen": -88.8939208984375, "logps/ref_rejected": -102.57278442382812, "logps/rejected": -191.09796142578125, "loss": 1.0165, "margin_dpo/margin_mean": 35.39748764038086, "margin_dpo/margin_std": 40.216529846191406, "step": 244 }, { "KL/chosen_KL_mean": -45.166988372802734, "KL/mean": -64.09281158447266, "KL/rejected_KL_mean": -83.01862335205078, "KL/std": 37.48448944091797, "epoch": 0.35976505139500736, "fcm_dpo/beta": 0.014203433878719807, "fcm_dpo/delta": -0.1413463056087494, "fcm_dpo/margin": 37.85163879394531, "fcm_dpo/q_t": 0.3769792914390564, "grad_norm": 8.73080825805664, "learning_rate": 4.057290731287531e-07, "logits/chosen": -2.277449131011963, "logits/rejected": -1.4491236209869385, "logps/chosen": -149.36099243164062, "logps/ref_chosen": -104.19400024414062, "logps/ref_rejected": -92.65645599365234, "logps/rejected": -175.67507934570312, "loss": 0.9867, "margin_dpo/margin_mean": 37.85163879394531, "margin_dpo/margin_std": 38.26203918457031, "step": 245 }, { "KL/chosen_KL_mean": -58.21797180175781, "KL/mean": -73.08688354492188, "KL/rejected_KL_mean": -87.95578002929688, "KL/std": 36.32801818847656, "epoch": 0.36123348017621143, "fcm_dpo/beta": 0.014096453785896301, "fcm_dpo/delta": -0.01977512240409851, "fcm_dpo/margin": 29.737821578979492, "fcm_dpo/q_t": 0.40413278341293335, "grad_norm": 8.992693901062012, "learning_rate": 4.047230911780736e-07, "logits/chosen": -1.753974199295044, "logits/rejected": -1.2043837308883667, "logps/chosen": -161.43701171875, "logps/ref_chosen": -103.21904754638672, "logps/ref_rejected": -90.99221801757812, "logps/rejected": -178.947998046875, "loss": 1.0815, "margin_dpo/margin_mean": 29.737821578979492, "margin_dpo/margin_std": 39.16379165649414, "step": 246 }, { "KL/chosen_KL_mean": -52.50224304199219, "KL/mean": -76.59077453613281, "KL/rejected_KL_mean": -100.67929077148438, "KL/std": 40.87024688720703, "epoch": 0.36270190895741555, "fcm_dpo/beta": 0.01379719004034996, "fcm_dpo/delta": -0.27464771270751953, "fcm_dpo/margin": 48.17705535888672, "fcm_dpo/q_t": 0.3524268567562103, "grad_norm": 9.25114631652832, "learning_rate": 4.0371303213004814e-07, "logits/chosen": -2.296752452850342, "logits/rejected": -2.112170696258545, "logps/chosen": -139.49661254882812, "logps/ref_chosen": -86.99437713623047, "logps/ref_rejected": -111.33802795410156, "logps/rejected": -212.01731872558594, "loss": 0.9159, "margin_dpo/margin_mean": 48.17705535888672, "margin_dpo/margin_std": 45.65269470214844, "step": 247 }, { "KL/chosen_KL_mean": -60.68099594116211, "KL/mean": -76.72364807128906, "KL/rejected_KL_mean": -92.76630401611328, "KL/std": 35.815860748291016, "epoch": 0.3641703377386197, "fcm_dpo/beta": 0.013711996376514435, "fcm_dpo/delta": -0.040887363255023956, "fcm_dpo/margin": 32.085296630859375, "fcm_dpo/q_t": 0.3984089493751526, "grad_norm": 9.688079833984375, "learning_rate": 4.0269892260067197e-07, "logits/chosen": -1.8588383197784424, "logits/rejected": -1.8263883590698242, "logps/chosen": -135.46652221679688, "logps/ref_chosen": -74.7855224609375, "logps/ref_rejected": -98.27689361572266, "logps/rejected": -191.04319763183594, "loss": 1.0547, "margin_dpo/margin_mean": 32.08530044555664, "margin_dpo/margin_std": 37.043212890625, "step": 248 }, { "KL/chosen_KL_mean": -66.77388000488281, "KL/mean": -81.49119567871094, "KL/rejected_KL_mean": -96.20854187011719, "KL/std": 38.422542572021484, "epoch": 0.3656387665198238, "fcm_dpo/beta": 0.013647126965224743, "fcm_dpo/delta": -0.001898549497127533, "fcm_dpo/margin": 29.434659957885742, "fcm_dpo/q_t": 0.40885186195373535, "grad_norm": 10.69800090789795, "learning_rate": 4.0168078931267426e-07, "logits/chosen": -2.252659320831299, "logits/rejected": -1.7239189147949219, "logps/chosen": -162.4776611328125, "logps/ref_chosen": -95.70379638671875, "logps/ref_rejected": -87.14646911621094, "logps/rejected": -183.35501098632812, "loss": 1.11, "margin_dpo/margin_mean": 29.434661865234375, "margin_dpo/margin_std": 44.06612014770508, "step": 249 }, { "KL/chosen_KL_mean": -59.6246452331543, "KL/mean": -76.11438751220703, "KL/rejected_KL_mean": -92.6041259765625, "KL/std": 38.982086181640625, "epoch": 0.3671071953010279, "fcm_dpo/beta": 0.013586452230811119, "fcm_dpo/delta": -0.04960451275110245, "fcm_dpo/margin": 32.9794807434082, "fcm_dpo/q_t": 0.3966714143753052, "grad_norm": 9.97586727142334, "learning_rate": 4.006586590948141e-07, "logits/chosen": -1.920072078704834, "logits/rejected": -0.9889180660247803, "logps/chosen": -173.67684936523438, "logps/ref_chosen": -114.05221557617188, "logps/ref_rejected": -81.08767700195312, "logps/rejected": -173.69180297851562, "loss": 1.0681, "margin_dpo/margin_mean": 32.97947692871094, "margin_dpo/margin_std": 41.64201736450195, "step": 250 }, { "KL/chosen_KL_mean": -59.6796875, "KL/mean": -74.24447631835938, "KL/rejected_KL_mean": -88.80928039550781, "KL/std": 36.167442321777344, "epoch": 0.368575624082232, "fcm_dpo/beta": 0.013570677489042282, "fcm_dpo/delta": 0.004449483007192612, "fcm_dpo/margin": 29.129589080810547, "fcm_dpo/q_t": 0.41107043623924255, "grad_norm": 10.527535438537598, "learning_rate": 3.9963255888117325e-07, "logits/chosen": -1.83045494556427, "logits/rejected": -1.1271706819534302, "logps/chosen": -157.39097595214844, "logps/ref_chosen": -97.71128845214844, "logps/ref_rejected": -83.52742004394531, "logps/rejected": -172.33670043945312, "loss": 1.1146, "margin_dpo/margin_mean": 29.129589080810547, "margin_dpo/margin_std": 43.63068389892578, "step": 251 }, { "KL/chosen_KL_mean": -59.231910705566406, "KL/mean": -74.23173522949219, "KL/rejected_KL_mean": -89.23158264160156, "KL/std": 38.384971618652344, "epoch": 0.3700440528634361, "fcm_dpo/beta": 0.01361248642206192, "fcm_dpo/delta": -0.008556893095374107, "fcm_dpo/margin": 29.999683380126953, "fcm_dpo/q_t": 0.405634343624115, "grad_norm": 10.382631301879883, "learning_rate": 3.9860251571044666e-07, "logits/chosen": -2.1475400924682617, "logits/rejected": -1.6265931129455566, "logps/chosen": -168.218017578125, "logps/ref_chosen": -108.9861068725586, "logps/ref_rejected": -91.56423950195312, "logps/rejected": -180.7958221435547, "loss": 1.0926, "margin_dpo/margin_mean": 29.999683380126953, "margin_dpo/margin_std": 40.624610900878906, "step": 252 }, { "KL/chosen_KL_mean": -65.21946716308594, "KL/mean": -78.99172973632812, "KL/rejected_KL_mean": -92.76398468017578, "KL/std": 41.712711334228516, "epoch": 0.37151248164464024, "fcm_dpo/beta": 0.013668297789990902, "fcm_dpo/delta": 0.02347167208790779, "fcm_dpo/margin": 27.544523239135742, "fcm_dpo/q_t": 0.4119443893432617, "grad_norm": 10.293611526489258, "learning_rate": 3.9756855672522986e-07, "logits/chosen": -2.175955295562744, "logits/rejected": -1.9730327129364014, "logps/chosen": -165.43576049804688, "logps/ref_chosen": -100.21630096435547, "logps/ref_rejected": -105.67670440673828, "logps/rejected": -198.44068908691406, "loss": 1.1062, "margin_dpo/margin_mean": 27.544523239135742, "margin_dpo/margin_std": 36.181148529052734, "step": 253 }, { "KL/chosen_KL_mean": -63.12376022338867, "KL/mean": -77.42698669433594, "KL/rejected_KL_mean": -91.7302017211914, "KL/std": 39.1876335144043, "epoch": 0.37298091042584436, "fcm_dpo/beta": 0.013612687587738037, "fcm_dpo/delta": 0.010619394481182098, "fcm_dpo/margin": 28.606441497802734, "fcm_dpo/q_t": 0.4096386432647705, "grad_norm": 9.833754539489746, "learning_rate": 3.965307091713037e-07, "logits/chosen": -1.9349608421325684, "logits/rejected": -1.3964684009552002, "logps/chosen": -161.85894775390625, "logps/ref_chosen": -98.73518371582031, "logps/ref_rejected": -93.73825073242188, "logps/rejected": -185.46844482421875, "loss": 1.112, "margin_dpo/margin_mean": 28.606441497802734, "margin_dpo/margin_std": 41.547080993652344, "step": 254 }, { "KL/chosen_KL_mean": -63.74857711791992, "KL/mean": -85.19526672363281, "KL/rejected_KL_mean": -106.64195251464844, "KL/std": 44.287353515625, "epoch": 0.3744493392070485, "fcm_dpo/beta": 0.013470802456140518, "fcm_dpo/delta": -0.18299591541290283, "fcm_dpo/margin": 42.89338302612305, "fcm_dpo/q_t": 0.3691593408584595, "grad_norm": 9.76134204864502, "learning_rate": 3.954890003969163e-07, "logits/chosen": -1.923203945159912, "logits/rejected": -1.582334041595459, "logps/chosen": -154.1311492919922, "logps/ref_chosen": -90.382568359375, "logps/ref_rejected": -97.07624816894531, "logps/rejected": -203.71820068359375, "loss": 0.9738, "margin_dpo/margin_mean": 42.89338302612305, "margin_dpo/margin_std": 45.19108200073242, "step": 255 }, { "KL/chosen_KL_mean": -61.73622512817383, "KL/mean": -81.11106872558594, "KL/rejected_KL_mean": -100.48591613769531, "KL/std": 45.892906188964844, "epoch": 0.37591776798825255, "fcm_dpo/beta": 0.013298152945935726, "fcm_dpo/delta": -0.11826440691947937, "fcm_dpo/margin": 38.74969482421875, "fcm_dpo/q_t": 0.38547104597091675, "grad_norm": 10.44315242767334, "learning_rate": 3.944434578520628e-07, "logits/chosen": -1.9500672817230225, "logits/rejected": -1.6107311248779297, "logps/chosen": -150.48904418945312, "logps/ref_chosen": -88.7528076171875, "logps/ref_rejected": -98.49383544921875, "logps/rejected": -198.97975158691406, "loss": 1.0488, "margin_dpo/margin_mean": 38.749691009521484, "margin_dpo/margin_std": 53.081939697265625, "step": 256 }, { "KL/chosen_KL_mean": -63.659934997558594, "KL/mean": -83.0986328125, "KL/rejected_KL_mean": -102.53733825683594, "KL/std": 47.1826171875, "epoch": 0.37738619676945667, "fcm_dpo/beta": 0.01306983083486557, "fcm_dpo/delta": -0.11235986649990082, "fcm_dpo/margin": 38.877410888671875, "fcm_dpo/q_t": 0.38546422123908997, "grad_norm": 9.475180625915527, "learning_rate": 3.933941090877615e-07, "logits/chosen": -2.048408031463623, "logits/rejected": -1.662680983543396, "logps/chosen": -146.4634552001953, "logps/ref_chosen": -82.80352020263672, "logps/ref_rejected": -85.8677978515625, "logps/rejected": -188.40513610839844, "loss": 1.0311, "margin_dpo/margin_mean": 38.877410888671875, "margin_dpo/margin_std": 45.871917724609375, "step": 257 }, { "KL/chosen_KL_mean": -66.15030670166016, "KL/mean": -84.09268188476562, "KL/rejected_KL_mean": -102.03507232666016, "KL/std": 44.96271514892578, "epoch": 0.3788546255506608, "fcm_dpo/beta": 0.013067160733044147, "fcm_dpo/delta": -0.07075870037078857, "fcm_dpo/margin": 35.884765625, "fcm_dpo/q_t": 0.3921199440956116, "grad_norm": 9.93346118927002, "learning_rate": 3.923409817553284e-07, "logits/chosen": -2.3188517093658447, "logits/rejected": -1.9730923175811768, "logps/chosen": -156.33804321289062, "logps/ref_chosen": -90.187744140625, "logps/ref_rejected": -103.47068786621094, "logps/rejected": -205.50576782226562, "loss": 1.0475, "margin_dpo/margin_mean": 35.884765625, "margin_dpo/margin_std": 42.541934967041016, "step": 258 }, { "KL/chosen_KL_mean": -71.41622924804688, "KL/mean": -86.82487487792969, "KL/rejected_KL_mean": -102.2335205078125, "KL/std": 44.89874267578125, "epoch": 0.3803230543318649, "fcm_dpo/beta": 0.012963949702680111, "fcm_dpo/delta": 0.0001517757773399353, "fcm_dpo/margin": 30.817291259765625, "fcm_dpo/q_t": 0.4086730480194092, "grad_norm": 9.912965774536133, "learning_rate": 3.9128410360564793e-07, "logits/chosen": -1.4859259128570557, "logits/rejected": -1.2412042617797852, "logps/chosen": -162.18878173828125, "logps/ref_chosen": -90.77254486083984, "logps/ref_rejected": -94.58816528320312, "logps/rejected": -196.82168579101562, "loss": 1.1264, "margin_dpo/margin_mean": 30.81728744506836, "margin_dpo/margin_std": 50.30376434326172, "step": 259 }, { "KL/chosen_KL_mean": -73.35704040527344, "KL/mean": -86.64198303222656, "KL/rejected_KL_mean": -99.92691802978516, "KL/std": 45.77082061767578, "epoch": 0.38179148311306904, "fcm_dpo/beta": 0.013048084452748299, "fcm_dpo/delta": 0.05427170544862747, "fcm_dpo/margin": 26.569873809814453, "fcm_dpo/q_t": 0.4206811785697937, "grad_norm": 11.850217819213867, "learning_rate": 3.9022350248844246e-07, "logits/chosen": -1.5750935077667236, "logits/rejected": -1.5985513925552368, "logps/chosen": -148.94973754882812, "logps/ref_chosen": -75.59269714355469, "logps/ref_rejected": -100.84554290771484, "logps/rejected": -200.7724609375, "loss": 1.1699, "margin_dpo/margin_mean": 26.569873809814453, "margin_dpo/margin_std": 50.08036804199219, "step": 260 }, { "KL/chosen_KL_mean": -56.44215774536133, "KL/mean": -78.9533462524414, "KL/rejected_KL_mean": -101.46454620361328, "KL/std": 46.25511932373047, "epoch": 0.3832599118942731, "fcm_dpo/beta": 0.012858699075877666, "fcm_dpo/delta": -0.1850547343492508, "fcm_dpo/margin": 45.02238464355469, "fcm_dpo/q_t": 0.3689221441745758, "grad_norm": 9.221338272094727, "learning_rate": 3.891592063515376e-07, "logits/chosen": -1.8019177913665771, "logits/rejected": -1.4230998754501343, "logps/chosen": -150.6970672607422, "logps/ref_chosen": -94.25491333007812, "logps/ref_rejected": -93.65699005126953, "logps/rejected": -195.1215362548828, "loss": 0.9646, "margin_dpo/margin_mean": 45.02238464355469, "margin_dpo/margin_std": 44.906150817871094, "step": 261 }, { "KL/chosen_KL_mean": -61.256263732910156, "KL/mean": -82.1096420288086, "KL/rejected_KL_mean": -102.9630126953125, "KL/std": 48.38385009765625, "epoch": 0.38472834067547723, "fcm_dpo/beta": 0.012683868408203125, "fcm_dpo/delta": -0.13286834955215454, "fcm_dpo/margin": 41.706748962402344, "fcm_dpo/q_t": 0.38015425205230713, "grad_norm": 9.280346870422363, "learning_rate": 3.880912432401264e-07, "logits/chosen": -1.9459564685821533, "logits/rejected": -1.42217218875885, "logps/chosen": -146.5235595703125, "logps/ref_chosen": -85.26730346679688, "logps/ref_rejected": -90.82609558105469, "logps/rejected": -193.7891082763672, "loss": 1.01, "margin_dpo/margin_mean": 41.706748962402344, "margin_dpo/margin_std": 47.85508728027344, "step": 262 }, { "KL/chosen_KL_mean": -49.172874450683594, "KL/mean": -77.51377868652344, "KL/rejected_KL_mean": -105.85470581054688, "KL/std": 53.117530822753906, "epoch": 0.38619676945668135, "fcm_dpo/beta": 0.012384702451527119, "fcm_dpo/delta": -0.31269872188568115, "fcm_dpo/margin": 56.68181228637695, "fcm_dpo/q_t": 0.3443869352340698, "grad_norm": 9.009037971496582, "learning_rate": 3.870196412960302e-07, "logits/chosen": -1.912790060043335, "logits/rejected": -1.339303970336914, "logps/chosen": -144.9307861328125, "logps/ref_chosen": -95.75790405273438, "logps/ref_rejected": -101.83377075195312, "logps/rejected": -207.6884765625, "loss": 0.9088, "margin_dpo/margin_mean": 56.68181228637695, "margin_dpo/margin_std": 55.93214416503906, "step": 263 }, { "KL/chosen_KL_mean": -57.167823791503906, "KL/mean": -80.86551666259766, "KL/rejected_KL_mean": -104.56320190429688, "KL/std": 49.984439849853516, "epoch": 0.3876651982378855, "fcm_dpo/beta": 0.012100162915885448, "fcm_dpo/delta": -0.17880746722221375, "fcm_dpo/margin": 47.39539337158203, "fcm_dpo/q_t": 0.37050220370292664, "grad_norm": 8.927352905273438, "learning_rate": 3.8594442875695665e-07, "logits/chosen": -2.1370432376861572, "logits/rejected": -1.8541063070297241, "logps/chosen": -147.7904815673828, "logps/ref_chosen": -90.6226577758789, "logps/ref_rejected": -100.32554626464844, "logps/rejected": -204.8887481689453, "loss": 0.979, "margin_dpo/margin_mean": 47.39539337158203, "margin_dpo/margin_std": 50.73942947387695, "step": 264 }, { "KL/chosen_KL_mean": -60.32719421386719, "KL/mean": -81.72406005859375, "KL/rejected_KL_mean": -103.12092590332031, "KL/std": 51.06011962890625, "epoch": 0.3891336270190896, "fcm_dpo/beta": 0.011925960890948772, "fcm_dpo/delta": -0.11374220252037048, "fcm_dpo/margin": 42.79374313354492, "fcm_dpo/q_t": 0.3866499066352844, "grad_norm": 9.076062202453613, "learning_rate": 3.848656339557562e-07, "logits/chosen": -2.0397727489471436, "logits/rejected": -1.6357043981552124, "logps/chosen": -152.69952392578125, "logps/ref_chosen": -92.37232971191406, "logps/ref_rejected": -94.62757873535156, "logps/rejected": -197.74850463867188, "loss": 1.0393, "margin_dpo/margin_mean": 42.793739318847656, "margin_dpo/margin_std": 55.075286865234375, "step": 265 }, { "KL/chosen_KL_mean": -70.41415405273438, "KL/mean": -89.57887268066406, "KL/rejected_KL_mean": -108.74357604980469, "KL/std": 49.03712463378906, "epoch": 0.39060205580029367, "fcm_dpo/beta": 0.011857118457555771, "fcm_dpo/delta": -0.05585385859012604, "fcm_dpo/margin": 38.32942199707031, "fcm_dpo/q_t": 0.39746397733688354, "grad_norm": 11.182104110717773, "learning_rate": 3.8378328531967507e-07, "logits/chosen": -2.228628635406494, "logits/rejected": -1.4090077877044678, "logps/chosen": -172.6141815185547, "logps/ref_chosen": -102.20002746582031, "logps/ref_rejected": -74.36642456054688, "logps/rejected": -183.1099853515625, "loss": 1.071, "margin_dpo/margin_mean": 38.32942199707031, "margin_dpo/margin_std": 51.46796798706055, "step": 266 }, { "KL/chosen_KL_mean": -62.00703048706055, "KL/mean": -84.433837890625, "KL/rejected_KL_mean": -106.86064910888672, "KL/std": 49.8448371887207, "epoch": 0.3920704845814978, "fcm_dpo/beta": 0.01171448640525341, "fcm_dpo/delta": -0.12928348779678345, "fcm_dpo/margin": 44.85362243652344, "fcm_dpo/q_t": 0.37999919056892395, "grad_norm": 9.662176132202148, "learning_rate": 3.8269741136960646e-07, "logits/chosen": -1.5270602703094482, "logits/rejected": -0.879061222076416, "logps/chosen": -166.29302978515625, "logps/ref_chosen": -104.28599548339844, "logps/ref_rejected": -95.98719024658203, "logps/rejected": -202.84783935546875, "loss": 0.9983, "margin_dpo/margin_mean": 44.85362243652344, "margin_dpo/margin_std": 46.648719787597656, "step": 267 }, { "KL/chosen_KL_mean": -60.79924011230469, "KL/mean": -86.91950988769531, "KL/rejected_KL_mean": -113.03978729248047, "KL/std": 49.87260818481445, "epoch": 0.3935389133627019, "fcm_dpo/beta": 0.011496243998408318, "fcm_dpo/delta": -0.20743036270141602, "fcm_dpo/margin": 52.24053955078125, "fcm_dpo/q_t": 0.36282259225845337, "grad_norm": 9.734203338623047, "learning_rate": 3.8160804071933894e-07, "logits/chosen": -1.6752374172210693, "logits/rejected": -1.5151844024658203, "logps/chosen": -147.49545288085938, "logps/ref_chosen": -86.69622039794922, "logps/ref_rejected": -109.19184112548828, "logps/rejected": -222.23162841796875, "loss": 0.946, "margin_dpo/margin_mean": 52.24053955078125, "margin_dpo/margin_std": 48.81372833251953, "step": 268 }, { "KL/chosen_KL_mean": -64.9466552734375, "KL/mean": -90.62802124023438, "KL/rejected_KL_mean": -116.30937194824219, "KL/std": 53.49858856201172, "epoch": 0.39500734214390604, "fcm_dpo/beta": 0.011317870579659939, "fcm_dpo/delta": -0.1865673065185547, "fcm_dpo/margin": 51.362728118896484, "fcm_dpo/q_t": 0.36795780062675476, "grad_norm": 9.121478080749512, "learning_rate": 3.8051520207480204e-07, "logits/chosen": -2.126469135284424, "logits/rejected": -1.688768982887268, "logps/chosen": -169.91845703125, "logps/ref_chosen": -104.97181701660156, "logps/ref_rejected": -112.47647094726562, "logps/rejected": -228.7858428955078, "loss": 0.9635, "margin_dpo/margin_mean": 51.362728118896484, "margin_dpo/margin_std": 50.99756622314453, "step": 269 }, { "KL/chosen_KL_mean": -72.34004211425781, "KL/mean": -94.1907958984375, "KL/rejected_KL_mean": -116.04155731201172, "KL/std": 53.31135559082031, "epoch": 0.3964757709251101, "fcm_dpo/beta": 0.011191281490027905, "fcm_dpo/delta": -0.0912686288356781, "fcm_dpo/margin": 43.701515197753906, "fcm_dpo/q_t": 0.38979530334472656, "grad_norm": 9.89825439453125, "learning_rate": 3.794189242333106e-07, "logits/chosen": -2.4456238746643066, "logits/rejected": -2.0327701568603516, "logps/chosen": -173.41387939453125, "logps/ref_chosen": -101.07383728027344, "logps/ref_rejected": -117.75289154052734, "logps/rejected": -233.79444885253906, "loss": 1.0621, "margin_dpo/margin_mean": 43.701515197753906, "margin_dpo/margin_std": 60.226234436035156, "step": 270 }, { "KL/chosen_KL_mean": -59.0078010559082, "KL/mean": -86.10270690917969, "KL/rejected_KL_mean": -113.19761657714844, "KL/std": 50.22154998779297, "epoch": 0.39794419970631423, "fcm_dpo/beta": 0.010974034667015076, "fcm_dpo/delta": -0.2007962167263031, "fcm_dpo/margin": 54.18981170654297, "fcm_dpo/q_t": 0.363855242729187, "grad_norm": 9.167208671569824, "learning_rate": 3.7831923608280514e-07, "logits/chosen": -1.4656410217285156, "logits/rejected": -1.009652853012085, "logps/chosen": -155.73239135742188, "logps/ref_chosen": -96.72459411621094, "logps/ref_rejected": -98.5244140625, "logps/rejected": -211.72203063964844, "loss": 0.969, "margin_dpo/margin_mean": 54.18981170654297, "margin_dpo/margin_std": 57.05503463745117, "step": 271 }, { "KL/chosen_KL_mean": -50.96781921386719, "KL/mean": -76.56758117675781, "KL/rejected_KL_mean": -102.16735076904297, "KL/std": 55.12840270996094, "epoch": 0.39941262848751835, "fcm_dpo/beta": 0.010796170681715012, "fcm_dpo/delta": -0.15712842345237732, "fcm_dpo/margin": 51.199520111083984, "fcm_dpo/q_t": 0.3763847351074219, "grad_norm": 9.237937927246094, "learning_rate": 3.772161666010912e-07, "logits/chosen": -1.6019370555877686, "logits/rejected": -1.4854650497436523, "logps/chosen": -131.94503784179688, "logps/ref_chosen": -80.97721862792969, "logps/ref_rejected": -108.55535888671875, "logps/rejected": -210.72271728515625, "loss": 1.0023, "margin_dpo/margin_mean": 51.19952392578125, "margin_dpo/margin_std": 59.645423889160156, "step": 272 }, { "KL/chosen_KL_mean": -70.91657257080078, "KL/mean": -97.5293197631836, "KL/rejected_KL_mean": -124.14205169677734, "KL/std": 57.019798278808594, "epoch": 0.4008810572687225, "fcm_dpo/beta": 0.010575462132692337, "fcm_dpo/delta": -0.16903503239154816, "fcm_dpo/margin": 53.2254753112793, "fcm_dpo/q_t": 0.373455286026001, "grad_norm": 8.83747673034668, "learning_rate": 3.761097448550755e-07, "logits/chosen": -1.629859209060669, "logits/rejected": -1.2232038974761963, "logps/chosen": -163.14117431640625, "logps/ref_chosen": -92.22460174560547, "logps/ref_rejected": -97.3630599975586, "logps/rejected": -221.50511169433594, "loss": 1.01, "margin_dpo/margin_mean": 53.22547149658203, "margin_dpo/margin_std": 64.04597473144531, "step": 273 }, { "KL/chosen_KL_mean": -72.77104187011719, "KL/mean": -93.42359924316406, "KL/rejected_KL_mean": -114.07615661621094, "KL/std": 55.70240020751953, "epoch": 0.4023494860499266, "fcm_dpo/beta": 0.010533277876675129, "fcm_dpo/delta": -0.03596518561244011, "fcm_dpo/margin": 41.30513000488281, "fcm_dpo/q_t": 0.40027397871017456, "grad_norm": 13.462495803833008, "learning_rate": 3.75e-07, "logits/chosen": -2.035810947418213, "logits/rejected": -1.6436954736709595, "logps/chosen": -155.03192138671875, "logps/ref_chosen": -82.26089477539062, "logps/ref_rejected": -83.87699890136719, "logps/rejected": -197.95315551757812, "loss": 1.0659, "margin_dpo/margin_mean": 41.30512237548828, "margin_dpo/margin_std": 51.27845001220703, "step": 274 }, { "KL/chosen_KL_mean": -61.56467056274414, "KL/mean": -84.73492431640625, "KL/rejected_KL_mean": -107.90518188476562, "KL/std": 54.85707092285156, "epoch": 0.40381791483113066, "fcm_dpo/beta": 0.010496959090232849, "fcm_dpo/delta": -0.08866167813539505, "fcm_dpo/margin": 46.34049987792969, "fcm_dpo/q_t": 0.3900657892227173, "grad_norm": 9.999125480651855, "learning_rate": 3.738869612786737e-07, "logits/chosen": -2.123258352279663, "logits/rejected": -1.7999460697174072, "logps/chosen": -141.2516326904297, "logps/ref_chosen": -79.68696594238281, "logps/ref_rejected": -98.7509765625, "logps/rejected": -206.65615844726562, "loss": 1.0686, "margin_dpo/margin_mean": 46.34049987792969, "margin_dpo/margin_std": 64.74117279052734, "step": 275 }, { "KL/chosen_KL_mean": -74.02082824707031, "KL/mean": -98.64605712890625, "KL/rejected_KL_mean": -123.27128601074219, "KL/std": 55.285240173339844, "epoch": 0.4052863436123348, "fcm_dpo/beta": 0.010318556800484657, "fcm_dpo/delta": -0.11198265850543976, "fcm_dpo/margin": 49.25046157836914, "fcm_dpo/q_t": 0.3860994577407837, "grad_norm": 10.843255043029785, "learning_rate": 3.7277065802070204e-07, "logits/chosen": -2.1557772159576416, "logits/rejected": -1.5924997329711914, "logps/chosen": -160.560546875, "logps/ref_chosen": -86.53970336914062, "logps/ref_rejected": -77.85394287109375, "logps/rejected": -201.12522888183594, "loss": 1.0377, "margin_dpo/margin_mean": 49.250457763671875, "margin_dpo/margin_std": 62.01918029785156, "step": 276 }, { "KL/chosen_KL_mean": -80.3585205078125, "KL/mean": -98.63389587402344, "KL/rejected_KL_mean": -116.90927124023438, "KL/std": 53.80008316040039, "epoch": 0.4067547723935389, "fcm_dpo/beta": 0.010325425304472446, "fcm_dpo/delta": 0.02292863465845585, "fcm_dpo/margin": 36.550743103027344, "fcm_dpo/q_t": 0.41344523429870605, "grad_norm": 10.320710182189941, "learning_rate": 3.71651119641714e-07, "logits/chosen": -1.9516935348510742, "logits/rejected": -1.5289661884307861, "logps/chosen": -164.60263061523438, "logps/ref_chosen": -84.24411010742188, "logps/ref_rejected": -98.83421325683594, "logps/rejected": -215.7434844970703, "loss": 1.1199, "margin_dpo/margin_mean": 36.550743103027344, "margin_dpo/margin_std": 54.53224182128906, "step": 277 }, { "KL/chosen_KL_mean": -77.01026153564453, "KL/mean": -100.06043243408203, "KL/rejected_KL_mean": -123.11061096191406, "KL/std": 57.67681121826172, "epoch": 0.40822320117474303, "fcm_dpo/beta": 0.010279444977641106, "fcm_dpo/delta": -0.07584388554096222, "fcm_dpo/margin": 46.10035705566406, "fcm_dpo/q_t": 0.3908214569091797, "grad_norm": 11.485541343688965, "learning_rate": 3.705283756425872e-07, "logits/chosen": -2.2394917011260986, "logits/rejected": -2.0310826301574707, "logps/chosen": -159.442138671875, "logps/ref_chosen": -82.431884765625, "logps/ref_rejected": -97.85691833496094, "logps/rejected": -220.967529296875, "loss": 1.052, "margin_dpo/margin_mean": 46.10035705566406, "margin_dpo/margin_std": 57.73957061767578, "step": 278 }, { "KL/chosen_KL_mean": -80.40826416015625, "KL/mean": -108.73068237304688, "KL/rejected_KL_mean": -137.0531005859375, "KL/std": 60.78718948364258, "epoch": 0.40969162995594716, "fcm_dpo/beta": 0.010119625367224216, "fcm_dpo/delta": -0.17875683307647705, "fcm_dpo/margin": 56.64482879638672, "fcm_dpo/q_t": 0.37046751379966736, "grad_norm": 10.808032035827637, "learning_rate": 3.6940245560867e-07, "logits/chosen": -1.67910897731781, "logits/rejected": -1.4588512182235718, "logps/chosen": -165.57626342773438, "logps/ref_chosen": -85.16799926757812, "logps/ref_rejected": -94.12664794921875, "logps/rejected": -231.17974853515625, "loss": 0.9772, "margin_dpo/margin_mean": 56.64482879638672, "margin_dpo/margin_std": 60.275611877441406, "step": 279 }, { "KL/chosen_KL_mean": -83.28511810302734, "KL/mean": -111.74252319335938, "KL/rejected_KL_mean": -140.19992065429688, "KL/std": 55.801429748535156, "epoch": 0.4111600587371512, "fcm_dpo/beta": 0.009972771629691124, "fcm_dpo/delta": -0.1723557710647583, "fcm_dpo/margin": 56.91481018066406, "fcm_dpo/q_t": 0.3707793951034546, "grad_norm": 9.63039779663086, "learning_rate": 3.6827338920900253e-07, "logits/chosen": -2.03043270111084, "logits/rejected": -1.836248755455017, "logps/chosen": -169.14154052734375, "logps/ref_chosen": -85.85641479492188, "logps/ref_rejected": -104.11859130859375, "logps/rejected": -244.31851196289062, "loss": 0.9751, "margin_dpo/margin_mean": 56.91481018066406, "margin_dpo/margin_std": 58.20543670654297, "step": 280 }, { "KL/chosen_KL_mean": -88.35736083984375, "KL/mean": -116.92742156982422, "KL/rejected_KL_mean": -145.49746704101562, "KL/std": 65.55399322509766, "epoch": 0.41262848751835535, "fcm_dpo/beta": 0.009787246584892273, "fcm_dpo/delta": -0.16405034065246582, "fcm_dpo/margin": 57.14012145996094, "fcm_dpo/q_t": 0.3752574920654297, "grad_norm": 8.035417556762695, "learning_rate": 3.6714120619553435e-07, "logits/chosen": -2.2044858932495117, "logits/rejected": -1.4791982173919678, "logps/chosen": -174.90818786621094, "logps/ref_chosen": -86.55082702636719, "logps/ref_rejected": -88.62866973876953, "logps/rejected": -234.1261444091797, "loss": 0.9979, "margin_dpo/margin_mean": 57.14012145996094, "margin_dpo/margin_std": 66.53482055664062, "step": 281 }, { "KL/chosen_KL_mean": -86.35165405273438, "KL/mean": -121.68316650390625, "KL/rejected_KL_mean": -157.01470947265625, "KL/std": 63.29301452636719, "epoch": 0.41409691629955947, "fcm_dpo/beta": 0.009547384455800056, "fcm_dpo/delta": -0.284402072429657, "fcm_dpo/margin": 70.66304779052734, "fcm_dpo/q_t": 0.34778422117233276, "grad_norm": 8.942418098449707, "learning_rate": 3.660059364023408e-07, "logits/chosen": -1.9472167491912842, "logits/rejected": -1.5769758224487305, "logps/chosen": -191.45675659179688, "logps/ref_chosen": -105.10511016845703, "logps/ref_rejected": -102.85336303710938, "logps/rejected": -259.8680725097656, "loss": 0.9005, "margin_dpo/margin_mean": 70.66304779052734, "margin_dpo/margin_std": 62.99449157714844, "step": 282 }, { "KL/chosen_KL_mean": -100.85879516601562, "KL/mean": -125.83067321777344, "KL/rejected_KL_mean": -150.80255126953125, "KL/std": 62.785491943359375, "epoch": 0.4155653450807636, "fcm_dpo/beta": 0.009429804980754852, "fcm_dpo/delta": -0.07269223034381866, "fcm_dpo/margin": 49.943763732910156, "fcm_dpo/q_t": 0.3940184712409973, "grad_norm": 11.605323791503906, "learning_rate": 3.6486760974483685e-07, "logits/chosen": -2.504361629486084, "logits/rejected": -2.1848487854003906, "logps/chosen": -195.91139221191406, "logps/ref_chosen": -95.05259704589844, "logps/ref_rejected": -103.54454803466797, "logps/rejected": -254.34710693359375, "loss": 1.0681, "margin_dpo/margin_mean": 49.94375991821289, "margin_dpo/margin_std": 69.24406433105469, "step": 283 }, { "KL/chosen_KL_mean": -90.96278381347656, "KL/mean": -123.07139587402344, "KL/rejected_KL_mean": -155.18002319335938, "KL/std": 62.662872314453125, "epoch": 0.4170337738619677, "fcm_dpo/beta": 0.009253207594156265, "fcm_dpo/delta": -0.20085221529006958, "fcm_dpo/margin": 64.21722412109375, "fcm_dpo/q_t": 0.3629169166088104, "grad_norm": 10.228921890258789, "learning_rate": 3.6372625621898863e-07, "logits/chosen": -1.6387763023376465, "logits/rejected": -1.2992234230041504, "logps/chosen": -178.6292724609375, "logps/ref_chosen": -87.66647338867188, "logps/ref_rejected": -98.75103759765625, "logps/rejected": -253.93106079101562, "loss": 0.9389, "margin_dpo/margin_mean": 64.21722412109375, "margin_dpo/margin_std": 55.879920959472656, "step": 284 }, { "KL/chosen_KL_mean": -101.90450286865234, "KL/mean": -124.77113342285156, "KL/rejected_KL_mean": -147.63775634765625, "KL/std": 56.90541458129883, "epoch": 0.4185022026431718, "fcm_dpo/beta": 0.00922366976737976, "fcm_dpo/delta": -0.022332999855279922, "fcm_dpo/margin": 45.733253479003906, "fcm_dpo/q_t": 0.40109506249427795, "grad_norm": 11.351130485534668, "learning_rate": 3.625819059005228e-07, "logits/chosen": -2.3379592895507812, "logits/rejected": -2.057554244995117, "logps/chosen": -196.33753967285156, "logps/ref_chosen": -94.43303680419922, "logps/ref_rejected": -104.07194519042969, "logps/rejected": -251.709716796875, "loss": 1.0672, "margin_dpo/margin_mean": 45.733253479003906, "margin_dpo/margin_std": 53.72636795043945, "step": 285 }, { "KL/chosen_KL_mean": -106.91999816894531, "KL/mean": -137.54165649414062, "KL/rejected_KL_mean": -168.163330078125, "KL/std": 60.53830337524414, "epoch": 0.4199706314243759, "fcm_dpo/beta": 0.00908946618437767, "fcm_dpo/delta": -0.1613137274980545, "fcm_dpo/margin": 61.24333953857422, "fcm_dpo/q_t": 0.3724822998046875, "grad_norm": 8.834715843200684, "learning_rate": 3.614345889441346e-07, "logits/chosen": -1.9713590145111084, "logits/rejected": -1.5100107192993164, "logps/chosen": -210.64039611816406, "logps/ref_chosen": -103.72039794921875, "logps/ref_rejected": -96.25775909423828, "logps/rejected": -264.42108154296875, "loss": 0.9811, "margin_dpo/margin_mean": 61.24333953857422, "margin_dpo/margin_std": 62.48735427856445, "step": 286 }, { "KL/chosen_KL_mean": -123.29798889160156, "KL/mean": -140.94790649414062, "KL/rejected_KL_mean": -158.59780883789062, "KL/std": 55.81172180175781, "epoch": 0.42143906020558003, "fcm_dpo/beta": 0.009136696346104145, "fcm_dpo/delta": 0.07826828211545944, "fcm_dpo/margin": 35.299827575683594, "fcm_dpo/q_t": 0.42753660678863525, "grad_norm": 8.85141658782959, "learning_rate": 3.6028433558269275e-07, "logits/chosen": -2.2600173950195312, "logits/rejected": -1.6714916229248047, "logps/chosen": -217.1878662109375, "logps/ref_chosen": -93.88988494873047, "logps/ref_rejected": -83.33365631103516, "logps/rejected": -241.9314727783203, "loss": 1.1737, "margin_dpo/margin_mean": 35.299827575683594, "margin_dpo/margin_std": 64.06967163085938, "step": 287 }, { "KL/chosen_KL_mean": -117.12236022949219, "KL/mean": -148.90567016601562, "KL/rejected_KL_mean": -180.6890106201172, "KL/std": 63.171295166015625, "epoch": 0.42290748898678415, "fcm_dpo/beta": 0.00900285691022873, "fcm_dpo/delta": -0.17754358053207397, "fcm_dpo/margin": 63.56664276123047, "fcm_dpo/q_t": 0.37273240089416504, "grad_norm": 8.488272666931152, "learning_rate": 3.5913117612644327e-07, "logits/chosen": -1.835127592086792, "logits/rejected": -1.4236299991607666, "logps/chosen": -205.27838134765625, "logps/ref_chosen": -88.15602111816406, "logps/ref_rejected": -93.28195190429688, "logps/rejected": -273.970947265625, "loss": 0.9824, "margin_dpo/margin_mean": 63.56664276123047, "margin_dpo/margin_std": 70.30465698242188, "step": 288 }, { "KL/chosen_KL_mean": -113.85980224609375, "KL/mean": -146.16720581054688, "KL/rejected_KL_mean": -178.47459411621094, "KL/std": 66.67284393310547, "epoch": 0.4243759177679883, "fcm_dpo/beta": 0.00886271707713604, "fcm_dpo/delta": -0.17759893834590912, "fcm_dpo/margin": 64.61478424072266, "fcm_dpo/q_t": 0.37523603439331055, "grad_norm": 8.102302551269531, "learning_rate": 3.5797514096221024e-07, "logits/chosen": -2.062039375305176, "logits/rejected": -1.8636343479156494, "logps/chosen": -189.2527313232422, "logps/ref_chosen": -75.39292907714844, "logps/ref_rejected": -93.15428161621094, "logps/rejected": -271.6288757324219, "loss": 0.9953, "margin_dpo/margin_mean": 64.61478424072266, "margin_dpo/margin_std": 76.791015625, "step": 289 }, { "KL/chosen_KL_mean": -123.56502532958984, "KL/mean": -157.1913604736328, "KL/rejected_KL_mean": -190.81771850585938, "KL/std": 70.12358856201172, "epoch": 0.42584434654919234, "fcm_dpo/beta": 0.008718375116586685, "fcm_dpo/delta": -0.1917933225631714, "fcm_dpo/margin": 67.25267791748047, "fcm_dpo/q_t": 0.3702944815158844, "grad_norm": 9.027359962463379, "learning_rate": 3.568162605525952e-07, "logits/chosen": -1.4107613563537598, "logits/rejected": -1.4502681493759155, "logps/chosen": -211.60699462890625, "logps/ref_chosen": -88.04197692871094, "logps/ref_rejected": -123.212158203125, "logps/rejected": -314.0298767089844, "loss": 0.9764, "margin_dpo/margin_mean": 67.25267791748047, "margin_dpo/margin_std": 73.35262298583984, "step": 290 }, { "KL/chosen_KL_mean": -116.6624526977539, "KL/mean": -145.22256469726562, "KL/rejected_KL_mean": -173.78265380859375, "KL/std": 66.55136108398438, "epoch": 0.42731277533039647, "fcm_dpo/beta": 0.008590752258896828, "fcm_dpo/delta": -0.09294363856315613, "fcm_dpo/margin": 57.120208740234375, "fcm_dpo/q_t": 0.3867456018924713, "grad_norm": 9.705760955810547, "learning_rate": 3.5565456543517485e-07, "logits/chosen": -2.387730121612549, "logits/rejected": -1.9285943508148193, "logps/chosen": -210.7576904296875, "logps/ref_chosen": -94.09524536132812, "logps/ref_rejected": -96.05006408691406, "logps/rejected": -269.83270263671875, "loss": 1.0295, "margin_dpo/margin_mean": 57.120208740234375, "margin_dpo/margin_std": 65.94600677490234, "step": 291 }, { "KL/chosen_KL_mean": -120.08438873291016, "KL/mean": -149.91424560546875, "KL/rejected_KL_mean": -179.74407958984375, "KL/std": 64.5474624633789, "epoch": 0.4287812041116006, "fcm_dpo/beta": 0.008493432775139809, "fcm_dpo/delta": -0.10951417684555054, "fcm_dpo/margin": 59.65968322753906, "fcm_dpo/q_t": 0.38304364681243896, "grad_norm": 9.334427833557129, "learning_rate": 3.5449008622169583e-07, "logits/chosen": -2.1967124938964844, "logits/rejected": -1.774399995803833, "logps/chosen": -208.33480834960938, "logps/ref_chosen": -88.25041198730469, "logps/ref_rejected": -96.41764068603516, "logps/rejected": -276.1617126464844, "loss": 1.0014, "margin_dpo/margin_mean": 59.65968322753906, "margin_dpo/margin_std": 59.97696304321289, "step": 292 }, { "KL/chosen_KL_mean": -120.69432067871094, "KL/mean": -144.45950317382812, "KL/rejected_KL_mean": -168.22471618652344, "KL/std": 68.25572204589844, "epoch": 0.4302496328928047, "fcm_dpo/beta": 0.008488042280077934, "fcm_dpo/delta": -0.0036994032561779022, "fcm_dpo/margin": 47.53040313720703, "fcm_dpo/q_t": 0.40978381037712097, "grad_norm": 8.71912670135498, "learning_rate": 3.5332285359726846e-07, "logits/chosen": -2.4486050605773926, "logits/rejected": -1.8899152278900146, "logps/chosen": -208.07086181640625, "logps/ref_chosen": -87.37654113769531, "logps/ref_rejected": -85.75579833984375, "logps/rejected": -253.98052978515625, "loss": 1.1089, "margin_dpo/margin_mean": 47.53040313720703, "margin_dpo/margin_std": 71.50122833251953, "step": 293 }, { "KL/chosen_KL_mean": -123.61724853515625, "KL/mean": -143.39108276367188, "KL/rejected_KL_mean": -163.16494750976562, "KL/std": 64.72129821777344, "epoch": 0.43171806167400884, "fcm_dpo/beta": 0.008522875607013702, "fcm_dpo/delta": 0.06380850076675415, "fcm_dpo/margin": 39.54769515991211, "fcm_dpo/q_t": 0.42438316345214844, "grad_norm": 9.144161224365234, "learning_rate": 3.5215289831955786e-07, "logits/chosen": -1.729644536972046, "logits/rejected": -1.5599689483642578, "logps/chosen": -197.1251983642578, "logps/ref_chosen": -73.50794982910156, "logps/ref_rejected": -88.08877563476562, "logps/rejected": -251.25372314453125, "loss": 1.1774, "margin_dpo/margin_mean": 39.547691345214844, "margin_dpo/margin_std": 75.64398193359375, "step": 294 }, { "KL/chosen_KL_mean": -117.77323913574219, "KL/mean": -147.7166290283203, "KL/rejected_KL_mean": -177.6600341796875, "KL/std": 66.19148254394531, "epoch": 0.4331864904552129, "fcm_dpo/beta": 0.008433820679783821, "fcm_dpo/delta": -0.108289934694767, "fcm_dpo/margin": 59.886783599853516, "fcm_dpo/q_t": 0.3858569860458374, "grad_norm": 8.878125190734863, "learning_rate": 3.509802512179737e-07, "logits/chosen": -1.4714245796203613, "logits/rejected": -1.313106656074524, "logps/chosen": -195.53872680664062, "logps/ref_chosen": -77.76548767089844, "logps/ref_rejected": -94.24726867675781, "logps/rejected": -271.90728759765625, "loss": 1.027, "margin_dpo/margin_mean": 59.88678741455078, "margin_dpo/margin_std": 71.35919189453125, "step": 295 }, { "KL/chosen_KL_mean": -126.7281265258789, "KL/mean": -158.93380737304688, "KL/rejected_KL_mean": -191.13946533203125, "KL/std": 66.49462127685547, "epoch": 0.434654919236417, "fcm_dpo/beta": 0.008328838273882866, "fcm_dpo/delta": -0.1405864655971527, "fcm_dpo/margin": 64.41133880615234, "fcm_dpo/q_t": 0.3767910599708557, "grad_norm": 11.354016304016113, "learning_rate": 3.498049431928577e-07, "logits/chosen": -2.486717700958252, "logits/rejected": -2.0031280517578125, "logps/chosen": -224.58453369140625, "logps/ref_chosen": -97.85641479492188, "logps/ref_rejected": -100.81631469726562, "logps/rejected": -291.9557800292969, "loss": 0.9929, "margin_dpo/margin_mean": 64.41133880615234, "margin_dpo/margin_std": 67.1482162475586, "step": 296 }, { "KL/chosen_KL_mean": -128.646728515625, "KL/mean": -155.52529907226562, "KL/rejected_KL_mean": -182.4038848876953, "KL/std": 66.30714416503906, "epoch": 0.43612334801762115, "fcm_dpo/beta": 0.008292067795991898, "fcm_dpo/delta": -0.046792879700660706, "fcm_dpo/margin": 53.75714874267578, "fcm_dpo/q_t": 0.39970090985298157, "grad_norm": 12.310261726379395, "learning_rate": 3.486270052146694e-07, "logits/chosen": -1.2713685035705566, "logits/rejected": -0.9616923332214355, "logps/chosen": -217.2125701904297, "logps/ref_chosen": -88.56583404541016, "logps/ref_rejected": -101.55656433105469, "logps/rejected": -283.96044921875, "loss": 1.0691, "margin_dpo/margin_mean": 53.75714874267578, "margin_dpo/margin_std": 71.3289566040039, "step": 297 }, { "KL/chosen_KL_mean": -109.19425964355469, "KL/mean": -143.54034423828125, "KL/rejected_KL_mean": -177.8864288330078, "KL/std": 72.62641143798828, "epoch": 0.43759177679882527, "fcm_dpo/beta": 0.008183633908629417, "fcm_dpo/delta": -0.16668759286403656, "fcm_dpo/margin": 68.69216918945312, "fcm_dpo/q_t": 0.37451574206352234, "grad_norm": 13.161422729492188, "learning_rate": 3.474464683231698e-07, "logits/chosen": -2.142805337905884, "logits/rejected": -2.1097333431243896, "logps/chosen": -204.07470703125, "logps/ref_chosen": -94.88043975830078, "logps/ref_rejected": -122.31101989746094, "logps/rejected": -300.19744873046875, "loss": 0.9822, "margin_dpo/margin_mean": 68.69216918945312, "margin_dpo/margin_std": 73.83678436279297, "step": 298 }, { "KL/chosen_KL_mean": -113.2430419921875, "KL/mean": -147.8337860107422, "KL/rejected_KL_mean": -182.42453002929688, "KL/std": 70.03121185302734, "epoch": 0.4390602055800294, "fcm_dpo/beta": 0.008035982958972454, "fcm_dpo/delta": -0.16051000356674194, "fcm_dpo/margin": 69.18148803710938, "fcm_dpo/q_t": 0.3739425539970398, "grad_norm": 10.072239875793457, "learning_rate": 3.462633636266041e-07, "logits/chosen": -0.9450910091400146, "logits/rejected": -0.7160622477531433, "logps/chosen": -193.65139770507812, "logps/ref_chosen": -80.40835571289062, "logps/ref_rejected": -89.53717041015625, "logps/rejected": -271.9617004394531, "loss": 0.9843, "margin_dpo/margin_mean": 69.18148803710938, "margin_dpo/margin_std": 73.23422241210938, "step": 299 }, { "KL/chosen_KL_mean": -118.50556945800781, "KL/mean": -147.16018676757812, "KL/rejected_KL_mean": -175.8148193359375, "KL/std": 73.78291320800781, "epoch": 0.44052863436123346, "fcm_dpo/beta": 0.007956327870488167, "fcm_dpo/delta": -0.05759502202272415, "fcm_dpo/margin": 57.309268951416016, "fcm_dpo/q_t": 0.39639079570770264, "grad_norm": 9.214487075805664, "learning_rate": 3.4507772230088147e-07, "logits/chosen": -1.3223415613174438, "logits/rejected": -1.1025092601776123, "logps/chosen": -206.66445922851562, "logps/ref_chosen": -88.15890502929688, "logps/ref_rejected": -100.93919372558594, "logps/rejected": -276.7540283203125, "loss": 1.0719, "margin_dpo/margin_mean": 57.30927276611328, "margin_dpo/margin_std": 77.91091918945312, "step": 300 }, { "KL/chosen_KL_mean": -108.6796875, "KL/mean": -144.73355102539062, "KL/rejected_KL_mean": -180.78741455078125, "KL/std": 72.26156616210938, "epoch": 0.4419970631424376, "fcm_dpo/beta": 0.007838016375899315, "fcm_dpo/delta": -0.17084385454654694, "fcm_dpo/margin": 72.10772705078125, "fcm_dpo/q_t": 0.3733501136302948, "grad_norm": 10.67741584777832, "learning_rate": 3.4388957558875316e-07, "logits/chosen": -1.7678828239440918, "logits/rejected": -1.3351057767868042, "logps/chosen": -194.18460083007812, "logps/ref_chosen": -85.50491333007812, "logps/ref_rejected": -98.91683959960938, "logps/rejected": -279.70428466796875, "loss": 0.9909, "margin_dpo/margin_mean": 72.10773468017578, "margin_dpo/margin_std": 80.86177062988281, "step": 301 }, { "KL/chosen_KL_mean": -104.71505737304688, "KL/mean": -135.44473266601562, "KL/rejected_KL_mean": -166.1744384765625, "KL/std": 74.48359680175781, "epoch": 0.4434654919236417, "fcm_dpo/beta": 0.007772143464535475, "fcm_dpo/delta": -0.07970935851335526, "fcm_dpo/margin": 61.459381103515625, "fcm_dpo/q_t": 0.39007091522216797, "grad_norm": 9.153247833251953, "learning_rate": 3.426989547989902e-07, "logits/chosen": -1.3073691129684448, "logits/rejected": -1.1296212673187256, "logps/chosen": -184.16546630859375, "logps/ref_chosen": -79.45040893554688, "logps/ref_rejected": -103.0591049194336, "logps/rejected": -269.2335205078125, "loss": 1.0396, "margin_dpo/margin_mean": 61.459381103515625, "margin_dpo/margin_std": 72.98757934570312, "step": 302 }, { "KL/chosen_KL_mean": -97.7174072265625, "KL/mean": -131.5491180419922, "KL/rejected_KL_mean": -165.38082885742188, "KL/std": 69.20824432373047, "epoch": 0.44493392070484583, "fcm_dpo/beta": 0.00770445354282856, "fcm_dpo/delta": -0.12446106225252151, "fcm_dpo/margin": 67.66342163085938, "fcm_dpo/q_t": 0.3802735507488251, "grad_norm": 8.945985794067383, "learning_rate": 3.4150589130555773e-07, "logits/chosen": -2.0669796466827393, "logits/rejected": -1.626727819442749, "logps/chosen": -193.31570434570312, "logps/ref_chosen": -95.59829711914062, "logps/ref_rejected": -92.0645751953125, "logps/rejected": -257.4454040527344, "loss": 1.0062, "margin_dpo/margin_mean": 67.66342163085938, "margin_dpo/margin_std": 72.93284606933594, "step": 303 }, { "KL/chosen_KL_mean": -98.12828826904297, "KL/mean": -133.3216552734375, "KL/rejected_KL_mean": -168.5150146484375, "KL/std": 72.27722930908203, "epoch": 0.44640234948604995, "fcm_dpo/beta": 0.007592825219035149, "fcm_dpo/delta": -0.1380854845046997, "fcm_dpo/margin": 70.3867416381836, "fcm_dpo/q_t": 0.37734657526016235, "grad_norm": 9.544572830200195, "learning_rate": 3.403104165467883e-07, "logits/chosen": -1.8312063217163086, "logits/rejected": -1.3906562328338623, "logps/chosen": -197.58143615722656, "logps/ref_chosen": -99.4531478881836, "logps/ref_rejected": -98.08226013183594, "logps/rejected": -266.5972900390625, "loss": 0.9955, "margin_dpo/margin_mean": 70.3867416381836, "margin_dpo/margin_std": 74.65216064453125, "step": 304 }, { "KL/chosen_KL_mean": -105.89790344238281, "KL/mean": -139.5634002685547, "KL/rejected_KL_mean": -173.22891235351562, "KL/std": 74.91799926757812, "epoch": 0.447870778267254, "fcm_dpo/beta": 0.007480375934392214, "fcm_dpo/delta": -0.10706125944852829, "fcm_dpo/margin": 67.33100891113281, "fcm_dpo/q_t": 0.3846391439437866, "grad_norm": 8.392979621887207, "learning_rate": 3.391125620245535e-07, "logits/chosen": -1.777998924255371, "logits/rejected": -1.2655748128890991, "logps/chosen": -209.1613006591797, "logps/ref_chosen": -103.26339721679688, "logps/ref_rejected": -97.82897186279297, "logps/rejected": -271.057861328125, "loss": 1.0309, "margin_dpo/margin_mean": 67.33100128173828, "margin_dpo/margin_std": 79.23941040039062, "step": 305 }, { "KL/chosen_KL_mean": -95.70555114746094, "KL/mean": -135.34469604492188, "KL/rejected_KL_mean": -174.98382568359375, "KL/std": 71.4386978149414, "epoch": 0.44933920704845814, "fcm_dpo/beta": 0.0073669226840138435, "fcm_dpo/delta": -0.18995031714439392, "fcm_dpo/margin": 79.27827453613281, "fcm_dpo/q_t": 0.36386871337890625, "grad_norm": 9.94462776184082, "learning_rate": 3.3791235930343417e-07, "logits/chosen": -2.288588523864746, "logits/rejected": -1.7396411895751953, "logps/chosen": -196.0093994140625, "logps/ref_chosen": -100.3038330078125, "logps/ref_rejected": -92.23086547851562, "logps/rejected": -267.2146911621094, "loss": 0.9403, "margin_dpo/margin_mean": 79.27827453613281, "margin_dpo/margin_std": 65.4382553100586, "step": 306 }, { "KL/chosen_KL_mean": -101.53524780273438, "KL/mean": -131.15138244628906, "KL/rejected_KL_mean": -160.76751708984375, "KL/std": 72.34031677246094, "epoch": 0.45080763582966227, "fcm_dpo/beta": 0.007311197463423014, "fcm_dpo/delta": -0.03397107869386673, "fcm_dpo/margin": 59.232269287109375, "fcm_dpo/q_t": 0.40201082825660706, "grad_norm": 9.580766677856445, "learning_rate": 3.367098400098881e-07, "logits/chosen": -1.873453140258789, "logits/rejected": -1.525220274925232, "logps/chosen": -198.1850128173828, "logps/ref_chosen": -96.64976501464844, "logps/ref_rejected": -93.34860229492188, "logps/rejected": -254.11611938476562, "loss": 1.0896, "margin_dpo/margin_mean": 59.232269287109375, "margin_dpo/margin_std": 85.08269500732422, "step": 307 }, { "KL/chosen_KL_mean": -106.727783203125, "KL/mean": -136.04434204101562, "KL/rejected_KL_mean": -165.3609161376953, "KL/std": 73.83590698242188, "epoch": 0.4522760646108664, "fcm_dpo/beta": 0.007277283351868391, "fcm_dpo/delta": -0.027848266065120697, "fcm_dpo/margin": 58.633121490478516, "fcm_dpo/q_t": 0.40433669090270996, "grad_norm": 11.220791816711426, "learning_rate": 3.355050358314172e-07, "logits/chosen": -1.6847771406173706, "logits/rejected": -1.4062997102737427, "logps/chosen": -192.45245361328125, "logps/ref_chosen": -85.72467041015625, "logps/ref_rejected": -86.42317962646484, "logps/rejected": -251.78408813476562, "loss": 1.1, "margin_dpo/margin_mean": 58.63311767578125, "margin_dpo/margin_std": 85.84576416015625, "step": 308 }, { "KL/chosen_KL_mean": -119.58414459228516, "KL/mean": -146.57559204101562, "KL/rejected_KL_mean": -173.56704711914062, "KL/std": 78.19351196289062, "epoch": 0.45374449339207046, "fcm_dpo/beta": 0.0073021575808525085, "fcm_dpo/delta": 0.005921725183725357, "fcm_dpo/margin": 53.98289108276367, "fcm_dpo/q_t": 0.40943771600723267, "grad_norm": 13.742324829101562, "learning_rate": 3.3429797851573183e-07, "logits/chosen": -1.6337424516677856, "logits/rejected": -1.1967337131500244, "logps/chosen": -202.735595703125, "logps/ref_chosen": -83.15145874023438, "logps/ref_rejected": -86.58602142333984, "logps/rejected": -260.153076171875, "loss": 1.1083, "margin_dpo/margin_mean": 53.982887268066406, "margin_dpo/margin_std": 77.1131820678711, "step": 309 }, { "KL/chosen_KL_mean": -107.12490844726562, "KL/mean": -136.91375732421875, "KL/rejected_KL_mean": -166.70257568359375, "KL/std": 71.72038269042969, "epoch": 0.4552129221732746, "fcm_dpo/beta": 0.007280152291059494, "fcm_dpo/delta": -0.03450014069676399, "fcm_dpo/margin": 59.57769012451172, "fcm_dpo/q_t": 0.4000013470649719, "grad_norm": 9.500658988952637, "learning_rate": 3.3308869986991487e-07, "logits/chosen": -2.2264373302459717, "logits/rejected": -1.8096086978912354, "logps/chosen": -196.86289978027344, "logps/ref_chosen": -89.73799133300781, "logps/ref_rejected": -87.07083129882812, "logps/rejected": -253.77340698242188, "loss": 1.0685, "margin_dpo/margin_mean": 59.57769012451172, "margin_dpo/margin_std": 74.78469848632812, "step": 310 }, { "KL/chosen_KL_mean": -126.81550598144531, "KL/mean": -165.03680419921875, "KL/rejected_KL_mean": -203.25811767578125, "KL/std": 72.69265747070312, "epoch": 0.4566813509544787, "fcm_dpo/beta": 0.007193974684923887, "fcm_dpo/delta": -0.15414974093437195, "fcm_dpo/margin": 76.442626953125, "fcm_dpo/q_t": 0.37421339750289917, "grad_norm": 9.76922607421875, "learning_rate": 3.3187723175958346e-07, "logits/chosen": -1.393498182296753, "logits/rejected": -0.6349947452545166, "logps/chosen": -215.34194946289062, "logps/ref_chosen": -88.52644348144531, "logps/ref_rejected": -82.26608276367188, "logps/rejected": -285.5242004394531, "loss": 0.9733, "margin_dpo/margin_mean": 76.442626953125, "margin_dpo/margin_std": 73.83804321289062, "step": 311 }, { "KL/chosen_KL_mean": -122.10618591308594, "KL/mean": -154.15554809570312, "KL/rejected_KL_mean": -186.2049102783203, "KL/std": 80.05805969238281, "epoch": 0.4581497797356828, "fcm_dpo/beta": 0.007146069779992104, "fcm_dpo/delta": -0.0594366230070591, "fcm_dpo/margin": 64.09873962402344, "fcm_dpo/q_t": 0.39458873867988586, "grad_norm": 9.575690269470215, "learning_rate": 3.306636061080487e-07, "logits/chosen": -1.735209584236145, "logits/rejected": -1.2201147079467773, "logps/chosen": -202.50686645507812, "logps/ref_chosen": -80.40069580078125, "logps/ref_rejected": -82.4471435546875, "logps/rejected": -268.65203857421875, "loss": 1.0517, "margin_dpo/margin_mean": 64.09873962402344, "margin_dpo/margin_std": 77.16641235351562, "step": 312 }, { "KL/chosen_KL_mean": -115.78581237792969, "KL/mean": -147.08123779296875, "KL/rejected_KL_mean": -178.37669372558594, "KL/std": 78.87162780761719, "epoch": 0.45961820851688695, "fcm_dpo/beta": 0.007107880897819996, "fcm_dpo/delta": -0.04592150077223778, "fcm_dpo/margin": 62.590877532958984, "fcm_dpo/q_t": 0.39836472272872925, "grad_norm": 10.458137512207031, "learning_rate": 3.2944785489547537e-07, "logits/chosen": -2.418043375015259, "logits/rejected": -2.0126242637634277, "logps/chosen": -199.92771911621094, "logps/ref_chosen": -84.14190673828125, "logps/ref_rejected": -93.42684173583984, "logps/rejected": -271.80352783203125, "loss": 1.0745, "margin_dpo/margin_mean": 62.59087371826172, "margin_dpo/margin_std": 84.58625793457031, "step": 313 }, { "KL/chosen_KL_mean": -118.02828216552734, "KL/mean": -148.421875, "KL/rejected_KL_mean": -178.81546020507812, "KL/std": 78.34324645996094, "epoch": 0.461086637298091, "fcm_dpo/beta": 0.007063527125865221, "fcm_dpo/delta": -0.030151233077049255, "fcm_dpo/margin": 60.787174224853516, "fcm_dpo/q_t": 0.4020995497703552, "grad_norm": 10.606527328491211, "learning_rate": 3.2823001015803857e-07, "logits/chosen": -1.8365545272827148, "logits/rejected": -1.6150078773498535, "logps/chosen": -206.8380126953125, "logps/ref_chosen": -88.80972290039062, "logps/ref_rejected": -104.15892028808594, "logps/rejected": -282.974365234375, "loss": 1.0742, "margin_dpo/margin_mean": 60.787174224853516, "margin_dpo/margin_std": 78.30892181396484, "step": 314 }, { "KL/chosen_KL_mean": -113.82754516601562, "KL/mean": -145.38717651367188, "KL/rejected_KL_mean": -176.94680786132812, "KL/std": 82.81309509277344, "epoch": 0.46255506607929514, "fcm_dpo/beta": 0.007042215671390295, "fcm_dpo/delta": -0.04553193598985672, "fcm_dpo/margin": 63.11928176879883, "fcm_dpo/q_t": 0.400357723236084, "grad_norm": 9.304170608520508, "learning_rate": 3.270101039870797e-07, "logits/chosen": -1.2571425437927246, "logits/rejected": -1.0764639377593994, "logps/chosen": -194.62124633789062, "logps/ref_chosen": -80.793701171875, "logps/ref_rejected": -92.13711547851562, "logps/rejected": -269.08392333984375, "loss": 1.093, "margin_dpo/margin_mean": 63.11928176879883, "margin_dpo/margin_std": 94.76844787597656, "step": 315 }, { "KL/chosen_KL_mean": -117.46308898925781, "KL/mean": -151.2448272705078, "KL/rejected_KL_mean": -185.02658081054688, "KL/std": 81.0359878540039, "epoch": 0.46402349486049926, "fcm_dpo/beta": 0.007001618854701519, "fcm_dpo/delta": -0.0747932493686676, "fcm_dpo/margin": 67.5634994506836, "fcm_dpo/q_t": 0.3928653597831726, "grad_norm": 9.245851516723633, "learning_rate": 3.2578816852826086e-07, "logits/chosen": -1.742434024810791, "logits/rejected": -1.7421129941940308, "logps/chosen": -201.6473388671875, "logps/ref_chosen": -84.18424987792969, "logps/ref_rejected": -105.6908187866211, "logps/rejected": -290.7174072265625, "loss": 1.0469, "margin_dpo/margin_mean": 67.5634994506836, "margin_dpo/margin_std": 84.18782043457031, "step": 316 }, { "KL/chosen_KL_mean": -122.06709289550781, "KL/mean": -163.79750061035156, "KL/rejected_KL_mean": -205.52789306640625, "KL/std": 90.0140151977539, "epoch": 0.4654919236417034, "fcm_dpo/beta": 0.0069023724645376205, "fcm_dpo/delta": -0.18110814690589905, "fcm_dpo/margin": 83.46080017089844, "fcm_dpo/q_t": 0.3717363476753235, "grad_norm": 10.878752708435059, "learning_rate": 3.2456423598071783e-07, "logits/chosen": -1.9291088581085205, "logits/rejected": -1.4499439001083374, "logps/chosen": -209.94058227539062, "logps/ref_chosen": -87.87348937988281, "logps/ref_rejected": -108.26519775390625, "logps/rejected": -313.7930908203125, "loss": 0.9911, "margin_dpo/margin_mean": 83.46080017089844, "margin_dpo/margin_std": 95.65278625488281, "step": 317 }, { "KL/chosen_KL_mean": -117.95266723632812, "KL/mean": -155.4097442626953, "KL/rejected_KL_mean": -192.86683654785156, "KL/std": 75.55243682861328, "epoch": 0.4669603524229075, "fcm_dpo/beta": 0.006797453388571739, "fcm_dpo/delta": -0.11216254532337189, "fcm_dpo/margin": 74.9141616821289, "fcm_dpo/q_t": 0.3833470046520233, "grad_norm": 10.667691230773926, "learning_rate": 3.233383385962115e-07, "logits/chosen": -2.1375832557678223, "logits/rejected": -1.4198200702667236, "logps/chosen": -216.158203125, "logps/ref_chosen": -98.20553588867188, "logps/ref_rejected": -88.13629150390625, "logps/rejected": -281.00311279296875, "loss": 1.0134, "margin_dpo/margin_mean": 74.9141616821289, "margin_dpo/margin_std": 82.85549926757812, "step": 318 }, { "KL/chosen_KL_mean": -101.78440856933594, "KL/mean": -142.19290161132812, "KL/rejected_KL_mean": -182.6013946533203, "KL/std": 78.31090545654297, "epoch": 0.4684287812041116, "fcm_dpo/beta": 0.00670228898525238, "fcm_dpo/delta": -0.1457849144935608, "fcm_dpo/margin": 80.81695556640625, "fcm_dpo/q_t": 0.3765423893928528, "grad_norm": 10.612443923950195, "learning_rate": 3.2211050867827805e-07, "logits/chosen": -1.7281568050384521, "logits/rejected": -1.6550695896148682, "logps/chosen": -184.28778076171875, "logps/ref_chosen": -82.50337219238281, "logps/ref_rejected": -118.65068054199219, "logps/rejected": -301.2520751953125, "loss": 0.9956, "margin_dpo/margin_mean": 80.81696319580078, "margin_dpo/margin_std": 87.9461669921875, "step": 319 }, { "KL/chosen_KL_mean": -104.30291748046875, "KL/mean": -137.49179077148438, "KL/rejected_KL_mean": -170.6806640625, "KL/std": 80.15591430664062, "epoch": 0.4698972099853157, "fcm_dpo/beta": 0.006657521706074476, "fcm_dpo/delta": -0.04291648045182228, "fcm_dpo/margin": 66.37773132324219, "fcm_dpo/q_t": 0.39813435077667236, "grad_norm": 9.277299880981445, "learning_rate": 3.208807785813777e-07, "logits/chosen": -2.0051350593566895, "logits/rejected": -1.8158900737762451, "logps/chosen": -191.4401397705078, "logps/ref_chosen": -87.13722229003906, "logps/ref_rejected": -103.22412109375, "logps/rejected": -273.90478515625, "loss": 1.0544, "margin_dpo/margin_mean": 66.37773132324219, "margin_dpo/margin_std": 77.81114196777344, "step": 320 }, { "KL/chosen_KL_mean": -117.33633422851562, "KL/mean": -155.84814453125, "KL/rejected_KL_mean": -194.35995483398438, "KL/std": 82.23031616210938, "epoch": 0.4713656387665198, "fcm_dpo/beta": 0.006587624549865723, "fcm_dpo/delta": -0.11057978123426437, "fcm_dpo/margin": 77.02359008789062, "fcm_dpo/q_t": 0.38211143016815186, "grad_norm": 9.296162605285645, "learning_rate": 3.1964918071004217e-07, "logits/chosen": -1.7100443840026855, "logits/rejected": -1.302589774131775, "logps/chosen": -210.44100952148438, "logps/ref_chosen": -93.10466766357422, "logps/ref_rejected": -97.4681396484375, "logps/rejected": -291.8280944824219, "loss": 1.0104, "margin_dpo/margin_mean": 77.02359008789062, "margin_dpo/margin_std": 81.70606994628906, "step": 321 }, { "KL/chosen_KL_mean": -109.07298278808594, "KL/mean": -150.5196533203125, "KL/rejected_KL_mean": -191.96632385253906, "KL/std": 82.40232849121094, "epoch": 0.47283406754772395, "fcm_dpo/beta": 0.006489833816885948, "fcm_dpo/delta": -0.14276990294456482, "fcm_dpo/margin": 82.89335632324219, "fcm_dpo/q_t": 0.37723398208618164, "grad_norm": 13.532444953918457, "learning_rate": 3.184157475180207e-07, "logits/chosen": -1.8743815422058105, "logits/rejected": -1.52549147605896, "logps/chosen": -201.26589965820312, "logps/ref_chosen": -92.19291687011719, "logps/ref_rejected": -100.89321899414062, "logps/rejected": -292.85955810546875, "loss": 1.0, "margin_dpo/margin_mean": 82.89334869384766, "margin_dpo/margin_std": 89.18544006347656, "step": 322 }, { "KL/chosen_KL_mean": -126.39403533935547, "KL/mean": -161.65341186523438, "KL/rejected_KL_mean": -196.9127960205078, "KL/std": 82.48175048828125, "epoch": 0.47430249632892807, "fcm_dpo/beta": 0.006455026566982269, "fcm_dpo/delta": -0.05658616125583649, "fcm_dpo/margin": 70.51874542236328, "fcm_dpo/q_t": 0.39641073346138, "grad_norm": 11.275273323059082, "learning_rate": 3.171805115074251e-07, "logits/chosen": -2.009451389312744, "logits/rejected": -1.6307833194732666, "logps/chosen": -206.6661376953125, "logps/ref_chosen": -80.27210235595703, "logps/ref_rejected": -80.3033447265625, "logps/rejected": -277.21612548828125, "loss": 1.0523, "margin_dpo/margin_mean": 70.51874542236328, "margin_dpo/margin_std": 85.12706756591797, "step": 323 }, { "KL/chosen_KL_mean": -124.86747741699219, "KL/mean": -161.69384765625, "KL/rejected_KL_mean": -198.52020263671875, "KL/std": 82.39739990234375, "epoch": 0.47577092511013214, "fcm_dpo/beta": 0.006420278921723366, "fcm_dpo/delta": -0.07460595667362213, "fcm_dpo/margin": 73.65271759033203, "fcm_dpo/q_t": 0.3895666301250458, "grad_norm": 9.341920852661133, "learning_rate": 3.1594350522787295e-07, "logits/chosen": -1.6295100450515747, "logits/rejected": -0.9726321697235107, "logps/chosen": -220.59683227539062, "logps/ref_chosen": -95.72935485839844, "logps/ref_rejected": -94.16924285888672, "logps/rejected": -292.689453125, "loss": 1.0259, "margin_dpo/margin_mean": 73.65271759033203, "margin_dpo/margin_std": 77.02046203613281, "step": 324 }, { "KL/chosen_KL_mean": -114.83171081542969, "KL/mean": -154.26937866210938, "KL/rejected_KL_mean": -193.70703125, "KL/std": 78.82008361816406, "epoch": 0.47723935389133626, "fcm_dpo/beta": 0.006358121521770954, "fcm_dpo/delta": -0.104066863656044, "fcm_dpo/margin": 78.87532043457031, "fcm_dpo/q_t": 0.3835049867630005, "grad_norm": 8.144291877746582, "learning_rate": 3.147047612756302e-07, "logits/chosen": -1.9565669298171997, "logits/rejected": -1.2486205101013184, "logps/chosen": -213.53857421875, "logps/ref_chosen": -98.70686340332031, "logps/ref_rejected": -76.40809631347656, "logps/rejected": -270.1151123046875, "loss": 1.0091, "margin_dpo/margin_mean": 78.87532043457031, "margin_dpo/margin_std": 80.9867172241211, "step": 325 }, { "KL/chosen_KL_mean": -128.91058349609375, "KL/mean": -164.34329223632812, "KL/rejected_KL_mean": -199.77603149414062, "KL/std": 78.5462646484375, "epoch": 0.4787077826725404, "fcm_dpo/beta": 0.006313212215900421, "fcm_dpo/delta": -0.048492252826690674, "fcm_dpo/margin": 70.86544036865234, "fcm_dpo/q_t": 0.39595282077789307, "grad_norm": 8.289813995361328, "learning_rate": 3.134643122927519e-07, "logits/chosen": -2.399857759475708, "logits/rejected": -1.7276756763458252, "logps/chosen": -233.5718994140625, "logps/ref_chosen": -104.66130828857422, "logps/ref_rejected": -86.82537841796875, "logps/rejected": -286.6014099121094, "loss": 1.0548, "margin_dpo/margin_mean": 70.86544799804688, "margin_dpo/margin_std": 84.1456298828125, "step": 326 }, { "KL/chosen_KL_mean": -124.8133544921875, "KL/mean": -159.3769073486328, "KL/rejected_KL_mean": -193.94046020507812, "KL/std": 72.26637268066406, "epoch": 0.4801762114537445, "fcm_dpo/beta": 0.00628191651776433, "fcm_dpo/delta": -0.03514429181814194, "fcm_dpo/margin": 69.12710571289062, "fcm_dpo/q_t": 0.4001389145851135, "grad_norm": 9.0681791305542, "learning_rate": 3.1222219096622264e-07, "logits/chosen": -1.7196844816207886, "logits/rejected": -1.1237709522247314, "logps/chosen": -232.25360107421875, "logps/ref_chosen": -107.44023895263672, "logps/ref_rejected": -117.05323028564453, "logps/rejected": -310.9936828613281, "loss": 1.059, "margin_dpo/margin_mean": 69.12710571289062, "margin_dpo/margin_std": 81.40105438232422, "step": 327 }, { "KL/chosen_KL_mean": -121.76203918457031, "KL/mean": -166.88275146484375, "KL/rejected_KL_mean": -212.00347900390625, "KL/std": 85.75923156738281, "epoch": 0.48164464023494863, "fcm_dpo/beta": 0.0062103550881147385, "fcm_dpo/delta": -0.16501006484031677, "fcm_dpo/margin": 90.24147033691406, "fcm_dpo/q_t": 0.3704659342765808, "grad_norm": 10.541464805603027, "learning_rate": 3.1097843002709427e-07, "logits/chosen": -1.63505220413208, "logits/rejected": -1.3367586135864258, "logps/chosen": -208.37274169921875, "logps/ref_chosen": -86.6106948852539, "logps/ref_rejected": -97.40188598632812, "logps/rejected": -309.4053955078125, "loss": 0.9621, "margin_dpo/margin_mean": 90.24147033691406, "margin_dpo/margin_std": 82.0784912109375, "step": 328 }, { "KL/chosen_KL_mean": -127.21353149414062, "KL/mean": -162.9486846923828, "KL/rejected_KL_mean": -198.68385314941406, "KL/std": 82.5337905883789, "epoch": 0.4831130690161527, "fcm_dpo/beta": 0.006151432171463966, "fcm_dpo/delta": -0.040776610374450684, "fcm_dpo/margin": 71.47030639648438, "fcm_dpo/q_t": 0.3990474343299866, "grad_norm": 10.253419876098633, "learning_rate": 3.0973306224962437e-07, "logits/chosen": -1.500509262084961, "logits/rejected": -0.99690842628479, "logps/chosen": -227.82659912109375, "logps/ref_chosen": -100.61308288574219, "logps/ref_rejected": -104.95370483398438, "logps/rejected": -303.6375732421875, "loss": 1.0621, "margin_dpo/margin_mean": 71.47030639648438, "margin_dpo/margin_std": 87.02228546142578, "step": 329 }, { "KL/chosen_KL_mean": -109.98770141601562, "KL/mean": -147.79891967773438, "KL/rejected_KL_mean": -185.61012268066406, "KL/std": 80.75257873535156, "epoch": 0.4845814977973568, "fcm_dpo/beta": 0.006122858263552189, "fcm_dpo/delta": -0.06459493935108185, "fcm_dpo/margin": 75.62242126464844, "fcm_dpo/q_t": 0.3901899456977844, "grad_norm": 8.56578540802002, "learning_rate": 3.084861204504122e-07, "logits/chosen": -1.8679529428482056, "logits/rejected": -1.789703607559204, "logps/chosen": -186.94613647460938, "logps/ref_chosen": -76.95843505859375, "logps/ref_rejected": -92.81494140625, "logps/rejected": -278.425048828125, "loss": 1.0217, "margin_dpo/margin_mean": 75.62242126464844, "margin_dpo/margin_std": 70.45252227783203, "step": 330 }, { "KL/chosen_KL_mean": -118.14378356933594, "KL/mean": -160.84979248046875, "KL/rejected_KL_mean": -203.5557861328125, "KL/std": 79.2252197265625, "epoch": 0.48604992657856094, "fcm_dpo/beta": 0.0060738250613212585, "fcm_dpo/delta": -0.12185509502887726, "fcm_dpo/margin": 85.41200256347656, "fcm_dpo/q_t": 0.37913966178894043, "grad_norm": 10.115684509277344, "learning_rate": 3.072376374875335e-07, "logits/chosen": -1.8524417877197266, "logits/rejected": -1.5475966930389404, "logps/chosen": -201.2032012939453, "logps/ref_chosen": -83.05941772460938, "logps/ref_rejected": -94.13113403320312, "logps/rejected": -297.6869201660156, "loss": 0.9863, "margin_dpo/margin_mean": 85.41200256347656, "margin_dpo/margin_std": 77.58705139160156, "step": 331 }, { "KL/chosen_KL_mean": -125.71873474121094, "KL/mean": -155.18905639648438, "KL/rejected_KL_mean": -184.65936279296875, "KL/std": 87.41764831542969, "epoch": 0.48751835535976507, "fcm_dpo/beta": 0.006066532805562019, "fcm_dpo/delta": 0.043170031160116196, "fcm_dpo/margin": 58.940643310546875, "fcm_dpo/q_t": 0.4167279303073883, "grad_norm": 8.334808349609375, "learning_rate": 3.059876462596758e-07, "logits/chosen": -2.020550012588501, "logits/rejected": -1.5871708393096924, "logps/chosen": -207.64962768554688, "logps/ref_chosen": -81.93089294433594, "logps/ref_rejected": -82.84590148925781, "logps/rejected": -267.5052795410156, "loss": 1.1228, "margin_dpo/margin_mean": 58.940643310546875, "margin_dpo/margin_std": 84.65032196044922, "step": 332 }, { "KL/chosen_KL_mean": -145.59490966796875, "KL/mean": -186.91554260253906, "KL/rejected_KL_mean": -228.23619079589844, "KL/std": 91.00726318359375, "epoch": 0.4889867841409692, "fcm_dpo/beta": 0.0060072895139455795, "fcm_dpo/delta": -0.09953628480434418, "fcm_dpo/margin": 82.64126586914062, "fcm_dpo/q_t": 0.3870406150817871, "grad_norm": 8.61490249633789, "learning_rate": 3.0473617970527015e-07, "logits/chosen": -1.580399990081787, "logits/rejected": -1.2435898780822754, "logps/chosen": -234.1274871826172, "logps/ref_chosen": -88.53257751464844, "logps/ref_rejected": -102.46601867675781, "logps/rejected": -330.70220947265625, "loss": 1.0318, "margin_dpo/margin_mean": 82.64126586914062, "margin_dpo/margin_std": 97.16705322265625, "step": 333 }, { "KL/chosen_KL_mean": -132.01080322265625, "KL/mean": -168.78675842285156, "KL/rejected_KL_mean": -205.5626983642578, "KL/std": 85.84302520751953, "epoch": 0.49045521292217326, "fcm_dpo/beta": 0.0059740133583545685, "fcm_dpo/delta": -0.04074270650744438, "fcm_dpo/margin": 73.5518798828125, "fcm_dpo/q_t": 0.3978947401046753, "grad_norm": 9.13284683227539, "learning_rate": 3.034832708016243e-07, "logits/chosen": -1.4505441188812256, "logits/rejected": -1.1564712524414062, "logps/chosen": -229.76858520507812, "logps/ref_chosen": -97.75778198242188, "logps/ref_rejected": -100.0058364868164, "logps/rejected": -305.56854248046875, "loss": 1.0736, "margin_dpo/margin_mean": 73.5518798828125, "margin_dpo/margin_std": 95.15580749511719, "step": 334 }, { "KL/chosen_KL_mean": -131.37571716308594, "KL/mean": -163.61793518066406, "KL/rejected_KL_mean": -195.86013793945312, "KL/std": 85.87240600585938, "epoch": 0.4919236417033774, "fcm_dpo/beta": 0.005991844460368156, "fcm_dpo/delta": 0.013886312022805214, "fcm_dpo/margin": 64.48443603515625, "fcm_dpo/q_t": 0.41106393933296204, "grad_norm": 12.144055366516113, "learning_rate": 3.022289525640531e-07, "logits/chosen": -2.1533265113830566, "logits/rejected": -1.7214093208312988, "logps/chosen": -227.75173950195312, "logps/ref_chosen": -96.37603759765625, "logps/ref_rejected": -94.92401123046875, "logps/rejected": -290.7841491699219, "loss": 1.111, "margin_dpo/margin_mean": 64.48443603515625, "margin_dpo/margin_std": 93.58976745605469, "step": 335 }, { "KL/chosen_KL_mean": -134.54507446289062, "KL/mean": -175.42034912109375, "KL/rejected_KL_mean": -216.2956085205078, "KL/std": 88.1512680053711, "epoch": 0.4933920704845815, "fcm_dpo/beta": 0.005960237700492144, "fcm_dpo/delta": -0.08938218653202057, "fcm_dpo/margin": 81.75054931640625, "fcm_dpo/q_t": 0.3880557417869568, "grad_norm": 10.74407958984375, "learning_rate": 3.009732580450086e-07, "logits/chosen": -1.58468759059906, "logits/rejected": -1.429211139678955, "logps/chosen": -223.07461547851562, "logps/ref_chosen": -88.52952575683594, "logps/ref_rejected": -109.5557861328125, "logps/rejected": -325.85137939453125, "loss": 1.0257, "margin_dpo/margin_mean": 81.75054931640625, "margin_dpo/margin_std": 92.3271484375, "step": 336 }, { "KL/chosen_KL_mean": -132.97000122070312, "KL/mean": -171.70199584960938, "KL/rejected_KL_mean": -210.4340057373047, "KL/std": 88.75144958496094, "epoch": 0.4948604992657856, "fcm_dpo/beta": 0.005918422248214483, "fcm_dpo/delta": -0.05981440469622612, "fcm_dpo/margin": 77.46400451660156, "fcm_dpo/q_t": 0.39598459005355835, "grad_norm": 9.37641716003418, "learning_rate": 2.9971622033320914e-07, "logits/chosen": -2.385852098464966, "logits/rejected": -2.1027543544769287, "logps/chosen": -230.28237915039062, "logps/ref_chosen": -97.3123779296875, "logps/ref_rejected": -110.13134765625, "logps/rejected": -320.56536865234375, "loss": 1.0656, "margin_dpo/margin_mean": 77.46400451660156, "margin_dpo/margin_std": 103.07540893554688, "step": 337 }, { "KL/chosen_KL_mean": -137.79493713378906, "KL/mean": -173.38400268554688, "KL/rejected_KL_mean": -208.97308349609375, "KL/std": 90.3515396118164, "epoch": 0.49632892804698975, "fcm_dpo/beta": 0.0058928728103637695, "fcm_dpo/delta": -0.019881822168827057, "fcm_dpo/margin": 71.17814636230469, "fcm_dpo/q_t": 0.4019433856010437, "grad_norm": 9.315040588378906, "learning_rate": 2.984578725527675e-07, "logits/chosen": -1.7536665201187134, "logits/rejected": -1.5958571434020996, "logps/chosen": -221.425048828125, "logps/ref_chosen": -83.63011169433594, "logps/ref_rejected": -93.69235229492188, "logps/rejected": -302.6654357910156, "loss": 1.0738, "margin_dpo/margin_mean": 71.17814636230469, "margin_dpo/margin_std": 87.8560791015625, "step": 338 }, { "KL/chosen_KL_mean": -146.3248748779297, "KL/mean": -184.35818481445312, "KL/rejected_KL_mean": -222.3914794921875, "KL/std": 89.30668640136719, "epoch": 0.4977973568281938, "fcm_dpo/beta": 0.005891101434826851, "fcm_dpo/delta": -0.049510497599840164, "fcm_dpo/margin": 76.06661987304688, "fcm_dpo/q_t": 0.39708083868026733, "grad_norm": 8.576578140258789, "learning_rate": 2.9719824786231796e-07, "logits/chosen": -2.8634228706359863, "logits/rejected": -2.4320549964904785, "logps/chosen": -246.17669677734375, "logps/ref_chosen": -99.85182189941406, "logps/ref_rejected": -109.0743637084961, "logps/rejected": -331.4658508300781, "loss": 1.0705, "margin_dpo/margin_mean": 76.06661987304688, "margin_dpo/margin_std": 98.60176086425781, "step": 339 }, { "KL/chosen_KL_mean": -158.1726837158203, "KL/mean": -191.24658203125, "KL/rejected_KL_mean": -224.3204803466797, "KL/std": 81.87571716308594, "epoch": 0.49926578560939794, "fcm_dpo/beta": 0.005870448425412178, "fcm_dpo/delta": 0.011909442953765392, "fcm_dpo/margin": 66.14778137207031, "fcm_dpo/q_t": 0.4094269275665283, "grad_norm": 10.3517427444458, "learning_rate": 2.959373794541426e-07, "logits/chosen": -1.3551547527313232, "logits/rejected": -0.8668117523193359, "logps/chosen": -243.1466522216797, "logps/ref_chosen": -84.97396850585938, "logps/ref_rejected": -78.80033874511719, "logps/rejected": -303.1208190917969, "loss": 1.0999, "margin_dpo/margin_mean": 66.14778137207031, "margin_dpo/margin_std": 89.91426086425781, "step": 340 }, { "KL/chosen_KL_mean": -164.95297241210938, "KL/mean": -197.22988891601562, "KL/rejected_KL_mean": -229.50682067871094, "KL/std": 85.81216430664062, "epoch": 0.5007342143906021, "fcm_dpo/beta": 0.005873257294297218, "fcm_dpo/delta": 0.02123020775616169, "fcm_dpo/margin": 64.5538330078125, "fcm_dpo/q_t": 0.413163423538208, "grad_norm": 9.430603981018066, "learning_rate": 2.946753005532965e-07, "logits/chosen": -1.2441785335540771, "logits/rejected": -0.9830518960952759, "logps/chosen": -250.2169647216797, "logps/ref_chosen": -85.26399230957031, "logps/ref_rejected": -106.08595275878906, "logps/rejected": -335.5927734375, "loss": 1.1219, "margin_dpo/margin_mean": 64.5538330078125, "margin_dpo/margin_std": 97.63438415527344, "step": 341 }, { "KL/chosen_KL_mean": -132.4700164794922, "KL/mean": -179.40594482421875, "KL/rejected_KL_mean": -226.34185791015625, "KL/std": 86.9466552734375, "epoch": 0.5022026431718062, "fcm_dpo/beta": 0.005801278166472912, "fcm_dpo/delta": -0.1493971049785614, "fcm_dpo/margin": 93.8718490600586, "fcm_dpo/q_t": 0.3729795515537262, "grad_norm": 8.251336097717285, "learning_rate": 2.934120444167326e-07, "logits/chosen": -1.327622413635254, "logits/rejected": -0.5616747736930847, "logps/chosen": -233.50860595703125, "logps/ref_chosen": -101.03860473632812, "logps/ref_rejected": -98.89851379394531, "logps/rejected": -325.2403564453125, "loss": 0.9867, "margin_dpo/margin_mean": 93.87185668945312, "margin_dpo/margin_std": 92.29365539550781, "step": 342 }, { "KL/chosen_KL_mean": -143.25442504882812, "KL/mean": -182.144287109375, "KL/rejected_KL_mean": -221.0341339111328, "KL/std": 90.14166259765625, "epoch": 0.5036710719530103, "fcm_dpo/beta": 0.00579288462176919, "fcm_dpo/delta": -0.052006687968969345, "fcm_dpo/margin": 77.77969360351562, "fcm_dpo/q_t": 0.3962807059288025, "grad_norm": 10.580329895019531, "learning_rate": 2.9214764433242476e-07, "logits/chosen": -2.2103781700134277, "logits/rejected": -2.0971498489379883, "logps/chosen": -227.00180053710938, "logps/ref_chosen": -83.74736022949219, "logps/ref_rejected": -116.57908630371094, "logps/rejected": -337.61322021484375, "loss": 1.059, "margin_dpo/margin_mean": 77.77969360351562, "margin_dpo/margin_std": 93.73761749267578, "step": 343 }, { "KL/chosen_KL_mean": -144.05709838867188, "KL/mean": -184.96490478515625, "KL/rejected_KL_mean": -225.87274169921875, "KL/std": 95.94554138183594, "epoch": 0.5051395007342144, "fcm_dpo/beta": 0.005749419331550598, "fcm_dpo/delta": -0.07215927541255951, "fcm_dpo/margin": 81.81562805175781, "fcm_dpo/q_t": 0.3928288221359253, "grad_norm": 10.057609558105469, "learning_rate": 2.9088213361849126e-07, "logits/chosen": -1.6097278594970703, "logits/rejected": -1.357997179031372, "logps/chosen": -222.31106567382812, "logps/ref_chosen": -78.25396728515625, "logps/ref_rejected": -95.42870330810547, "logps/rejected": -321.30145263671875, "loss": 1.0519, "margin_dpo/margin_mean": 81.81562805175781, "margin_dpo/margin_std": 101.76969146728516, "step": 344 }, { "KL/chosen_KL_mean": -146.94740295410156, "KL/mean": -190.9511260986328, "KL/rejected_KL_mean": -234.95484924316406, "KL/std": 96.3075942993164, "epoch": 0.5066079295154186, "fcm_dpo/beta": 0.005676139611750841, "fcm_dpo/delta": -0.10221190750598907, "fcm_dpo/margin": 88.00743103027344, "fcm_dpo/q_t": 0.38573622703552246, "grad_norm": 9.586256980895996, "learning_rate": 2.896155456223163e-07, "logits/chosen": -1.506721019744873, "logits/rejected": -1.2078109979629517, "logps/chosen": -245.02688598632812, "logps/ref_chosen": -98.0794906616211, "logps/ref_rejected": -104.03477478027344, "logps/rejected": -338.9896240234375, "loss": 1.0205, "margin_dpo/margin_mean": 88.00743103027344, "margin_dpo/margin_std": 98.95065307617188, "step": 345 }, { "KL/chosen_KL_mean": -163.97592163085938, "KL/mean": -198.06011962890625, "KL/rejected_KL_mean": -232.14431762695312, "KL/std": 89.05767822265625, "epoch": 0.5080763582966226, "fcm_dpo/beta": 0.0056710317730903625, "fcm_dpo/delta": 0.013680306263267994, "fcm_dpo/margin": 68.16841125488281, "fcm_dpo/q_t": 0.41154944896698, "grad_norm": 8.16605281829834, "learning_rate": 2.883479137196714e-07, "logits/chosen": -1.7322882413864136, "logits/rejected": -1.4139642715454102, "logps/chosen": -244.65939331054688, "logps/ref_chosen": -80.68348693847656, "logps/ref_rejected": -85.89260864257812, "logps/rejected": -318.03692626953125, "loss": 1.1128, "margin_dpo/margin_mean": 68.16841125488281, "margin_dpo/margin_std": 100.79698181152344, "step": 346 }, { "KL/chosen_KL_mean": -154.59002685546875, "KL/mean": -187.125732421875, "KL/rejected_KL_mean": -219.6614532470703, "KL/std": 88.116943359375, "epoch": 0.5095447870778267, "fcm_dpo/beta": 0.005695726256817579, "fcm_dpo/delta": 0.02977931872010231, "fcm_dpo/margin": 65.07142639160156, "fcm_dpo/q_t": 0.4148285388946533, "grad_norm": 8.723408699035645, "learning_rate": 2.8707927131383614e-07, "logits/chosen": -1.648805856704712, "logits/rejected": -1.313896894454956, "logps/chosen": -248.26939392089844, "logps/ref_chosen": -93.67938232421875, "logps/ref_rejected": -97.54251098632812, "logps/rejected": -317.2039794921875, "loss": 1.1188, "margin_dpo/margin_mean": 65.07142639160156, "margin_dpo/margin_std": 94.70013427734375, "step": 347 }, { "KL/chosen_KL_mean": -133.38235473632812, "KL/mean": -170.54116821289062, "KL/rejected_KL_mean": -207.69996643066406, "KL/std": 83.91085815429688, "epoch": 0.5110132158590308, "fcm_dpo/beta": 0.005656718276441097, "fcm_dpo/delta": -0.02172429859638214, "fcm_dpo/margin": 74.317626953125, "fcm_dpo/q_t": 0.40178561210632324, "grad_norm": 8.032180786132812, "learning_rate": 2.858096518347179e-07, "logits/chosen": -1.9268500804901123, "logits/rejected": -1.6486040353775024, "logps/chosen": -223.63461303710938, "logps/ref_chosen": -90.25224304199219, "logps/ref_rejected": -95.77027893066406, "logps/rejected": -303.4702453613281, "loss": 1.0727, "margin_dpo/margin_mean": 74.317626953125, "margin_dpo/margin_std": 85.79255676269531, "step": 348 }, { "KL/chosen_KL_mean": -143.9263916015625, "KL/mean": -178.8564910888672, "KL/rejected_KL_mean": -213.7865753173828, "KL/std": 91.517822265625, "epoch": 0.5124816446402349, "fcm_dpo/beta": 0.00569108035415411, "fcm_dpo/delta": 0.0023026615381240845, "fcm_dpo/margin": 69.86019897460938, "fcm_dpo/q_t": 0.408700168132782, "grad_norm": 9.844830513000488, "learning_rate": 2.845390887379706e-07, "logits/chosen": -2.0407090187072754, "logits/rejected": -1.8150700330734253, "logps/chosen": -235.98446655273438, "logps/ref_chosen": -92.05809020996094, "logps/ref_rejected": -103.96527099609375, "logps/rejected": -317.7518310546875, "loss": 1.0964, "margin_dpo/margin_mean": 69.86019897460938, "margin_dpo/margin_std": 93.69661712646484, "step": 349 }, { "KL/chosen_KL_mean": -152.74566650390625, "KL/mean": -181.65223693847656, "KL/rejected_KL_mean": -210.55880737304688, "KL/std": 84.24983978271484, "epoch": 0.5139500734214391, "fcm_dpo/beta": 0.005707222037017345, "fcm_dpo/delta": 0.07124963402748108, "fcm_dpo/margin": 57.813148498535156, "fcm_dpo/q_t": 0.42351120710372925, "grad_norm": 10.26970100402832, "learning_rate": 2.8326761550411346e-07, "logits/chosen": -1.8634589910507202, "logits/rejected": -1.6376185417175293, "logps/chosen": -239.79986572265625, "logps/ref_chosen": -87.05419921875, "logps/ref_rejected": -95.72193145751953, "logps/rejected": -306.2807312011719, "loss": 1.1476, "margin_dpo/margin_mean": 57.813148498535156, "margin_dpo/margin_std": 90.6649169921875, "step": 350 }, { "KL/chosen_KL_mean": -152.50033569335938, "KL/mean": -194.06781005859375, "KL/rejected_KL_mean": -235.63526916503906, "KL/std": 94.23765563964844, "epoch": 0.5154185022026432, "fcm_dpo/beta": 0.005695220082998276, "fcm_dpo/delta": -0.0752205103635788, "fcm_dpo/margin": 83.13493347167969, "fcm_dpo/q_t": 0.3907047510147095, "grad_norm": 8.515031814575195, "learning_rate": 2.819952656376487e-07, "logits/chosen": -1.7712879180908203, "logits/rejected": -1.4669511318206787, "logps/chosen": -241.66937255859375, "logps/ref_chosen": -89.16903686523438, "logps/ref_rejected": -108.29801940917969, "logps/rejected": -343.93328857421875, "loss": 1.0429, "margin_dpo/margin_mean": 83.13493347167969, "margin_dpo/margin_std": 99.15927124023438, "step": 351 }, { "KL/chosen_KL_mean": -139.89956665039062, "KL/mean": -179.33883666992188, "KL/rejected_KL_mean": -218.778076171875, "KL/std": 93.02790069580078, "epoch": 0.5168869309838473, "fcm_dpo/beta": 0.005658067762851715, "fcm_dpo/delta": -0.04735404998064041, "fcm_dpo/margin": 78.87853240966797, "fcm_dpo/q_t": 0.39896896481513977, "grad_norm": 8.825704574584961, "learning_rate": 2.8072207266617854e-07, "logits/chosen": -1.929368019104004, "logits/rejected": -1.3850340843200684, "logps/chosen": -236.71104431152344, "logps/ref_chosen": -96.81147766113281, "logps/ref_rejected": -84.40483093261719, "logps/rejected": -303.18292236328125, "loss": 1.0641, "margin_dpo/margin_mean": 78.87853240966797, "margin_dpo/margin_std": 99.80470275878906, "step": 352 }, { "KL/chosen_KL_mean": -159.1074981689453, "KL/mean": -184.49716186523438, "KL/rejected_KL_mean": -209.88687133789062, "KL/std": 82.40461730957031, "epoch": 0.5183553597650514, "fcm_dpo/beta": 0.00568999582901597, "fcm_dpo/delta": 0.1127714216709137, "fcm_dpo/margin": 50.77938461303711, "fcm_dpo/q_t": 0.4323740303516388, "grad_norm": 10.180861473083496, "learning_rate": 2.794480701395219e-07, "logits/chosen": -1.9871768951416016, "logits/rejected": -1.5933310985565186, "logps/chosen": -244.469970703125, "logps/ref_chosen": -85.36248779296875, "logps/ref_rejected": -87.36732482910156, "logps/rejected": -297.2541809082031, "loss": 1.1852, "margin_dpo/margin_mean": 50.77938461303711, "margin_dpo/margin_std": 91.58820343017578, "step": 353 }, { "KL/chosen_KL_mean": -139.350830078125, "KL/mean": -179.1335906982422, "KL/rejected_KL_mean": -218.9163360595703, "KL/std": 83.9892578125, "epoch": 0.5198237885462555, "fcm_dpo/beta": 0.005698390305042267, "fcm_dpo/delta": -0.054624903947114944, "fcm_dpo/margin": 79.56549072265625, "fcm_dpo/q_t": 0.39372730255126953, "grad_norm": 8.844533920288086, "learning_rate": 2.781732916288303e-07, "logits/chosen": -1.7616510391235352, "logits/rejected": -1.2819842100143433, "logps/chosen": -235.15200805664062, "logps/ref_chosen": -95.8011703491211, "logps/ref_rejected": -94.86614227294922, "logps/rejected": -313.782470703125, "loss": 1.0282, "margin_dpo/margin_mean": 79.56548309326172, "margin_dpo/margin_std": 76.62129211425781, "step": 354 }, { "KL/chosen_KL_mean": -137.7054443359375, "KL/mean": -174.47357177734375, "KL/rejected_KL_mean": -211.24172973632812, "KL/std": 85.3939208984375, "epoch": 0.5212922173274597, "fcm_dpo/beta": 0.005666209850460291, "fcm_dpo/delta": -0.017194081097841263, "fcm_dpo/margin": 73.53628540039062, "fcm_dpo/q_t": 0.4029642343521118, "grad_norm": 9.882235527038574, "learning_rate": 2.7689777072570284e-07, "logits/chosen": -2.5044636726379395, "logits/rejected": -2.0279130935668945, "logps/chosen": -223.8909912109375, "logps/ref_chosen": -86.185546875, "logps/ref_rejected": -88.4615707397461, "logps/rejected": -299.70330810546875, "loss": 1.0776, "margin_dpo/margin_mean": 73.53628540039062, "margin_dpo/margin_std": 89.85978698730469, "step": 355 }, { "KL/chosen_KL_mean": -162.29611206054688, "KL/mean": -190.62353515625, "KL/rejected_KL_mean": -218.950927734375, "KL/std": 83.63676452636719, "epoch": 0.5227606461086637, "fcm_dpo/beta": 0.005715552251785994, "fcm_dpo/delta": 0.07723444700241089, "fcm_dpo/margin": 56.65481948852539, "fcm_dpo/q_t": 0.4248383641242981, "grad_norm": 9.32064151763916, "learning_rate": 2.7562154104130176e-07, "logits/chosen": -1.6554985046386719, "logits/rejected": -1.175462007522583, "logps/chosen": -253.11422729492188, "logps/ref_chosen": -90.818115234375, "logps/ref_rejected": -82.1273193359375, "logps/rejected": -301.0782470703125, "loss": 1.1591, "margin_dpo/margin_mean": 56.65481948852539, "margin_dpo/margin_std": 94.94267272949219, "step": 356 }, { "KL/chosen_KL_mean": -134.91741943359375, "KL/mean": -181.00082397460938, "KL/rejected_KL_mean": -227.08419799804688, "KL/std": 92.35015106201172, "epoch": 0.5242290748898678, "fcm_dpo/beta": 0.005645669065415859, "fcm_dpo/delta": -0.12447288632392883, "fcm_dpo/margin": 92.16676330566406, "fcm_dpo/q_t": 0.3788727819919586, "grad_norm": 8.428705215454102, "learning_rate": 2.7434463620546594e-07, "logits/chosen": -1.9340600967407227, "logits/rejected": -1.5638494491577148, "logps/chosen": -222.42218017578125, "logps/ref_chosen": -87.5047607421875, "logps/ref_rejected": -92.24937438964844, "logps/rejected": -319.33355712890625, "loss": 0.9895, "margin_dpo/margin_mean": 92.16676330566406, "margin_dpo/margin_std": 84.73847198486328, "step": 357 }, { "KL/chosen_KL_mean": -146.48899841308594, "KL/mean": -180.29013061523438, "KL/rejected_KL_mean": -214.0912628173828, "KL/std": 85.75318908691406, "epoch": 0.5256975036710719, "fcm_dpo/beta": 0.0056540099903941154, "fcm_dpo/delta": 0.01812445931136608, "fcm_dpo/margin": 67.60226440429688, "fcm_dpo/q_t": 0.41270163655281067, "grad_norm": 8.836957931518555, "learning_rate": 2.730670898658255e-07, "logits/chosen": -1.5799734592437744, "logits/rejected": -1.101548671722412, "logps/chosen": -241.54446411132812, "logps/ref_chosen": -95.05546569824219, "logps/ref_rejected": -96.47611999511719, "logps/rejected": -310.5673828125, "loss": 1.1023, "margin_dpo/margin_mean": 67.60226440429688, "margin_dpo/margin_std": 91.72412109375, "step": 358 }, { "KL/chosen_KL_mean": -163.58352661132812, "KL/mean": -195.0387725830078, "KL/rejected_KL_mean": -226.4940185546875, "KL/std": 80.65864562988281, "epoch": 0.527165932452276, "fcm_dpo/beta": 0.005661042407155037, "fcm_dpo/delta": 0.04447634518146515, "fcm_dpo/margin": 62.9105224609375, "fcm_dpo/q_t": 0.4183029234409332, "grad_norm": 9.111247062683105, "learning_rate": 2.717889356869146e-07, "logits/chosen": -1.6120582818984985, "logits/rejected": -1.0486648082733154, "logps/chosen": -250.95684814453125, "logps/ref_chosen": -87.37332916259766, "logps/ref_rejected": -88.77188110351562, "logps/rejected": -315.2658996582031, "loss": 1.1297, "margin_dpo/margin_mean": 62.910518646240234, "margin_dpo/margin_std": 93.03585815429688, "step": 359 }, { "KL/chosen_KL_mean": -150.69784545898438, "KL/mean": -191.13604736328125, "KL/rejected_KL_mean": -231.5742645263672, "KL/std": 81.5629653930664, "epoch": 0.5286343612334802, "fcm_dpo/beta": 0.005654939450323582, "fcm_dpo/delta": -0.05873660743236542, "fcm_dpo/margin": 80.87641143798828, "fcm_dpo/q_t": 0.3925561308860779, "grad_norm": 11.699244499206543, "learning_rate": 2.7051020734928443e-07, "logits/chosen": -1.5691527128219604, "logits/rejected": -1.2215745449066162, "logps/chosen": -225.92129516601562, "logps/ref_chosen": -75.22344970703125, "logps/ref_rejected": -76.09400177001953, "logps/rejected": -307.66827392578125, "loss": 1.0345, "margin_dpo/margin_mean": 80.87641143798828, "margin_dpo/margin_std": 84.70951843261719, "step": 360 }, { "KL/chosen_KL_mean": -154.71302795410156, "KL/mean": -189.95858764648438, "KL/rejected_KL_mean": -225.20413208007812, "KL/std": 92.02262878417969, "epoch": 0.5301027900146843, "fcm_dpo/beta": 0.005652287509292364, "fcm_dpo/delta": 0.0015900330618023872, "fcm_dpo/margin": 70.4910888671875, "fcm_dpo/q_t": 0.4088614583015442, "grad_norm": 10.40850830078125, "learning_rate": 2.6923093854861593e-07, "logits/chosen": -1.7217049598693848, "logits/rejected": -1.4749481678009033, "logps/chosen": -236.66636657714844, "logps/ref_chosen": -81.95333099365234, "logps/ref_rejected": -96.67919921875, "logps/rejected": -321.8833312988281, "loss": 1.0975, "margin_dpo/margin_mean": 70.4910888671875, "margin_dpo/margin_std": 97.18360137939453, "step": 361 }, { "KL/chosen_KL_mean": -160.30938720703125, "KL/mean": -196.97291564941406, "KL/rejected_KL_mean": -233.63645935058594, "KL/std": 86.3531265258789, "epoch": 0.5315712187958884, "fcm_dpo/beta": 0.005639141425490379, "fcm_dpo/delta": -0.013838537037372589, "fcm_dpo/margin": 73.32706451416016, "fcm_dpo/q_t": 0.4035763740539551, "grad_norm": 9.496636390686035, "learning_rate": 2.679511629948319e-07, "logits/chosen": -1.8173277378082275, "logits/rejected": -1.6120038032531738, "logps/chosen": -249.64910888671875, "logps/ref_chosen": -89.33971405029297, "logps/ref_rejected": -110.50196838378906, "logps/rejected": -344.138427734375, "loss": 1.0785, "margin_dpo/margin_mean": 73.32707214355469, "margin_dpo/margin_std": 90.49586486816406, "step": 362 }, { "KL/chosen_KL_mean": -149.59332275390625, "KL/mean": -197.52212524414062, "KL/rejected_KL_mean": -245.45095825195312, "KL/std": 91.43606567382812, "epoch": 0.5330396475770925, "fcm_dpo/beta": 0.0055956486612558365, "fcm_dpo/delta": -0.14002801477909088, "fcm_dpo/margin": 95.85762023925781, "fcm_dpo/q_t": 0.37761062383651733, "grad_norm": 10.82379150390625, "learning_rate": 2.6667091441120816e-07, "logits/chosen": -1.6798560619354248, "logits/rejected": -1.2348127365112305, "logps/chosen": -230.36831665039062, "logps/ref_chosen": -80.77499389648438, "logps/ref_rejected": -79.43267822265625, "logps/rejected": -324.88360595703125, "loss": 0.9919, "margin_dpo/margin_mean": 95.85762023925781, "margin_dpo/margin_std": 99.33370971679688, "step": 363 }, { "KL/chosen_KL_mean": -147.55364990234375, "KL/mean": -186.46624755859375, "KL/rejected_KL_mean": -225.37884521484375, "KL/std": 85.24311828613281, "epoch": 0.5345080763582967, "fcm_dpo/beta": 0.00556713342666626, "fcm_dpo/delta": -0.03427809849381447, "fcm_dpo/margin": 77.82520294189453, "fcm_dpo/q_t": 0.3981122672557831, "grad_norm": 11.939802169799805, "learning_rate": 2.6539022653348575e-07, "logits/chosen": -1.9894155263900757, "logits/rejected": -1.843038558959961, "logps/chosen": -224.083251953125, "logps/ref_chosen": -76.52960205078125, "logps/ref_rejected": -97.56173706054688, "logps/rejected": -322.9405822753906, "loss": 1.0508, "margin_dpo/margin_mean": 77.8251953125, "margin_dpo/margin_std": 80.68634796142578, "step": 364 }, { "KL/chosen_KL_mean": -144.2945098876953, "KL/mean": -186.32528686523438, "KL/rejected_KL_mean": -228.3560333251953, "KL/std": 90.98835754394531, "epoch": 0.5359765051395007, "fcm_dpo/beta": 0.005528109148144722, "fcm_dpo/delta": -0.06625748425722122, "fcm_dpo/margin": 84.0615234375, "fcm_dpo/q_t": 0.3943924605846405, "grad_norm": 8.978610038757324, "learning_rate": 2.641091331089811e-07, "logits/chosen": -1.7142645120620728, "logits/rejected": -1.5845690965652466, "logps/chosen": -224.8394012451172, "logps/ref_chosen": -80.54489135742188, "logps/ref_rejected": -99.32316589355469, "logps/rejected": -327.67919921875, "loss": 1.0508, "margin_dpo/margin_mean": 84.0615234375, "margin_dpo/margin_std": 103.22001647949219, "step": 365 }, { "KL/chosen_KL_mean": -145.9812774658203, "KL/mean": -181.06365966796875, "KL/rejected_KL_mean": -216.14602661132812, "KL/std": 94.70976257324219, "epoch": 0.5374449339207048, "fcm_dpo/beta": 0.005514367483556271, "fcm_dpo/delta": 0.013342992402613163, "fcm_dpo/margin": 70.16474914550781, "fcm_dpo/q_t": 0.41204941272735596, "grad_norm": 8.517179489135742, "learning_rate": 2.6282766789569736e-07, "logits/chosen": -1.614109754562378, "logits/rejected": -1.646314263343811, "logps/chosen": -218.21934509277344, "logps/ref_chosen": -72.23806762695312, "logps/ref_rejected": -91.67182922363281, "logps/rejected": -307.81787109375, "loss": 1.0943, "margin_dpo/margin_mean": 70.16474914550781, "margin_dpo/margin_std": 91.62953186035156, "step": 366 }, { "KL/chosen_KL_mean": -132.9840087890625, "KL/mean": -175.03427124023438, "KL/rejected_KL_mean": -217.08453369140625, "KL/std": 83.69141387939453, "epoch": 0.5389133627019089, "fcm_dpo/beta": 0.0054914457723498344, "fcm_dpo/delta": -0.0632803663611412, "fcm_dpo/margin": 84.10052490234375, "fcm_dpo/q_t": 0.3923204839229584, "grad_norm": 8.39808464050293, "learning_rate": 2.615458646614349e-07, "logits/chosen": -1.7484509944915771, "logits/rejected": -1.4032632112503052, "logps/chosen": -218.40602111816406, "logps/ref_chosen": -85.42201232910156, "logps/ref_rejected": -83.51779174804688, "logps/rejected": -300.6023254394531, "loss": 1.0292, "margin_dpo/margin_mean": 84.10052490234375, "margin_dpo/margin_std": 86.51387786865234, "step": 367 }, { "KL/chosen_KL_mean": -124.29361724853516, "KL/mean": -170.45367431640625, "KL/rejected_KL_mean": -216.61373901367188, "KL/std": 92.17273712158203, "epoch": 0.540381791483113, "fcm_dpo/beta": 0.005449716001749039, "fcm_dpo/delta": -0.10573888570070267, "fcm_dpo/margin": 92.32012939453125, "fcm_dpo/q_t": 0.38275134563446045, "grad_norm": 9.48375415802002, "learning_rate": 2.6026375718290083e-07, "logits/chosen": -1.3949027061462402, "logits/rejected": -1.3586883544921875, "logps/chosen": -198.13473510742188, "logps/ref_chosen": -73.84112548828125, "logps/ref_rejected": -104.69528198242188, "logps/rejected": -321.30902099609375, "loss": 1.009, "margin_dpo/margin_mean": 92.32012176513672, "margin_dpo/margin_std": 93.4371337890625, "step": 368 }, { "KL/chosen_KL_mean": -137.7772674560547, "KL/mean": -181.25323486328125, "KL/rejected_KL_mean": -224.72921752929688, "KL/std": 84.56979370117188, "epoch": 0.5418502202643172, "fcm_dpo/beta": 0.005399555899202824, "fcm_dpo/delta": -0.07114151865243912, "fcm_dpo/margin": 86.95196533203125, "fcm_dpo/q_t": 0.39026662707328796, "grad_norm": 9.18364429473877, "learning_rate": 2.589813792448196e-07, "logits/chosen": -1.7773559093475342, "logits/rejected": -1.4143869876861572, "logps/chosen": -236.23922729492188, "logps/ref_chosen": -98.46195983886719, "logps/ref_rejected": -99.15672302246094, "logps/rejected": -323.88592529296875, "loss": 1.0195, "margin_dpo/margin_mean": 86.95196533203125, "margin_dpo/margin_std": 84.35160827636719, "step": 369 }, { "KL/chosen_KL_mean": -154.00531005859375, "KL/mean": -183.43630981445312, "KL/rejected_KL_mean": -212.8673095703125, "KL/std": 82.20635986328125, "epoch": 0.5433186490455213, "fcm_dpo/beta": 0.0054197171702980995, "fcm_dpo/delta": 0.08227226138114929, "fcm_dpo/margin": 58.861976623535156, "fcm_dpo/q_t": 0.42515408992767334, "grad_norm": 8.826552391052246, "learning_rate": 2.5769876463904263e-07, "logits/chosen": -1.9563322067260742, "logits/rejected": -1.7574396133422852, "logps/chosen": -250.7810516357422, "logps/ref_chosen": -96.77574157714844, "logps/ref_rejected": -102.74411010742188, "logps/rejected": -315.6114196777344, "loss": 1.1556, "margin_dpo/margin_mean": 58.861976623535156, "margin_dpo/margin_std": 94.48282623291016, "step": 370 }, { "KL/chosen_KL_mean": -147.05093383789062, "KL/mean": -178.96517944335938, "KL/rejected_KL_mean": -210.87942504882812, "KL/std": 82.5028076171875, "epoch": 0.5447870778267254, "fcm_dpo/beta": 0.005458069033920765, "fcm_dpo/delta": 0.05239345133304596, "fcm_dpo/margin": 63.82848358154297, "fcm_dpo/q_t": 0.41823816299438477, "grad_norm": 11.277172088623047, "learning_rate": 2.5641594716365744e-07, "logits/chosen": -1.833635926246643, "logits/rejected": -1.464059829711914, "logps/chosen": -250.0411376953125, "logps/ref_chosen": -102.99020385742188, "logps/ref_rejected": -105.35062408447266, "logps/rejected": -316.23004150390625, "loss": 1.1251, "margin_dpo/margin_mean": 63.82848358154297, "margin_dpo/margin_std": 88.89564514160156, "step": 371 }, { "KL/chosen_KL_mean": -124.31694030761719, "KL/mean": -172.72836303710938, "KL/rejected_KL_mean": -221.13978576660156, "KL/std": 95.9947509765625, "epoch": 0.5462555066079295, "fcm_dpo/beta": 0.0054099359549582005, "fcm_dpo/delta": -0.1270708441734314, "fcm_dpo/margin": 96.82284545898438, "fcm_dpo/q_t": 0.3796873390674591, "grad_norm": 7.84365701675415, "learning_rate": 2.551329606220976e-07, "logits/chosen": -1.9173626899719238, "logits/rejected": -1.2888731956481934, "logps/chosen": -215.34317016601562, "logps/ref_chosen": -91.02623748779297, "logps/ref_rejected": -85.6102294921875, "logps/rejected": -306.75, "loss": 0.9947, "margin_dpo/margin_mean": 96.82284545898438, "margin_dpo/margin_std": 97.77336120605469, "step": 372 }, { "KL/chosen_KL_mean": -143.43479919433594, "KL/mean": -179.27552795410156, "KL/rejected_KL_mean": -215.1162567138672, "KL/std": 91.15010070800781, "epoch": 0.5477239353891337, "fcm_dpo/beta": 0.005391295067965984, "fcm_dpo/delta": 0.013803921639919281, "fcm_dpo/margin": 71.68144989013672, "fcm_dpo/q_t": 0.4103190302848816, "grad_norm": 10.874719619750977, "learning_rate": 2.538498388222517e-07, "logits/chosen": -2.0248820781707764, "logits/rejected": -1.493084192276001, "logps/chosen": -243.50787353515625, "logps/ref_chosen": -100.07307434082031, "logps/ref_rejected": -91.11634826660156, "logps/rejected": -306.23260498046875, "loss": 1.102, "margin_dpo/margin_mean": 71.68144989013672, "margin_dpo/margin_std": 97.31051635742188, "step": 373 }, { "KL/chosen_KL_mean": -124.77534484863281, "KL/mean": -164.08541870117188, "KL/rejected_KL_mean": -203.39549255371094, "KL/std": 86.46551513671875, "epoch": 0.5491923641703378, "fcm_dpo/beta": 0.005380621179938316, "fcm_dpo/delta": -0.0236080139875412, "fcm_dpo/margin": 78.6201400756836, "fcm_dpo/q_t": 0.4011400043964386, "grad_norm": 8.345185279846191, "learning_rate": 2.525666155755725e-07, "logits/chosen": -2.243584394454956, "logits/rejected": -1.8547483682632446, "logps/chosen": -222.78768920898438, "logps/ref_chosen": -98.01234436035156, "logps/ref_rejected": -100.75224304199219, "logps/rejected": -304.1477355957031, "loss": 1.0547, "margin_dpo/margin_mean": 78.6201400756836, "margin_dpo/margin_std": 83.22883605957031, "step": 374 }, { "KL/chosen_KL_mean": -127.82306671142578, "KL/mean": -172.7537841796875, "KL/rejected_KL_mean": -217.68447875976562, "KL/std": 84.24693298339844, "epoch": 0.5506607929515418, "fcm_dpo/beta": 0.005334075540304184, "fcm_dpo/delta": -0.08193912357091904, "fcm_dpo/margin": 89.86143493652344, "fcm_dpo/q_t": 0.3876490890979767, "grad_norm": 9.127181053161621, "learning_rate": 2.512833246961859e-07, "logits/chosen": -1.533527135848999, "logits/rejected": -1.2978014945983887, "logps/chosen": -212.21934509277344, "logps/ref_chosen": -84.39627838134766, "logps/ref_rejected": -95.7188491821289, "logps/rejected": -313.4033203125, "loss": 1.0197, "margin_dpo/margin_mean": 89.86143493652344, "margin_dpo/margin_std": 88.99827575683594, "step": 375 }, { "KL/chosen_KL_mean": -122.32588958740234, "KL/mean": -171.61663818359375, "KL/rejected_KL_mean": -220.90737915039062, "KL/std": 95.63398742675781, "epoch": 0.5521292217327459, "fcm_dpo/beta": 0.005288714077323675, "fcm_dpo/delta": -0.12466256320476532, "fcm_dpo/margin": 98.58151245117188, "fcm_dpo/q_t": 0.3797568380832672, "grad_norm": 10.029169082641602, "learning_rate": 2.5e-07, "logits/chosen": -1.7209749221801758, "logits/rejected": -1.40751051902771, "logps/chosen": -218.0635986328125, "logps/ref_chosen": -95.73770141601562, "logps/ref_rejected": -111.97331237792969, "logps/rejected": -332.88067626953125, "loss": 0.9978, "margin_dpo/margin_mean": 98.58151245117188, "margin_dpo/margin_std": 101.38668823242188, "step": 376 }, { "KL/chosen_KL_mean": -130.382080078125, "KL/mean": -181.6572723388672, "KL/rejected_KL_mean": -232.93246459960938, "KL/std": 96.98780822753906, "epoch": 0.55359765051395, "fcm_dpo/beta": 0.005209389142692089, "fcm_dpo/delta": -0.13833145797252655, "fcm_dpo/margin": 102.55036163330078, "fcm_dpo/q_t": 0.3759406507015228, "grad_norm": 8.391050338745117, "learning_rate": 2.487166753038141e-07, "logits/chosen": -1.5231859683990479, "logits/rejected": -1.4043909311294556, "logps/chosen": -209.3090362548828, "logps/ref_chosen": -78.92694854736328, "logps/ref_rejected": -105.89892578125, "logps/rejected": -338.8313903808594, "loss": 0.9811, "margin_dpo/margin_mean": 102.55035400390625, "margin_dpo/margin_std": 96.33819580078125, "step": 377 }, { "KL/chosen_KL_mean": -126.89985656738281, "KL/mean": -167.79168701171875, "KL/rejected_KL_mean": -208.68353271484375, "KL/std": 92.03629302978516, "epoch": 0.5550660792951542, "fcm_dpo/beta": 0.0052024684846401215, "fcm_dpo/delta": -0.026166213676333427, "fcm_dpo/margin": 81.78369903564453, "fcm_dpo/q_t": 0.39973193407058716, "grad_norm": 9.367769241333008, "learning_rate": 2.4743338442442754e-07, "logits/chosen": -1.9060044288635254, "logits/rejected": -1.6208364963531494, "logps/chosen": -202.27224731445312, "logps/ref_chosen": -75.37240600585938, "logps/ref_rejected": -93.70409393310547, "logps/rejected": -302.38763427734375, "loss": 1.0641, "margin_dpo/margin_mean": 81.78369903564453, "margin_dpo/margin_std": 93.89768981933594, "step": 378 }, { "KL/chosen_KL_mean": -149.19711303710938, "KL/mean": -185.47186279296875, "KL/rejected_KL_mean": -221.74664306640625, "KL/std": 95.45217895507812, "epoch": 0.5565345080763583, "fcm_dpo/beta": 0.005195148754864931, "fcm_dpo/delta": 0.023541312664747238, "fcm_dpo/margin": 72.54954528808594, "fcm_dpo/q_t": 0.41411858797073364, "grad_norm": 9.454183578491211, "learning_rate": 2.461501611777483e-07, "logits/chosen": -1.9638092517852783, "logits/rejected": -1.8794009685516357, "logps/chosen": -238.05279541015625, "logps/ref_chosen": -88.85568237304688, "logps/ref_rejected": -121.38114166259766, "logps/rejected": -343.1278076171875, "loss": 1.1199, "margin_dpo/margin_mean": 72.54954528808594, "margin_dpo/margin_std": 108.92121887207031, "step": 379 }, { "KL/chosen_KL_mean": -163.71939086914062, "KL/mean": -207.33065795898438, "KL/rejected_KL_mean": -250.94189453125, "KL/std": 97.46766662597656, "epoch": 0.5580029368575624, "fcm_dpo/beta": 0.005178709514439106, "fcm_dpo/delta": -0.052902355790138245, "fcm_dpo/margin": 87.22249603271484, "fcm_dpo/q_t": 0.39754214882850647, "grad_norm": 12.316363334655762, "learning_rate": 2.4486703937790243e-07, "logits/chosen": -1.2050063610076904, "logits/rejected": -1.1780513525009155, "logps/chosen": -243.55259704589844, "logps/ref_chosen": -79.83321380615234, "logps/ref_rejected": -110.7447509765625, "logps/rejected": -361.6866455078125, "loss": 1.0585, "margin_dpo/margin_mean": 87.22249603271484, "margin_dpo/margin_std": 110.45135498046875, "step": 380 }, { "KL/chosen_KL_mean": -143.37591552734375, "KL/mean": -181.6533203125, "KL/rejected_KL_mean": -219.9307403564453, "KL/std": 93.62744140625, "epoch": 0.5594713656387665, "fcm_dpo/beta": 0.005175272934138775, "fcm_dpo/delta": 0.003881353884935379, "fcm_dpo/margin": 76.55481719970703, "fcm_dpo/q_t": 0.4071226716041565, "grad_norm": 9.783101081848145, "learning_rate": 2.435840528363426e-07, "logits/chosen": -1.7776129245758057, "logits/rejected": -1.2259242534637451, "logps/chosen": -229.9427490234375, "logps/ref_chosen": -86.56683349609375, "logps/ref_rejected": -84.416748046875, "logps/rejected": -304.34747314453125, "loss": 1.0875, "margin_dpo/margin_mean": 76.55481719970703, "margin_dpo/margin_std": 95.85802459716797, "step": 381 }, { "KL/chosen_KL_mean": -141.72714233398438, "KL/mean": -185.39215087890625, "KL/rejected_KL_mean": -229.05714416503906, "KL/std": 91.73875427246094, "epoch": 0.5609397944199707, "fcm_dpo/beta": 0.005140431225299835, "fcm_dpo/delta": -0.05050382763147354, "fcm_dpo/margin": 87.33000183105469, "fcm_dpo/q_t": 0.3953739106655121, "grad_norm": 9.143963813781738, "learning_rate": 2.4230123536095745e-07, "logits/chosen": -1.864398717880249, "logits/rejected": -1.5961270332336426, "logps/chosen": -246.18893432617188, "logps/ref_chosen": -104.46180725097656, "logps/ref_rejected": -114.60223388671875, "logps/rejected": -343.65936279296875, "loss": 1.0438, "margin_dpo/margin_mean": 87.33000183105469, "margin_dpo/margin_std": 93.35982513427734, "step": 382 }, { "KL/chosen_KL_mean": -153.2938232421875, "KL/mean": -190.36862182617188, "KL/rejected_KL_mean": -227.44345092773438, "KL/std": 94.58848571777344, "epoch": 0.5624082232011748, "fcm_dpo/beta": 0.005150897894054651, "fcm_dpo/delta": 0.018398616462945938, "fcm_dpo/margin": 74.14964294433594, "fcm_dpo/q_t": 0.41075003147125244, "grad_norm": 9.582107543945312, "learning_rate": 2.4101862075518037e-07, "logits/chosen": -1.8412387371063232, "logits/rejected": -1.7237522602081299, "logps/chosen": -235.40988159179688, "logps/ref_chosen": -82.11607360839844, "logps/ref_rejected": -98.77716064453125, "logps/rejected": -326.2206115722656, "loss": 1.0955, "margin_dpo/margin_mean": 74.1496353149414, "margin_dpo/margin_std": 91.98062133789062, "step": 383 }, { "KL/chosen_KL_mean": -164.7509765625, "KL/mean": -201.65774536132812, "KL/rejected_KL_mean": -238.5645294189453, "KL/std": 98.34101867675781, "epoch": 0.5638766519823789, "fcm_dpo/beta": 0.005181390792131424, "fcm_dpo/delta": 0.01740475371479988, "fcm_dpo/margin": 73.81354522705078, "fcm_dpo/q_t": 0.4112620949745178, "grad_norm": 10.982965469360352, "learning_rate": 2.397362428170992e-07, "logits/chosen": -1.8433072566986084, "logits/rejected": -1.6198350191116333, "logps/chosen": -245.33621215820312, "logps/ref_chosen": -80.58525085449219, "logps/ref_rejected": -91.44789123535156, "logps/rejected": -330.0124206542969, "loss": 1.1212, "margin_dpo/margin_mean": 73.81354522705078, "margin_dpo/margin_std": 108.63145446777344, "step": 384 }, { "KL/chosen_KL_mean": -137.7400665283203, "KL/mean": -185.21299743652344, "KL/rejected_KL_mean": -232.6859130859375, "KL/std": 88.93122863769531, "epoch": 0.5653450807635829, "fcm_dpo/beta": 0.005119113251566887, "fcm_dpo/delta": -0.0887046754360199, "fcm_dpo/margin": 94.94586181640625, "fcm_dpo/q_t": 0.3881973624229431, "grad_norm": 10.41855239868164, "learning_rate": 2.3845413533856514e-07, "logits/chosen": -2.226529121398926, "logits/rejected": -1.6185941696166992, "logps/chosen": -230.50022888183594, "logps/ref_chosen": -92.76016235351562, "logps/ref_rejected": -84.45997619628906, "logps/rejected": -317.1458740234375, "loss": 1.025, "margin_dpo/margin_mean": 94.94586181640625, "margin_dpo/margin_std": 102.96405029296875, "step": 385 }, { "KL/chosen_KL_mean": -145.50119018554688, "KL/mean": -190.98117065429688, "KL/rejected_KL_mean": -236.46116638183594, "KL/std": 90.67424774169922, "epoch": 0.566813509544787, "fcm_dpo/beta": 0.005096007138490677, "fcm_dpo/delta": -0.06504582613706589, "fcm_dpo/margin": 90.95999145507812, "fcm_dpo/q_t": 0.3906695544719696, "grad_norm": 7.827576637268066, "learning_rate": 2.3717233210430254e-07, "logits/chosen": -1.6459641456604004, "logits/rejected": -1.3595550060272217, "logps/chosen": -230.86013793945312, "logps/ref_chosen": -85.35896301269531, "logps/ref_rejected": -96.99065399169922, "logps/rejected": -333.4518127441406, "loss": 1.0354, "margin_dpo/margin_mean": 90.95999145507812, "margin_dpo/margin_std": 98.32936096191406, "step": 386 }, { "KL/chosen_KL_mean": -168.51785278320312, "KL/mean": -209.60140991210938, "KL/rejected_KL_mean": -250.6849365234375, "KL/std": 87.74478149414062, "epoch": 0.5682819383259912, "fcm_dpo/beta": 0.005085395649075508, "fcm_dpo/delta": -0.018246542662382126, "fcm_dpo/margin": 82.1671142578125, "fcm_dpo/q_t": 0.40200328826904297, "grad_norm": 10.508011817932129, "learning_rate": 2.3589086689101889e-07, "logits/chosen": -2.450582981109619, "logits/rejected": -1.7666316032409668, "logps/chosen": -271.20599365234375, "logps/ref_chosen": -102.68814086914062, "logps/ref_rejected": -99.42858123779297, "logps/rejected": -350.113525390625, "loss": 1.0619, "margin_dpo/margin_mean": 82.1671142578125, "margin_dpo/margin_std": 90.64164733886719, "step": 387 }, { "KL/chosen_KL_mean": -170.2499542236328, "KL/mean": -215.51138305664062, "KL/rejected_KL_mean": -260.7728271484375, "KL/std": 97.5136489868164, "epoch": 0.5697503671071953, "fcm_dpo/beta": 0.00505464943125844, "fcm_dpo/delta": -0.05893528461456299, "fcm_dpo/margin": 90.5228500366211, "fcm_dpo/q_t": 0.39332252740859985, "grad_norm": 8.827988624572754, "learning_rate": 2.3460977346651428e-07, "logits/chosen": -1.7942323684692383, "logits/rejected": -1.6745786666870117, "logps/chosen": -253.83518981933594, "logps/ref_chosen": -83.58523559570312, "logps/ref_rejected": -110.10037231445312, "logps/rejected": -370.8731689453125, "loss": 1.0406, "margin_dpo/margin_mean": 90.5228500366211, "margin_dpo/margin_std": 100.05789947509766, "step": 388 }, { "KL/chosen_KL_mean": -175.6865692138672, "KL/mean": -214.46148681640625, "KL/rejected_KL_mean": -253.23638916015625, "KL/std": 99.87646484375, "epoch": 0.5712187958883994, "fcm_dpo/beta": 0.00505096185952425, "fcm_dpo/delta": 0.008464265614748001, "fcm_dpo/margin": 77.54985046386719, "fcm_dpo/q_t": 0.4094657897949219, "grad_norm": 11.04814624786377, "learning_rate": 2.3332908558879177e-07, "logits/chosen": -2.4872093200683594, "logits/rejected": -2.0473787784576416, "logps/chosen": -265.50885009765625, "logps/ref_chosen": -89.822265625, "logps/ref_rejected": -84.85376739501953, "logps/rejected": -338.09014892578125, "loss": 1.1017, "margin_dpo/margin_mean": 77.54985046386719, "margin_dpo/margin_std": 107.62672424316406, "step": 389 }, { "KL/chosen_KL_mean": -176.50376892089844, "KL/mean": -213.9547119140625, "KL/rejected_KL_mean": -251.4056854248047, "KL/std": 97.05670166015625, "epoch": 0.5726872246696035, "fcm_dpo/beta": 0.005049366038292646, "fcm_dpo/delta": 0.02203134074807167, "fcm_dpo/margin": 74.90190124511719, "fcm_dpo/q_t": 0.4146232306957245, "grad_norm": 8.85820484161377, "learning_rate": 2.320488370051681e-07, "logits/chosen": -1.5099680423736572, "logits/rejected": -1.2983508110046387, "logps/chosen": -254.0362091064453, "logps/ref_chosen": -77.53244018554688, "logps/ref_rejected": -92.58392333984375, "logps/rejected": -343.9895935058594, "loss": 1.1181, "margin_dpo/margin_mean": 74.90190124511719, "margin_dpo/margin_std": 111.27388000488281, "step": 390 }, { "KL/chosen_KL_mean": -159.51177978515625, "KL/mean": -200.08975219726562, "KL/rejected_KL_mean": -240.667724609375, "KL/std": 93.39694213867188, "epoch": 0.5741556534508077, "fcm_dpo/beta": 0.0050530098378658295, "fcm_dpo/delta": -0.010394874960184097, "fcm_dpo/margin": 81.15596008300781, "fcm_dpo/q_t": 0.4045126438140869, "grad_norm": 9.436186790466309, "learning_rate": 2.3076906145138405e-07, "logits/chosen": -2.031592845916748, "logits/rejected": -1.586551547050476, "logps/chosen": -247.7447052001953, "logps/ref_chosen": -88.2329330444336, "logps/ref_rejected": -93.00756072998047, "logps/rejected": -333.67529296875, "loss": 1.0772, "margin_dpo/margin_mean": 81.15596008300781, "margin_dpo/margin_std": 98.28289794921875, "step": 391 }, { "KL/chosen_KL_mean": -161.844482421875, "KL/mean": -211.0378875732422, "KL/rejected_KL_mean": -260.23126220703125, "KL/std": 104.6351318359375, "epoch": 0.5756240822320118, "fcm_dpo/beta": 0.005030553787946701, "fcm_dpo/delta": -0.09729719907045364, "fcm_dpo/margin": 98.38677978515625, "fcm_dpo/q_t": 0.3859652280807495, "grad_norm": 9.742138862609863, "learning_rate": 2.294897926507156e-07, "logits/chosen": -1.7425212860107422, "logits/rejected": -1.6242263317108154, "logps/chosen": -249.58889770507812, "logps/ref_chosen": -87.74442291259766, "logps/ref_rejected": -108.57071685791016, "logps/rejected": -368.802001953125, "loss": 1.0278, "margin_dpo/margin_mean": 98.38677978515625, "margin_dpo/margin_std": 113.29537963867188, "step": 392 }, { "KL/chosen_KL_mean": -143.92857360839844, "KL/mean": -188.2545166015625, "KL/rejected_KL_mean": -232.58047485351562, "KL/std": 98.93472290039062, "epoch": 0.5770925110132159, "fcm_dpo/beta": 0.004992286674678326, "fcm_dpo/delta": -0.043556347489356995, "fcm_dpo/margin": 88.65188598632812, "fcm_dpo/q_t": 0.39683884382247925, "grad_norm": 8.555660247802734, "learning_rate": 2.2821106431308543e-07, "logits/chosen": -1.750241756439209, "logits/rejected": -1.430694580078125, "logps/chosen": -220.49240112304688, "logps/ref_chosen": -76.56381225585938, "logps/ref_rejected": -79.72618103027344, "logps/rejected": -312.306640625, "loss": 1.0472, "margin_dpo/margin_mean": 88.65188598632812, "margin_dpo/margin_std": 97.14595794677734, "step": 393 }, { "KL/chosen_KL_mean": -151.9507293701172, "KL/mean": -200.47891235351562, "KL/rejected_KL_mean": -249.00709533691406, "KL/std": 94.06358337402344, "epoch": 0.57856093979442, "fcm_dpo/beta": 0.004957845434546471, "fcm_dpo/delta": -0.08318550139665604, "fcm_dpo/margin": 97.05638122558594, "fcm_dpo/q_t": 0.38672322034835815, "grad_norm": 8.391071319580078, "learning_rate": 2.2693291013417452e-07, "logits/chosen": -1.7843585014343262, "logits/rejected": -1.5593231916427612, "logps/chosen": -236.64938354492188, "logps/ref_chosen": -84.69866180419922, "logps/ref_rejected": -97.457763671875, "logps/rejected": -346.46484375, "loss": 1.0101, "margin_dpo/margin_mean": 97.05638122558594, "margin_dpo/margin_std": 90.96719360351562, "step": 394 }, { "KL/chosen_KL_mean": -158.53370666503906, "KL/mean": -195.54624938964844, "KL/rejected_KL_mean": -232.5587921142578, "KL/std": 98.96717071533203, "epoch": 0.580029368575624, "fcm_dpo/beta": 0.004948120564222336, "fcm_dpo/delta": 0.034203484654426575, "fcm_dpo/margin": 74.02508544921875, "fcm_dpo/q_t": 0.4181096851825714, "grad_norm": 9.102099418640137, "learning_rate": 2.2565536379453404e-07, "logits/chosen": -2.6051642894744873, "logits/rejected": -2.209625005722046, "logps/chosen": -247.42135620117188, "logps/ref_chosen": -88.88763427734375, "logps/ref_rejected": -91.0455322265625, "logps/rejected": -323.60430908203125, "loss": 1.1572, "margin_dpo/margin_mean": 74.02508544921875, "margin_dpo/margin_std": 133.750244140625, "step": 395 }, { "KL/chosen_KL_mean": -167.6079559326172, "KL/mean": -200.68048095703125, "KL/rejected_KL_mean": -233.7530059814453, "KL/std": 87.22335815429688, "epoch": 0.5814977973568282, "fcm_dpo/beta": 0.004989763256162405, "fcm_dpo/delta": 0.0711345374584198, "fcm_dpo/margin": 66.14505004882812, "fcm_dpo/q_t": 0.42322269082069397, "grad_norm": 11.415595054626465, "learning_rate": 2.2437845895869825e-07, "logits/chosen": -1.938049077987671, "logits/rejected": -1.4490585327148438, "logps/chosen": -270.07928466796875, "logps/ref_chosen": -102.47132873535156, "logps/ref_rejected": -93.23478698730469, "logps/rejected": -326.98779296875, "loss": 1.1467, "margin_dpo/margin_mean": 66.1450424194336, "margin_dpo/margin_std": 103.30195617675781, "step": 396 }, { "KL/chosen_KL_mean": -150.33509826660156, "KL/mean": -209.33294677734375, "KL/rejected_KL_mean": -268.330810546875, "KL/std": 101.42700958251953, "epoch": 0.5829662261380323, "fcm_dpo/beta": 0.004926600493490696, "fcm_dpo/delta": -0.18690262734889984, "fcm_dpo/margin": 117.99571228027344, "fcm_dpo/q_t": 0.3672063946723938, "grad_norm": 8.721650123596191, "learning_rate": 2.2310222927429716e-07, "logits/chosen": -2.457662582397461, "logits/rejected": -1.8766531944274902, "logps/chosen": -251.3795166015625, "logps/ref_chosen": -101.04441833496094, "logps/ref_rejected": -108.8511734008789, "logps/rejected": -377.1819763183594, "loss": 0.9687, "margin_dpo/margin_mean": 117.99571990966797, "margin_dpo/margin_std": 120.7520751953125, "step": 397 }, { "KL/chosen_KL_mean": -158.80148315429688, "KL/mean": -201.41180419921875, "KL/rejected_KL_mean": -244.0221405029297, "KL/std": 99.4446029663086, "epoch": 0.5844346549192364, "fcm_dpo/beta": 0.004896920640021563, "fcm_dpo/delta": -0.01771967113018036, "fcm_dpo/margin": 85.22065734863281, "fcm_dpo/q_t": 0.40429913997650146, "grad_norm": 8.529611587524414, "learning_rate": 2.2182670837116972e-07, "logits/chosen": -2.036698579788208, "logits/rejected": -1.7508046627044678, "logps/chosen": -253.8507080078125, "logps/ref_chosen": -95.04922485351562, "logps/ref_rejected": -114.00563049316406, "logps/rejected": -358.02777099609375, "loss": 1.0889, "margin_dpo/margin_mean": 85.22065734863281, "margin_dpo/margin_std": 117.54795837402344, "step": 398 }, { "KL/chosen_KL_mean": -139.36349487304688, "KL/mean": -193.91470336914062, "KL/rejected_KL_mean": -248.46588134765625, "KL/std": 90.28276062011719, "epoch": 0.5859030837004405, "fcm_dpo/beta": 0.0048548500053584576, "fcm_dpo/delta": -0.1331082433462143, "fcm_dpo/margin": 109.1023941040039, "fcm_dpo/q_t": 0.3763774633407593, "grad_norm": 8.733227729797363, "learning_rate": 2.2055192986047804e-07, "logits/chosen": -2.118957757949829, "logits/rejected": -1.3204925060272217, "logps/chosen": -236.00204467773438, "logps/ref_chosen": -96.63854217529297, "logps/ref_rejected": -83.53645324707031, "logps/rejected": -332.0023193359375, "loss": 0.9769, "margin_dpo/margin_mean": 109.1023941040039, "margin_dpo/margin_std": 96.88218688964844, "step": 399 }, { "KL/chosen_KL_mean": -145.23931884765625, "KL/mean": -200.82565307617188, "KL/rejected_KL_mean": -256.4120178222656, "KL/std": 99.74657440185547, "epoch": 0.5873715124816447, "fcm_dpo/beta": 0.004783862270414829, "fcm_dpo/delta": -0.1354629099369049, "fcm_dpo/margin": 111.1727066040039, "fcm_dpo/q_t": 0.3774910569190979, "grad_norm": 12.675237655639648, "learning_rate": 2.192779273338215e-07, "logits/chosen": -2.337460994720459, "logits/rejected": -1.8641526699066162, "logps/chosen": -250.54246520996094, "logps/ref_chosen": -105.30314636230469, "logps/ref_rejected": -108.6555404663086, "logps/rejected": -365.06756591796875, "loss": 0.9941, "margin_dpo/margin_mean": 111.1727066040039, "margin_dpo/margin_std": 113.88356018066406, "step": 400 }, { "KL/chosen_KL_mean": -160.87066650390625, "KL/mean": -195.5791015625, "KL/rejected_KL_mean": -230.2875213623047, "KL/std": 100.72047424316406, "epoch": 0.5888399412628488, "fcm_dpo/beta": 0.004786025732755661, "fcm_dpo/delta": 0.06892874836921692, "fcm_dpo/margin": 69.41685485839844, "fcm_dpo/q_t": 0.4258604347705841, "grad_norm": 11.414010047912598, "learning_rate": 2.1800473436235136e-07, "logits/chosen": -1.4528486728668213, "logits/rejected": -1.2064399719238281, "logps/chosen": -245.1598358154297, "logps/ref_chosen": -84.28916931152344, "logps/ref_rejected": -90.943115234375, "logps/rejected": -321.23065185546875, "loss": 1.1611, "margin_dpo/margin_mean": 69.41685485839844, "margin_dpo/margin_std": 122.73370361328125, "step": 401 }, { "KL/chosen_KL_mean": -143.14768981933594, "KL/mean": -207.9290771484375, "KL/rejected_KL_mean": -272.71044921875, "KL/std": 107.3772964477539, "epoch": 0.5903083700440529, "fcm_dpo/beta": 0.004728003870695829, "fcm_dpo/delta": -0.2191193848848343, "fcm_dpo/margin": 129.562744140625, "fcm_dpo/q_t": 0.35737913846969604, "grad_norm": 8.555963516235352, "learning_rate": 2.1673238449588665e-07, "logits/chosen": -1.9182894229888916, "logits/rejected": -1.450634241104126, "logps/chosen": -226.74081420898438, "logps/ref_chosen": -83.59312438964844, "logps/ref_rejected": -87.81027221679688, "logps/rejected": -360.5207214355469, "loss": 0.9217, "margin_dpo/margin_mean": 129.562744140625, "margin_dpo/margin_std": 104.06205749511719, "step": 402 }, { "KL/chosen_KL_mean": -146.33766174316406, "KL/mean": -193.99560546875, "KL/rejected_KL_mean": -241.653564453125, "KL/std": 103.49179077148438, "epoch": 0.591776798825257, "fcm_dpo/beta": 0.004675738979130983, "fcm_dpo/delta": -0.046812623739242554, "fcm_dpo/margin": 95.31590270996094, "fcm_dpo/q_t": 0.39642834663391113, "grad_norm": 9.09311294555664, "learning_rate": 2.154609112620295e-07, "logits/chosen": -2.3327059745788574, "logits/rejected": -1.9421412944793701, "logps/chosen": -220.0907440185547, "logps/ref_chosen": -73.75308227539062, "logps/ref_rejected": -83.92012786865234, "logps/rejected": -325.5736999511719, "loss": 1.0517, "margin_dpo/margin_mean": 95.31590270996094, "margin_dpo/margin_std": 109.45540618896484, "step": 403 }, { "KL/chosen_KL_mean": -173.8556671142578, "KL/mean": -214.64608764648438, "KL/rejected_KL_mean": -255.43649291992188, "KL/std": 99.79122924804688, "epoch": 0.593245227606461, "fcm_dpo/beta": 0.004676331765949726, "fcm_dpo/delta": 0.01882590353488922, "fcm_dpo/margin": 81.58082580566406, "fcm_dpo/q_t": 0.4122175872325897, "grad_norm": 12.997926712036133, "learning_rate": 2.1419034816528218e-07, "logits/chosen": -2.1320300102233887, "logits/rejected": -1.801403284072876, "logps/chosen": -253.53184509277344, "logps/ref_chosen": -79.67617797851562, "logps/ref_rejected": -84.280517578125, "logps/rejected": -339.7170104980469, "loss": 1.1167, "margin_dpo/margin_mean": 81.58082580566406, "margin_dpo/margin_std": 121.57772827148438, "step": 404 }, { "KL/chosen_KL_mean": -177.12628173828125, "KL/mean": -218.49700927734375, "KL/rejected_KL_mean": -259.8677062988281, "KL/std": 93.47978210449219, "epoch": 0.5947136563876652, "fcm_dpo/beta": 0.004699420183897018, "fcm_dpo/delta": 0.011119663715362549, "fcm_dpo/margin": 82.74143981933594, "fcm_dpo/q_t": 0.4095121920108795, "grad_norm": 8.282407760620117, "learning_rate": 2.129207286861638e-07, "logits/chosen": -1.3658430576324463, "logits/rejected": -0.878475546836853, "logps/chosen": -273.5882263183594, "logps/ref_chosen": -96.46195220947266, "logps/ref_rejected": -92.87071228027344, "logps/rejected": -352.7384033203125, "loss": 1.0878, "margin_dpo/margin_mean": 82.74143981933594, "margin_dpo/margin_std": 99.27070617675781, "step": 405 }, { "KL/chosen_KL_mean": -167.82861328125, "KL/mean": -206.00262451171875, "KL/rejected_KL_mean": -244.17665100097656, "KL/std": 100.68778228759766, "epoch": 0.5961820851688693, "fcm_dpo/beta": 0.004703653044998646, "fcm_dpo/delta": 0.04163544625043869, "fcm_dpo/margin": 76.3480224609375, "fcm_dpo/q_t": 0.4170023798942566, "grad_norm": 15.066657066345215, "learning_rate": 2.1165208628032861e-07, "logits/chosen": -1.674580693244934, "logits/rejected": -1.5004230737686157, "logps/chosen": -245.96258544921875, "logps/ref_chosen": -78.13396453857422, "logps/ref_rejected": -98.28359985351562, "logps/rejected": -342.46026611328125, "loss": 1.132, "margin_dpo/margin_mean": 76.3480224609375, "margin_dpo/margin_std": 117.81017303466797, "step": 406 }, { "KL/chosen_KL_mean": -175.41339111328125, "KL/mean": -208.70529174804688, "KL/rejected_KL_mean": -241.99716186523438, "KL/std": 98.84358215332031, "epoch": 0.5976505139500734, "fcm_dpo/beta": 0.004739393480122089, "fcm_dpo/delta": 0.08578373491764069, "fcm_dpo/margin": 66.58377075195312, "fcm_dpo/q_t": 0.4263036251068115, "grad_norm": 10.013971328735352, "learning_rate": 2.1038445437768375e-07, "logits/chosen": -2.348097324371338, "logits/rejected": -1.8438570499420166, "logps/chosen": -259.42620849609375, "logps/ref_chosen": -84.01283264160156, "logps/ref_rejected": -82.78103637695312, "logps/rejected": -324.7781982421875, "loss": 1.1548, "margin_dpo/margin_mean": 66.58377075195312, "margin_dpo/margin_std": 105.86473083496094, "step": 407 }, { "KL/chosen_KL_mean": -167.69586181640625, "KL/mean": -201.46151733398438, "KL/rejected_KL_mean": -235.2271728515625, "KL/std": 94.01092529296875, "epoch": 0.5991189427312775, "fcm_dpo/beta": 0.004774310626089573, "fcm_dpo/delta": 0.0788908377289772, "fcm_dpo/margin": 67.53133392333984, "fcm_dpo/q_t": 0.4260936379432678, "grad_norm": 9.59772777557373, "learning_rate": 2.0911786638150872e-07, "logits/chosen": -2.2610366344451904, "logits/rejected": -1.6420804262161255, "logps/chosen": -272.1576232910156, "logps/ref_chosen": -104.46174621582031, "logps/ref_rejected": -96.37217712402344, "logps/rejected": -331.599365234375, "loss": 1.1629, "margin_dpo/margin_mean": 67.53132629394531, "margin_dpo/margin_std": 116.539794921875, "step": 408 }, { "KL/chosen_KL_mean": -179.12255859375, "KL/mean": -214.22048950195312, "KL/rejected_KL_mean": -249.31842041015625, "KL/std": 96.73359680175781, "epoch": 0.6005873715124816, "fcm_dpo/beta": 0.004808701109141111, "fcm_dpo/delta": 0.06353440135717392, "fcm_dpo/margin": 70.19586944580078, "fcm_dpo/q_t": 0.42004379630088806, "grad_norm": 9.173608779907227, "learning_rate": 2.0785235566757517e-07, "logits/chosen": -1.6094869375228882, "logits/rejected": -1.1763787269592285, "logps/chosen": -276.7908630371094, "logps/ref_chosen": -97.66830444335938, "logps/ref_rejected": -90.04583740234375, "logps/rejected": -339.3642578125, "loss": 1.1414, "margin_dpo/margin_mean": 70.19587707519531, "margin_dpo/margin_std": 106.02630615234375, "step": 409 }, { "KL/chosen_KL_mean": -173.44845581054688, "KL/mean": -215.58717346191406, "KL/rejected_KL_mean": -257.72589111328125, "KL/std": 101.90505981445312, "epoch": 0.6020558002936858, "fcm_dpo/beta": 0.004814858548343182, "fcm_dpo/delta": -0.0059051611460745335, "fcm_dpo/margin": 84.27743530273438, "fcm_dpo/q_t": 0.40437474846839905, "grad_norm": 8.52559757232666, "learning_rate": 2.065879555832674e-07, "logits/chosen": -1.7588629722595215, "logits/rejected": -1.6279422044754028, "logps/chosen": -249.91769409179688, "logps/ref_chosen": -76.46923828125, "logps/ref_rejected": -88.64064025878906, "logps/rejected": -346.36651611328125, "loss": 1.0793, "margin_dpo/margin_mean": 84.27743530273438, "margin_dpo/margin_std": 102.83348083496094, "step": 410 }, { "KL/chosen_KL_mean": -185.77816772460938, "KL/mean": -225.36993408203125, "KL/rejected_KL_mean": -264.96173095703125, "KL/std": 94.49539184570312, "epoch": 0.6035242290748899, "fcm_dpo/beta": 0.004813498817384243, "fcm_dpo/delta": 0.01913302391767502, "fcm_dpo/margin": 79.18352508544922, "fcm_dpo/q_t": 0.4122669994831085, "grad_norm": 8.587647438049316, "learning_rate": 2.0532469944670343e-07, "logits/chosen": -2.399257183074951, "logits/rejected": -1.9989705085754395, "logps/chosen": -272.9444580078125, "logps/ref_chosen": -87.16630554199219, "logps/ref_rejected": -87.09603118896484, "logps/rejected": -352.0577392578125, "loss": 1.1003, "margin_dpo/margin_mean": 79.18352508544922, "margin_dpo/margin_std": 102.8643569946289, "step": 411 }, { "KL/chosen_KL_mean": -175.91085815429688, "KL/mean": -214.44985961914062, "KL/rejected_KL_mean": -252.98886108398438, "KL/std": 95.19498443603516, "epoch": 0.604992657856094, "fcm_dpo/beta": 0.00482857134193182, "fcm_dpo/delta": 0.02832910418510437, "fcm_dpo/margin": 77.07798767089844, "fcm_dpo/q_t": 0.4154488444328308, "grad_norm": 10.215325355529785, "learning_rate": 2.0406262054585738e-07, "logits/chosen": -2.296231746673584, "logits/rejected": -2.0764575004577637, "logps/chosen": -254.85821533203125, "logps/ref_chosen": -78.94734191894531, "logps/ref_rejected": -106.10554504394531, "logps/rejected": -359.09442138671875, "loss": 1.1309, "margin_dpo/margin_mean": 77.07798767089844, "margin_dpo/margin_std": 122.54806518554688, "step": 412 }, { "KL/chosen_KL_mean": -187.9027557373047, "KL/mean": -231.10401916503906, "KL/rejected_KL_mean": -274.3052978515625, "KL/std": 100.91322326660156, "epoch": 0.6064610866372981, "fcm_dpo/beta": 0.004823585972189903, "fcm_dpo/delta": -0.017261624336242676, "fcm_dpo/margin": 86.40254974365234, "fcm_dpo/q_t": 0.4031721353530884, "grad_norm": 9.71200942993164, "learning_rate": 2.0280175213768205e-07, "logits/chosen": -1.306992769241333, "logits/rejected": -1.0508956909179688, "logps/chosen": -283.59747314453125, "logps/ref_chosen": -95.69470977783203, "logps/ref_rejected": -107.96085357666016, "logps/rejected": -382.2661437988281, "loss": 1.0901, "margin_dpo/margin_mean": 86.40254211425781, "margin_dpo/margin_std": 119.45939636230469, "step": 413 }, { "KL/chosen_KL_mean": -173.376220703125, "KL/mean": -213.64059448242188, "KL/rejected_KL_mean": -253.90493774414062, "KL/std": 98.22486114501953, "epoch": 0.6079295154185022, "fcm_dpo/beta": 0.004836251959204674, "fcm_dpo/delta": 0.010729165747761726, "fcm_dpo/margin": 80.52872467041016, "fcm_dpo/q_t": 0.4110715687274933, "grad_norm": 13.348609924316406, "learning_rate": 2.0154212744723247e-07, "logits/chosen": -1.5461112260818481, "logits/rejected": -1.187941074371338, "logps/chosen": -261.65289306640625, "logps/ref_chosen": -88.27667236328125, "logps/ref_rejected": -92.87004089355469, "logps/rejected": -346.7749938964844, "loss": 1.1206, "margin_dpo/margin_mean": 80.52872467041016, "margin_dpo/margin_std": 125.7408447265625, "step": 414 }, { "KL/chosen_KL_mean": -188.40792846679688, "KL/mean": -222.40643310546875, "KL/rejected_KL_mean": -256.4049377441406, "KL/std": 88.91867065429688, "epoch": 0.6093979441997063, "fcm_dpo/beta": 0.0048626502975821495, "fcm_dpo/delta": 0.07047092169523239, "fcm_dpo/margin": 67.99700927734375, "fcm_dpo/q_t": 0.42346203327178955, "grad_norm": 10.771480560302734, "learning_rate": 2.002837796667909e-07, "logits/chosen": -1.4849743843078613, "logits/rejected": -1.098928451538086, "logps/chosen": -297.3238220214844, "logps/ref_chosen": -108.91590118408203, "logps/ref_rejected": -107.47135925292969, "logps/rejected": -363.87628173828125, "loss": 1.1505, "margin_dpo/margin_mean": 67.99700927734375, "margin_dpo/margin_std": 109.11955261230469, "step": 415 }, { "KL/chosen_KL_mean": -186.81390380859375, "KL/mean": -233.7891845703125, "KL/rejected_KL_mean": -280.76446533203125, "KL/std": 103.83106994628906, "epoch": 0.6108663729809104, "fcm_dpo/beta": 0.004865242168307304, "fcm_dpo/delta": -0.05875661224126816, "fcm_dpo/margin": 93.9505615234375, "fcm_dpo/q_t": 0.3949504792690277, "grad_norm": 9.113449096679688, "learning_rate": 1.990267419549914e-07, "logits/chosen": -1.7630290985107422, "logits/rejected": -1.2539470195770264, "logps/chosen": -280.2127990722656, "logps/ref_chosen": -93.39888000488281, "logps/ref_rejected": -97.6729736328125, "logps/rejected": -378.43743896484375, "loss": 1.0527, "margin_dpo/margin_mean": 93.95055389404297, "margin_dpo/margin_std": 109.44353485107422, "step": 416 }, { "KL/chosen_KL_mean": -173.53857421875, "KL/mean": -217.29107666015625, "KL/rejected_KL_mean": -261.0435485839844, "KL/std": 96.80854797363281, "epoch": 0.6123348017621145, "fcm_dpo/beta": 0.004834542982280254, "fcm_dpo/delta": -0.023545043542981148, "fcm_dpo/margin": 87.50498962402344, "fcm_dpo/q_t": 0.4011887311935425, "grad_norm": 9.152027130126953, "learning_rate": 1.9777104743594686e-07, "logits/chosen": -2.349184513092041, "logits/rejected": -1.4558703899383545, "logps/chosen": -257.07391357421875, "logps/ref_chosen": -83.53533172607422, "logps/ref_rejected": -74.44184875488281, "logps/rejected": -335.48541259765625, "loss": 1.0629, "margin_dpo/margin_mean": 87.50498962402344, "margin_dpo/margin_std": 99.91358947753906, "step": 417 }, { "KL/chosen_KL_mean": -165.89976501464844, "KL/mean": -220.84170532226562, "KL/rejected_KL_mean": -275.78363037109375, "KL/std": 103.19744873046875, "epoch": 0.6138032305433186, "fcm_dpo/beta": 0.004803690128028393, "fcm_dpo/delta": -0.13156190514564514, "fcm_dpo/margin": 109.88385772705078, "fcm_dpo/q_t": 0.37834036350250244, "grad_norm": 10.972209930419922, "learning_rate": 1.965167291983757e-07, "logits/chosen": -2.258859157562256, "logits/rejected": -1.6114673614501953, "logps/chosen": -274.12127685546875, "logps/ref_chosen": -108.22152709960938, "logps/ref_rejected": -111.8646469116211, "logps/rejected": -387.64825439453125, "loss": 0.9886, "margin_dpo/margin_mean": 109.88385009765625, "margin_dpo/margin_std": 100.96171569824219, "step": 418 }, { "KL/chosen_KL_mean": -172.44215393066406, "KL/mean": -220.4755859375, "KL/rejected_KL_mean": -268.5090026855469, "KL/std": 92.15483093261719, "epoch": 0.6152716593245228, "fcm_dpo/beta": 0.00475567951798439, "fcm_dpo/delta": -0.05826590582728386, "fcm_dpo/margin": 96.06684875488281, "fcm_dpo/q_t": 0.3935830295085907, "grad_norm": 10.959362030029297, "learning_rate": 1.9526382029472988e-07, "logits/chosen": -1.4935133457183838, "logits/rejected": -1.0479406118392944, "logps/chosen": -269.62542724609375, "logps/ref_chosen": -97.18328094482422, "logps/ref_rejected": -98.18531799316406, "logps/rejected": -366.6943359375, "loss": 1.0371, "margin_dpo/margin_mean": 96.06684875488281, "margin_dpo/margin_std": 101.3607406616211, "step": 419 }, { "KL/chosen_KL_mean": -176.37557983398438, "KL/mean": -219.51361083984375, "KL/rejected_KL_mean": -262.65167236328125, "KL/std": 97.74342346191406, "epoch": 0.6167400881057269, "fcm_dpo/beta": 0.0047423625364899635, "fcm_dpo/delta": -0.009435174986720085, "fcm_dpo/margin": 86.27606201171875, "fcm_dpo/q_t": 0.4055350422859192, "grad_norm": 14.201628684997559, "learning_rate": 1.9401235374032425e-07, "logits/chosen": -2.254940986633301, "logits/rejected": -1.1628015041351318, "logps/chosen": -290.68408203125, "logps/ref_chosen": -114.30848693847656, "logps/ref_rejected": -75.68355560302734, "logps/rejected": -338.335205078125, "loss": 1.0866, "margin_dpo/margin_mean": 86.27605438232422, "margin_dpo/margin_std": 112.82630920410156, "step": 420 }, { "KL/chosen_KL_mean": -168.66363525390625, "KL/mean": -215.98492431640625, "KL/rejected_KL_mean": -263.3062438964844, "KL/std": 100.22111511230469, "epoch": 0.618208516886931, "fcm_dpo/beta": 0.004713702015578747, "fcm_dpo/delta": -0.04721851646900177, "fcm_dpo/margin": 94.64258575439453, "fcm_dpo/q_t": 0.3961656093597412, "grad_norm": 8.292851448059082, "learning_rate": 1.9276236251246653e-07, "logits/chosen": -1.853539228439331, "logits/rejected": -1.4831633567810059, "logps/chosen": -254.54348754882812, "logps/ref_chosen": -85.87985229492188, "logps/ref_rejected": -96.33648681640625, "logps/rejected": -359.6427307128906, "loss": 1.0535, "margin_dpo/margin_mean": 94.64259338378906, "margin_dpo/margin_std": 110.34892272949219, "step": 421 }, { "KL/chosen_KL_mean": -165.330078125, "KL/mean": -210.84902954101562, "KL/rejected_KL_mean": -256.36798095703125, "KL/std": 93.14848327636719, "epoch": 0.6196769456681351, "fcm_dpo/beta": 0.004704089369624853, "fcm_dpo/delta": -0.028866756707429886, "fcm_dpo/margin": 91.03791046142578, "fcm_dpo/q_t": 0.4004102349281311, "grad_norm": 11.493514060974121, "learning_rate": 1.9151387954958792e-07, "logits/chosen": -2.522101879119873, "logits/rejected": -2.1626193523406982, "logps/chosen": -265.8106689453125, "logps/ref_chosen": -100.48060607910156, "logps/ref_rejected": -94.40821838378906, "logps/rejected": -350.7762145996094, "loss": 1.0592, "margin_dpo/margin_mean": 91.03790283203125, "margin_dpo/margin_std": 103.98231506347656, "step": 422 }, { "KL/chosen_KL_mean": -154.11300659179688, "KL/mean": -200.14254760742188, "KL/rejected_KL_mean": -246.17213439941406, "KL/std": 110.77723693847656, "epoch": 0.6211453744493393, "fcm_dpo/beta": 0.004694974981248379, "fcm_dpo/delta": -0.03300972282886505, "fcm_dpo/margin": 92.05912017822266, "fcm_dpo/q_t": 0.4020630121231079, "grad_norm": 10.145715713500977, "learning_rate": 1.902669377503756e-07, "logits/chosen": -1.9930813312530518, "logits/rejected": -1.875953197479248, "logps/chosen": -232.56292724609375, "logps/ref_chosen": -78.44993591308594, "logps/ref_rejected": -92.04652404785156, "logps/rejected": -338.2186584472656, "loss": 1.0819, "margin_dpo/margin_mean": 92.05912780761719, "margin_dpo/margin_std": 125.51560974121094, "step": 423 }, { "KL/chosen_KL_mean": -173.57057189941406, "KL/mean": -218.7734375, "KL/rejected_KL_mean": -263.9762878417969, "KL/std": 98.9072265625, "epoch": 0.6226138032305433, "fcm_dpo/beta": 0.00467345304787159, "fcm_dpo/delta": -0.0229966938495636, "fcm_dpo/margin": 90.40572357177734, "fcm_dpo/q_t": 0.4008086919784546, "grad_norm": 8.867308616638184, "learning_rate": 1.890215699729057e-07, "logits/chosen": -1.8452506065368652, "logits/rejected": -1.2578388452529907, "logps/chosen": -261.2129211425781, "logps/ref_chosen": -87.64236450195312, "logps/ref_rejected": -72.36566162109375, "logps/rejected": -336.3419494628906, "loss": 1.0673, "margin_dpo/margin_mean": 90.40573120117188, "margin_dpo/margin_std": 106.90797424316406, "step": 424 }, { "KL/chosen_KL_mean": -178.9095458984375, "KL/mean": -209.4991912841797, "KL/rejected_KL_mean": -240.08883666992188, "KL/std": 101.8576889038086, "epoch": 0.6240822320117474, "fcm_dpo/beta": 0.0047068363055586815, "fcm_dpo/delta": 0.11374793946743011, "fcm_dpo/margin": 61.179283142089844, "fcm_dpo/q_t": 0.4357501268386841, "grad_norm": 10.427700996398926, "learning_rate": 1.8777780903377732e-07, "logits/chosen": -1.8461852073669434, "logits/rejected": -1.7806973457336426, "logps/chosen": -257.4293212890625, "logps/ref_chosen": -78.51979064941406, "logps/ref_rejected": -102.74864196777344, "logps/rejected": -342.83746337890625, "loss": 1.2024, "margin_dpo/margin_mean": 61.179283142089844, "margin_dpo/margin_std": 126.70219421386719, "step": 425 }, { "KL/chosen_KL_mean": -153.753662109375, "KL/mean": -198.8845672607422, "KL/rejected_KL_mean": -244.01548767089844, "KL/std": 94.11798095703125, "epoch": 0.6255506607929515, "fcm_dpo/beta": 0.004721187986433506, "fcm_dpo/delta": -0.026739971712231636, "fcm_dpo/margin": 90.2618179321289, "fcm_dpo/q_t": 0.4008149802684784, "grad_norm": 10.677534103393555, "learning_rate": 1.8653568770724803e-07, "logits/chosen": -1.785016655921936, "logits/rejected": -1.0579626560211182, "logps/chosen": -262.2594909667969, "logps/ref_chosen": -108.50582885742188, "logps/ref_rejected": -88.300048828125, "logps/rejected": -332.3155517578125, "loss": 1.0687, "margin_dpo/margin_mean": 90.26181030273438, "margin_dpo/margin_std": 109.3831558227539, "step": 426 }, { "KL/chosen_KL_mean": -126.3250732421875, "KL/mean": -167.93746948242188, "KL/rejected_KL_mean": -209.5498504638672, "KL/std": 89.25233459472656, "epoch": 0.6270190895741556, "fcm_dpo/beta": 0.004717155359685421, "fcm_dpo/delta": 0.007551322225481272, "fcm_dpo/margin": 83.22476959228516, "fcm_dpo/q_t": 0.4077058732509613, "grad_norm": 8.4463529586792, "learning_rate": 1.8529523872436977e-07, "logits/chosen": -2.3506836891174316, "logits/rejected": -1.7098972797393799, "logps/chosen": -225.44554138183594, "logps/ref_chosen": -99.12046813964844, "logps/ref_rejected": -85.724609375, "logps/rejected": -295.27447509765625, "loss": 1.0811, "margin_dpo/margin_mean": 83.22476959228516, "margin_dpo/margin_std": 96.59994506835938, "step": 427 }, { "KL/chosen_KL_mean": -181.33168029785156, "KL/mean": -217.00277709960938, "KL/rejected_KL_mean": -252.6738739013672, "KL/std": 101.35346221923828, "epoch": 0.6284875183553598, "fcm_dpo/beta": 0.0047279344871640205, "fcm_dpo/delta": 0.06371945142745972, "fcm_dpo/margin": 71.34217834472656, "fcm_dpo/q_t": 0.4220263361930847, "grad_norm": 9.733904838562012, "learning_rate": 1.8405649477212697e-07, "logits/chosen": -1.7347946166992188, "logits/rejected": -1.4749555587768555, "logps/chosen": -287.3009338378906, "logps/ref_chosen": -105.96925354003906, "logps/ref_rejected": -109.10218048095703, "logps/rejected": -361.77606201171875, "loss": 1.1455, "margin_dpo/margin_mean": 71.3421859741211, "margin_dpo/margin_std": 112.17533874511719, "step": 428 }, { "KL/chosen_KL_mean": -164.83837890625, "KL/mean": -210.51641845703125, "KL/rejected_KL_mean": -256.1944885253906, "KL/std": 105.25552368164062, "epoch": 0.6299559471365639, "fcm_dpo/beta": 0.0047499933280050755, "fcm_dpo/delta": -0.03505821153521538, "fcm_dpo/margin": 91.35610961914062, "fcm_dpo/q_t": 0.4010747969150543, "grad_norm": 8.392938613891602, "learning_rate": 1.828194884925749e-07, "logits/chosen": -1.8830163478851318, "logits/rejected": -1.1597661972045898, "logps/chosen": -278.38323974609375, "logps/ref_chosen": -113.54487609863281, "logps/ref_rejected": -98.24201965332031, "logps/rejected": -354.4364929199219, "loss": 1.0777, "margin_dpo/margin_mean": 91.35610961914062, "margin_dpo/margin_std": 119.05657958984375, "step": 429 }, { "KL/chosen_KL_mean": -158.03033447265625, "KL/mean": -193.03497314453125, "KL/rejected_KL_mean": -228.03961181640625, "KL/std": 97.89982604980469, "epoch": 0.631424375917768, "fcm_dpo/beta": 0.0047600772231817245, "fcm_dpo/delta": 0.06770406663417816, "fcm_dpo/margin": 70.00929260253906, "fcm_dpo/q_t": 0.4233488440513611, "grad_norm": 9.014313697814941, "learning_rate": 1.8158425248197928e-07, "logits/chosen": -1.4146814346313477, "logits/rejected": -1.3777351379394531, "logps/chosen": -249.34970092773438, "logps/ref_chosen": -91.31936645507812, "logps/ref_rejected": -110.1096420288086, "logps/rejected": -338.1492614746094, "loss": 1.1504, "margin_dpo/margin_mean": 70.00928497314453, "margin_dpo/margin_std": 113.6490478515625, "step": 430 }, { "KL/chosen_KL_mean": -159.05381774902344, "KL/mean": -200.48519897460938, "KL/rejected_KL_mean": -241.9165802001953, "KL/std": 99.53968048095703, "epoch": 0.6328928046989721, "fcm_dpo/beta": 0.004760223440825939, "fcm_dpo/delta": 0.005651640705764294, "fcm_dpo/margin": 82.86277770996094, "fcm_dpo/q_t": 0.4084901809692383, "grad_norm": 10.219233512878418, "learning_rate": 1.8035081928995788e-07, "logits/chosen": -1.8844611644744873, "logits/rejected": -1.496817946434021, "logps/chosen": -252.2350311279297, "logps/ref_chosen": -93.18122100830078, "logps/ref_rejected": -98.13226318359375, "logps/rejected": -340.048828125, "loss": 1.0996, "margin_dpo/margin_mean": 82.86277770996094, "margin_dpo/margin_std": 113.59375, "step": 431 }, { "KL/chosen_KL_mean": -139.7103271484375, "KL/mean": -190.18319702148438, "KL/rejected_KL_mean": -240.6560821533203, "KL/std": 104.056884765625, "epoch": 0.6343612334801763, "fcm_dpo/beta": 0.004742786753922701, "fcm_dpo/delta": -0.08066726475954056, "fcm_dpo/margin": 100.94577026367188, "fcm_dpo/q_t": 0.3882020115852356, "grad_norm": 8.932520866394043, "learning_rate": 1.791192214186223e-07, "logits/chosen": -1.461732029914856, "logits/rejected": -0.9035096764564514, "logps/chosen": -244.1450958251953, "logps/ref_chosen": -104.43478393554688, "logps/ref_rejected": -105.08955383300781, "logps/rejected": -345.7456359863281, "loss": 1.0206, "margin_dpo/margin_mean": 100.94577026367188, "margin_dpo/margin_std": 100.34437561035156, "step": 432 }, { "KL/chosen_KL_mean": -167.08853149414062, "KL/mean": -206.53819274902344, "KL/rejected_KL_mean": -245.98785400390625, "KL/std": 98.2428207397461, "epoch": 0.6358296622613803, "fcm_dpo/beta": 0.004745253827422857, "fcm_dpo/delta": 0.02584647573530674, "fcm_dpo/margin": 78.89933013916016, "fcm_dpo/q_t": 0.4134621024131775, "grad_norm": 8.986966133117676, "learning_rate": 1.7788949132172193e-07, "logits/chosen": -1.633807897567749, "logits/rejected": -1.3091528415679932, "logps/chosen": -256.9317626953125, "logps/ref_chosen": -89.84322357177734, "logps/ref_rejected": -101.73345947265625, "logps/rejected": -347.7213134765625, "loss": 1.1149, "margin_dpo/margin_mean": 78.89933013916016, "margin_dpo/margin_std": 111.78718566894531, "step": 433 }, { "KL/chosen_KL_mean": -144.32949829101562, "KL/mean": -189.07864379882812, "KL/rejected_KL_mean": -233.8277587890625, "KL/std": 109.24855041503906, "epoch": 0.6372980910425844, "fcm_dpo/beta": 0.004731958732008934, "fcm_dpo/delta": -0.024008650332689285, "fcm_dpo/margin": 89.49827575683594, "fcm_dpo/q_t": 0.4053788185119629, "grad_norm": 11.974830627441406, "learning_rate": 1.7666166140378853e-07, "logits/chosen": -2.403743028640747, "logits/rejected": -1.7312242984771729, "logps/chosen": -242.0220184326172, "logps/ref_chosen": -97.6925277709961, "logps/ref_rejected": -84.09129333496094, "logps/rejected": -317.9190673828125, "loss": 1.0875, "margin_dpo/margin_mean": 89.49827575683594, "margin_dpo/margin_std": 126.85468292236328, "step": 434 }, { "KL/chosen_KL_mean": -142.92691040039062, "KL/mean": -189.04196166992188, "KL/rejected_KL_mean": -235.15701293945312, "KL/std": 101.40711975097656, "epoch": 0.6387665198237885, "fcm_dpo/beta": 0.0047174254432320595, "fcm_dpo/delta": -0.03586380183696747, "fcm_dpo/margin": 92.2301025390625, "fcm_dpo/q_t": 0.4001331329345703, "grad_norm": 9.90157699584961, "learning_rate": 1.7543576401928218e-07, "logits/chosen": -2.3470094203948975, "logits/rejected": -1.8930422067642212, "logps/chosen": -229.0988311767578, "logps/ref_chosen": -86.17192077636719, "logps/ref_rejected": -93.751708984375, "logps/rejected": -328.9087219238281, "loss": 1.0635, "margin_dpo/margin_mean": 92.2301025390625, "margin_dpo/margin_std": 112.92974853515625, "step": 435 }, { "KL/chosen_KL_mean": -117.54508209228516, "KL/mean": -166.2962646484375, "KL/rejected_KL_mean": -215.04742431640625, "KL/std": 86.28128051757812, "epoch": 0.6402349486049926, "fcm_dpo/beta": 0.004687583073973656, "fcm_dpo/delta": -0.058411478996276855, "fcm_dpo/margin": 97.50234985351562, "fcm_dpo/q_t": 0.391975998878479, "grad_norm": 10.353282928466797, "learning_rate": 1.742118314717391e-07, "logits/chosen": -1.6869323253631592, "logits/rejected": -0.8123513460159302, "logps/chosen": -223.33218383789062, "logps/ref_chosen": -105.78710174560547, "logps/ref_rejected": -88.62471008300781, "logps/rejected": -303.672119140625, "loss": 1.0223, "margin_dpo/margin_mean": 97.50234985351562, "margin_dpo/margin_std": 89.54705810546875, "step": 436 }, { "KL/chosen_KL_mean": -138.62945556640625, "KL/mean": -177.51483154296875, "KL/rejected_KL_mean": -216.4002227783203, "KL/std": 92.57539367675781, "epoch": 0.6417033773861968, "fcm_dpo/beta": 0.004697813652455807, "fcm_dpo/delta": 0.03525166213512421, "fcm_dpo/margin": 77.77076721191406, "fcm_dpo/q_t": 0.4128236174583435, "grad_norm": 10.114171981811523, "learning_rate": 1.7298989601292036e-07, "logits/chosen": -2.1233909130096436, "logits/rejected": -1.48197340965271, "logps/chosen": -234.69149780273438, "logps/ref_chosen": -96.06204223632812, "logps/ref_rejected": -89.01220703125, "logps/rejected": -305.41241455078125, "loss": 1.0986, "margin_dpo/margin_mean": 77.77076721191406, "margin_dpo/margin_std": 91.54072570800781, "step": 437 }, { "KL/chosen_KL_mean": -143.12850952148438, "KL/mean": -182.71168518066406, "KL/rejected_KL_mean": -222.29486083984375, "KL/std": 93.2328872680664, "epoch": 0.6431718061674009, "fcm_dpo/beta": 0.004699420183897018, "fcm_dpo/delta": 0.02838689088821411, "fcm_dpo/margin": 79.16636657714844, "fcm_dpo/q_t": 0.4126254618167877, "grad_norm": 9.714757919311523, "learning_rate": 1.7176998984196144e-07, "logits/chosen": -2.0807719230651855, "logits/rejected": -1.258175253868103, "logps/chosen": -244.98388671875, "logps/ref_chosen": -101.85537719726562, "logps/ref_rejected": -89.44766235351562, "logps/rejected": -311.7425231933594, "loss": 1.1029, "margin_dpo/margin_mean": 79.16636657714844, "margin_dpo/margin_std": 99.94515991210938, "step": 438 }, { "KL/chosen_KL_mean": -154.33096313476562, "KL/mean": -193.7960968017578, "KL/rejected_KL_mean": -233.26124572753906, "KL/std": 101.01506042480469, "epoch": 0.644640234948605, "fcm_dpo/beta": 0.004707925021648407, "fcm_dpo/delta": 0.028646834194660187, "fcm_dpo/margin": 78.9302978515625, "fcm_dpo/q_t": 0.41347455978393555, "grad_norm": 10.10484504699707, "learning_rate": 1.7055214510452458e-07, "logits/chosen": -1.4271914958953857, "logits/rejected": -1.1817679405212402, "logps/chosen": -236.08657836914062, "logps/ref_chosen": -81.75563049316406, "logps/ref_rejected": -90.58636474609375, "logps/rejected": -323.84759521484375, "loss": 1.1035, "margin_dpo/margin_mean": 78.93029022216797, "margin_dpo/margin_std": 98.69255065917969, "step": 439 }, { "KL/chosen_KL_mean": -137.7785186767578, "KL/mean": -187.16180419921875, "KL/rejected_KL_mean": -236.5450897216797, "KL/std": 102.16831970214844, "epoch": 0.6461086637298091, "fcm_dpo/beta": 0.0046911099925637245, "fcm_dpo/delta": -0.06524041295051575, "fcm_dpo/margin": 98.76656341552734, "fcm_dpo/q_t": 0.3934011459350586, "grad_norm": 7.743350505828857, "learning_rate": 1.6933639389195134e-07, "logits/chosen": -2.421034097671509, "logits/rejected": -1.923110008239746, "logps/chosen": -243.4196014404297, "logps/ref_chosen": -105.64108276367188, "logps/ref_rejected": -103.4010009765625, "logps/rejected": -339.94610595703125, "loss": 1.0424, "margin_dpo/margin_mean": 98.76657104492188, "margin_dpo/margin_std": 112.55791473388672, "step": 440 }, { "KL/chosen_KL_mean": -177.23562622070312, "KL/mean": -219.63995361328125, "KL/rejected_KL_mean": -262.04425048828125, "KL/std": 97.11551666259766, "epoch": 0.6475770925110133, "fcm_dpo/beta": 0.004692320246249437, "fcm_dpo/delta": 0.002071728929877281, "fcm_dpo/margin": 84.80866241455078, "fcm_dpo/q_t": 0.4085714817047119, "grad_norm": 8.650904655456543, "learning_rate": 1.681227682404166e-07, "logits/chosen": -1.7390990257263184, "logits/rejected": -1.3782991170883179, "logps/chosen": -268.76513671875, "logps/ref_chosen": -91.529541015625, "logps/ref_rejected": -103.619384765625, "logps/rejected": -365.66363525390625, "loss": 1.083, "margin_dpo/margin_mean": 84.80865478515625, "margin_dpo/margin_std": 105.1185302734375, "step": 441 }, { "KL/chosen_KL_mean": -173.149658203125, "KL/mean": -226.041015625, "KL/rejected_KL_mean": -278.932373046875, "KL/std": 102.69546508789062, "epoch": 0.6490455212922174, "fcm_dpo/beta": 0.004663519561290741, "fcm_dpo/delta": -0.09566286206245422, "fcm_dpo/margin": 105.78274536132812, "fcm_dpo/q_t": 0.38478708267211914, "grad_norm": 11.034876823425293, "learning_rate": 1.669113001300851e-07, "logits/chosen": -1.560469627380371, "logits/rejected": -1.0817285776138306, "logps/chosen": -257.9272155761719, "logps/ref_chosen": -84.77755737304688, "logps/ref_rejected": -83.82415771484375, "logps/rejected": -362.75653076171875, "loss": 1.0066, "margin_dpo/margin_mean": 105.7827377319336, "margin_dpo/margin_std": 103.17422485351562, "step": 442 }, { "KL/chosen_KL_mean": -168.7570343017578, "KL/mean": -202.0042724609375, "KL/rejected_KL_mean": -235.25152587890625, "KL/std": 100.0272216796875, "epoch": 0.6505139500734214, "fcm_dpo/beta": 0.004673696123063564, "fcm_dpo/delta": 0.09065397083759308, "fcm_dpo/margin": 66.49447631835938, "fcm_dpo/q_t": 0.42698436975479126, "grad_norm": 10.281853675842285, "learning_rate": 1.6570202148426815e-07, "logits/chosen": -1.9659523963928223, "logits/rejected": -1.4209094047546387, "logps/chosen": -271.40631103515625, "logps/ref_chosen": -102.64927673339844, "logps/ref_rejected": -93.03807067871094, "logps/rejected": -328.28961181640625, "loss": 1.1508, "margin_dpo/margin_mean": 66.49447631835938, "margin_dpo/margin_std": 96.69249725341797, "step": 443 }, { "KL/chosen_KL_mean": -173.14263916015625, "KL/mean": -212.30007934570312, "KL/rejected_KL_mean": -251.45753479003906, "KL/std": 97.48283386230469, "epoch": 0.6519823788546255, "fcm_dpo/beta": 0.004706915467977524, "fcm_dpo/delta": 0.0319465696811676, "fcm_dpo/margin": 78.31489562988281, "fcm_dpo/q_t": 0.4147818088531494, "grad_norm": 10.789939880371094, "learning_rate": 1.6449496416858282e-07, "logits/chosen": -1.5742313861846924, "logits/rejected": -1.4047833681106567, "logps/chosen": -261.0623779296875, "logps/ref_chosen": -87.91971588134766, "logps/ref_rejected": -103.32345581054688, "logps/rejected": -354.781005859375, "loss": 1.1182, "margin_dpo/margin_mean": 78.31490325927734, "margin_dpo/margin_std": 113.07164001464844, "step": 444 }, { "KL/chosen_KL_mean": -152.49105834960938, "KL/mean": -197.5020294189453, "KL/rejected_KL_mean": -242.5130157470703, "KL/std": 95.07347869873047, "epoch": 0.6534508076358296, "fcm_dpo/beta": 0.004693006165325642, "fcm_dpo/delta": -0.023100951686501503, "fcm_dpo/margin": 90.02197265625, "fcm_dpo/q_t": 0.40098679065704346, "grad_norm": 8.540969848632812, "learning_rate": 1.6329015999011182e-07, "logits/chosen": -1.9734916687011719, "logits/rejected": -1.5597445964813232, "logps/chosen": -253.89190673828125, "logps/ref_chosen": -101.40086364746094, "logps/ref_rejected": -99.03790283203125, "logps/rejected": -341.5509033203125, "loss": 1.0614, "margin_dpo/margin_mean": 90.02197265625, "margin_dpo/margin_std": 100.81184387207031, "step": 445 }, { "KL/chosen_KL_mean": -146.49798583984375, "KL/mean": -186.83102416992188, "KL/rejected_KL_mean": -227.16404724121094, "KL/std": 100.24253845214844, "epoch": 0.6549192364170338, "fcm_dpo/beta": 0.0047082798555493355, "fcm_dpo/delta": 0.02055959962308407, "fcm_dpo/margin": 80.66606903076172, "fcm_dpo/q_t": 0.41088733077049255, "grad_norm": 12.138264656066895, "learning_rate": 1.6208764069656578e-07, "logits/chosen": -1.8087987899780273, "logits/rejected": -1.6642836332321167, "logps/chosen": -233.9203338623047, "logps/ref_chosen": -87.4223403930664, "logps/ref_rejected": -106.70075988769531, "logps/rejected": -333.86480712890625, "loss": 1.0907, "margin_dpo/margin_mean": 80.66607666015625, "margin_dpo/margin_std": 95.730224609375, "step": 446 }, { "KL/chosen_KL_mean": -144.6820831298828, "KL/mean": -197.67042541503906, "KL/rejected_KL_mean": -250.65878295898438, "KL/std": 104.49948120117188, "epoch": 0.6563876651982379, "fcm_dpo/beta": 0.004673209972679615, "fcm_dpo/delta": -0.09767167270183563, "fcm_dpo/margin": 105.9766845703125, "fcm_dpo/q_t": 0.3860313892364502, "grad_norm": 8.628990173339844, "learning_rate": 1.608874379754465e-07, "logits/chosen": -2.2289130687713623, "logits/rejected": -2.1847023963928223, "logps/chosen": -228.29733276367188, "logps/ref_chosen": -83.6152572631836, "logps/ref_rejected": -104.91239929199219, "logps/rejected": -355.5711669921875, "loss": 1.0242, "margin_dpo/margin_mean": 105.9766845703125, "margin_dpo/margin_std": 119.16873168945312, "step": 447 }, { "KL/chosen_KL_mean": -156.11453247070312, "KL/mean": -207.7711181640625, "KL/rejected_KL_mean": -259.42767333984375, "KL/std": 109.33413696289062, "epoch": 0.657856093979442, "fcm_dpo/beta": 0.0046438174322247505, "fcm_dpo/delta": -0.08174840360879898, "fcm_dpo/margin": 103.31315612792969, "fcm_dpo/q_t": 0.39050740003585815, "grad_norm": 10.02404499053955, "learning_rate": 1.5968958345321177e-07, "logits/chosen": -1.691270351409912, "logits/rejected": -1.420040249824524, "logps/chosen": -248.69032287597656, "logps/ref_chosen": -92.5757827758789, "logps/ref_rejected": -107.68977355957031, "logps/rejected": -367.1174621582031, "loss": 1.0469, "margin_dpo/margin_mean": 103.31315612792969, "margin_dpo/margin_std": 127.27349853515625, "step": 448 }, { "KL/chosen_KL_mean": -154.34991455078125, "KL/mean": -207.43212890625, "KL/rejected_KL_mean": -260.51434326171875, "KL/std": 101.9240951538086, "epoch": 0.6593245227606461, "fcm_dpo/beta": 0.0045927721075713634, "fcm_dpo/delta": -0.0897803008556366, "fcm_dpo/margin": 106.1644287109375, "fcm_dpo/q_t": 0.3868679702281952, "grad_norm": 9.01496696472168, "learning_rate": 1.584941086944423e-07, "logits/chosen": -1.525179147720337, "logits/rejected": -1.0864238739013672, "logps/chosen": -256.74884033203125, "logps/ref_chosen": -102.39893341064453, "logps/ref_rejected": -95.14886474609375, "logps/rejected": -355.6632080078125, "loss": 1.023, "margin_dpo/margin_mean": 106.16442108154297, "margin_dpo/margin_std": 115.57987976074219, "step": 449 }, { "KL/chosen_KL_mean": -137.4766845703125, "KL/mean": -187.43618774414062, "KL/rejected_KL_mean": -237.39566040039062, "KL/std": 103.41763305664062, "epoch": 0.6607929515418502, "fcm_dpo/beta": 0.0045609744265675545, "fcm_dpo/delta": -0.057060033082962036, "fcm_dpo/margin": 99.9189682006836, "fcm_dpo/q_t": 0.3936775028705597, "grad_norm": 9.524036407470703, "learning_rate": 1.573010452010098e-07, "logits/chosen": -2.168283462524414, "logits/rejected": -1.9669804573059082, "logps/chosen": -224.46954345703125, "logps/ref_chosen": -86.99285888671875, "logps/ref_rejected": -108.53203582763672, "logps/rejected": -345.92767333984375, "loss": 1.0447, "margin_dpo/margin_mean": 99.91896057128906, "margin_dpo/margin_std": 113.42950439453125, "step": 450 }, { "KL/chosen_KL_mean": -155.76068115234375, "KL/mean": -196.02365112304688, "KL/rejected_KL_mean": -236.28663635253906, "KL/std": 102.81525421142578, "epoch": 0.6622613803230544, "fcm_dpo/beta": 0.0045709796249866486, "fcm_dpo/delta": 0.032467059791088104, "fcm_dpo/margin": 80.52595520019531, "fcm_dpo/q_t": 0.41586577892303467, "grad_norm": 14.505491256713867, "learning_rate": 1.5611042441124687e-07, "logits/chosen": -2.390557289123535, "logits/rejected": -1.884310245513916, "logps/chosen": -242.57196044921875, "logps/ref_chosen": -86.811279296875, "logps/ref_rejected": -79.8555908203125, "logps/rejected": -316.1422119140625, "loss": 1.1309, "margin_dpo/margin_mean": 80.52595520019531, "margin_dpo/margin_std": 126.62712097167969, "step": 451 }, { "KL/chosen_KL_mean": -136.00730895996094, "KL/mean": -180.67881774902344, "KL/rejected_KL_mean": -225.35032653808594, "KL/std": 98.49957275390625, "epoch": 0.6637298091042585, "fcm_dpo/beta": 0.004552492406219244, "fcm_dpo/delta": -0.007410183548927307, "fcm_dpo/margin": 89.343017578125, "fcm_dpo/q_t": 0.4043623208999634, "grad_norm": 8.390554428100586, "learning_rate": 1.549222776991186e-07, "logits/chosen": -1.7462925910949707, "logits/rejected": -1.6864103078842163, "logps/chosen": -215.38694763183594, "logps/ref_chosen": -79.379638671875, "logps/ref_rejected": -103.71539306640625, "logps/rejected": -329.06573486328125, "loss": 1.0674, "margin_dpo/margin_mean": 89.343017578125, "margin_dpo/margin_std": 91.57948303222656, "step": 452 }, { "KL/chosen_KL_mean": -164.1031494140625, "KL/mean": -218.38922119140625, "KL/rejected_KL_mean": -272.67529296875, "KL/std": 100.4375228881836, "epoch": 0.6651982378854625, "fcm_dpo/beta": 0.0045382254756987095, "fcm_dpo/delta": -0.09502498060464859, "fcm_dpo/margin": 108.5721664428711, "fcm_dpo/q_t": 0.3858945369720459, "grad_norm": 12.023402214050293, "learning_rate": 1.5373663637339584e-07, "logits/chosen": -1.9317398071289062, "logits/rejected": -1.460160493850708, "logps/chosen": -251.79833984375, "logps/ref_chosen": -87.6951904296875, "logps/ref_rejected": -90.0582275390625, "logps/rejected": -362.7335205078125, "loss": 1.0088, "margin_dpo/margin_mean": 108.57217407226562, "margin_dpo/margin_std": 106.9014892578125, "step": 453 }, { "KL/chosen_KL_mean": -172.17532348632812, "KL/mean": -216.41552734375, "KL/rejected_KL_mean": -260.6557312011719, "KL/std": 103.01646423339844, "epoch": 0.6666666666666666, "fcm_dpo/beta": 0.00453456025570631, "fcm_dpo/delta": -0.0014551468193531036, "fcm_dpo/margin": 88.48040771484375, "fcm_dpo/q_t": 0.40848463773727417, "grad_norm": 9.503375053405762, "learning_rate": 1.5255353167683017e-07, "logits/chosen": -1.8287334442138672, "logits/rejected": -1.3658068180084229, "logps/chosen": -261.7415771484375, "logps/ref_chosen": -89.56623840332031, "logps/ref_rejected": -92.92105102539062, "logps/rejected": -353.5767822265625, "loss": 1.1132, "margin_dpo/margin_mean": 88.48040771484375, "margin_dpo/margin_std": 133.8170166015625, "step": 454 }, { "KL/chosen_KL_mean": -162.19863891601562, "KL/mean": -203.32666015625, "KL/rejected_KL_mean": -244.4546661376953, "KL/std": 100.7164535522461, "epoch": 0.6681350954478708, "fcm_dpo/beta": 0.004540126770734787, "fcm_dpo/delta": 0.0269131101667881, "fcm_dpo/margin": 82.25601196289062, "fcm_dpo/q_t": 0.4132026731967926, "grad_norm": 9.481156349182129, "learning_rate": 1.5137299478533064e-07, "logits/chosen": -1.7597609758377075, "logits/rejected": -1.7187235355377197, "logps/chosen": -239.82862854003906, "logps/ref_chosen": -77.62998962402344, "logps/ref_rejected": -118.97795104980469, "logps/rejected": -363.4326171875, "loss": 1.1129, "margin_dpo/margin_mean": 82.25601196289062, "margin_dpo/margin_std": 114.65438842773438, "step": 455 }, { "KL/chosen_KL_mean": -164.06820678710938, "KL/mean": -205.66725158691406, "KL/rejected_KL_mean": -247.2662811279297, "KL/std": 102.66464233398438, "epoch": 0.6696035242290749, "fcm_dpo/beta": 0.004547426477074623, "fcm_dpo/delta": 0.022034112364053726, "fcm_dpo/margin": 83.19805908203125, "fcm_dpo/q_t": 0.41143402457237244, "grad_norm": 11.320704460144043, "learning_rate": 1.5019505680714232e-07, "logits/chosen": -1.9902832508087158, "logits/rejected": -1.858022928237915, "logps/chosen": -253.68508911132812, "logps/ref_chosen": -89.61686706542969, "logps/ref_rejected": -109.5597152709961, "logps/rejected": -356.82598876953125, "loss": 1.096, "margin_dpo/margin_mean": 83.19805908203125, "margin_dpo/margin_std": 103.41998291015625, "step": 456 }, { "KL/chosen_KL_mean": -132.9967803955078, "KL/mean": -188.26211547851562, "KL/rejected_KL_mean": -243.52743530273438, "KL/std": 102.64701843261719, "epoch": 0.671071953010279, "fcm_dpo/beta": 0.004495893605053425, "fcm_dpo/delta": -0.100301094353199, "fcm_dpo/margin": 110.53064727783203, "fcm_dpo/q_t": 0.3827398419380188, "grad_norm": 10.286169052124023, "learning_rate": 1.4901974878202627e-07, "logits/chosen": -2.2586655616760254, "logits/rejected": -1.9511044025421143, "logps/chosen": -220.3184814453125, "logps/ref_chosen": -87.32168579101562, "logps/ref_rejected": -90.76660919189453, "logps/rejected": -334.2940368652344, "loss": 0.9938, "margin_dpo/margin_mean": 110.53064727783203, "margin_dpo/margin_std": 91.14752197265625, "step": 457 }, { "KL/chosen_KL_mean": -155.6369171142578, "KL/mean": -195.60855102539062, "KL/rejected_KL_mean": -235.5802001953125, "KL/std": 102.11534881591797, "epoch": 0.6725403817914831, "fcm_dpo/beta": 0.004518887959420681, "fcm_dpo/delta": 0.0393698550760746, "fcm_dpo/margin": 79.9432601928711, "fcm_dpo/q_t": 0.4175107181072235, "grad_norm": 9.641457557678223, "learning_rate": 1.4784710168044212e-07, "logits/chosen": -1.5261669158935547, "logits/rejected": -1.2823119163513184, "logps/chosen": -249.15737915039062, "logps/ref_chosen": -93.52044677734375, "logps/ref_rejected": -103.36898803710938, "logps/rejected": -338.9491882324219, "loss": 1.1241, "margin_dpo/margin_mean": 79.9432601928711, "margin_dpo/margin_std": 118.30757904052734, "step": 458 }, { "KL/chosen_KL_mean": -148.59414672851562, "KL/mean": -198.66152954101562, "KL/rejected_KL_mean": -248.72891235351562, "KL/std": 107.11579895019531, "epoch": 0.6740088105726872, "fcm_dpo/beta": 0.004491039551794529, "fcm_dpo/delta": -0.0511801615357399, "fcm_dpo/margin": 100.13475036621094, "fcm_dpo/q_t": 0.3964363932609558, "grad_norm": 7.2166666984558105, "learning_rate": 1.466771464027316e-07, "logits/chosen": -1.595045566558838, "logits/rejected": -1.3415178060531616, "logps/chosen": -224.2823486328125, "logps/ref_chosen": -75.68820190429688, "logps/ref_rejected": -92.17048645019531, "logps/rejected": -340.8993835449219, "loss": 1.0498, "margin_dpo/margin_mean": 100.13475036621094, "margin_dpo/margin_std": 114.70283508300781, "step": 459 }, { "KL/chosen_KL_mean": -150.5113525390625, "KL/mean": -197.10366821289062, "KL/rejected_KL_mean": -243.69598388671875, "KL/std": 103.70185852050781, "epoch": 0.6754772393538914, "fcm_dpo/beta": 0.0044807372614741325, "fcm_dpo/delta": -0.018095871433615685, "fcm_dpo/margin": 93.18464660644531, "fcm_dpo/q_t": 0.4033958613872528, "grad_norm": 10.2720947265625, "learning_rate": 1.4550991377830423e-07, "logits/chosen": -2.0562527179718018, "logits/rejected": -2.0793256759643555, "logps/chosen": -231.62924194335938, "logps/ref_chosen": -81.11788940429688, "logps/ref_rejected": -110.31238555908203, "logps/rejected": -354.00836181640625, "loss": 1.0798, "margin_dpo/margin_mean": 93.18463897705078, "margin_dpo/margin_std": 118.6648178100586, "step": 460 }, { "KL/chosen_KL_mean": -187.42837524414062, "KL/mean": -224.85317993164062, "KL/rejected_KL_mean": -262.2779846191406, "KL/std": 90.62449645996094, "epoch": 0.6769456681350955, "fcm_dpo/beta": 0.004502894356846809, "fcm_dpo/delta": 0.0640527606010437, "fcm_dpo/margin": 74.84961700439453, "fcm_dpo/q_t": 0.42167627811431885, "grad_norm": 8.665507316589355, "learning_rate": 1.4434543456482518e-07, "logits/chosen": -1.5992321968078613, "logits/rejected": -1.4055674076080322, "logps/chosen": -269.01190185546875, "logps/ref_chosen": -81.58352661132812, "logps/ref_rejected": -93.87710571289062, "logps/rejected": -356.15509033203125, "loss": 1.1351, "margin_dpo/margin_mean": 74.84962463378906, "margin_dpo/margin_std": 108.11590576171875, "step": 461 }, { "KL/chosen_KL_mean": -141.35569763183594, "KL/mean": -189.80584716796875, "KL/rejected_KL_mean": -238.25601196289062, "KL/std": 98.12359619140625, "epoch": 0.6784140969162996, "fcm_dpo/beta": 0.004507179372012615, "fcm_dpo/delta": -0.03756365552544594, "fcm_dpo/margin": 96.90032958984375, "fcm_dpo/q_t": 0.3980672359466553, "grad_norm": 8.83620548248291, "learning_rate": 1.4318373944740484e-07, "logits/chosen": -2.4595284461975098, "logits/rejected": -1.9736459255218506, "logps/chosen": -235.5542449951172, "logps/ref_chosen": -94.19854736328125, "logps/ref_rejected": -85.63162994384766, "logps/rejected": -323.88763427734375, "loss": 1.0492, "margin_dpo/margin_mean": 96.90032958984375, "margin_dpo/margin_std": 104.82482147216797, "step": 462 }, { "KL/chosen_KL_mean": -138.1318359375, "KL/mean": -194.96804809570312, "KL/rejected_KL_mean": -251.80426025390625, "KL/std": 105.51051330566406, "epoch": 0.6798825256975036, "fcm_dpo/beta": 0.0044762734323740005, "fcm_dpo/delta": -0.11172803491353989, "fcm_dpo/margin": 113.67243957519531, "fcm_dpo/q_t": 0.3813176155090332, "grad_norm": 13.596323013305664, "learning_rate": 1.4202485903778976e-07, "logits/chosen": -1.9420959949493408, "logits/rejected": -1.5202367305755615, "logps/chosen": -224.05657958984375, "logps/ref_chosen": -85.92474365234375, "logps/ref_rejected": -96.90184020996094, "logps/rejected": -348.70611572265625, "loss": 1.0068, "margin_dpo/margin_mean": 113.67243957519531, "margin_dpo/margin_std": 112.43539428710938, "step": 463 }, { "KL/chosen_KL_mean": -170.67959594726562, "KL/mean": -217.0213623046875, "KL/rejected_KL_mean": -263.3631591796875, "KL/std": 104.08018493652344, "epoch": 0.6813509544787077, "fcm_dpo/beta": 0.004440618678927422, "fcm_dpo/delta": -0.011873176321387291, "fcm_dpo/margin": 92.68354034423828, "fcm_dpo/q_t": 0.404552161693573, "grad_norm": 12.353242874145508, "learning_rate": 1.4086882387355658e-07, "logits/chosen": -1.769266963005066, "logits/rejected": -1.8489060401916504, "logps/chosen": -250.36880493164062, "logps/ref_chosen": -79.68919372558594, "logps/ref_rejected": -107.29232025146484, "logps/rejected": -370.65545654296875, "loss": 1.0816, "margin_dpo/margin_mean": 92.68354034423828, "margin_dpo/margin_std": 118.63115692138672, "step": 464 }, { "KL/chosen_KL_mean": -161.2972412109375, "KL/mean": -216.36190795898438, "KL/rejected_KL_mean": -271.42657470703125, "KL/std": 113.54704284667969, "epoch": 0.6828193832599119, "fcm_dpo/beta": 0.004412083886563778, "fcm_dpo/delta": -0.08810941874980927, "fcm_dpo/margin": 110.12930297851562, "fcm_dpo/q_t": 0.38634487986564636, "grad_norm": 9.132247924804688, "learning_rate": 1.3971566441730714e-07, "logits/chosen": -1.3953578472137451, "logits/rejected": -1.0809001922607422, "logps/chosen": -253.157470703125, "logps/ref_chosen": -91.8602294921875, "logps/ref_rejected": -118.71001434326172, "logps/rejected": -390.1365661621094, "loss": 1.0107, "margin_dpo/margin_mean": 110.12931823730469, "margin_dpo/margin_std": 107.18203735351562, "step": 465 }, { "KL/chosen_KL_mean": -144.19749450683594, "KL/mean": -200.17919921875, "KL/rejected_KL_mean": -256.160888671875, "KL/std": 108.34889221191406, "epoch": 0.684287812041116, "fcm_dpo/beta": 0.004373148083686829, "fcm_dpo/delta": -0.09192436933517456, "fcm_dpo/margin": 111.96339416503906, "fcm_dpo/q_t": 0.3860629200935364, "grad_norm": 7.248248100280762, "learning_rate": 1.3856541105586545e-07, "logits/chosen": -1.8160350322723389, "logits/rejected": -1.5093111991882324, "logps/chosen": -228.89889526367188, "logps/ref_chosen": -84.70140838623047, "logps/ref_rejected": -96.05084228515625, "logps/rejected": -352.21173095703125, "loss": 1.0129, "margin_dpo/margin_mean": 111.96339416503906, "margin_dpo/margin_std": 113.51492309570312, "step": 466 }, { "KL/chosen_KL_mean": -184.41482543945312, "KL/mean": -231.31777954101562, "KL/rejected_KL_mean": -278.220703125, "KL/std": 112.89960479736328, "epoch": 0.6857562408223201, "fcm_dpo/beta": 0.00435209833085537, "fcm_dpo/delta": -0.008647546172142029, "fcm_dpo/margin": 93.80587768554688, "fcm_dpo/q_t": 0.4054826498031616, "grad_norm": 12.664158821105957, "learning_rate": 1.3741809409947729e-07, "logits/chosen": -2.3796207904815674, "logits/rejected": -1.8078844547271729, "logps/chosen": -293.713134765625, "logps/ref_chosen": -109.29832458496094, "logps/ref_rejected": -108.8436508178711, "logps/rejected": -387.0643310546875, "loss": 1.0934, "margin_dpo/margin_mean": 93.80587005615234, "margin_dpo/margin_std": 127.40046691894531, "step": 467 }, { "KL/chosen_KL_mean": -144.43234252929688, "KL/mean": -187.2611541748047, "KL/rejected_KL_mean": -230.08999633789062, "KL/std": 110.3796157836914, "epoch": 0.6872246696035242, "fcm_dpo/beta": 0.0043672965839505196, "fcm_dpo/delta": 0.026404431089758873, "fcm_dpo/margin": 85.65763092041016, "fcm_dpo/q_t": 0.4147111475467682, "grad_norm": 9.516472816467285, "learning_rate": 1.362737437810114e-07, "logits/chosen": -1.7444417476654053, "logits/rejected": -1.5374441146850586, "logps/chosen": -242.7539825439453, "logps/ref_chosen": -98.32164764404297, "logps/ref_rejected": -106.68048095703125, "logps/rejected": -336.77044677734375, "loss": 1.1178, "margin_dpo/margin_mean": 85.65763092041016, "margin_dpo/margin_std": 126.364990234375, "step": 468 }, { "KL/chosen_KL_mean": -177.84457397460938, "KL/mean": -230.97900390625, "KL/rejected_KL_mean": -284.1134338378906, "KL/std": 110.55096435546875, "epoch": 0.6886930983847284, "fcm_dpo/beta": 0.004341612569987774, "fcm_dpo/delta": -0.06320111453533173, "fcm_dpo/margin": 106.26884460449219, "fcm_dpo/q_t": 0.3931505084037781, "grad_norm": 10.600975036621094, "learning_rate": 1.351323902551631e-07, "logits/chosen": -2.2757620811462402, "logits/rejected": -1.8573853969573975, "logps/chosen": -274.6087646484375, "logps/ref_chosen": -96.76420593261719, "logps/ref_rejected": -109.59500885009766, "logps/rejected": -393.70843505859375, "loss": 1.0423, "margin_dpo/margin_mean": 106.26884460449219, "margin_dpo/margin_std": 119.9745101928711, "step": 469 }, { "KL/chosen_KL_mean": -162.87403869628906, "KL/mean": -211.81552124023438, "KL/rejected_KL_mean": -260.7569580078125, "KL/std": 103.23440551757812, "epoch": 0.6901615271659325, "fcm_dpo/beta": 0.004335965495556593, "fcm_dpo/delta": -0.02495286986231804, "fcm_dpo/margin": 97.88294982910156, "fcm_dpo/q_t": 0.40029242634773254, "grad_norm": 9.35530948638916, "learning_rate": 1.339940635976592e-07, "logits/chosen": -1.653503656387329, "logits/rejected": -1.2327951192855835, "logps/chosen": -246.37069702148438, "logps/ref_chosen": -83.49665832519531, "logps/ref_rejected": -88.48578643798828, "logps/rejected": -349.2427673339844, "loss": 1.0534, "margin_dpo/margin_mean": 97.88294982910156, "margin_dpo/margin_std": 102.92399597167969, "step": 470 }, { "KL/chosen_KL_mean": -188.07757568359375, "KL/mean": -234.3421630859375, "KL/rejected_KL_mean": -280.60675048828125, "KL/std": 101.43286895751953, "epoch": 0.6916299559471366, "fcm_dpo/beta": 0.004333192948251963, "fcm_dpo/delta": -0.0009668557904660702, "fcm_dpo/margin": 92.52915954589844, "fcm_dpo/q_t": 0.4056793451309204, "grad_norm": 9.222952842712402, "learning_rate": 1.3285879380446563e-07, "logits/chosen": -1.6367769241333008, "logits/rejected": -1.226168155670166, "logps/chosen": -276.5518798828125, "logps/ref_chosen": -88.47430419921875, "logps/ref_rejected": -90.48171997070312, "logps/rejected": -371.08843994140625, "loss": 1.0717, "margin_dpo/margin_mean": 92.52915954589844, "margin_dpo/margin_std": 102.62583923339844, "step": 471 }, { "KL/chosen_KL_mean": -175.8319549560547, "KL/mean": -230.28292846679688, "KL/rejected_KL_mean": -284.73388671875, "KL/std": 102.81556701660156, "epoch": 0.6930983847283406, "fcm_dpo/beta": 0.004305423237383366, "fcm_dpo/delta": -0.07067485898733139, "fcm_dpo/margin": 108.90191650390625, "fcm_dpo/q_t": 0.39110201597213745, "grad_norm": 7.875804424285889, "learning_rate": 1.317266107909975e-07, "logits/chosen": -2.215156316757202, "logits/rejected": -1.5237432718276978, "logps/chosen": -301.065673828125, "logps/ref_chosen": -125.23370361328125, "logps/ref_rejected": -121.05349731445312, "logps/rejected": -405.787353515625, "loss": 1.0233, "margin_dpo/margin_mean": 108.90191650390625, "margin_dpo/margin_std": 108.75623321533203, "step": 472 }, { "KL/chosen_KL_mean": -199.05072021484375, "KL/mean": -233.37893676757812, "KL/rejected_KL_mean": -267.7071228027344, "KL/std": 103.5265121459961, "epoch": 0.6945668135095447, "fcm_dpo/beta": 0.004339549690485001, "fcm_dpo/delta": 0.10351648926734924, "fcm_dpo/margin": 68.65638732910156, "fcm_dpo/q_t": 0.431788831949234, "grad_norm": 10.161319732666016, "learning_rate": 1.3059754439133002e-07, "logits/chosen": -1.7749607563018799, "logits/rejected": -1.2788692712783813, "logps/chosen": -294.662109375, "logps/ref_chosen": -95.61137390136719, "logps/ref_rejected": -88.15115356445312, "logps/rejected": -355.8582763671875, "loss": 1.174, "margin_dpo/margin_mean": 68.65637969970703, "margin_dpo/margin_std": 119.888671875, "step": 473 }, { "KL/chosen_KL_mean": -187.4449920654297, "KL/mean": -222.13156127929688, "KL/rejected_KL_mean": -256.81817626953125, "KL/std": 98.05271911621094, "epoch": 0.6960352422907489, "fcm_dpo/beta": 0.004344789776951075, "fcm_dpo/delta": -0.014370165765285492, "fcm_dpo/margin": 69.3731689453125, "fcm_dpo/q_t": 0.42997315526008606, "grad_norm": 14.437629699707031, "learning_rate": 1.2947162435741277e-07, "logits/chosen": -1.3510382175445557, "logits/rejected": -1.1564675569534302, "logps/chosen": -268.92474365234375, "logps/ref_chosen": -81.47975158691406, "logps/ref_rejected": -96.46562957763672, "logps/rejected": -353.2837829589844, "loss": 1.1776, "margin_dpo/margin_mean": 69.3731689453125, "margin_dpo/margin_std": 119.30937957763672, "step": 474 }, { "KL/chosen_KL_mean": -140.66171264648438, "KL/mean": -194.95770263671875, "KL/rejected_KL_mean": -249.25369262695312, "KL/std": 105.5289535522461, "epoch": 0.697503671071953, "fcm_dpo/beta": 0.004320449195802212, "fcm_dpo/delta": -0.07080624252557755, "fcm_dpo/margin": 108.59196472167969, "fcm_dpo/q_t": 0.39168640971183777, "grad_norm": 8.487594604492188, "learning_rate": 1.2834888035828596e-07, "logits/chosen": -2.346311092376709, "logits/rejected": -2.2934858798980713, "logps/chosen": -214.85769653320312, "logps/ref_chosen": -74.19598388671875, "logps/ref_rejected": -94.69242095947266, "logps/rejected": -343.94610595703125, "loss": 1.0336, "margin_dpo/margin_mean": 108.59196472167969, "margin_dpo/margin_std": 119.8677978515625, "step": 475 }, { "KL/chosen_KL_mean": -164.2483367919922, "KL/mean": -212.70899963378906, "KL/rejected_KL_mean": -261.169677734375, "KL/std": 107.41903686523438, "epoch": 0.6989720998531571, "fcm_dpo/beta": 0.004299451597034931, "fcm_dpo/delta": -0.017168425023555756, "fcm_dpo/margin": 96.92134094238281, "fcm_dpo/q_t": 0.4038906693458557, "grad_norm": 9.994364738464355, "learning_rate": 1.2722934197929802e-07, "logits/chosen": -2.041455030441284, "logits/rejected": -1.6665287017822266, "logps/chosen": -236.2194366455078, "logps/ref_chosen": -71.97109985351562, "logps/ref_rejected": -80.26224517822266, "logps/rejected": -341.4319152832031, "loss": 1.0714, "margin_dpo/margin_mean": 96.92134094238281, "margin_dpo/margin_std": 116.35638427734375, "step": 476 }, { "KL/chosen_KL_mean": -164.52719116210938, "KL/mean": -212.98812866210938, "KL/rejected_KL_mean": -261.4490661621094, "KL/std": 95.23974609375, "epoch": 0.7004405286343612, "fcm_dpo/beta": 0.004299557767808437, "fcm_dpo/delta": -0.017075784504413605, "fcm_dpo/margin": 96.92189025878906, "fcm_dpo/q_t": 0.40237241983413696, "grad_norm": 8.734857559204102, "learning_rate": 1.2611303872132631e-07, "logits/chosen": -2.049647808074951, "logits/rejected": -1.113594889640808, "logps/chosen": -269.5327453613281, "logps/ref_chosen": -105.00555419921875, "logps/ref_rejected": -81.87843322753906, "logps/rejected": -343.3275146484375, "loss": 1.0731, "margin_dpo/margin_mean": 96.92189025878906, "margin_dpo/margin_std": 117.73863220214844, "step": 477 }, { "KL/chosen_KL_mean": -160.11184692382812, "KL/mean": -210.2587127685547, "KL/rejected_KL_mean": -260.40557861328125, "KL/std": 103.36613464355469, "epoch": 0.7019089574155654, "fcm_dpo/beta": 0.004286094568669796, "fcm_dpo/delta": -0.030527781695127487, "fcm_dpo/margin": 100.29373168945312, "fcm_dpo/q_t": 0.3995046615600586, "grad_norm": 8.388529777526855, "learning_rate": 1.2500000000000005e-07, "logits/chosen": -1.548363447189331, "logits/rejected": -1.3709733486175537, "logps/chosen": -236.90005493164062, "logps/ref_chosen": -76.7882080078125, "logps/ref_rejected": -90.43994140625, "logps/rejected": -350.84552001953125, "loss": 1.058, "margin_dpo/margin_mean": 100.2937240600586, "margin_dpo/margin_std": 113.66435241699219, "step": 478 }, { "KL/chosen_KL_mean": -162.942138671875, "KL/mean": -212.99124145507812, "KL/rejected_KL_mean": -263.040283203125, "KL/std": 104.16551208496094, "epoch": 0.7033773861967695, "fcm_dpo/beta": 0.004281010944396257, "fcm_dpo/delta": -0.02920958399772644, "fcm_dpo/margin": 100.09815216064453, "fcm_dpo/q_t": 0.39909622073173523, "grad_norm": 11.315074920654297, "learning_rate": 1.2389025514492456e-07, "logits/chosen": -1.3584550619125366, "logits/rejected": -1.2152658700942993, "logps/chosen": -244.30445861816406, "logps/ref_chosen": -81.3623046875, "logps/ref_rejected": -101.09114074707031, "logps/rejected": -364.1314392089844, "loss": 1.0543, "margin_dpo/margin_mean": 100.09815216064453, "margin_dpo/margin_std": 107.10712432861328, "step": 479 }, { "KL/chosen_KL_mean": -189.47189331054688, "KL/mean": -228.690673828125, "KL/rejected_KL_mean": -267.909423828125, "KL/std": 99.26670837402344, "epoch": 0.7048458149779736, "fcm_dpo/beta": 0.004291826859116554, "fcm_dpo/delta": 0.0644366443157196, "fcm_dpo/margin": 78.43753051757812, "fcm_dpo/q_t": 0.42118948698043823, "grad_norm": 10.010136604309082, "learning_rate": 1.227838333989088e-07, "logits/chosen": -1.7310407161712646, "logits/rejected": -1.0485994815826416, "logps/chosen": -286.24578857421875, "logps/ref_chosen": -96.7739028930664, "logps/ref_rejected": -86.40473937988281, "logps/rejected": -354.31414794921875, "loss": 1.1307, "margin_dpo/margin_mean": 78.43753051757812, "margin_dpo/margin_std": 109.64742279052734, "step": 480 }, { "KL/chosen_KL_mean": -183.91741943359375, "KL/mean": -230.33399963378906, "KL/rejected_KL_mean": -276.75054931640625, "KL/std": 103.16033935546875, "epoch": 0.7063142437591777, "fcm_dpo/beta": 0.004297881852835417, "fcm_dpo/delta": 0.0010334514081478119, "fcm_dpo/margin": 92.83319091796875, "fcm_dpo/q_t": 0.4065982699394226, "grad_norm": 11.734601020812988, "learning_rate": 1.2168076391719489e-07, "logits/chosen": -1.8699932098388672, "logits/rejected": -1.427708387374878, "logps/chosen": -275.58758544921875, "logps/ref_chosen": -91.670166015625, "logps/ref_rejected": -98.69490051269531, "logps/rejected": -375.4454650878906, "loss": 1.0835, "margin_dpo/margin_mean": 92.83319091796875, "margin_dpo/margin_std": 114.13086700439453, "step": 481 }, { "KL/chosen_KL_mean": -178.9447479248047, "KL/mean": -223.60183715820312, "KL/rejected_KL_mean": -268.2589111328125, "KL/std": 101.00959777832031, "epoch": 0.7077826725403817, "fcm_dpo/beta": 0.004298394545912743, "fcm_dpo/delta": 0.016359619796276093, "fcm_dpo/margin": 89.31414794921875, "fcm_dpo/q_t": 0.4094899594783783, "grad_norm": 12.075055122375488, "learning_rate": 1.2058107576668938e-07, "logits/chosen": -1.8125064373016357, "logits/rejected": -1.4661319255828857, "logps/chosen": -277.46484375, "logps/ref_chosen": -98.52011108398438, "logps/ref_rejected": -94.82945251464844, "logps/rejected": -363.0883483886719, "loss": 1.0955, "margin_dpo/margin_mean": 89.31414794921875, "margin_dpo/margin_std": 112.89973449707031, "step": 482 }, { "KL/chosen_KL_mean": -183.4427032470703, "KL/mean": -231.00625610351562, "KL/rejected_KL_mean": -278.56982421875, "KL/std": 107.516357421875, "epoch": 0.7092511013215859, "fcm_dpo/beta": 0.004314601421356201, "fcm_dpo/delta": -0.010952681303024292, "fcm_dpo/margin": 95.12712097167969, "fcm_dpo/q_t": 0.40353137254714966, "grad_norm": 7.654962062835693, "learning_rate": 1.194847979251979e-07, "logits/chosen": -1.9605950117111206, "logits/rejected": -1.3095484972000122, "logps/chosen": -290.5613098144531, "logps/ref_chosen": -107.11860656738281, "logps/ref_rejected": -101.11499786376953, "logps/rejected": -379.684814453125, "loss": 1.0703, "margin_dpo/margin_mean": 95.12712097167969, "margin_dpo/margin_std": 106.72515869140625, "step": 483 }, { "KL/chosen_KL_mean": -167.53448486328125, "KL/mean": -220.8016815185547, "KL/rejected_KL_mean": -274.0688781738281, "KL/std": 109.21299743652344, "epoch": 0.71071953010279, "fcm_dpo/beta": 0.004288649186491966, "fcm_dpo/delta": -0.058219149708747864, "fcm_dpo/margin": 106.53436279296875, "fcm_dpo/q_t": 0.39327579736709595, "grad_norm": 8.433719635009766, "learning_rate": 1.1839195928066101e-07, "logits/chosen": -2.3360369205474854, "logits/rejected": -1.8512699604034424, "logps/chosen": -254.514404296875, "logps/ref_chosen": -86.97991943359375, "logps/ref_rejected": -90.72367095947266, "logps/rejected": -364.79254150390625, "loss": 1.0422, "margin_dpo/margin_mean": 106.53436279296875, "margin_dpo/margin_std": 118.3977279663086, "step": 484 }, { "KL/chosen_KL_mean": -159.260498046875, "KL/mean": -216.86544799804688, "KL/rejected_KL_mean": -274.4703674316406, "KL/std": 113.40766906738281, "epoch": 0.7121879588839941, "fcm_dpo/beta": 0.004247823264449835, "fcm_dpo/delta": -0.09161669760942459, "fcm_dpo/margin": 115.20988464355469, "fcm_dpo/q_t": 0.3861236572265625, "grad_norm": 9.152985572814941, "learning_rate": 1.1730258863039347e-07, "logits/chosen": -1.783046841621399, "logits/rejected": -1.3642797470092773, "logps/chosen": -253.31924438476562, "logps/ref_chosen": -94.05874633789062, "logps/ref_rejected": -108.56298828125, "logps/rejected": -383.0333557128906, "loss": 1.0105, "margin_dpo/margin_mean": 115.20988464355469, "margin_dpo/margin_std": 114.2982177734375, "step": 485 }, { "KL/chosen_KL_mean": -182.59701538085938, "KL/mean": -233.10520935058594, "KL/rejected_KL_mean": -283.6133728027344, "KL/std": 109.70758819580078, "epoch": 0.7136563876651982, "fcm_dpo/beta": 0.004233483225107193, "fcm_dpo/delta": -0.028283368796110153, "fcm_dpo/margin": 101.01638793945312, "fcm_dpo/q_t": 0.40003499388694763, "grad_norm": 10.350366592407227, "learning_rate": 1.1621671468032493e-07, "logits/chosen": -2.1053478717803955, "logits/rejected": -1.4079363346099854, "logps/chosen": -276.3428955078125, "logps/ref_chosen": -93.74588012695312, "logps/ref_rejected": -98.07064819335938, "logps/rejected": -381.68402099609375, "loss": 1.0629, "margin_dpo/margin_mean": 101.01638793945312, "margin_dpo/margin_std": 117.03358459472656, "step": 486 }, { "KL/chosen_KL_mean": -175.9898223876953, "KL/mean": -222.04298400878906, "KL/rejected_KL_mean": -268.0961608886719, "KL/std": 104.520263671875, "epoch": 0.7151248164464024, "fcm_dpo/beta": 0.004227600060403347, "fcm_dpo/delta": 0.010822284035384655, "fcm_dpo/margin": 92.10633850097656, "fcm_dpo/q_t": 0.40871530771255493, "grad_norm": 9.570023536682129, "learning_rate": 1.1513436604424378e-07, "logits/chosen": -1.90761399269104, "logits/rejected": -1.5849990844726562, "logps/chosen": -264.02337646484375, "logps/ref_chosen": -88.0335693359375, "logps/ref_rejected": -98.47209930419922, "logps/rejected": -366.5682373046875, "loss": 1.0867, "margin_dpo/margin_mean": 92.10633850097656, "margin_dpo/margin_std": 111.05476379394531, "step": 487 }, { "KL/chosen_KL_mean": -159.51055908203125, "KL/mean": -210.24594116210938, "KL/rejected_KL_mean": -260.9813232421875, "KL/std": 110.86837768554688, "epoch": 0.7165932452276065, "fcm_dpo/beta": 0.004223078954964876, "fcm_dpo/delta": -0.029140576720237732, "fcm_dpo/margin": 101.47074890136719, "fcm_dpo/q_t": 0.39857885241508484, "grad_norm": 9.156325340270996, "learning_rate": 1.1405557124304335e-07, "logits/chosen": -1.6006579399108887, "logits/rejected": -1.3385562896728516, "logps/chosen": -244.3002166748047, "logps/ref_chosen": -84.78964233398438, "logps/ref_rejected": -90.2734603881836, "logps/rejected": -351.2547912597656, "loss": 1.0462, "margin_dpo/margin_mean": 101.47075653076172, "margin_dpo/margin_std": 98.6378173828125, "step": 488 }, { "KL/chosen_KL_mean": -179.03553771972656, "KL/mean": -225.57130432128906, "KL/rejected_KL_mean": -272.1070556640625, "KL/std": 101.32999420166016, "epoch": 0.7180616740088106, "fcm_dpo/beta": 0.004220565780997276, "fcm_dpo/delta": 0.007326389662921429, "fcm_dpo/margin": 93.07154083251953, "fcm_dpo/q_t": 0.4082677960395813, "grad_norm": 8.935827255249023, "learning_rate": 1.1298035870396985e-07, "logits/chosen": -1.643143892288208, "logits/rejected": -1.1533708572387695, "logps/chosen": -269.50482177734375, "logps/ref_chosen": -90.46929931640625, "logps/ref_rejected": -86.39761352539062, "logps/rejected": -358.50469970703125, "loss": 1.0847, "margin_dpo/margin_mean": 93.07154083251953, "margin_dpo/margin_std": 112.48039245605469, "step": 489 }, { "KL/chosen_KL_mean": -180.14649963378906, "KL/mean": -223.5347137451172, "KL/rejected_KL_mean": -266.92291259765625, "KL/std": 109.46475219726562, "epoch": 0.7195301027900147, "fcm_dpo/beta": 0.004244981333613396, "fcm_dpo/delta": 0.03165161982178688, "fcm_dpo/margin": 86.77641296386719, "fcm_dpo/q_t": 0.41641145944595337, "grad_norm": 9.038500785827637, "learning_rate": 1.1190875675987355e-07, "logits/chosen": -2.1497085094451904, "logits/rejected": -2.2259225845336914, "logps/chosen": -265.46661376953125, "logps/ref_chosen": -85.32012939453125, "logps/ref_rejected": -115.99385070800781, "logps/rejected": -382.916748046875, "loss": 1.1429, "margin_dpo/margin_mean": 86.77641296386719, "margin_dpo/margin_std": 143.5731658935547, "step": 490 }, { "KL/chosen_KL_mean": -182.928955078125, "KL/mean": -223.3063507080078, "KL/rejected_KL_mean": -263.6837463378906, "KL/std": 111.60299682617188, "epoch": 0.7209985315712188, "fcm_dpo/beta": 0.004251599311828613, "fcm_dpo/delta": 0.057659976184368134, "fcm_dpo/margin": 80.75479125976562, "fcm_dpo/q_t": 0.4193815588951111, "grad_norm": 10.477417945861816, "learning_rate": 1.1084079364846241e-07, "logits/chosen": -1.9827344417572021, "logits/rejected": -1.501983880996704, "logps/chosen": -269.0724792480469, "logps/ref_chosen": -86.14352416992188, "logps/ref_rejected": -80.67945861816406, "logps/rejected": -344.36322021484375, "loss": 1.1268, "margin_dpo/margin_mean": 80.75479125976562, "margin_dpo/margin_std": 112.4705810546875, "step": 491 }, { "KL/chosen_KL_mean": -171.31777954101562, "KL/mean": -204.08111572265625, "KL/rejected_KL_mean": -236.84442138671875, "KL/std": 109.46138000488281, "epoch": 0.7224669603524229, "fcm_dpo/beta": 0.004294299520552158, "fcm_dpo/delta": 0.12036000192165375, "fcm_dpo/margin": 65.52665710449219, "fcm_dpo/q_t": 0.4351654350757599, "grad_norm": 13.580964088439941, "learning_rate": 1.097764975115576e-07, "logits/chosen": -2.052213668823242, "logits/rejected": -1.6659529209136963, "logps/chosen": -252.42535400390625, "logps/ref_chosen": -81.10757446289062, "logps/ref_rejected": -80.75199890136719, "logps/rejected": -317.596435546875, "loss": 1.2057, "margin_dpo/margin_mean": 65.52665710449219, "margin_dpo/margin_std": 135.97732543945312, "step": 492 }, { "KL/chosen_KL_mean": -169.51980590820312, "KL/mean": -212.3175048828125, "KL/rejected_KL_mean": -255.11517333984375, "KL/std": 99.52519989013672, "epoch": 0.723935389133627, "fcm_dpo/beta": 0.004311233758926392, "fcm_dpo/delta": 0.031454842537641525, "fcm_dpo/margin": 85.59535217285156, "fcm_dpo/q_t": 0.41295337677001953, "grad_norm": 14.967672348022461, "learning_rate": 1.0871589639435203e-07, "logits/chosen": -2.505528450012207, "logits/rejected": -1.784529447555542, "logps/chosen": -281.7271728515625, "logps/ref_chosen": -112.20733642578125, "logps/ref_rejected": -93.60719299316406, "logps/rejected": -348.72235107421875, "loss": 1.1019, "margin_dpo/margin_mean": 85.59535217285156, "margin_dpo/margin_std": 104.9568862915039, "step": 493 }, { "KL/chosen_KL_mean": -163.8925323486328, "KL/mean": -221.25074768066406, "KL/rejected_KL_mean": -278.60894775390625, "KL/std": 109.71672058105469, "epoch": 0.7254038179148311, "fcm_dpo/beta": 0.004281867295503616, "fcm_dpo/delta": -0.0940324068069458, "fcm_dpo/margin": 114.7164306640625, "fcm_dpo/q_t": 0.3857485353946686, "grad_norm": 10.091239929199219, "learning_rate": 1.0765901824467166e-07, "logits/chosen": -1.4631227254867554, "logits/rejected": -1.384997844696045, "logps/chosen": -237.00743103027344, "logps/ref_chosen": -73.11489868164062, "logps/ref_rejected": -92.16300201416016, "logps/rejected": -370.77197265625, "loss": 1.0159, "margin_dpo/margin_mean": 114.7164306640625, "margin_dpo/margin_std": 117.69435119628906, "step": 494 }, { "KL/chosen_KL_mean": -168.63095092773438, "KL/mean": -217.2857208251953, "KL/rejected_KL_mean": -265.94049072265625, "KL/std": 104.12474060058594, "epoch": 0.7268722466960352, "fcm_dpo/beta": 0.004276534542441368, "fcm_dpo/delta": -0.016499020159244537, "fcm_dpo/margin": 97.3095703125, "fcm_dpo/q_t": 0.403533011674881, "grad_norm": 10.774046897888184, "learning_rate": 1.0660589091223854e-07, "logits/chosen": -2.1831910610198975, "logits/rejected": -1.714792251586914, "logps/chosen": -268.1512451171875, "logps/ref_chosen": -99.52031707763672, "logps/ref_rejected": -97.93089294433594, "logps/rejected": -363.87139892578125, "loss": 1.0788, "margin_dpo/margin_mean": 97.30955505371094, "margin_dpo/margin_std": 124.45156860351562, "step": 495 }, { "KL/chosen_KL_mean": -187.17742919921875, "KL/mean": -228.65325927734375, "KL/rejected_KL_mean": -270.12908935546875, "KL/std": 104.17898559570312, "epoch": 0.7283406754772394, "fcm_dpo/beta": 0.004295707680284977, "fcm_dpo/delta": 0.04433218017220497, "fcm_dpo/margin": 82.95169067382812, "fcm_dpo/q_t": 0.4157905578613281, "grad_norm": 8.726499557495117, "learning_rate": 1.0555654214793722e-07, "logits/chosen": -2.221287965774536, "logits/rejected": -1.583255648612976, "logps/chosen": -295.0341796875, "logps/ref_chosen": -107.85675048828125, "logps/ref_rejected": -92.77056121826172, "logps/rejected": -362.899658203125, "loss": 1.1118, "margin_dpo/margin_mean": 82.9516830444336, "margin_dpo/margin_std": 105.19441986083984, "step": 496 }, { "KL/chosen_KL_mean": -201.352294921875, "KL/mean": -228.52642822265625, "KL/rejected_KL_mean": -255.70054626464844, "KL/std": 106.59402465820312, "epoch": 0.7298091042584435, "fcm_dpo/beta": 0.00431904848664999, "fcm_dpo/delta": 0.05586153268814087, "fcm_dpo/margin": 54.348236083984375, "fcm_dpo/q_t": 0.44529393315315247, "grad_norm": 8.359515190124512, "learning_rate": 1.0451099960308374e-07, "logits/chosen": -1.7472078800201416, "logits/rejected": -1.2990825176239014, "logps/chosen": -293.435546875, "logps/ref_chosen": -92.08322143554688, "logps/ref_rejected": -81.79503631591797, "logps/rejected": -337.4955749511719, "loss": 1.225, "margin_dpo/margin_mean": 54.348236083984375, "margin_dpo/margin_std": 113.9087142944336, "step": 497 }, { "KL/chosen_KL_mean": -169.301025390625, "KL/mean": -214.54832458496094, "KL/rejected_KL_mean": -259.79559326171875, "KL/std": 103.0816650390625, "epoch": 0.7312775330396476, "fcm_dpo/beta": 0.004320803098380566, "fcm_dpo/delta": 0.009169381111860275, "fcm_dpo/margin": 90.49458312988281, "fcm_dpo/q_t": 0.4081946015357971, "grad_norm": 11.365286827087402, "learning_rate": 1.0346929082869641e-07, "logits/chosen": -1.7868220806121826, "logits/rejected": -1.2878594398498535, "logps/chosen": -267.49542236328125, "logps/ref_chosen": -98.19437408447266, "logps/ref_rejected": -90.68745422363281, "logps/rejected": -350.4830627441406, "loss": 1.0885, "margin_dpo/margin_mean": 90.49458312988281, "margin_dpo/margin_std": 111.96771240234375, "step": 498 }, { "KL/chosen_KL_mean": -178.27432250976562, "KL/mean": -225.7821044921875, "KL/rejected_KL_mean": -273.28985595703125, "KL/std": 113.1290283203125, "epoch": 0.7327459618208517, "fcm_dpo/beta": 0.004321195185184479, "fcm_dpo/delta": -0.010804334655404091, "fcm_dpo/margin": 95.01554870605469, "fcm_dpo/q_t": 0.4047010540962219, "grad_norm": 10.275177001953125, "learning_rate": 1.0243144327477013e-07, "logits/chosen": -2.10418701171875, "logits/rejected": -2.0533480644226074, "logps/chosen": -259.3142395019531, "logps/ref_chosen": -81.0399169921875, "logps/ref_rejected": -106.92170715332031, "logps/rejected": -380.2115783691406, "loss": 1.0809, "margin_dpo/margin_mean": 95.01554107666016, "margin_dpo/margin_std": 120.46728515625, "step": 499 }, { "KL/chosen_KL_mean": -198.32318115234375, "KL/mean": -239.4547882080078, "KL/rejected_KL_mean": -280.58636474609375, "KL/std": 108.84161376953125, "epoch": 0.7342143906020558, "fcm_dpo/beta": 0.004330311436206102, "fcm_dpo/delta": 0.04458080977201462, "fcm_dpo/margin": 82.26321411132812, "fcm_dpo/q_t": 0.41844114661216736, "grad_norm": 11.698296546936035, "learning_rate": 1.0139748428955333e-07, "logits/chosen": -1.834825038909912, "logits/rejected": -1.6684452295303345, "logps/chosen": -287.57122802734375, "logps/ref_chosen": -89.248046875, "logps/ref_rejected": -100.41021728515625, "logps/rejected": -380.99658203125, "loss": 1.1281, "margin_dpo/margin_mean": 82.26319885253906, "margin_dpo/margin_std": 122.76046752929688, "step": 500 }, { "KL/chosen_KL_mean": -173.25103759765625, "KL/mean": -218.40542602539062, "KL/rejected_KL_mean": -263.559814453125, "KL/std": 118.0541000366211, "epoch": 0.73568281938326, "fcm_dpo/beta": 0.004355341196060181, "fcm_dpo/delta": 0.006262555718421936, "fcm_dpo/margin": 90.30876159667969, "fcm_dpo/q_t": 0.40993058681488037, "grad_norm": 9.389232635498047, "learning_rate": 1.0036744111882672e-07, "logits/chosen": -1.5474653244018555, "logits/rejected": -1.10724937915802, "logps/chosen": -263.84796142578125, "logps/ref_chosen": -90.596923828125, "logps/ref_rejected": -85.2972640991211, "logps/rejected": -348.8570861816406, "loss": 1.1131, "margin_dpo/margin_mean": 90.30876159667969, "margin_dpo/margin_std": 131.63571166992188, "step": 501 }, { "KL/chosen_KL_mean": -174.43557739257812, "KL/mean": -214.81771850585938, "KL/rejected_KL_mean": -255.1998748779297, "KL/std": 98.42835998535156, "epoch": 0.737151248164464, "fcm_dpo/beta": 0.00436469865962863, "fcm_dpo/delta": 0.04808322340250015, "fcm_dpo/margin": 80.76429748535156, "fcm_dpo/q_t": 0.41761714220046997, "grad_norm": 9.52171516418457, "learning_rate": 9.934134090518592e-08, "logits/chosen": -1.9332880973815918, "logits/rejected": -1.2001639604568481, "logps/chosen": -277.90631103515625, "logps/ref_chosen": -103.47073364257812, "logps/ref_rejected": -89.04415893554688, "logps/rejected": -344.2440185546875, "loss": 1.1334, "margin_dpo/margin_mean": 80.76428985595703, "margin_dpo/margin_std": 121.49900817871094, "step": 502 }, { "KL/chosen_KL_mean": -173.05178833007812, "KL/mean": -219.31829833984375, "KL/rejected_KL_mean": -265.58477783203125, "KL/std": 104.69598388671875, "epoch": 0.7386196769456681, "fcm_dpo/beta": 0.004369216971099377, "fcm_dpo/delta": -0.0045638903975486755, "fcm_dpo/margin": 92.53302001953125, "fcm_dpo/q_t": 0.4057713747024536, "grad_norm": 9.045948028564453, "learning_rate": 9.831921068732571e-08, "logits/chosen": -1.4325740337371826, "logits/rejected": -0.9047003984451294, "logps/chosen": -266.7817687988281, "logps/ref_chosen": -93.72998809814453, "logps/ref_rejected": -87.60896301269531, "logps/rejected": -353.1937561035156, "loss": 1.0839, "margin_dpo/margin_mean": 92.53302001953125, "margin_dpo/margin_std": 114.78254699707031, "step": 503 }, { "KL/chosen_KL_mean": -174.3057861328125, "KL/mean": -233.30015563964844, "KL/rejected_KL_mean": -292.2945251464844, "KL/std": 108.99729919433594, "epoch": 0.7400881057268722, "fcm_dpo/beta": 0.004325555171817541, "fcm_dpo/delta": -0.11319869011640549, "fcm_dpo/margin": 117.98873901367188, "fcm_dpo/q_t": 0.3803535997867584, "grad_norm": 10.324762344360352, "learning_rate": 9.730107739932805e-08, "logits/chosen": -2.1881322860717773, "logits/rejected": -1.8581938743591309, "logps/chosen": -266.59765625, "logps/ref_chosen": -92.2918701171875, "logps/ref_rejected": -109.03765106201172, "logps/rejected": -401.3321533203125, "loss": 0.9932, "margin_dpo/margin_mean": 117.98873901367188, "margin_dpo/margin_std": 109.99915313720703, "step": 504 }, { "KL/chosen_KL_mean": -191.95904541015625, "KL/mean": -234.58697509765625, "KL/rejected_KL_mean": -277.21490478515625, "KL/std": 108.44577026367188, "epoch": 0.7415565345080763, "fcm_dpo/beta": 0.0043198708444833755, "fcm_dpo/delta": 0.03230484575033188, "fcm_dpo/margin": 85.25588989257812, "fcm_dpo/q_t": 0.4135434627532959, "grad_norm": 8.236823081970215, "learning_rate": 9.628696786995188e-08, "logits/chosen": -2.4303483963012695, "logits/rejected": -1.8763779401779175, "logps/chosen": -298.2359619140625, "logps/ref_chosen": -106.27693176269531, "logps/ref_rejected": -94.20611572265625, "logps/rejected": -371.4210205078125, "loss": 1.1014, "margin_dpo/margin_mean": 85.25588989257812, "margin_dpo/margin_std": 105.20756530761719, "step": 505 }, { "KL/chosen_KL_mean": -185.80029296875, "KL/mean": -233.56759643554688, "KL/rejected_KL_mean": -281.3349304199219, "KL/std": 102.73516845703125, "epoch": 0.7430249632892805, "fcm_dpo/beta": 0.004312439821660519, "fcm_dpo/delta": -0.012391077354550362, "fcm_dpo/margin": 95.53462219238281, "fcm_dpo/q_t": 0.40235525369644165, "grad_norm": 9.178267478942871, "learning_rate": 9.527690882192635e-08, "logits/chosen": -2.01747465133667, "logits/rejected": -1.659229040145874, "logps/chosen": -267.24114990234375, "logps/ref_chosen": -81.44086456298828, "logps/ref_rejected": -85.66439819335938, "logps/rejected": -366.99932861328125, "loss": 1.0686, "margin_dpo/margin_mean": 95.53462219238281, "margin_dpo/margin_std": 107.47976684570312, "step": 506 }, { "KL/chosen_KL_mean": -195.15692138671875, "KL/mean": -243.4122772216797, "KL/rejected_KL_mean": -291.6676330566406, "KL/std": 106.70512390136719, "epoch": 0.7444933920704846, "fcm_dpo/beta": 0.004314038436859846, "fcm_dpo/delta": -0.016697021201252937, "fcm_dpo/margin": 96.5107192993164, "fcm_dpo/q_t": 0.4039158225059509, "grad_norm": 9.299942970275879, "learning_rate": 9.427092687124691e-08, "logits/chosen": -2.117767810821533, "logits/rejected": -1.6730823516845703, "logps/chosen": -294.5838623046875, "logps/ref_chosen": -99.42694854736328, "logps/ref_rejected": -102.25296020507812, "logps/rejected": -393.92059326171875, "loss": 1.0847, "margin_dpo/margin_mean": 96.51071166992188, "margin_dpo/margin_std": 126.58059692382812, "step": 507 }, { "KL/chosen_KL_mean": -189.24838256835938, "KL/mean": -229.632568359375, "KL/rejected_KL_mean": -270.0167236328125, "KL/std": 115.06764221191406, "epoch": 0.7459618208516887, "fcm_dpo/beta": 0.0043282099068164825, "fcm_dpo/delta": 0.051314108073711395, "fcm_dpo/margin": 80.76837921142578, "fcm_dpo/q_t": 0.4202200770378113, "grad_norm": 15.795506477355957, "learning_rate": 9.326904852647344e-08, "logits/chosen": -1.964473009109497, "logits/rejected": -1.4738422632217407, "logps/chosen": -287.1640319824219, "logps/ref_chosen": -97.9156494140625, "logps/ref_rejected": -103.79782104492188, "logps/rejected": -373.8145751953125, "loss": 1.1489, "margin_dpo/margin_mean": 80.76837921142578, "margin_dpo/margin_std": 133.6261749267578, "step": 508 }, { "KL/chosen_KL_mean": -175.0009765625, "KL/mean": -213.0543212890625, "KL/rejected_KL_mean": -251.10763549804688, "KL/std": 90.32614135742188, "epoch": 0.7474302496328928, "fcm_dpo/beta": 0.004359084647148848, "fcm_dpo/delta": 0.06933551281690598, "fcm_dpo/margin": 76.10665893554688, "fcm_dpo/q_t": 0.4213906526565552, "grad_norm": 15.046432495117188, "learning_rate": 9.227130018803195e-08, "logits/chosen": -2.1193723678588867, "logits/rejected": -1.4645274877548218, "logps/chosen": -275.7308349609375, "logps/ref_chosen": -100.72984313964844, "logps/ref_rejected": -86.943359375, "logps/rejected": -338.0509948730469, "loss": 1.1474, "margin_dpo/margin_mean": 76.10665893554688, "margin_dpo/margin_std": 119.12760925292969, "step": 509 }, { "KL/chosen_KL_mean": -208.4959716796875, "KL/mean": -253.7179718017578, "KL/rejected_KL_mean": -298.94000244140625, "KL/std": 108.95396423339844, "epoch": 0.748898678414097, "fcm_dpo/beta": 0.004369454458355904, "fcm_dpo/delta": 0.0048688193783164024, "fcm_dpo/margin": 90.44403839111328, "fcm_dpo/q_t": 0.40750959515571594, "grad_norm": 12.02993106842041, "learning_rate": 9.127770814751932e-08, "logits/chosen": -1.562624454498291, "logits/rejected": -1.3781036138534546, "logps/chosen": -297.66534423828125, "logps/ref_chosen": -89.16938781738281, "logps/ref_rejected": -107.91939544677734, "logps/rejected": -406.859375, "loss": 1.0869, "margin_dpo/margin_mean": 90.44403076171875, "margin_dpo/margin_std": 109.98480987548828, "step": 510 }, { "KL/chosen_KL_mean": -190.4791259765625, "KL/mean": -239.8486328125, "KL/rejected_KL_mean": -289.2181396484375, "KL/std": 106.54054260253906, "epoch": 0.750367107195301, "fcm_dpo/beta": 0.004360137972980738, "fcm_dpo/delta": -0.03119494952261448, "fcm_dpo/margin": 98.73902130126953, "fcm_dpo/q_t": 0.40058374404907227, "grad_norm": 11.539670944213867, "learning_rate": 9.028829858700973e-08, "logits/chosen": -2.3088908195495605, "logits/rejected": -1.7543621063232422, "logps/chosen": -280.2289123535156, "logps/ref_chosen": -89.74979400634766, "logps/ref_rejected": -97.73127746582031, "logps/rejected": -386.9494323730469, "loss": 1.0643, "margin_dpo/margin_mean": 98.73902893066406, "margin_dpo/margin_std": 118.61759948730469, "step": 511 }, { "KL/chosen_KL_mean": -180.93002319335938, "KL/mean": -228.41734313964844, "KL/rejected_KL_mean": -275.9046630859375, "KL/std": 104.64901733398438, "epoch": 0.7518355359765051, "fcm_dpo/beta": 0.004358671605587006, "fcm_dpo/delta": -0.014475390315055847, "fcm_dpo/margin": 94.97463989257812, "fcm_dpo/q_t": 0.40367159247398376, "grad_norm": 14.001425743103027, "learning_rate": 8.930309757836516e-08, "logits/chosen": -1.8658201694488525, "logits/rejected": -1.4259169101715088, "logps/chosen": -266.6651611328125, "logps/ref_chosen": -85.73515319824219, "logps/ref_rejected": -87.6248779296875, "logps/rejected": -363.529541015625, "loss": 1.0859, "margin_dpo/margin_mean": 94.97463989257812, "margin_dpo/margin_std": 123.84884643554688, "step": 512 }, { "KL/chosen_KL_mean": -191.44728088378906, "KL/mean": -231.55633544921875, "KL/rejected_KL_mean": -271.6654052734375, "KL/std": 100.33480072021484, "epoch": 0.7533039647577092, "fcm_dpo/beta": 0.004353723954409361, "fcm_dpo/delta": 0.051581744104623795, "fcm_dpo/margin": 80.21810913085938, "fcm_dpo/q_t": 0.41749680042266846, "grad_norm": 11.111223220825195, "learning_rate": 8.832213108254863e-08, "logits/chosen": -2.4596195220947266, "logits/rejected": -1.7820154428482056, "logps/chosen": -287.6377868652344, "logps/ref_chosen": -96.19050598144531, "logps/ref_rejected": -82.91773986816406, "logps/rejected": -354.5831298828125, "loss": 1.1276, "margin_dpo/margin_mean": 80.21810913085938, "margin_dpo/margin_std": 112.85614013671875, "step": 513 }, { "KL/chosen_KL_mean": -185.30006408691406, "KL/mean": -229.09481811523438, "KL/rejected_KL_mean": -272.8895568847656, "KL/std": 100.63801574707031, "epoch": 0.7547723935389133, "fcm_dpo/beta": 0.004374968819320202, "fcm_dpo/delta": 0.017122941091656685, "fcm_dpo/margin": 87.5894775390625, "fcm_dpo/q_t": 0.4115946888923645, "grad_norm": 11.036741256713867, "learning_rate": 8.734542494893954e-08, "logits/chosen": -1.8064651489257812, "logits/rejected": -1.2432509660720825, "logps/chosen": -283.12371826171875, "logps/ref_chosen": -97.82363891601562, "logps/ref_rejected": -85.36982727050781, "logps/rejected": -358.2593994140625, "loss": 1.1017, "margin_dpo/margin_mean": 87.5894775390625, "margin_dpo/margin_std": 117.62734985351562, "step": 514 }, { "KL/chosen_KL_mean": -178.0718994140625, "KL/mean": -220.32586669921875, "KL/rejected_KL_mean": -262.579833984375, "KL/std": 104.15945434570312, "epoch": 0.7562408223201175, "fcm_dpo/beta": 0.004385504871606827, "fcm_dpo/delta": 0.029947441071271896, "fcm_dpo/margin": 84.50794982910156, "fcm_dpo/q_t": 0.4135631322860718, "grad_norm": 12.009663581848145, "learning_rate": 8.637300491465272e-08, "logits/chosen": -1.6973528861999512, "logits/rejected": -1.4620082378387451, "logps/chosen": -260.00543212890625, "logps/ref_chosen": -81.93355560302734, "logps/ref_rejected": -93.57656860351562, "logps/rejected": -356.1564025878906, "loss": 1.1104, "margin_dpo/margin_mean": 84.50796508789062, "margin_dpo/margin_std": 113.95793151855469, "step": 515 }, { "KL/chosen_KL_mean": -182.89695739746094, "KL/mean": -223.70980834960938, "KL/rejected_KL_mean": -264.52264404296875, "KL/std": 120.03334045410156, "epoch": 0.7577092511013216, "fcm_dpo/beta": 0.004405728541314602, "fcm_dpo/delta": 0.04107276350259781, "fcm_dpo/margin": 81.62566375732422, "fcm_dpo/q_t": 0.41856199502944946, "grad_norm": 11.645223617553711, "learning_rate": 8.540489660386064e-08, "logits/chosen": -2.290118455886841, "logits/rejected": -2.1101884841918945, "logps/chosen": -279.9886474609375, "logps/ref_chosen": -97.09166717529297, "logps/ref_rejected": -118.55667114257812, "logps/rejected": -383.0793151855469, "loss": 1.1493, "margin_dpo/margin_mean": 81.62567138671875, "margin_dpo/margin_std": 141.0421142578125, "step": 516 }, { "KL/chosen_KL_mean": -195.79498291015625, "KL/mean": -245.7969970703125, "KL/rejected_KL_mean": -295.7990417480469, "KL/std": 107.49064636230469, "epoch": 0.7591776798825257, "fcm_dpo/beta": 0.004379909951239824, "fcm_dpo/delta": -0.03942904993891716, "fcm_dpo/margin": 100.00407409667969, "fcm_dpo/q_t": 0.3989553451538086, "grad_norm": 9.119050025939941, "learning_rate": 8.444112552711752e-08, "logits/chosen": -1.8042175769805908, "logits/rejected": -1.371110439300537, "logps/chosen": -288.9815368652344, "logps/ref_chosen": -93.18656921386719, "logps/ref_rejected": -95.5927963256836, "logps/rejected": -391.391845703125, "loss": 1.0679, "margin_dpo/margin_mean": 100.00407409667969, "margin_dpo/margin_std": 122.32931518554688, "step": 517 }, { "KL/chosen_KL_mean": -203.226806640625, "KL/mean": -240.835205078125, "KL/rejected_KL_mean": -278.4435729980469, "KL/std": 102.02003479003906, "epoch": 0.7606461086637298, "fcm_dpo/beta": 0.004411557223647833, "fcm_dpo/delta": 0.06934365630149841, "fcm_dpo/margin": 75.21675872802734, "fcm_dpo/q_t": 0.42382487654685974, "grad_norm": 9.899693489074707, "learning_rate": 8.348171708068747e-08, "logits/chosen": -2.000744342803955, "logits/rejected": -1.6752307415008545, "logps/chosen": -290.2481689453125, "logps/ref_chosen": -87.0213623046875, "logps/ref_rejected": -91.24456024169922, "logps/rejected": -369.6881408691406, "loss": 1.1541, "margin_dpo/margin_mean": 75.21676635742188, "margin_dpo/margin_std": 125.54377746582031, "step": 518 }, { "KL/chosen_KL_mean": -189.05581665039062, "KL/mean": -229.63177490234375, "KL/rejected_KL_mean": -270.2077331542969, "KL/std": 95.0452880859375, "epoch": 0.762114537444934, "fcm_dpo/beta": 0.004429070744663477, "fcm_dpo/delta": 0.041308604180812836, "fcm_dpo/margin": 81.15189361572266, "fcm_dpo/q_t": 0.4158271849155426, "grad_norm": 10.189618110656738, "learning_rate": 8.25266965458755e-08, "logits/chosen": -2.021937370300293, "logits/rejected": -1.6529254913330078, "logps/chosen": -289.7433776855469, "logps/ref_chosen": -100.68755340576172, "logps/ref_rejected": -109.03042602539062, "logps/rejected": -379.2381591796875, "loss": 1.1056, "margin_dpo/margin_mean": 81.15189361572266, "margin_dpo/margin_std": 99.58242797851562, "step": 519 }, { "KL/chosen_KL_mean": -175.2655029296875, "KL/mean": -224.4226531982422, "KL/rejected_KL_mean": -273.57977294921875, "KL/std": 106.88964080810547, "epoch": 0.7635829662261381, "fcm_dpo/beta": 0.004423252306878567, "fcm_dpo/delta": -0.03570302203297615, "fcm_dpo/margin": 98.31427001953125, "fcm_dpo/q_t": 0.3976425528526306, "grad_norm": 9.430007934570312, "learning_rate": 8.15760890883607e-08, "logits/chosen": -1.6200438737869263, "logits/rejected": -1.1836421489715576, "logps/chosen": -274.04315185546875, "logps/ref_chosen": -98.77762603759766, "logps/ref_rejected": -106.9457015991211, "logps/rejected": -380.5254821777344, "loss": 1.0481, "margin_dpo/margin_mean": 98.31427001953125, "margin_dpo/margin_std": 103.47676086425781, "step": 520 }, { "KL/chosen_KL_mean": -176.67758178710938, "KL/mean": -226.67919921875, "KL/rejected_KL_mean": -276.6808166503906, "KL/std": 100.22225952148438, "epoch": 0.7650513950073421, "fcm_dpo/beta": 0.004418683238327503, "fcm_dpo/delta": -0.04293358698487282, "fcm_dpo/margin": 100.00321960449219, "fcm_dpo/q_t": 0.39646491408348083, "grad_norm": 10.494758605957031, "learning_rate": 8.062991975753378e-08, "logits/chosen": -1.9101364612579346, "logits/rejected": -1.5698418617248535, "logps/chosen": -263.0286865234375, "logps/ref_chosen": -86.35108947753906, "logps/ref_rejected": -90.39317321777344, "logps/rejected": -367.073974609375, "loss": 1.042, "margin_dpo/margin_mean": 100.00321960449219, "margin_dpo/margin_std": 101.9517822265625, "step": 521 }, { "KL/chosen_KL_mean": -183.92930603027344, "KL/mean": -228.66050720214844, "KL/rejected_KL_mean": -273.3917236328125, "KL/std": 109.4559326171875, "epoch": 0.7665198237885462, "fcm_dpo/beta": 0.004409389104694128, "fcm_dpo/delta": 0.005606852471828461, "fcm_dpo/margin": 89.46241760253906, "fcm_dpo/q_t": 0.4081873297691345, "grad_norm": 9.874151229858398, "learning_rate": 7.968821348583643e-08, "logits/chosen": -1.723008155822754, "logits/rejected": -1.355360507965088, "logps/chosen": -259.9373474121094, "logps/ref_chosen": -76.008056640625, "logps/ref_rejected": -74.44125366210938, "logps/rejected": -347.8329772949219, "loss": 1.0908, "margin_dpo/margin_mean": 89.46241760253906, "margin_dpo/margin_std": 114.49295806884766, "step": 522 }, { "KL/chosen_KL_mean": -205.46885681152344, "KL/mean": -237.66238403320312, "KL/rejected_KL_mean": -269.85589599609375, "KL/std": 107.57886505126953, "epoch": 0.7679882525697503, "fcm_dpo/beta": 0.004441681317985058, "fcm_dpo/delta": 0.11572961509227753, "fcm_dpo/margin": 64.38703918457031, "fcm_dpo/q_t": 0.4343034625053406, "grad_norm": 12.052689552307129, "learning_rate": 7.875099508810484e-08, "logits/chosen": -1.9469399452209473, "logits/rejected": -1.240065336227417, "logps/chosen": -301.96514892578125, "logps/ref_chosen": -96.49627685546875, "logps/ref_rejected": -91.0152816772461, "logps/rejected": -360.8711853027344, "loss": 1.1942, "margin_dpo/margin_mean": 64.38703918457031, "margin_dpo/margin_std": 125.32069396972656, "step": 523 }, { "KL/chosen_KL_mean": -186.62509155273438, "KL/mean": -232.78353881835938, "KL/rejected_KL_mean": -278.9419860839844, "KL/std": 107.5989761352539, "epoch": 0.7694566813509545, "fcm_dpo/beta": 0.004452683497220278, "fcm_dpo/delta": -0.011297235265374184, "fcm_dpo/margin": 92.31688690185547, "fcm_dpo/q_t": 0.40502554178237915, "grad_norm": 12.514791488647461, "learning_rate": 7.781828926091535e-08, "logits/chosen": -2.059178590774536, "logits/rejected": -1.3506228923797607, "logps/chosen": -290.03057861328125, "logps/ref_chosen": -103.40550994873047, "logps/ref_rejected": -87.91015625, "logps/rejected": -366.85211181640625, "loss": 1.0777, "margin_dpo/margin_mean": 92.31688690185547, "margin_dpo/margin_std": 114.35617065429688, "step": 524 }, { "KL/chosen_KL_mean": -174.86947631835938, "KL/mean": -227.27371215820312, "KL/rejected_KL_mean": -279.6779479980469, "KL/std": 107.77822875976562, "epoch": 0.7709251101321586, "fcm_dpo/beta": 0.004410797264426947, "fcm_dpo/delta": -0.06493005156517029, "fcm_dpo/margin": 104.80848693847656, "fcm_dpo/q_t": 0.3954683840274811, "grad_norm": 10.79555892944336, "learning_rate": 7.689012058193384e-08, "logits/chosen": -1.5946438312530518, "logits/rejected": -1.5078279972076416, "logps/chosen": -259.0759582519531, "logps/ref_chosen": -84.20648193359375, "logps/ref_rejected": -106.26158905029297, "logps/rejected": -385.9395446777344, "loss": 1.0544, "margin_dpo/margin_mean": 104.80848693847656, "margin_dpo/margin_std": 125.32443237304688, "step": 525 }, { "KL/chosen_KL_mean": -180.89666748046875, "KL/mean": -228.4936065673828, "KL/rejected_KL_mean": -276.0905456542969, "KL/std": 102.58419036865234, "epoch": 0.7723935389133627, "fcm_dpo/beta": 0.004417727701365948, "fcm_dpo/delta": -0.020979641005396843, "fcm_dpo/margin": 95.19389343261719, "fcm_dpo/q_t": 0.4024355411529541, "grad_norm": 8.173274040222168, "learning_rate": 7.596651350926836e-08, "logits/chosen": -1.8941781520843506, "logits/rejected": -1.254791259765625, "logps/chosen": -283.75079345703125, "logps/ref_chosen": -102.85411071777344, "logps/ref_rejected": -93.65145874023438, "logps/rejected": -369.74200439453125, "loss": 1.0759, "margin_dpo/margin_mean": 95.19389343261719, "margin_dpo/margin_std": 120.099365234375, "step": 526 }, { "KL/chosen_KL_mean": -182.9990234375, "KL/mean": -231.366455078125, "KL/rejected_KL_mean": -279.73388671875, "KL/std": 107.65803527832031, "epoch": 0.7738619676945668, "fcm_dpo/beta": 0.0044051269069314, "fcm_dpo/delta": -0.026704099029302597, "fcm_dpo/margin": 96.73486328125, "fcm_dpo/q_t": 0.39909684658050537, "grad_norm": 11.291439056396484, "learning_rate": 7.504749238082414e-08, "logits/chosen": -2.989976406097412, "logits/rejected": -2.1545581817626953, "logps/chosen": -279.76446533203125, "logps/ref_chosen": -96.76543426513672, "logps/ref_rejected": -85.39518737792969, "logps/rejected": -365.12908935546875, "loss": 1.0463, "margin_dpo/margin_mean": 96.73486328125, "margin_dpo/margin_std": 94.80209350585938, "step": 527 }, { "KL/chosen_KL_mean": -171.50094604492188, "KL/mean": -219.40159606933594, "KL/rejected_KL_mean": -267.30224609375, "KL/std": 109.06422424316406, "epoch": 0.775330396475771, "fcm_dpo/beta": 0.004398823715746403, "fcm_dpo/delta": -0.02187850885093212, "fcm_dpo/margin": 95.8012924194336, "fcm_dpo/q_t": 0.40233707427978516, "grad_norm": 9.183014869689941, "learning_rate": 7.413308141366254e-08, "logits/chosen": -2.2333383560180664, "logits/rejected": -1.7298436164855957, "logps/chosen": -265.6658630371094, "logps/ref_chosen": -94.1649169921875, "logps/ref_rejected": -100.372314453125, "logps/rejected": -367.674560546875, "loss": 1.077, "margin_dpo/margin_mean": 95.8012924194336, "margin_dpo/margin_std": 122.46337127685547, "step": 528 }, { "KL/chosen_KL_mean": -199.2355499267578, "KL/mean": -242.46295166015625, "KL/rejected_KL_mean": -285.69036865234375, "KL/std": 107.2583999633789, "epoch": 0.7767988252569751, "fcm_dpo/beta": 0.004404405131936073, "fcm_dpo/delta": 0.019524242728948593, "fcm_dpo/margin": 86.454833984375, "fcm_dpo/q_t": 0.4126690626144409, "grad_norm": 10.917290687561035, "learning_rate": 7.322330470336313e-08, "logits/chosen": -1.7952885627746582, "logits/rejected": -1.6185481548309326, "logps/chosen": -280.0123291015625, "logps/ref_chosen": -80.77680206298828, "logps/ref_rejected": -97.191162109375, "logps/rejected": -382.88153076171875, "loss": 1.1145, "margin_dpo/margin_mean": 86.454833984375, "margin_dpo/margin_std": 126.48206329345703, "step": 529 }, { "KL/chosen_KL_mean": -183.493896484375, "KL/mean": -226.79876708984375, "KL/rejected_KL_mean": -270.1036682128906, "KL/std": 104.29458618164062, "epoch": 0.7782672540381792, "fcm_dpo/beta": 0.004401109181344509, "fcm_dpo/delta": 0.01911301724612713, "fcm_dpo/margin": 86.60977172851562, "fcm_dpo/q_t": 0.4109833240509033, "grad_norm": 9.695281028747559, "learning_rate": 7.231818622338822e-08, "logits/chosen": -1.643599033355713, "logits/rejected": -1.3254046440124512, "logps/chosen": -263.2696228027344, "logps/ref_chosen": -79.7757339477539, "logps/ref_rejected": -92.05390167236328, "logps/rejected": -362.1575622558594, "loss": 1.0874, "margin_dpo/margin_mean": 86.60977172851562, "margin_dpo/margin_std": 100.26891326904297, "step": 530 }, { "KL/chosen_KL_mean": -175.6118927001953, "KL/mean": -229.8429412841797, "KL/rejected_KL_mean": -284.073974609375, "KL/std": 109.40030670166016, "epoch": 0.7797356828193832, "fcm_dpo/beta": 0.004383610561490059, "fcm_dpo/delta": -0.0773196667432785, "fcm_dpo/margin": 108.46211242675781, "fcm_dpo/q_t": 0.38822096586227417, "grad_norm": 11.18143367767334, "learning_rate": 7.141774982445147e-08, "logits/chosen": -1.80515718460083, "logits/rejected": -1.1994197368621826, "logps/chosen": -254.93109130859375, "logps/ref_chosen": -79.31919860839844, "logps/ref_rejected": -78.10832214355469, "logps/rejected": -362.18231201171875, "loss": 1.0205, "margin_dpo/margin_mean": 108.46211242675781, "margin_dpo/margin_std": 109.42643737792969, "step": 531 }, { "KL/chosen_KL_mean": -189.3814697265625, "KL/mean": -248.40243530273438, "KL/rejected_KL_mean": -307.4233703613281, "KL/std": 104.97398376464844, "epoch": 0.7812041116005873, "fcm_dpo/beta": 0.004344305023550987, "fcm_dpo/delta": -0.1156969666481018, "fcm_dpo/margin": 118.04192352294922, "fcm_dpo/q_t": 0.38102883100509644, "grad_norm": 11.231595993041992, "learning_rate": 7.052201923388953e-08, "logits/chosen": -1.7117230892181396, "logits/rejected": -1.089500904083252, "logps/chosen": -291.01837158203125, "logps/ref_chosen": -101.63691711425781, "logps/ref_rejected": -93.34539031982422, "logps/rejected": -400.7687683105469, "loss": 0.9999, "margin_dpo/margin_mean": 118.04193115234375, "margin_dpo/margin_std": 116.68435668945312, "step": 532 }, { "KL/chosen_KL_mean": -183.0260467529297, "KL/mean": -217.306396484375, "KL/rejected_KL_mean": -251.58676147460938, "KL/std": 110.54348754882812, "epoch": 0.7826725403817915, "fcm_dpo/beta": 0.004361595027148724, "fcm_dpo/delta": 0.10243275761604309, "fcm_dpo/margin": 68.56072235107422, "fcm_dpo/q_t": 0.4316903054714203, "grad_norm": 11.220250129699707, "learning_rate": 6.963101805503646e-08, "logits/chosen": -2.3838000297546387, "logits/rejected": -1.7806295156478882, "logps/chosen": -273.772705078125, "logps/ref_chosen": -90.74664306640625, "logps/ref_rejected": -82.88848876953125, "logps/rejected": -334.47528076171875, "loss": 1.1722, "margin_dpo/margin_mean": 68.56072235107422, "margin_dpo/margin_std": 118.21772766113281, "step": 533 }, { "KL/chosen_KL_mean": -164.00794982910156, "KL/mean": -214.61422729492188, "KL/rejected_KL_mean": -265.22052001953125, "KL/std": 104.31243133544922, "epoch": 0.7841409691629956, "fcm_dpo/beta": 0.004359339363873005, "fcm_dpo/delta": -0.04213901236653328, "fcm_dpo/margin": 101.21258544921875, "fcm_dpo/q_t": 0.39604300260543823, "grad_norm": 9.108848571777344, "learning_rate": 6.874476976660184e-08, "logits/chosen": -2.0334296226501465, "logits/rejected": -1.6431384086608887, "logps/chosen": -250.6387939453125, "logps/ref_chosen": -86.6308364868164, "logps/ref_rejected": -84.87196350097656, "logps/rejected": -350.0924987792969, "loss": 1.0402, "margin_dpo/margin_mean": 101.21257781982422, "margin_dpo/margin_std": 102.01959228515625, "step": 534 }, { "KL/chosen_KL_mean": -165.27149963378906, "KL/mean": -218.83226013183594, "KL/rejected_KL_mean": -272.39300537109375, "KL/std": 104.81996154785156, "epoch": 0.7856093979441997, "fcm_dpo/beta": 0.0043401820585131645, "fcm_dpo/delta": -0.06652242690324783, "fcm_dpo/margin": 107.12152099609375, "fcm_dpo/q_t": 0.3924905061721802, "grad_norm": 8.032547950744629, "learning_rate": 6.786329772205246e-08, "logits/chosen": -1.6907243728637695, "logits/rejected": -1.5026648044586182, "logps/chosen": -255.61688232421875, "logps/ref_chosen": -90.34539031982422, "logps/ref_rejected": -100.7779769897461, "logps/rejected": -373.1709899902344, "loss": 1.0388, "margin_dpo/margin_mean": 107.12152099609375, "margin_dpo/margin_std": 119.40167236328125, "step": 535 }, { "KL/chosen_KL_mean": -164.34039306640625, "KL/mean": -217.17098999023438, "KL/rejected_KL_mean": -270.0015869140625, "KL/std": 106.03883361816406, "epoch": 0.7870778267254038, "fcm_dpo/beta": 0.0043008942157030106, "fcm_dpo/delta": -0.05580340325832367, "fcm_dpo/margin": 105.66120910644531, "fcm_dpo/q_t": 0.3933941125869751, "grad_norm": 8.979888916015625, "learning_rate": 6.698662514899638e-08, "logits/chosen": -1.9817721843719482, "logits/rejected": -1.7338640689849854, "logps/chosen": -243.29995727539062, "logps/ref_chosen": -78.95956420898438, "logps/ref_rejected": -93.08779907226562, "logps/rejected": -363.08941650390625, "loss": 1.037, "margin_dpo/margin_mean": 105.66120910644531, "margin_dpo/margin_std": 111.1412353515625, "step": 536 }, { "KL/chosen_KL_mean": -144.0820770263672, "KL/mean": -199.3745880126953, "KL/rejected_KL_mean": -254.6671142578125, "KL/std": 109.31318664550781, "epoch": 0.788546255506608, "fcm_dpo/beta": 0.004287478979676962, "fcm_dpo/delta": -0.07605566084384918, "fcm_dpo/margin": 110.58503723144531, "fcm_dpo/q_t": 0.38893699645996094, "grad_norm": 10.103387832641602, "learning_rate": 6.611477514857114e-08, "logits/chosen": -1.640284538269043, "logits/rejected": -0.9562705755233765, "logps/chosen": -230.79147338867188, "logps/ref_chosen": -86.70939636230469, "logps/ref_rejected": -78.90766906738281, "logps/rejected": -333.57476806640625, "loss": 1.0201, "margin_dpo/margin_mean": 110.58503723144531, "margin_dpo/margin_std": 106.64178466796875, "step": 537 }, { "KL/chosen_KL_mean": -196.79312133789062, "KL/mean": -242.23422241210938, "KL/rejected_KL_mean": -287.6753234863281, "KL/std": 97.90306091308594, "epoch": 0.7900146842878121, "fcm_dpo/beta": 0.004270879551768303, "fcm_dpo/delta": 0.012086473405361176, "fcm_dpo/margin": 90.8821792602539, "fcm_dpo/q_t": 0.4087004065513611, "grad_norm": 9.069642066955566, "learning_rate": 6.524777069483525e-08, "logits/chosen": -1.8402318954467773, "logits/rejected": -1.3024128675460815, "logps/chosen": -295.23529052734375, "logps/ref_chosen": -98.44214630126953, "logps/ref_rejected": -91.09465026855469, "logps/rejected": -378.76995849609375, "loss": 1.0826, "margin_dpo/margin_mean": 90.88218688964844, "margin_dpo/margin_std": 104.44237518310547, "step": 538 }, { "KL/chosen_KL_mean": -171.3607635498047, "KL/mean": -218.93344116210938, "KL/rejected_KL_mean": -266.506103515625, "KL/std": 107.00035095214844, "epoch": 0.7914831130690162, "fcm_dpo/beta": 0.004270798526704311, "fcm_dpo/delta": -0.0064790137112140656, "fcm_dpo/margin": 95.14533996582031, "fcm_dpo/q_t": 0.4053787589073181, "grad_norm": 8.477505683898926, "learning_rate": 6.438563463416221e-08, "logits/chosen": -2.4220075607299805, "logits/rejected": -1.9417331218719482, "logps/chosen": -262.2928771972656, "logps/ref_chosen": -90.93211364746094, "logps/ref_rejected": -97.44546508789062, "logps/rejected": -363.9515686035156, "loss": 1.0848, "margin_dpo/margin_mean": 95.14533996582031, "margin_dpo/margin_std": 122.10641479492188, "step": 539 }, { "KL/chosen_KL_mean": -183.33441162109375, "KL/mean": -241.50558471679688, "KL/rejected_KL_mean": -299.6767578125, "KL/std": 116.54806518554688, "epoch": 0.7929515418502202, "fcm_dpo/beta": 0.004247043281793594, "fcm_dpo/delta": -0.09646552801132202, "fcm_dpo/margin": 116.34233856201172, "fcm_dpo/q_t": 0.3877401351928711, "grad_norm": 10.630231857299805, "learning_rate": 6.352838968463919e-08, "logits/chosen": -2.0335841178894043, "logits/rejected": -1.7958580255508423, "logps/chosen": -291.2043151855469, "logps/ref_chosen": -107.86990356445312, "logps/ref_rejected": -121.39350128173828, "logps/rejected": -421.07025146484375, "loss": 1.0295, "margin_dpo/margin_mean": 116.34233093261719, "margin_dpo/margin_std": 136.109375, "step": 540 }, { "KL/chosen_KL_mean": -203.08187866210938, "KL/mean": -239.1380615234375, "KL/rejected_KL_mean": -275.1942443847656, "KL/std": 96.72978210449219, "epoch": 0.7944199706314243, "fcm_dpo/beta": 0.004223422612994909, "fcm_dpo/delta": -0.01101903896778822, "fcm_dpo/margin": 72.11237335205078, "fcm_dpo/q_t": 0.42861074209213257, "grad_norm": 9.924826622009277, "learning_rate": 6.267605843546767e-08, "logits/chosen": -2.4393324851989746, "logits/rejected": -1.981326937675476, "logps/chosen": -309.64935302734375, "logps/ref_chosen": -106.56748962402344, "logps/ref_rejected": -109.06633758544922, "logps/rejected": -384.2605895996094, "loss": 1.156, "margin_dpo/margin_mean": 72.11236572265625, "margin_dpo/margin_std": 103.96543884277344, "step": 541 }, { "KL/chosen_KL_mean": -173.12039184570312, "KL/mean": -228.54608154296875, "KL/rejected_KL_mean": -283.97174072265625, "KL/std": 109.05888366699219, "epoch": 0.7958883994126285, "fcm_dpo/beta": 0.0042023384012281895, "fcm_dpo/delta": -0.06740282475948334, "fcm_dpo/margin": 110.85132598876953, "fcm_dpo/q_t": 0.39074161648750305, "grad_norm": 10.293766975402832, "learning_rate": 6.182866334636888e-08, "logits/chosen": -2.1644296646118164, "logits/rejected": -2.0390355587005615, "logps/chosen": -260.48968505859375, "logps/ref_chosen": -87.36929321289062, "logps/ref_rejected": -102.82034301757812, "logps/rejected": -386.7920837402344, "loss": 1.0316, "margin_dpo/margin_mean": 110.85132598876953, "margin_dpo/margin_std": 117.49456024169922, "step": 542 }, { "KL/chosen_KL_mean": -198.0701904296875, "KL/mean": -227.83261108398438, "KL/rejected_KL_mean": -257.5950012207031, "KL/std": 115.06761169433594, "epoch": 0.7973568281938326, "fcm_dpo/beta": 0.004211590625345707, "fcm_dpo/delta": 0.039348069578409195, "fcm_dpo/margin": 59.524818420410156, "fcm_dpo/q_t": 0.44260284304618835, "grad_norm": 12.425097465515137, "learning_rate": 6.098622674699147e-08, "logits/chosen": -1.8456076383590698, "logits/rejected": -1.8888741731643677, "logps/chosen": -285.2227783203125, "logps/ref_chosen": -87.152587890625, "logps/ref_rejected": -111.53172302246094, "logps/rejected": -369.126708984375, "loss": 1.2339, "margin_dpo/margin_mean": 59.524810791015625, "margin_dpo/margin_std": 139.34402465820312, "step": 543 }, { "KL/chosen_KL_mean": -207.41098022460938, "KL/mean": -244.48147583007812, "KL/rejected_KL_mean": -281.5519714355469, "KL/std": 107.93679809570312, "epoch": 0.7988252569750367, "fcm_dpo/beta": 0.004237482324242592, "fcm_dpo/delta": 0.08722686767578125, "fcm_dpo/margin": 74.14102172851562, "fcm_dpo/q_t": 0.4270614981651306, "grad_norm": 9.349100112915039, "learning_rate": 6.01487708363232e-08, "logits/chosen": -1.62346351146698, "logits/rejected": -1.5277810096740723, "logps/chosen": -293.5578918457031, "logps/ref_chosen": -86.14691162109375, "logps/ref_rejected": -106.3939208984375, "logps/rejected": -387.9458923339844, "loss": 1.1506, "margin_dpo/margin_mean": 74.14102172851562, "margin_dpo/margin_std": 113.228759765625, "step": 544 }, { "KL/chosen_KL_mean": -184.37823486328125, "KL/mean": -229.91763305664062, "KL/rejected_KL_mean": -275.4570007324219, "KL/std": 107.56153869628906, "epoch": 0.8002936857562408, "fcm_dpo/beta": 0.004255616571754217, "fcm_dpo/delta": 0.012617932632565498, "fcm_dpo/margin": 91.07875061035156, "fcm_dpo/q_t": 0.4101194441318512, "grad_norm": 9.380599021911621, "learning_rate": 5.9316317682106294e-08, "logits/chosen": -1.5111886262893677, "logits/rejected": -1.0829861164093018, "logps/chosen": -280.802490234375, "logps/ref_chosen": -96.42424011230469, "logps/ref_rejected": -100.73405456542969, "logps/rejected": -376.1910400390625, "loss": 1.1123, "margin_dpo/margin_mean": 91.07874298095703, "margin_dpo/margin_std": 133.35379028320312, "step": 545 }, { "KL/chosen_KL_mean": -189.38973999023438, "KL/mean": -231.05819702148438, "KL/rejected_KL_mean": -272.72662353515625, "KL/std": 108.98518371582031, "epoch": 0.801762114537445, "fcm_dpo/beta": 0.004264291375875473, "fcm_dpo/delta": 0.04544122517108917, "fcm_dpo/margin": 83.33690643310547, "fcm_dpo/q_t": 0.41621696949005127, "grad_norm": 8.938708305358887, "learning_rate": 5.848888922025552e-08, "logits/chosen": -1.4880073070526123, "logits/rejected": -1.1289596557617188, "logps/chosen": -275.410888671875, "logps/ref_chosen": -86.02112579345703, "logps/ref_rejected": -87.37263488769531, "logps/rejected": -360.0992736816406, "loss": 1.1244, "margin_dpo/margin_mean": 83.33690643310547, "margin_dpo/margin_std": 119.4713134765625, "step": 546 }, { "KL/chosen_KL_mean": -182.38621520996094, "KL/mean": -232.0530242919922, "KL/rejected_KL_mean": -281.7198486328125, "KL/std": 106.35832214355469, "epoch": 0.8032305433186491, "fcm_dpo/beta": 0.0042706625536084175, "fcm_dpo/delta": -0.024777617305517197, "fcm_dpo/margin": 99.33365631103516, "fcm_dpo/q_t": 0.3994084596633911, "grad_norm": 9.093918800354004, "learning_rate": 5.7666507254280265e-08, "logits/chosen": -1.9151792526245117, "logits/rejected": -1.4566993713378906, "logps/chosen": -283.0372314453125, "logps/ref_chosen": -100.6510009765625, "logps/ref_rejected": -97.9117431640625, "logps/rejected": -379.631591796875, "loss": 1.046, "margin_dpo/margin_mean": 99.33365631103516, "margin_dpo/margin_std": 94.48394775390625, "step": 547 }, { "KL/chosen_KL_mean": -179.056884765625, "KL/mean": -219.66635131835938, "KL/rejected_KL_mean": -260.27581787109375, "KL/std": 108.4866943359375, "epoch": 0.8046989720998532, "fcm_dpo/beta": 0.004280437249690294, "fcm_dpo/delta": 0.05326255410909653, "fcm_dpo/margin": 81.21894073486328, "fcm_dpo/q_t": 0.4195561408996582, "grad_norm": 10.500082969665527, "learning_rate": 5.684919345471029e-08, "logits/chosen": -2.185760974884033, "logits/rejected": -1.824209451675415, "logps/chosen": -275.08697509765625, "logps/ref_chosen": -96.03010559082031, "logps/ref_rejected": -98.41322326660156, "logps/rejected": -358.68902587890625, "loss": 1.1493, "margin_dpo/margin_mean": 81.21894073486328, "margin_dpo/margin_std": 135.64102172851562, "step": 548 }, { "KL/chosen_KL_mean": -168.52032470703125, "KL/mean": -211.0810546875, "KL/rejected_KL_mean": -253.64183044433594, "KL/std": 103.16926574707031, "epoch": 0.8061674008810573, "fcm_dpo/beta": 0.004305689129978418, "fcm_dpo/delta": 0.03385113924741745, "fcm_dpo/margin": 85.12151336669922, "fcm_dpo/q_t": 0.4143906533718109, "grad_norm": 10.721205711364746, "learning_rate": 5.603696935852426e-08, "logits/chosen": -1.8859022855758667, "logits/rejected": -1.4682605266571045, "logps/chosen": -251.97128295898438, "logps/ref_chosen": -83.45095825195312, "logps/ref_rejected": -81.38023376464844, "logps/rejected": -335.0220642089844, "loss": 1.1272, "margin_dpo/margin_mean": 85.12151336669922, "margin_dpo/margin_std": 126.7396240234375, "step": 549 }, { "KL/chosen_KL_mean": -188.04991149902344, "KL/mean": -229.26483154296875, "KL/rejected_KL_mean": -270.479736328125, "KL/std": 98.27571868896484, "epoch": 0.8076358296622613, "fcm_dpo/beta": 0.0043181851506233215, "fcm_dpo/delta": 0.04477984458208084, "fcm_dpo/margin": 82.4298095703125, "fcm_dpo/q_t": 0.416768878698349, "grad_norm": 9.057662010192871, "learning_rate": 5.5229856368582376e-08, "logits/chosen": -1.8359873294830322, "logits/rejected": -1.626725673675537, "logps/chosen": -277.6442565917969, "logps/ref_chosen": -89.59434509277344, "logps/ref_rejected": -101.31549835205078, "logps/rejected": -371.79522705078125, "loss": 1.1137, "margin_dpo/margin_mean": 82.42981719970703, "margin_dpo/margin_std": 106.25439453125, "step": 550 }, { "KL/chosen_KL_mean": -168.37030029296875, "KL/mean": -218.2247314453125, "KL/rejected_KL_mean": -268.07916259765625, "KL/std": 112.21926879882812, "epoch": 0.8091042584434655, "fcm_dpo/beta": 0.004299826920032501, "fcm_dpo/delta": -0.02964065968990326, "fcm_dpo/margin": 99.70887756347656, "fcm_dpo/q_t": 0.40124115347862244, "grad_norm": 11.02514934539795, "learning_rate": 5.4427875753062734e-08, "logits/chosen": -1.654052972793579, "logits/rejected": -1.5122342109680176, "logps/chosen": -263.45245361328125, "logps/ref_chosen": -95.08216094970703, "logps/ref_rejected": -117.7192611694336, "logps/rejected": -385.79840087890625, "loss": 1.0597, "margin_dpo/margin_mean": 99.70887756347656, "margin_dpo/margin_std": 113.82139587402344, "step": 551 }, { "KL/chosen_KL_mean": -158.564453125, "KL/mean": -230.02291870117188, "KL/rejected_KL_mean": -301.48138427734375, "KL/std": 121.185302734375, "epoch": 0.8105726872246696, "fcm_dpo/beta": 0.004224185831844807, "fcm_dpo/delta": -0.21129748225212097, "fcm_dpo/margin": 142.9169158935547, "fcm_dpo/q_t": 0.3604808449745178, "grad_norm": 10.975080490112305, "learning_rate": 5.363104864490034e-08, "logits/chosen": -2.1447153091430664, "logits/rejected": -1.785742998123169, "logps/chosen": -259.9736328125, "logps/ref_chosen": -101.40918731689453, "logps/ref_rejected": -110.83843994140625, "logps/rejected": -412.31982421875, "loss": 0.9399, "margin_dpo/margin_mean": 142.91693115234375, "margin_dpo/margin_std": 125.75274658203125, "step": 552 }, { "KL/chosen_KL_mean": -173.323486328125, "KL/mean": -216.4599151611328, "KL/rejected_KL_mean": -259.5963439941406, "KL/std": 112.45269775390625, "epoch": 0.8120411160058737, "fcm_dpo/beta": 0.004230804275721312, "fcm_dpo/delta": 0.035598188638687134, "fcm_dpo/margin": 86.27285766601562, "fcm_dpo/q_t": 0.4143606722354889, "grad_norm": 9.267971992492676, "learning_rate": 5.2839396041230415e-08, "logits/chosen": -1.7019909620285034, "logits/rejected": -1.3816907405853271, "logps/chosen": -271.2873840332031, "logps/ref_chosen": -97.96388244628906, "logps/ref_rejected": -103.57719421386719, "logps/rejected": -363.17352294921875, "loss": 1.1178, "margin_dpo/margin_mean": 86.27285766601562, "margin_dpo/margin_std": 121.21662902832031, "step": 553 }, { "KL/chosen_KL_mean": -182.293701171875, "KL/mean": -229.29205322265625, "KL/rejected_KL_mean": -276.2904052734375, "KL/std": 112.57914733886719, "epoch": 0.8135095447870778, "fcm_dpo/beta": 0.004239792004227638, "fcm_dpo/delta": 0.0013606362044811249, "fcm_dpo/margin": 93.99668884277344, "fcm_dpo/q_t": 0.40841352939605713, "grad_norm": 9.318767547607422, "learning_rate": 5.205293880283551e-08, "logits/chosen": -1.8114211559295654, "logits/rejected": -1.0258686542510986, "logps/chosen": -285.98052978515625, "logps/ref_chosen": -103.68680572509766, "logps/ref_rejected": -92.99210357666016, "logps/rejected": -369.2825012207031, "loss": 1.1162, "margin_dpo/margin_mean": 93.99668884277344, "margin_dpo/margin_std": 144.33462524414062, "step": 554 }, { "KL/chosen_KL_mean": -191.05401611328125, "KL/mean": -234.1144561767578, "KL/rejected_KL_mean": -277.1748962402344, "KL/std": 116.19886016845703, "epoch": 0.8149779735682819, "fcm_dpo/beta": 0.0042431107722222805, "fcm_dpo/delta": 0.03522365912795067, "fcm_dpo/margin": 86.12091064453125, "fcm_dpo/q_t": 0.4163702726364136, "grad_norm": 9.264857292175293, "learning_rate": 5.127169765359515e-08, "logits/chosen": -1.9149367809295654, "logits/rejected": -1.823670744895935, "logps/chosen": -277.25762939453125, "logps/ref_chosen": -86.20362854003906, "logps/ref_rejected": -114.54719543457031, "logps/rejected": -391.72210693359375, "loss": 1.1301, "margin_dpo/margin_mean": 86.12091064453125, "margin_dpo/margin_std": 134.2264404296875, "step": 555 }, { "KL/chosen_KL_mean": -171.4479522705078, "KL/mean": -218.99276733398438, "KL/rejected_KL_mean": -266.53759765625, "KL/std": 103.26528930664062, "epoch": 0.8164464023494861, "fcm_dpo/beta": 0.004240359179675579, "fcm_dpo/delta": -0.0033538993448019028, "fcm_dpo/margin": 95.08967590332031, "fcm_dpo/q_t": 0.40582576394081116, "grad_norm": 10.22527027130127, "learning_rate": 5.049569317994012e-08, "logits/chosen": -1.5934078693389893, "logits/rejected": -1.290602684020996, "logps/chosen": -259.8514404296875, "logps/ref_chosen": -88.40348815917969, "logps/ref_rejected": -106.63911437988281, "logps/rejected": -373.1767272949219, "loss": 1.082, "margin_dpo/margin_mean": 95.08966827392578, "margin_dpo/margin_std": 117.21231079101562, "step": 556 }, { "KL/chosen_KL_mean": -183.21473693847656, "KL/mean": -237.11883544921875, "KL/rejected_KL_mean": -291.0229797363281, "KL/std": 119.51036834716797, "epoch": 0.8179148311306902, "fcm_dpo/beta": 0.004218678921461105, "fcm_dpo/delta": -0.056388579308986664, "fcm_dpo/margin": 107.8082504272461, "fcm_dpo/q_t": 0.3957628607749939, "grad_norm": 7.976409912109375, "learning_rate": 4.9724945830310144e-08, "logits/chosen": -1.9251445531845093, "logits/rejected": -1.7509065866470337, "logps/chosen": -282.04998779296875, "logps/ref_chosen": -98.83523559570312, "logps/ref_rejected": -115.64888763427734, "logps/rejected": -406.671875, "loss": 1.0542, "margin_dpo/margin_mean": 107.80825805664062, "margin_dpo/margin_std": 130.3460693359375, "step": 557 }, { "KL/chosen_KL_mean": -181.15966796875, "KL/mean": -238.71969604492188, "KL/rejected_KL_mean": -296.27972412109375, "KL/std": 102.74571228027344, "epoch": 0.8193832599118943, "fcm_dpo/beta": 0.004191510379314423, "fcm_dpo/delta": -0.08466774225234985, "fcm_dpo/margin": 115.12003326416016, "fcm_dpo/q_t": 0.3868030309677124, "grad_norm": 11.017813682556152, "learning_rate": 4.8959475914614554e-08, "logits/chosen": -2.434208869934082, "logits/rejected": -1.832306146621704, "logps/chosen": -287.1715087890625, "logps/ref_chosen": -106.01183319091797, "logps/ref_rejected": -108.39131927490234, "logps/rejected": -404.6710205078125, "loss": 1.0081, "margin_dpo/margin_mean": 115.12003326416016, "margin_dpo/margin_std": 105.36422729492188, "step": 558 }, { "KL/chosen_KL_mean": -195.71725463867188, "KL/mean": -241.516357421875, "KL/rejected_KL_mean": -287.31549072265625, "KL/std": 105.34667205810547, "epoch": 0.8208516886930984, "fcm_dpo/beta": 0.004193184897303581, "fcm_dpo/delta": 0.01620602048933506, "fcm_dpo/margin": 91.59820556640625, "fcm_dpo/q_t": 0.409600168466568, "grad_norm": 10.939364433288574, "learning_rate": 4.8199303603697614e-08, "logits/chosen": -2.4441542625427246, "logits/rejected": -1.9754533767700195, "logps/chosen": -296.1419982910156, "logps/ref_chosen": -100.42474365234375, "logps/ref_rejected": -100.41270446777344, "logps/rejected": -387.7281799316406, "loss": 1.099, "margin_dpo/margin_mean": 91.59820556640625, "margin_dpo/margin_std": 119.55643463134766, "step": 559 }, { "KL/chosen_KL_mean": -185.2583465576172, "KL/mean": -230.53582763671875, "KL/rejected_KL_mean": -275.8133239746094, "KL/std": 104.00809478759766, "epoch": 0.8223201174743024, "fcm_dpo/beta": 0.004197395406663418, "fcm_dpo/delta": 0.02029186673462391, "fcm_dpo/margin": 90.55494689941406, "fcm_dpo/q_t": 0.4112550616264343, "grad_norm": 12.303892135620117, "learning_rate": 4.7444448928806615e-08, "logits/chosen": -1.384832739830017, "logits/rejected": -0.9386763572692871, "logps/chosen": -272.89385986328125, "logps/ref_chosen": -87.63551330566406, "logps/ref_rejected": -87.37985229492188, "logps/rejected": -363.19317626953125, "loss": 1.1017, "margin_dpo/margin_mean": 90.55494689941406, "margin_dpo/margin_std": 119.59379577636719, "step": 560 }, { "KL/chosen_KL_mean": -195.02647399902344, "KL/mean": -237.00106811523438, "KL/rejected_KL_mean": -278.97564697265625, "KL/std": 102.20809936523438, "epoch": 0.8237885462555066, "fcm_dpo/beta": 0.00421981792896986, "fcm_dpo/delta": 0.046500034630298615, "fcm_dpo/margin": 83.94918823242188, "fcm_dpo/q_t": 0.41637569665908813, "grad_norm": 8.328563690185547, "learning_rate": 4.669493178106432e-08, "logits/chosen": -1.869408369064331, "logits/rejected": -1.8405293226242065, "logps/chosen": -269.9078674316406, "logps/ref_chosen": -74.88140106201172, "logps/ref_rejected": -106.04796600341797, "logps/rejected": -385.02362060546875, "loss": 1.1041, "margin_dpo/margin_mean": 83.94918823242188, "margin_dpo/margin_std": 98.23512268066406, "step": 561 }, { "KL/chosen_KL_mean": -177.64889526367188, "KL/mean": -219.72433471679688, "KL/rejected_KL_mean": -261.7998046875, "KL/std": 103.27224731445312, "epoch": 0.8252569750367107, "fcm_dpo/beta": 0.00422816863283515, "fcm_dpo/delta": 0.04495958983898163, "fcm_dpo/margin": 84.15090942382812, "fcm_dpo/q_t": 0.41772592067718506, "grad_norm": 12.563915252685547, "learning_rate": 4.5950771910944596e-08, "logits/chosen": -2.1260461807250977, "logits/rejected": -1.672189474105835, "logps/chosen": -278.3121032714844, "logps/ref_chosen": -100.66322326660156, "logps/ref_rejected": -102.24087524414062, "logps/rejected": -364.0406494140625, "loss": 1.131, "margin_dpo/margin_mean": 84.15090942382812, "margin_dpo/margin_std": 127.79916381835938, "step": 562 }, { "KL/chosen_KL_mean": -187.5830078125, "KL/mean": -223.40481567382812, "KL/rejected_KL_mean": -259.22662353515625, "KL/std": 104.31047058105469, "epoch": 0.8267254038179148, "fcm_dpo/beta": 0.004276228602975607, "fcm_dpo/delta": 0.09487748146057129, "fcm_dpo/margin": 71.64361572265625, "fcm_dpo/q_t": 0.42809247970581055, "grad_norm": 10.187355041503906, "learning_rate": 4.521198892775202e-08, "logits/chosen": -1.6545624732971191, "logits/rejected": -1.3139811754226685, "logps/chosen": -283.84222412109375, "logps/ref_chosen": -96.25921630859375, "logps/ref_rejected": -101.20503997802734, "logps/rejected": -360.4316711425781, "loss": 1.1574, "margin_dpo/margin_mean": 71.64361572265625, "margin_dpo/margin_std": 110.49944305419922, "step": 563 }, { "KL/chosen_KL_mean": -183.77444458007812, "KL/mean": -223.22848510742188, "KL/rejected_KL_mean": -262.68255615234375, "KL/std": 104.37713623046875, "epoch": 0.8281938325991189, "fcm_dpo/beta": 0.004304289817810059, "fcm_dpo/delta": 0.061300523579120636, "fcm_dpo/margin": 78.90811157226562, "fcm_dpo/q_t": 0.41939252614974976, "grad_norm": 10.905845642089844, "learning_rate": 4.447860229910544e-08, "logits/chosen": -2.345728635787964, "logits/rejected": -1.776198148727417, "logps/chosen": -296.7348327636719, "logps/ref_chosen": -112.96040344238281, "logps/ref_rejected": -99.52803802490234, "logps/rejected": -362.2105712890625, "loss": 1.1162, "margin_dpo/margin_mean": 78.90811157226562, "margin_dpo/margin_std": 94.8890380859375, "step": 564 }, { "KL/chosen_KL_mean": -179.577880859375, "KL/mean": -228.5552978515625, "KL/rejected_KL_mean": -277.5327453613281, "KL/std": 113.28913879394531, "epoch": 0.8296622613803231, "fcm_dpo/beta": 0.004297152627259493, "fcm_dpo/delta": -0.021406136453151703, "fcm_dpo/margin": 97.95487213134766, "fcm_dpo/q_t": 0.4026171565055847, "grad_norm": 9.346363067626953, "learning_rate": 4.375063135042445e-08, "logits/chosen": -1.6725883483886719, "logits/rejected": -1.1641533374786377, "logps/chosen": -274.9434509277344, "logps/ref_chosen": -95.36558532714844, "logps/ref_rejected": -92.21737670898438, "logps/rejected": -369.7501220703125, "loss": 1.0759, "margin_dpo/margin_mean": 97.95486450195312, "margin_dpo/margin_std": 123.97798156738281, "step": 565 }, { "KL/chosen_KL_mean": -187.5893096923828, "KL/mean": -234.23489379882812, "KL/rejected_KL_mean": -280.8804931640625, "KL/std": 115.55814361572266, "epoch": 0.8311306901615272, "fcm_dpo/beta": 0.004301795735955238, "fcm_dpo/delta": -0.0013939663767814636, "fcm_dpo/margin": 93.29119873046875, "fcm_dpo/q_t": 0.4056416153907776, "grad_norm": 8.98111343383789, "learning_rate": 4.3028095264420525e-08, "logits/chosen": -1.9083738327026367, "logits/rejected": -1.4756141901016235, "logps/chosen": -286.5625, "logps/ref_chosen": -98.97320556640625, "logps/ref_rejected": -110.82090759277344, "logps/rejected": -391.701416015625, "loss": 1.083, "margin_dpo/margin_mean": 93.29119873046875, "margin_dpo/margin_std": 114.16362762451172, "step": 566 }, { "KL/chosen_KL_mean": -183.2840576171875, "KL/mean": -224.27145385742188, "KL/rejected_KL_mean": -265.25885009765625, "KL/std": 104.42694091796875, "epoch": 0.8325991189427313, "fcm_dpo/beta": 0.0043027363717556, "fcm_dpo/delta": 0.04803746938705444, "fcm_dpo/margin": 81.97478485107422, "fcm_dpo/q_t": 0.41656142473220825, "grad_norm": 9.797769546508789, "learning_rate": 4.231101308059165e-08, "logits/chosen": -2.3833329677581787, "logits/rejected": -1.7242250442504883, "logps/chosen": -275.9876403808594, "logps/ref_chosen": -92.7035903930664, "logps/ref_rejected": -91.22431945800781, "logps/rejected": -356.483154296875, "loss": 1.1208, "margin_dpo/margin_mean": 81.97478485107422, "margin_dpo/margin_std": 109.92363739013672, "step": 567 }, { "KL/chosen_KL_mean": -173.11724853515625, "KL/mean": -227.53045654296875, "KL/rejected_KL_mean": -281.94366455078125, "KL/std": 109.86747741699219, "epoch": 0.8340675477239354, "fcm_dpo/beta": 0.00428895466029644, "fcm_dpo/delta": -0.06854759901762009, "fcm_dpo/margin": 108.82640075683594, "fcm_dpo/q_t": 0.39120543003082275, "grad_norm": 8.776500701904297, "learning_rate": 4.1599403694720145e-08, "logits/chosen": -1.3140344619750977, "logits/rejected": -1.1812775135040283, "logps/chosen": -246.24951171875, "logps/ref_chosen": -73.13226318359375, "logps/ref_rejected": -94.95265197753906, "logps/rejected": -376.89630126953125, "loss": 1.0268, "margin_dpo/margin_mean": 108.82640075683594, "margin_dpo/margin_std": 111.12168884277344, "step": 568 }, { "KL/chosen_KL_mean": -183.662109375, "KL/mean": -227.85125732421875, "KL/rejected_KL_mean": -272.0404052734375, "KL/std": 111.59596252441406, "epoch": 0.8355359765051396, "fcm_dpo/beta": 0.0042973468080163, "fcm_dpo/delta": 0.020571105182170868, "fcm_dpo/margin": 88.37831115722656, "fcm_dpo/q_t": 0.41141578555107117, "grad_norm": 13.591999053955078, "learning_rate": 4.089328585837512e-08, "logits/chosen": -1.9653756618499756, "logits/rejected": -1.5891880989074707, "logps/chosen": -272.87921142578125, "logps/ref_chosen": -89.21708679199219, "logps/ref_rejected": -86.41105651855469, "logps/rejected": -358.45147705078125, "loss": 1.1216, "margin_dpo/margin_mean": 88.37831115722656, "margin_dpo/margin_std": 133.74679565429688, "step": 569 }, { "KL/chosen_KL_mean": -164.3401641845703, "KL/mean": -221.7617950439453, "KL/rejected_KL_mean": -279.18341064453125, "KL/std": 105.72899627685547, "epoch": 0.8370044052863436, "fcm_dpo/beta": 0.004266203381121159, "fcm_dpo/delta": -0.09223379194736481, "fcm_dpo/margin": 114.8432388305664, "fcm_dpo/q_t": 0.38477516174316406, "grad_norm": 9.2087984085083, "learning_rate": 4.019267817841834e-08, "logits/chosen": -2.3868305683135986, "logits/rejected": -1.711867094039917, "logps/chosen": -259.1029357910156, "logps/ref_chosen": -94.76277160644531, "logps/ref_rejected": -88.846923828125, "logps/rejected": -368.03033447265625, "loss": 1.0078, "margin_dpo/margin_mean": 114.84324645996094, "margin_dpo/margin_std": 110.63545227050781, "step": 570 }, { "KL/chosen_KL_mean": -171.70896911621094, "KL/mean": -222.963623046875, "KL/rejected_KL_mean": -274.2182922363281, "KL/std": 104.20111083984375, "epoch": 0.8384728340675477, "fcm_dpo/beta": 0.00425078347325325, "fcm_dpo/delta": -0.036552563309669495, "fcm_dpo/margin": 102.50930786132812, "fcm_dpo/q_t": 0.39907926321029663, "grad_norm": 12.149124145507812, "learning_rate": 3.9497599116513705e-08, "logits/chosen": -1.7392020225524902, "logits/rejected": -1.5820019245147705, "logps/chosen": -252.53025817871094, "logps/ref_chosen": -80.8212890625, "logps/ref_rejected": -97.01528930664062, "logps/rejected": -371.23358154296875, "loss": 1.0645, "margin_dpo/margin_mean": 102.50930786132812, "margin_dpo/margin_std": 125.06383514404297, "step": 571 }, { "KL/chosen_KL_mean": -176.1534881591797, "KL/mean": -227.5482177734375, "KL/rejected_KL_mean": -278.94293212890625, "KL/std": 111.404296875, "epoch": 0.8399412628487518, "fcm_dpo/beta": 0.004228315316140652, "fcm_dpo/delta": -0.03542909398674965, "fcm_dpo/margin": 102.78945922851562, "fcm_dpo/q_t": 0.3998091220855713, "grad_norm": 9.076624870300293, "learning_rate": 3.880806698864086e-08, "logits/chosen": -1.9218034744262695, "logits/rejected": -1.7577283382415771, "logps/chosen": -253.53341674804688, "logps/ref_chosen": -77.37992858886719, "logps/ref_rejected": -90.56597900390625, "logps/rejected": -369.5089111328125, "loss": 1.0576, "margin_dpo/margin_mean": 102.78945922851562, "margin_dpo/margin_std": 119.30110168457031, "step": 572 }, { "KL/chosen_KL_mean": -170.03018188476562, "KL/mean": -217.98114013671875, "KL/rejected_KL_mean": -265.93212890625, "KL/std": 110.15621948242188, "epoch": 0.8414096916299559, "fcm_dpo/beta": 0.0042373258620500565, "fcm_dpo/delta": -0.006832681596279144, "fcm_dpo/margin": 95.90192413330078, "fcm_dpo/q_t": 0.4045547842979431, "grad_norm": 8.94240951538086, "learning_rate": 3.812409996461275e-08, "logits/chosen": -2.0915260314941406, "logits/rejected": -1.8593183755874634, "logps/chosen": -253.73626708984375, "logps/ref_chosen": -83.7060775756836, "logps/ref_rejected": -90.69746398925781, "logps/rejected": -356.62957763671875, "loss": 1.0806, "margin_dpo/margin_mean": 95.90192413330078, "margin_dpo/margin_std": 115.402587890625, "step": 573 }, { "KL/chosen_KL_mean": -180.3427734375, "KL/mean": -232.21051025390625, "KL/rejected_KL_mean": -284.0782470703125, "KL/std": 109.18146514892578, "epoch": 0.8428781204111601, "fcm_dpo/beta": 0.0042181843891739845, "fcm_dpo/delta": -0.03844724968075752, "fcm_dpo/margin": 103.73548889160156, "fcm_dpo/q_t": 0.3981703519821167, "grad_norm": 9.558290481567383, "learning_rate": 3.74457160675965e-08, "logits/chosen": -2.1462769508361816, "logits/rejected": -1.8194911479949951, "logps/chosen": -265.4722595214844, "logps/ref_chosen": -85.12948608398438, "logps/ref_rejected": -99.05403137207031, "logps/rejected": -383.1322937011719, "loss": 1.0604, "margin_dpo/margin_mean": 103.73548889160156, "margin_dpo/margin_std": 123.2427978515625, "step": 574 }, { "KL/chosen_KL_mean": -185.52943420410156, "KL/mean": -233.59130859375, "KL/rejected_KL_mean": -281.6531677246094, "KL/std": 111.70156860351562, "epoch": 0.8443465491923642, "fcm_dpo/beta": 0.004206107929348946, "fcm_dpo/delta": -0.004396369215101004, "fcm_dpo/margin": 96.12374114990234, "fcm_dpo/q_t": 0.40646833181381226, "grad_norm": 11.440290451049805, "learning_rate": 3.677293317363864e-08, "logits/chosen": -1.3912854194641113, "logits/rejected": -0.9254108667373657, "logps/chosen": -291.30029296875, "logps/ref_chosen": -105.77084350585938, "logps/ref_rejected": -100.94944763183594, "logps/rejected": -382.60260009765625, "loss": 1.0927, "margin_dpo/margin_mean": 96.12374114990234, "margin_dpo/margin_std": 130.293212890625, "step": 575 }, { "KL/chosen_KL_mean": -184.7664337158203, "KL/mean": -230.8877410888672, "KL/rejected_KL_mean": -277.009033203125, "KL/std": 104.25774383544922, "epoch": 0.8458149779735683, "fcm_dpo/beta": 0.004212846048176289, "fcm_dpo/delta": 0.011590029112994671, "fcm_dpo/margin": 92.24259948730469, "fcm_dpo/q_t": 0.4093271493911743, "grad_norm": 10.916496276855469, "learning_rate": 3.6105769011194224e-08, "logits/chosen": -2.108309268951416, "logits/rejected": -2.04500150680542, "logps/chosen": -265.970947265625, "logps/ref_chosen": -81.20452880859375, "logps/ref_rejected": -105.40940856933594, "logps/rejected": -382.41845703125, "loss": 1.1026, "margin_dpo/margin_mean": 92.24259948730469, "margin_dpo/margin_std": 126.09615325927734, "step": 576 }, { "KL/chosen_KL_mean": -165.4770050048828, "KL/mean": -223.34164428710938, "KL/rejected_KL_mean": -281.2062683105469, "KL/std": 112.65083312988281, "epoch": 0.8472834067547724, "fcm_dpo/beta": 0.004177002236247063, "fcm_dpo/delta": -0.08572462946176529, "fcm_dpo/margin": 115.72924041748047, "fcm_dpo/q_t": 0.38782861828804016, "grad_norm": 8.09939956665039, "learning_rate": 3.5444241160659304e-08, "logits/chosen": -1.9048547744750977, "logits/rejected": -1.195278525352478, "logps/chosen": -266.94122314453125, "logps/ref_chosen": -101.46419525146484, "logps/ref_rejected": -89.32637023925781, "logps/rejected": -370.53265380859375, "loss": 1.0287, "margin_dpo/margin_mean": 115.72924041748047, "margin_dpo/margin_std": 128.42343139648438, "step": 577 }, { "KL/chosen_KL_mean": -196.70587158203125, "KL/mean": -238.6914520263672, "KL/rejected_KL_mean": -280.6770324707031, "KL/std": 106.98638153076172, "epoch": 0.8487518355359766, "fcm_dpo/beta": 0.004189492668956518, "fcm_dpo/delta": 0.04907448589801788, "fcm_dpo/margin": 83.97114562988281, "fcm_dpo/q_t": 0.4178633987903595, "grad_norm": 13.935843467712402, "learning_rate": 3.478836705390808e-08, "logits/chosen": -1.3720407485961914, "logits/rejected": -1.131575345993042, "logps/chosen": -273.998291015625, "logps/ref_chosen": -77.29241943359375, "logps/ref_rejected": -88.24701690673828, "logps/rejected": -368.9240417480469, "loss": 1.1278, "margin_dpo/margin_mean": 83.97116088867188, "margin_dpo/margin_std": 122.52455139160156, "step": 578 }, { "KL/chosen_KL_mean": -177.35621643066406, "KL/mean": -219.72396850585938, "KL/rejected_KL_mean": -262.09173583984375, "KL/std": 107.15487670898438, "epoch": 0.8502202643171806, "fcm_dpo/beta": 0.004200979135930538, "fcm_dpo/delta": 0.044748704880476, "fcm_dpo/margin": 84.73553466796875, "fcm_dpo/q_t": 0.4170425236225128, "grad_norm": 8.626152992248535, "learning_rate": 3.41381639738331e-08, "logits/chosen": -1.7011899948120117, "logits/rejected": -1.3987054824829102, "logps/chosen": -264.69927978515625, "logps/ref_chosen": -87.34305572509766, "logps/ref_rejected": -102.12100982666016, "logps/rejected": -364.2127685546875, "loss": 1.1165, "margin_dpo/margin_mean": 84.73553466796875, "margin_dpo/margin_std": 109.35496520996094, "step": 579 }, { "KL/chosen_KL_mean": -169.3138427734375, "KL/mean": -221.40972900390625, "KL/rejected_KL_mean": -273.505615234375, "KL/std": 106.29006958007812, "epoch": 0.8516886930983847, "fcm_dpo/beta": 0.004194262437522411, "fcm_dpo/delta": -0.03805701434612274, "fcm_dpo/margin": 104.1917724609375, "fcm_dpo/q_t": 0.39961639046669006, "grad_norm": 9.784381866455078, "learning_rate": 3.349364905389032e-08, "logits/chosen": -1.6212871074676514, "logits/rejected": -1.3128399848937988, "logps/chosen": -258.2426452636719, "logps/ref_chosen": -88.92879486083984, "logps/ref_rejected": -88.69661712646484, "logps/rejected": -362.2022399902344, "loss": 1.0643, "margin_dpo/margin_mean": 104.1917724609375, "margin_dpo/margin_std": 127.74482727050781, "step": 580 }, { "KL/chosen_KL_mean": -178.91464233398438, "KL/mean": -226.57461547851562, "KL/rejected_KL_mean": -274.2345886230469, "KL/std": 107.68406677246094, "epoch": 0.8531571218795888, "fcm_dpo/beta": 0.004200180061161518, "fcm_dpo/delta": -0.00037740543484687805, "fcm_dpo/margin": 95.31995391845703, "fcm_dpo/q_t": 0.40698665380477905, "grad_norm": 11.438523292541504, "learning_rate": 3.285483927764726e-08, "logits/chosen": -2.0745649337768555, "logits/rejected": -1.731683373451233, "logps/chosen": -271.13787841796875, "logps/ref_chosen": -92.22323608398438, "logps/ref_rejected": -99.38943481445312, "logps/rejected": -373.6240234375, "loss": 1.1021, "margin_dpo/margin_mean": 95.31995391845703, "margin_dpo/margin_std": 135.0211181640625, "step": 581 }, { "KL/chosen_KL_mean": -157.91864013671875, "KL/mean": -200.6662139892578, "KL/rejected_KL_mean": -243.41378784179688, "KL/std": 111.92686462402344, "epoch": 0.8546255506607929, "fcm_dpo/beta": 0.004184977151453495, "fcm_dpo/delta": -0.0628759041428566, "fcm_dpo/margin": 85.49515533447266, "fcm_dpo/q_t": 0.4179139733314514, "grad_norm": 8.963157653808594, "learning_rate": 3.222175147833556e-08, "logits/chosen": -1.9678072929382324, "logits/rejected": -1.9589488506317139, "logps/chosen": -245.2542724609375, "logps/ref_chosen": -87.33561706542969, "logps/ref_rejected": -115.1544189453125, "logps/rejected": -358.5682067871094, "loss": 1.1278, "margin_dpo/margin_mean": 85.49514770507812, "margin_dpo/margin_std": 122.86141967773438, "step": 582 }, { "KL/chosen_KL_mean": -191.21295166015625, "KL/mean": -238.563720703125, "KL/rejected_KL_mean": -285.91448974609375, "KL/std": 105.42802429199219, "epoch": 0.856093979441997, "fcm_dpo/beta": 0.004165738821029663, "fcm_dpo/delta": 0.0054777562618255615, "fcm_dpo/margin": 94.7015151977539, "fcm_dpo/q_t": 0.4089813828468323, "grad_norm": 10.466100692749023, "learning_rate": 3.159440233840763e-08, "logits/chosen": -1.444058895111084, "logits/rejected": -1.2077831029891968, "logps/chosen": -274.9300231933594, "logps/ref_chosen": -83.71707153320312, "logps/ref_rejected": -94.58465576171875, "logps/rejected": -380.4991455078125, "loss": 1.0948, "margin_dpo/margin_mean": 94.70150756835938, "margin_dpo/margin_std": 124.96258544921875, "step": 583 }, { "KL/chosen_KL_mean": -169.07260131835938, "KL/mean": -219.37368774414062, "KL/rejected_KL_mean": -269.6747741699219, "KL/std": 109.57730102539062, "epoch": 0.8575624082232012, "fcm_dpo/beta": 0.004181563388556242, "fcm_dpo/delta": -0.021508898586034775, "fcm_dpo/margin": 100.6021728515625, "fcm_dpo/q_t": 0.401737242937088, "grad_norm": 7.747732162475586, "learning_rate": 3.0972808389096635e-08, "logits/chosen": -1.9428331851959229, "logits/rejected": -1.4840140342712402, "logps/chosen": -263.3321533203125, "logps/ref_chosen": -94.25955963134766, "logps/ref_rejected": -104.48269653320312, "logps/rejected": -374.157470703125, "loss": 1.0776, "margin_dpo/margin_mean": 100.6021728515625, "margin_dpo/margin_std": 124.16036987304688, "step": 584 }, { "KL/chosen_KL_mean": -180.53451538085938, "KL/mean": -236.79672241210938, "KL/rejected_KL_mean": -293.05889892578125, "KL/std": 125.39028930664062, "epoch": 0.8590308370044053, "fcm_dpo/beta": 0.00415035430341959, "fcm_dpo/delta": -0.06861012428998947, "fcm_dpo/margin": 112.52438354492188, "fcm_dpo/q_t": 0.3936796486377716, "grad_norm": 8.805819511413574, "learning_rate": 3.035698600998121e-08, "logits/chosen": -1.9238547086715698, "logits/rejected": -1.648674726486206, "logps/chosen": -266.1712646484375, "logps/ref_chosen": -85.63676452636719, "logps/ref_rejected": -90.28547668457031, "logps/rejected": -383.3443603515625, "loss": 1.0597, "margin_dpo/margin_mean": 112.52439880371094, "margin_dpo/margin_std": 147.00514221191406, "step": 585 }, { "KL/chosen_KL_mean": -200.45510864257812, "KL/mean": -238.25338745117188, "KL/rejected_KL_mean": -276.0516662597656, "KL/std": 110.1264877319336, "epoch": 0.8604992657856094, "fcm_dpo/beta": 0.004163142293691635, "fcm_dpo/delta": 0.08665263652801514, "fcm_dpo/margin": 75.5965576171875, "fcm_dpo/q_t": 0.42570269107818604, "grad_norm": 9.690286636352539, "learning_rate": 2.974695142855388e-08, "logits/chosen": -1.9055235385894775, "logits/rejected": -1.644179344177246, "logps/chosen": -284.13623046875, "logps/ref_chosen": -83.68114471435547, "logps/ref_rejected": -98.35916137695312, "logps/rejected": -374.41082763671875, "loss": 1.1423, "margin_dpo/margin_mean": 75.5965576171875, "margin_dpo/margin_std": 104.54347229003906, "step": 586 }, { "KL/chosen_KL_mean": -161.44586181640625, "KL/mean": -203.44906616210938, "KL/rejected_KL_mean": -245.4522705078125, "KL/std": 106.53964233398438, "epoch": 0.8619676945668135, "fcm_dpo/beta": 0.004189962055534124, "fcm_dpo/delta": 0.048843298107385635, "fcm_dpo/margin": 84.00640869140625, "fcm_dpo/q_t": 0.41745859384536743, "grad_norm": 10.635544776916504, "learning_rate": 2.9142720719793122e-08, "logits/chosen": -2.402045726776123, "logits/rejected": -2.1180245876312256, "logps/chosen": -239.0233612060547, "logps/ref_chosen": -77.57750701904297, "logps/ref_rejected": -89.32032775878906, "logps/rejected": -334.7726135253906, "loss": 1.115, "margin_dpo/margin_mean": 84.00640869140625, "margin_dpo/margin_std": 109.501953125, "step": 587 }, { "KL/chosen_KL_mean": -176.60110473632812, "KL/mean": -214.95611572265625, "KL/rejected_KL_mean": -253.31112670898438, "KL/std": 107.96633911132812, "epoch": 0.8634361233480177, "fcm_dpo/beta": 0.004214026965200901, "fcm_dpo/delta": 0.07803996652364731, "fcm_dpo/margin": 76.71000671386719, "fcm_dpo/q_t": 0.4259260296821594, "grad_norm": 13.976094245910645, "learning_rate": 2.8544309805740018e-08, "logits/chosen": -1.7158441543579102, "logits/rejected": -1.6754772663116455, "logps/chosen": -260.4053649902344, "logps/ref_chosen": -83.80426025390625, "logps/ref_rejected": -113.11943054199219, "logps/rejected": -366.4305419921875, "loss": 1.1603, "margin_dpo/margin_mean": 76.71000671386719, "margin_dpo/margin_std": 129.92958068847656, "step": 588 }, { "KL/chosen_KL_mean": -189.71478271484375, "KL/mean": -240.4716796875, "KL/rejected_KL_mean": -291.2286376953125, "KL/std": 106.54381561279297, "epoch": 0.8649045521292217, "fcm_dpo/beta": 0.004217791371047497, "fcm_dpo/delta": -0.02877596952021122, "fcm_dpo/margin": 101.51384735107422, "fcm_dpo/q_t": 0.3988649547100067, "grad_norm": 8.614251136779785, "learning_rate": 2.7951734455078786e-08, "logits/chosen": -1.6511094570159912, "logits/rejected": -1.255927324295044, "logps/chosen": -286.1798095703125, "logps/ref_chosen": -96.46501922607422, "logps/ref_rejected": -116.03719329833984, "logps/rejected": -407.26580810546875, "loss": 1.0474, "margin_dpo/margin_mean": 101.51384735107422, "margin_dpo/margin_std": 102.04906463623047, "step": 589 }, { "KL/chosen_KL_mean": -173.4657745361328, "KL/mean": -224.2518310546875, "KL/rejected_KL_mean": -275.03790283203125, "KL/std": 99.48233795166016, "epoch": 0.8663729809104258, "fcm_dpo/beta": 0.004213081207126379, "fcm_dpo/delta": -0.02864237315952778, "fcm_dpo/margin": 101.57211303710938, "fcm_dpo/q_t": 0.3997061848640442, "grad_norm": 12.059309005737305, "learning_rate": 2.736501028272095e-08, "logits/chosen": -1.8667161464691162, "logits/rejected": -1.7173430919647217, "logps/chosen": -261.8936462402344, "logps/ref_chosen": -88.4278564453125, "logps/ref_rejected": -110.19876861572266, "logps/rejected": -385.2366638183594, "loss": 1.0491, "margin_dpo/margin_mean": 101.57211303710938, "margin_dpo/margin_std": 103.51896667480469, "step": 590 }, { "KL/chosen_KL_mean": -165.8837890625, "KL/mean": -219.65408325195312, "KL/rejected_KL_mean": -273.42437744140625, "KL/std": 107.0798110961914, "epoch": 0.8678414096916299, "fcm_dpo/beta": 0.004181294236332178, "fcm_dpo/delta": -0.050902556627988815, "fcm_dpo/margin": 107.54058074951172, "fcm_dpo/q_t": 0.3943061828613281, "grad_norm": 7.323884010314941, "learning_rate": 2.678415274939408e-08, "logits/chosen": -1.9463168382644653, "logits/rejected": -1.329752802848816, "logps/chosen": -253.73348999023438, "logps/ref_chosen": -87.84968566894531, "logps/ref_rejected": -89.57919311523438, "logps/rejected": -363.00360107421875, "loss": 1.0317, "margin_dpo/margin_mean": 107.54058837890625, "margin_dpo/margin_std": 104.0509262084961, "step": 591 }, { "KL/chosen_KL_mean": -195.7640380859375, "KL/mean": -237.59823608398438, "KL/rejected_KL_mean": -279.4324645996094, "KL/std": 103.68382263183594, "epoch": 0.869309838472834, "fcm_dpo/beta": 0.004196913912892342, "fcm_dpo/delta": 0.04971081763505936, "fcm_dpo/margin": 83.66842651367188, "fcm_dpo/q_t": 0.4180733859539032, "grad_norm": 13.464978218078613, "learning_rate": 2.6209177161234442e-08, "logits/chosen": -1.7920985221862793, "logits/rejected": -1.4633945226669312, "logps/chosen": -270.7397766113281, "logps/ref_chosen": -74.97573852539062, "logps/ref_rejected": -81.55262756347656, "logps/rejected": -360.985107421875, "loss": 1.1354, "margin_dpo/margin_mean": 83.66842651367188, "margin_dpo/margin_std": 128.9634552001953, "step": 592 }, { "KL/chosen_KL_mean": -178.82388305664062, "KL/mean": -214.72918701171875, "KL/rejected_KL_mean": -250.63449096679688, "KL/std": 109.00132751464844, "epoch": 0.8707782672540382, "fcm_dpo/beta": 0.004225343465805054, "fcm_dpo/delta": 0.0981161817908287, "fcm_dpo/margin": 71.81060028076172, "fcm_dpo/q_t": 0.42971232533454895, "grad_norm": 14.15564250946045, "learning_rate": 2.564009866938349e-08, "logits/chosen": -0.9875301122665405, "logits/rejected": -0.6912494897842407, "logps/chosen": -251.0629119873047, "logps/ref_chosen": -72.23902893066406, "logps/ref_rejected": -70.09652709960938, "logps/rejected": -320.73101806640625, "loss": 1.1684, "margin_dpo/margin_mean": 71.81059265136719, "margin_dpo/margin_std": 120.20622253417969, "step": 593 }, { "KL/chosen_KL_mean": -159.74139404296875, "KL/mean": -214.7735595703125, "KL/rejected_KL_mean": -269.80572509765625, "KL/std": 105.62200927734375, "epoch": 0.8722466960352423, "fcm_dpo/beta": 0.004228656180202961, "fcm_dpo/delta": -0.06698539853096008, "fcm_dpo/margin": 110.0643310546875, "fcm_dpo/q_t": 0.3907328248023987, "grad_norm": 9.069114685058594, "learning_rate": 2.5076932269588708e-08, "logits/chosen": -1.8708136081695557, "logits/rejected": -1.3399810791015625, "logps/chosen": -250.17910766601562, "logps/ref_chosen": -90.43771362304688, "logps/ref_rejected": -92.267578125, "logps/rejected": -362.07330322265625, "loss": 1.0195, "margin_dpo/margin_mean": 110.0643310546875, "margin_dpo/margin_std": 102.59178924560547, "step": 594 }, { "KL/chosen_KL_mean": -172.47482299804688, "KL/mean": -217.28160095214844, "KL/rejected_KL_mean": -262.08837890625, "KL/std": 105.78938293457031, "epoch": 0.8737151248164464, "fcm_dpo/beta": 0.004226146265864372, "fcm_dpo/delta": 0.021608756855130196, "fcm_dpo/margin": 89.61358642578125, "fcm_dpo/q_t": 0.41251981258392334, "grad_norm": 9.121129989624023, "learning_rate": 2.451969280180849e-08, "logits/chosen": -1.8399101495742798, "logits/rejected": -1.4325063228607178, "logps/chosen": -253.9896240234375, "logps/ref_chosen": -81.51480102539062, "logps/ref_rejected": -86.52047729492188, "logps/rejected": -348.60888671875, "loss": 1.1059, "margin_dpo/margin_mean": 89.61358642578125, "margin_dpo/margin_std": 121.52436828613281, "step": 595 }, { "KL/chosen_KL_mean": -179.64102172851562, "KL/mean": -219.46377563476562, "KL/rejected_KL_mean": -259.2865295410156, "KL/std": 104.24404907226562, "epoch": 0.8751835535976505, "fcm_dpo/beta": 0.0042449552565813065, "fcm_dpo/delta": 0.06294430792331696, "fcm_dpo/margin": 79.6455078125, "fcm_dpo/q_t": 0.42216452956199646, "grad_norm": 12.253155708312988, "learning_rate": 2.396839494982103e-08, "logits/chosen": -1.6848998069763184, "logits/rejected": -1.1086348295211792, "logps/chosen": -267.84906005859375, "logps/ref_chosen": -88.20805358886719, "logps/ref_rejected": -86.09375, "logps/rejected": -345.3802795410156, "loss": 1.1512, "margin_dpo/margin_mean": 79.6455078125, "margin_dpo/margin_std": 131.56802368164062, "step": 596 }, { "KL/chosen_KL_mean": -174.72183227539062, "KL/mean": -228.6036376953125, "KL/rejected_KL_mean": -282.4854736328125, "KL/std": 112.6059341430664, "epoch": 0.8766519823788547, "fcm_dpo/beta": 0.0042278952896595, "fcm_dpo/delta": -0.05700536072254181, "fcm_dpo/margin": 107.76363372802734, "fcm_dpo/q_t": 0.39465266466140747, "grad_norm": 8.95300579071045, "learning_rate": 2.3423053240837514e-08, "logits/chosen": -1.3638274669647217, "logits/rejected": -1.2469326257705688, "logps/chosen": -258.31182861328125, "logps/ref_chosen": -83.58998107910156, "logps/ref_rejected": -106.94439697265625, "logps/rejected": -389.42987060546875, "loss": 1.0438, "margin_dpo/margin_mean": 107.76363372802734, "margin_dpo/margin_std": 121.83272552490234, "step": 597 }, { "KL/chosen_KL_mean": -165.83795166015625, "KL/mean": -211.21551513671875, "KL/rejected_KL_mean": -256.59307861328125, "KL/std": 105.02983093261719, "epoch": 0.8781204111600588, "fcm_dpo/beta": 0.00422354182228446, "fcm_dpo/delta": 0.01691240258514881, "fcm_dpo/margin": 90.75508880615234, "fcm_dpo/q_t": 0.4097345471382141, "grad_norm": 9.187825202941895, "learning_rate": 2.2883682045119062e-08, "logits/chosen": -1.9689390659332275, "logits/rejected": -1.656595230102539, "logps/chosen": -256.02056884765625, "logps/ref_chosen": -90.18262481689453, "logps/ref_rejected": -94.79856872558594, "logps/rejected": -351.3916320800781, "loss": 1.1019, "margin_dpo/margin_mean": 90.75509643554688, "margin_dpo/margin_std": 119.14258575439453, "step": 598 }, { "KL/chosen_KL_mean": -168.7278289794922, "KL/mean": -208.91629028320312, "KL/rejected_KL_mean": -249.10475158691406, "KL/std": 109.91768646240234, "epoch": 0.8795888399412628, "fcm_dpo/beta": 0.004242206923663616, "fcm_dpo/delta": 0.059966474771499634, "fcm_dpo/margin": 80.3769302368164, "fcm_dpo/q_t": 0.42027875781059265, "grad_norm": 9.196270942687988, "learning_rate": 2.2350295575598367e-08, "logits/chosen": -1.7377703189849854, "logits/rejected": -1.6156129837036133, "logps/chosen": -246.57009887695312, "logps/ref_chosen": -77.84226989746094, "logps/ref_rejected": -89.1976318359375, "logps/rejected": -338.3023681640625, "loss": 1.1394, "margin_dpo/margin_mean": 80.3769302368164, "margin_dpo/margin_std": 120.65621185302734, "step": 599 }, { "KL/chosen_KL_mean": -187.6644287109375, "KL/mean": -232.8166046142578, "KL/rejected_KL_mean": -277.9687805175781, "KL/std": 103.58748626708984, "epoch": 0.8810572687224669, "fcm_dpo/beta": 0.0042645311914384365, "fcm_dpo/delta": 0.01518004946410656, "fcm_dpo/margin": 90.3043441772461, "fcm_dpo/q_t": 0.4103008806705475, "grad_norm": 9.53038501739502, "learning_rate": 2.1822907887504932e-08, "logits/chosen": -2.4354634284973145, "logits/rejected": -2.0924274921417236, "logps/chosen": -279.85223388671875, "logps/ref_chosen": -92.18781280517578, "logps/ref_rejected": -92.95429992675781, "logps/rejected": -370.923095703125, "loss": 1.0934, "margin_dpo/margin_mean": 90.3043441772461, "margin_dpo/margin_std": 112.5677261352539, "step": 600 }, { "KL/chosen_KL_mean": -189.81248474121094, "KL/mean": -238.33114624023438, "KL/rejected_KL_mean": -286.8498229980469, "KL/std": 105.1885986328125, "epoch": 0.882525697503671, "fcm_dpo/beta": 0.004253928083926439, "fcm_dpo/delta": -0.013187844306230545, "fcm_dpo/margin": 97.0373306274414, "fcm_dpo/q_t": 0.4028637111186981, "grad_norm": 9.240110397338867, "learning_rate": 2.1301532877994742e-08, "logits/chosen": -1.7433884143829346, "logits/rejected": -1.4168462753295898, "logps/chosen": -279.02862548828125, "logps/ref_chosen": -89.21614074707031, "logps/ref_rejected": -100.17054748535156, "logps/rejected": -387.0203857421875, "loss": 1.0594, "margin_dpo/margin_mean": 97.0373306274414, "margin_dpo/margin_std": 100.19400024414062, "step": 601 }, { "KL/chosen_KL_mean": -151.40528869628906, "KL/mean": -213.4713134765625, "KL/rejected_KL_mean": -275.537353515625, "KL/std": 115.55693054199219, "epoch": 0.8839941262848752, "fcm_dpo/beta": 0.00421971594914794, "fcm_dpo/delta": -0.12711726129055023, "fcm_dpo/margin": 124.132080078125, "fcm_dpo/q_t": 0.37969306111335754, "grad_norm": 9.733186721801758, "learning_rate": 2.0786184285784298e-08, "logits/chosen": -2.0520858764648438, "logits/rejected": -1.7885940074920654, "logps/chosen": -231.462890625, "logps/ref_chosen": -80.05760192871094, "logps/ref_rejected": -93.197509765625, "logps/rejected": -368.73486328125, "loss": 1.001, "margin_dpo/margin_mean": 124.13207244873047, "margin_dpo/margin_std": 132.10569763183594, "step": 602 }, { "KL/chosen_KL_mean": -161.75003051757812, "KL/mean": -213.3122100830078, "KL/rejected_KL_mean": -264.8743896484375, "KL/std": 103.96458435058594, "epoch": 0.8854625550660793, "fcm_dpo/beta": 0.004196738824248314, "fcm_dpo/delta": -0.03350426256656647, "fcm_dpo/margin": 103.1243667602539, "fcm_dpo/q_t": 0.3992677927017212, "grad_norm": 8.172650337219238, "learning_rate": 2.0276875690788204e-08, "logits/chosen": -1.9824209213256836, "logits/rejected": -1.3867383003234863, "logps/chosen": -264.05963134765625, "logps/ref_chosen": -102.30958557128906, "logps/ref_rejected": -108.06884765625, "logps/rejected": -372.9432373046875, "loss": 1.0577, "margin_dpo/margin_mean": 103.12435913085938, "margin_dpo/margin_std": 119.281494140625, "step": 603 }, { "KL/chosen_KL_mean": -168.54342651367188, "KL/mean": -221.8045654296875, "KL/rejected_KL_mean": -275.06573486328125, "KL/std": 109.0684585571289, "epoch": 0.8869309838472834, "fcm_dpo/beta": 0.004171429201960564, "fcm_dpo/delta": -0.04550330340862274, "fcm_dpo/margin": 106.52227783203125, "fcm_dpo/q_t": 0.3947213888168335, "grad_norm": 8.666648864746094, "learning_rate": 1.977362051376158e-08, "logits/chosen": -1.846801996231079, "logits/rejected": -1.6951457262039185, "logps/chosen": -246.717529296875, "logps/ref_chosen": -78.17408752441406, "logps/ref_rejected": -99.49610900878906, "logps/rejected": -374.56182861328125, "loss": 1.0418, "margin_dpo/margin_mean": 106.52227783203125, "margin_dpo/margin_std": 110.56468200683594, "step": 604 }, { "KL/chosen_KL_mean": -183.33782958984375, "KL/mean": -227.0364990234375, "KL/rejected_KL_mean": -270.73516845703125, "KL/std": 97.87216186523438, "epoch": 0.8883994126284875, "fcm_dpo/beta": 0.004182265140116215, "fcm_dpo/delta": 0.03512997552752495, "fcm_dpo/margin": 87.3973388671875, "fcm_dpo/q_t": 0.4144812524318695, "grad_norm": 9.476158142089844, "learning_rate": 1.9276432015946446e-08, "logits/chosen": -1.8394591808319092, "logits/rejected": -1.3417384624481201, "logps/chosen": -278.11114501953125, "logps/ref_chosen": -94.77332305908203, "logps/ref_rejected": -107.30490112304688, "logps/rejected": -378.04010009765625, "loss": 1.1044, "margin_dpo/margin_mean": 87.39733123779297, "margin_dpo/margin_std": 110.24339294433594, "step": 605 }, { "KL/chosen_KL_mean": -177.42254638671875, "KL/mean": -223.08975219726562, "KL/rejected_KL_mean": -268.7569580078125, "KL/std": 107.15673828125, "epoch": 0.8898678414096917, "fcm_dpo/beta": 0.004202789627015591, "fcm_dpo/delta": 0.01616329699754715, "fcm_dpo/margin": 91.33442687988281, "fcm_dpo/q_t": 0.40960532426834106, "grad_norm": 8.350889205932617, "learning_rate": 1.8785323298722093e-08, "logits/chosen": -1.9895682334899902, "logits/rejected": -1.6034722328186035, "logps/chosen": -265.1759338378906, "logps/ref_chosen": -87.7533950805664, "logps/ref_rejected": -104.2422103881836, "logps/rejected": -372.9991455078125, "loss": 1.0962, "margin_dpo/margin_mean": 91.33441925048828, "margin_dpo/margin_std": 114.53302001953125, "step": 606 }, { "KL/chosen_KL_mean": -172.34371948242188, "KL/mean": -223.76773071289062, "KL/rejected_KL_mean": -275.1917724609375, "KL/std": 114.09440612792969, "epoch": 0.8913362701908958, "fcm_dpo/beta": 0.00418511126190424, "fcm_dpo/delta": -0.031095139682292938, "fcm_dpo/margin": 102.8480453491211, "fcm_dpo/q_t": 0.40087151527404785, "grad_norm": 11.536015510559082, "learning_rate": 1.8300307303259904e-08, "logits/chosen": -1.7407681941986084, "logits/rejected": -1.208794355392456, "logps/chosen": -260.6727600097656, "logps/ref_chosen": -88.32904052734375, "logps/ref_rejected": -86.76810455322266, "logps/rejected": -361.9598693847656, "loss": 1.0826, "margin_dpo/margin_mean": 102.84803771972656, "margin_dpo/margin_std": 140.21351623535156, "step": 607 }, { "KL/chosen_KL_mean": -157.2107696533203, "KL/mean": -211.02716064453125, "KL/rejected_KL_mean": -264.84356689453125, "KL/std": 112.22077941894531, "epoch": 0.8928046989720999, "fcm_dpo/beta": 0.004153198562562466, "fcm_dpo/delta": -0.04861520603299141, "fcm_dpo/margin": 107.63280487060547, "fcm_dpo/q_t": 0.39496105909347534, "grad_norm": 9.324193000793457, "learning_rate": 1.7821396810182437e-08, "logits/chosen": -1.8130810260772705, "logits/rejected": -1.5153980255126953, "logps/chosen": -242.98013305664062, "logps/ref_chosen": -85.76937866210938, "logps/ref_rejected": -100.23281860351562, "logps/rejected": -365.07635498046875, "loss": 1.0527, "margin_dpo/margin_mean": 107.63280487060547, "margin_dpo/margin_std": 121.04887390136719, "step": 608 }, { "KL/chosen_KL_mean": -175.19119262695312, "KL/mean": -225.33331298828125, "KL/rejected_KL_mean": -275.47540283203125, "KL/std": 113.3258056640625, "epoch": 0.8942731277533039, "fcm_dpo/beta": 0.004146730061620474, "fcm_dpo/delta": -0.016407813876867294, "fcm_dpo/margin": 100.28421020507812, "fcm_dpo/q_t": 0.4036363661289215, "grad_norm": 10.994895935058594, "learning_rate": 1.7348604439226617e-08, "logits/chosen": -2.2698824405670166, "logits/rejected": -1.8502426147460938, "logps/chosen": -268.15777587890625, "logps/ref_chosen": -92.9665756225586, "logps/ref_rejected": -95.91818237304688, "logps/rejected": -371.3935852050781, "loss": 1.077, "margin_dpo/margin_mean": 100.28421020507812, "margin_dpo/margin_std": 120.16020202636719, "step": 609 }, { "KL/chosen_KL_mean": -160.09906005859375, "KL/mean": -214.26263427734375, "KL/rejected_KL_mean": -268.42620849609375, "KL/std": 107.76206970214844, "epoch": 0.895741556534508, "fcm_dpo/beta": 0.0041342913173139095, "fcm_dpo/delta": -0.04904966428875923, "fcm_dpo/margin": 108.3271484375, "fcm_dpo/q_t": 0.3942834436893463, "grad_norm": 8.671380996704102, "learning_rate": 1.6881942648911074e-08, "logits/chosen": -1.853963851928711, "logits/rejected": -1.3201584815979004, "logps/chosen": -254.79934692382812, "logps/ref_chosen": -94.70028686523438, "logps/ref_rejected": -89.68739318847656, "logps/rejected": -358.11358642578125, "loss": 1.0409, "margin_dpo/margin_mean": 108.3271484375, "margin_dpo/margin_std": 114.06095886230469, "step": 610 }, { "KL/chosen_KL_mean": -169.27125549316406, "KL/mean": -221.78567504882812, "KL/rejected_KL_mean": -274.300048828125, "KL/std": 111.56262969970703, "epoch": 0.8972099853157122, "fcm_dpo/beta": 0.004132473841309547, "fcm_dpo/delta": -0.034924089908599854, "fcm_dpo/margin": 105.02880859375, "fcm_dpo/q_t": 0.39899617433547974, "grad_norm": 9.575910568237305, "learning_rate": 1.6421423736208e-08, "logits/chosen": -2.1754343509674072, "logits/rejected": -1.836348056793213, "logps/chosen": -256.0545959472656, "logps/ref_chosen": -86.78334045410156, "logps/ref_rejected": -89.84307861328125, "logps/rejected": -364.14312744140625, "loss": 1.0518, "margin_dpo/margin_mean": 105.02880096435547, "margin_dpo/margin_std": 112.94429016113281, "step": 611 }, { "KL/chosen_KL_mean": -186.12014770507812, "KL/mean": -240.17996215820312, "KL/rejected_KL_mean": -294.23980712890625, "KL/std": 106.01173400878906, "epoch": 0.8986784140969163, "fcm_dpo/beta": 0.004102885723114014, "fcm_dpo/delta": -0.04463447630405426, "fcm_dpo/margin": 108.11961364746094, "fcm_dpo/q_t": 0.39586541056632996, "grad_norm": 9.0076904296875, "learning_rate": 1.5967059836219042e-08, "logits/chosen": -2.126946449279785, "logits/rejected": -1.2894513607025146, "logps/chosen": -287.14031982421875, "logps/ref_chosen": -101.02015686035156, "logps/ref_rejected": -93.78302764892578, "logps/rejected": -388.0228271484375, "loss": 1.0394, "margin_dpo/margin_mean": 108.11961364746094, "margin_dpo/margin_std": 110.65426635742188, "step": 612 }, { "KL/chosen_KL_mean": -148.8216552734375, "KL/mean": -209.75296020507812, "KL/rejected_KL_mean": -270.68426513671875, "KL/std": 108.37228393554688, "epoch": 0.9001468428781204, "fcm_dpo/beta": 0.004067935049533844, "fcm_dpo/delta": -0.09825228154659271, "fcm_dpo/margin": 121.86264038085938, "fcm_dpo/q_t": 0.3832070231437683, "grad_norm": 8.24657154083252, "learning_rate": 1.551886292185553e-08, "logits/chosen": -1.9012269973754883, "logits/rejected": -1.702202320098877, "logps/chosen": -237.81027221679688, "logps/ref_chosen": -88.98861694335938, "logps/ref_rejected": -109.99551391601562, "logps/rejected": -380.6798095703125, "loss": 0.9944, "margin_dpo/margin_mean": 121.8626480102539, "margin_dpo/margin_std": 106.0664291381836, "step": 613 }, { "KL/chosen_KL_mean": -186.1546630859375, "KL/mean": -236.07130432128906, "KL/rejected_KL_mean": -285.9879455566406, "KL/std": 102.84451293945312, "epoch": 0.9016152716593245, "fcm_dpo/beta": 0.004057104699313641, "fcm_dpo/delta": -0.0051508015021681786, "fcm_dpo/margin": 99.83326721191406, "fcm_dpo/q_t": 0.4050413966178894, "grad_norm": 10.427536964416504, "learning_rate": 1.507684480352292e-08, "logits/chosen": -1.3547275066375732, "logits/rejected": -1.3464431762695312, "logps/chosen": -266.354736328125, "logps/ref_chosen": -80.20005798339844, "logps/ref_rejected": -109.86239624023438, "logps/rejected": -395.850341796875, "loss": 1.0849, "margin_dpo/margin_mean": 99.83325958251953, "margin_dpo/margin_std": 126.01268005371094, "step": 614 }, { "KL/chosen_KL_mean": -175.25161743164062, "KL/mean": -216.08859252929688, "KL/rejected_KL_mean": -256.925537109375, "KL/std": 103.86686706542969, "epoch": 0.9030837004405287, "fcm_dpo/beta": 0.0040531461127102375, "fcm_dpo/delta": -0.02694574184715748, "fcm_dpo/margin": 81.67393493652344, "fcm_dpo/q_t": 0.4209776520729065, "grad_norm": 9.5346040725708, "learning_rate": 1.4641017128809801e-08, "logits/chosen": -2.0354623794555664, "logits/rejected": -1.6485373973846436, "logps/chosen": -275.6868896484375, "logps/ref_chosen": -100.43525695800781, "logps/ref_rejected": -101.1800537109375, "logps/rejected": -358.1055908203125, "loss": 1.1243, "margin_dpo/margin_mean": 81.67393493652344, "margin_dpo/margin_std": 98.24105834960938, "step": 615 }, { "KL/chosen_KL_mean": -177.67347717285156, "KL/mean": -232.94281005859375, "KL/rejected_KL_mean": -288.2121276855469, "KL/std": 103.70980834960938, "epoch": 0.9045521292217328, "fcm_dpo/beta": 0.004040713422000408, "fcm_dpo/delta": -0.04776977002620697, "fcm_dpo/margin": 110.53861999511719, "fcm_dpo/q_t": 0.3956100642681122, "grad_norm": 8.622490882873535, "learning_rate": 1.4211391382180637e-08, "logits/chosen": -1.7587127685546875, "logits/rejected": -1.2570774555206299, "logps/chosen": -270.1664123535156, "logps/ref_chosen": -92.49292755126953, "logps/ref_rejected": -82.06065368652344, "logps/rejected": -370.27276611328125, "loss": 1.0464, "margin_dpo/margin_mean": 110.53861999511719, "margin_dpo/margin_std": 121.771484375, "step": 616 }, { "KL/chosen_KL_mean": -183.70352172851562, "KL/mean": -227.9383544921875, "KL/rejected_KL_mean": -272.1731872558594, "KL/std": 101.46426391601562, "epoch": 0.9060205580029369, "fcm_dpo/beta": 0.004040570463985205, "fcm_dpo/delta": 0.04331379011273384, "fcm_dpo/margin": 88.46965026855469, "fcm_dpo/q_t": 0.4155511260032654, "grad_norm": 8.827617645263672, "learning_rate": 1.378797888467345e-08, "logits/chosen": -1.6747782230377197, "logits/rejected": -1.0403505563735962, "logps/chosen": -274.8005065917969, "logps/ref_chosen": -91.09699249267578, "logps/ref_rejected": -70.41004943847656, "logps/rejected": -342.583251953125, "loss": 1.1121, "margin_dpo/margin_mean": 88.46965026855469, "margin_dpo/margin_std": 113.83547973632812, "step": 617 }, { "KL/chosen_KL_mean": -187.313720703125, "KL/mean": -236.4571533203125, "KL/rejected_KL_mean": -285.6005859375, "KL/std": 111.92489624023438, "epoch": 0.9074889867841409, "fcm_dpo/beta": 0.004044072702527046, "fcm_dpo/delta": 0.0025570346042513847, "fcm_dpo/margin": 98.286865234375, "fcm_dpo/q_t": 0.4060271680355072, "grad_norm": 8.524399757385254, "learning_rate": 1.3370790793601371e-08, "logits/chosen": -2.071159839630127, "logits/rejected": -1.7124844789505005, "logps/chosen": -289.3343505859375, "logps/ref_chosen": -102.02059936523438, "logps/ref_rejected": -99.80119323730469, "logps/rejected": -385.40179443359375, "loss": 1.0883, "margin_dpo/margin_mean": 98.286865234375, "margin_dpo/margin_std": 123.62213134765625, "step": 618 }, { "KL/chosen_KL_mean": -181.24652099609375, "KL/mean": -224.62380981445312, "KL/rejected_KL_mean": -268.0010986328125, "KL/std": 110.29859924316406, "epoch": 0.908957415565345, "fcm_dpo/beta": 0.004060784354805946, "fcm_dpo/delta": 0.04857244715094566, "fcm_dpo/margin": 86.75454711914062, "fcm_dpo/q_t": 0.41928327083587646, "grad_norm": 8.90882396697998, "learning_rate": 1.2959838102258535e-08, "logits/chosen": -1.5276286602020264, "logits/rejected": -1.1612324714660645, "logps/chosen": -270.9879150390625, "logps/ref_chosen": -89.74136352539062, "logps/ref_rejected": -99.90138244628906, "logps/rejected": -367.9024658203125, "loss": 1.1324, "margin_dpo/margin_mean": 86.75454711914062, "margin_dpo/margin_std": 132.44285583496094, "step": 619 }, { "KL/chosen_KL_mean": -172.2540283203125, "KL/mean": -220.5706787109375, "KL/rejected_KL_mean": -268.8872985839844, "KL/std": 115.8946304321289, "epoch": 0.9104258443465492, "fcm_dpo/beta": 0.004051398951560259, "fcm_dpo/delta": 0.00793851912021637, "fcm_dpo/margin": 96.63325500488281, "fcm_dpo/q_t": 0.40886205434799194, "grad_norm": 9.311195373535156, "learning_rate": 1.2555131639630567e-08, "logits/chosen": -2.3657383918762207, "logits/rejected": -1.853982925415039, "logps/chosen": -257.37835693359375, "logps/ref_chosen": -85.12431335449219, "logps/ref_rejected": -85.41253662109375, "logps/rejected": -354.2998352050781, "loss": 1.1003, "margin_dpo/margin_mean": 96.63325500488281, "margin_dpo/margin_std": 124.00234985351562, "step": 620 }, { "KL/chosen_KL_mean": -175.43194580078125, "KL/mean": -222.54608154296875, "KL/rejected_KL_mean": -269.6602478027344, "KL/std": 112.26216125488281, "epoch": 0.9118942731277533, "fcm_dpo/beta": 0.0040911100804805756, "fcm_dpo/delta": 0.01403132826089859, "fcm_dpo/margin": 94.22830963134766, "fcm_dpo/q_t": 0.41055241227149963, "grad_norm": 9.403168678283691, "learning_rate": 1.2156682070109086e-08, "logits/chosen": -1.966080904006958, "logits/rejected": -1.7026954889297485, "logps/chosen": -264.68035888671875, "logps/ref_chosen": -89.24842071533203, "logps/ref_rejected": -95.46463775634766, "logps/rejected": -365.1248779296875, "loss": 1.1125, "margin_dpo/margin_mean": 94.22831726074219, "margin_dpo/margin_std": 131.06173706054688, "step": 621 }, { "KL/chosen_KL_mean": -174.87196350097656, "KL/mean": -224.781494140625, "KL/rejected_KL_mean": -274.6910400390625, "KL/std": 99.69038391113281, "epoch": 0.9133627019089574, "fcm_dpo/beta": 0.0040695141069591045, "fcm_dpo/delta": -0.006391493603587151, "fcm_dpo/margin": 99.81906127929688, "fcm_dpo/q_t": 0.40419769287109375, "grad_norm": 10.707737922668457, "learning_rate": 1.1764499893210878e-08, "logits/chosen": -1.482156753540039, "logits/rejected": -0.8214179277420044, "logps/chosen": -274.66607666015625, "logps/ref_chosen": -99.79412841796875, "logps/ref_rejected": -90.82821655273438, "logps/rejected": -365.51922607421875, "loss": 1.0681, "margin_dpo/margin_mean": 99.81907653808594, "margin_dpo/margin_std": 108.8629150390625, "step": 622 }, { "KL/chosen_KL_mean": -180.89132690429688, "KL/mean": -216.42132568359375, "KL/rejected_KL_mean": -251.95132446289062, "KL/std": 107.79739379882812, "epoch": 0.9148311306901615, "fcm_dpo/beta": 0.004100928083062172, "fcm_dpo/delta": 0.11026909947395325, "fcm_dpo/margin": 71.05996704101562, "fcm_dpo/q_t": 0.4328714907169342, "grad_norm": 9.613092422485352, "learning_rate": 1.1378595443300998e-08, "logits/chosen": -2.2701926231384277, "logits/rejected": -1.8339961767196655, "logps/chosen": -271.34686279296875, "logps/ref_chosen": -90.45555114746094, "logps/ref_rejected": -91.32276153564453, "logps/rejected": -343.2740783691406, "loss": 1.1869, "margin_dpo/margin_mean": 71.05996704101562, "margin_dpo/margin_std": 132.18682861328125, "step": 623 }, { "KL/chosen_KL_mean": -186.8487548828125, "KL/mean": -243.2112274169922, "KL/rejected_KL_mean": -299.5736999511719, "KL/std": 106.96835327148438, "epoch": 0.9162995594713657, "fcm_dpo/beta": 0.004098599776625633, "fcm_dpo/delta": -0.06348156183958054, "fcm_dpo/margin": 112.72494506835938, "fcm_dpo/q_t": 0.39251458644866943, "grad_norm": 9.84626579284668, "learning_rate": 1.0998978889320582e-08, "logits/chosen": -2.473254442214966, "logits/rejected": -1.542797565460205, "logps/chosen": -296.7239990234375, "logps/ref_chosen": -109.87522888183594, "logps/ref_rejected": -104.77320861816406, "logps/rejected": -404.346923828125, "loss": 1.0362, "margin_dpo/margin_mean": 112.72494506835938, "margin_dpo/margin_std": 122.802734375, "step": 624 }, { "KL/chosen_KL_mean": -192.9557647705078, "KL/mean": -233.96263122558594, "KL/rejected_KL_mean": -274.9695129394531, "KL/std": 106.32356262207031, "epoch": 0.9177679882525698, "fcm_dpo/beta": 0.0041158488020300865, "fcm_dpo/delta": 0.0634041354060173, "fcm_dpo/margin": 82.01374816894531, "fcm_dpo/q_t": 0.42098718881607056, "grad_norm": 8.71985912322998, "learning_rate": 1.0625660234518913e-08, "logits/chosen": -1.7170172929763794, "logits/rejected": -1.3425695896148682, "logps/chosen": -280.1239318847656, "logps/ref_chosen": -87.16815948486328, "logps/ref_rejected": -91.86148071289062, "logps/rejected": -366.83099365234375, "loss": 1.129, "margin_dpo/margin_mean": 82.01374053955078, "margin_dpo/margin_std": 113.38253784179688, "step": 625 }, { "KL/chosen_KL_mean": -184.22486877441406, "KL/mean": -225.16958618164062, "KL/rejected_KL_mean": -266.1142883300781, "KL/std": 117.6822509765625, "epoch": 0.9192364170337739, "fcm_dpo/beta": 0.004141476005315781, "fcm_dpo/delta": 0.061798568814992905, "fcm_dpo/margin": 81.88943481445312, "fcm_dpo/q_t": 0.4218093156814575, "grad_norm": 11.027982711791992, "learning_rate": 1.0258649316189721e-08, "logits/chosen": -1.242647409439087, "logits/rejected": -0.7952531576156616, "logps/chosen": -288.4490966796875, "logps/ref_chosen": -104.22421264648438, "logps/ref_rejected": -104.1774673461914, "logps/rejected": -370.291748046875, "loss": 1.1429, "margin_dpo/margin_mean": 81.88943481445312, "margin_dpo/margin_std": 128.6464385986328, "step": 626 }, { "KL/chosen_KL_mean": -161.93655395507812, "KL/mean": -213.656982421875, "KL/rejected_KL_mean": -265.3774108886719, "KL/std": 120.37371826171875, "epoch": 0.920704845814978, "fcm_dpo/beta": 0.004135278519243002, "fcm_dpo/delta": -0.02836085483431816, "fcm_dpo/margin": 103.44085693359375, "fcm_dpo/q_t": 0.40200987458229065, "grad_norm": 7.081504821777344, "learning_rate": 9.897955805412e-09, "logits/chosen": -1.532655954360962, "logits/rejected": -1.679688811302185, "logps/chosen": -236.8711700439453, "logps/ref_chosen": -74.93461608886719, "logps/ref_rejected": -112.57289123535156, "logps/rejected": -377.9502868652344, "loss": 1.0723, "margin_dpo/margin_mean": 103.44085693359375, "margin_dpo/margin_std": 132.03456115722656, "step": 627 }, { "KL/chosen_KL_mean": -172.147216796875, "KL/mean": -225.19296264648438, "KL/rejected_KL_mean": -278.23870849609375, "KL/std": 108.57937622070312, "epoch": 0.922173274596182, "fcm_dpo/beta": 0.0041205501183867455, "fcm_dpo/delta": -0.03798246383666992, "fcm_dpo/margin": 106.09148406982422, "fcm_dpo/q_t": 0.39822065830230713, "grad_norm": 11.217353820800781, "learning_rate": 9.543589206795238e-09, "logits/chosen": -2.225289821624756, "logits/rejected": -1.9694900512695312, "logps/chosen": -265.8382873535156, "logps/ref_chosen": -93.69107818603516, "logps/ref_rejected": -107.34395599365234, "logps/rejected": -385.5826416015625, "loss": 1.0555, "margin_dpo/margin_mean": 106.09147644042969, "margin_dpo/margin_std": 121.18578338623047, "step": 628 }, { "KL/chosen_KL_mean": -186.0232696533203, "KL/mean": -234.23904418945312, "KL/rejected_KL_mean": -282.4548034667969, "KL/std": 109.074951171875, "epoch": 0.9236417033773862, "fcm_dpo/beta": 0.004111516289412975, "fcm_dpo/delta": 0.00351644866168499, "fcm_dpo/margin": 96.43151092529297, "fcm_dpo/q_t": 0.40640905499458313, "grad_norm": 9.434202194213867, "learning_rate": 9.19555885822887e-09, "logits/chosen": -2.146562099456787, "logits/rejected": -1.718677043914795, "logps/chosen": -289.253662109375, "logps/ref_chosen": -103.23037719726562, "logps/ref_rejected": -97.16841888427734, "logps/rejected": -379.62322998046875, "loss": 1.0835, "margin_dpo/margin_mean": 96.4315185546875, "margin_dpo/margin_std": 115.64098358154297, "step": 629 }, { "KL/chosen_KL_mean": -168.41787719726562, "KL/mean": -199.5541534423828, "KL/rejected_KL_mean": -230.6904296875, "KL/std": 101.696533203125, "epoch": 0.9251101321585903, "fcm_dpo/beta": 0.004126053303480148, "fcm_dpo/delta": 0.0406528003513813, "fcm_dpo/margin": 62.27254104614258, "fcm_dpo/q_t": 0.4392763376235962, "grad_norm": 11.541455268859863, "learning_rate": 8.85387393063622e-09, "logits/chosen": -1.829874873161316, "logits/rejected": -1.2854524850845337, "logps/chosen": -262.3154296875, "logps/ref_chosen": -93.89755249023438, "logps/ref_rejected": -89.3743896484375, "logps/rejected": -320.0648193359375, "loss": 1.2009, "margin_dpo/margin_mean": 62.272544860839844, "margin_dpo/margin_std": 114.41587829589844, "step": 630 }, { "KL/chosen_KL_mean": -166.42706298828125, "KL/mean": -213.21585083007812, "KL/rejected_KL_mean": -260.004638671875, "KL/std": 101.66095733642578, "epoch": 0.9265785609397944, "fcm_dpo/beta": 0.004136594478040934, "fcm_dpo/delta": 0.01315954327583313, "fcm_dpo/margin": 93.57760620117188, "fcm_dpo/q_t": 0.4088446795940399, "grad_norm": 9.3220853805542, "learning_rate": 8.518543427732949e-09, "logits/chosen": -2.3428568840026855, "logits/rejected": -1.9033761024475098, "logps/chosen": -254.1978759765625, "logps/ref_chosen": -87.77082061767578, "logps/ref_rejected": -88.68241882324219, "logps/rejected": -348.68707275390625, "loss": 1.0915, "margin_dpo/margin_mean": 93.57760620117188, "margin_dpo/margin_std": 116.26165008544922, "step": 631 }, { "KL/chosen_KL_mean": -162.17013549804688, "KL/mean": -213.90036010742188, "KL/rejected_KL_mean": -265.630615234375, "KL/std": 106.58201599121094, "epoch": 0.9280469897209985, "fcm_dpo/beta": 0.004122081212699413, "fcm_dpo/delta": -0.027319464832544327, "fcm_dpo/margin": 103.4604721069336, "fcm_dpo/q_t": 0.40139931440353394, "grad_norm": 10.833718299865723, "learning_rate": 8.189576185789637e-09, "logits/chosen": -1.946033000946045, "logits/rejected": -1.512399435043335, "logps/chosen": -250.79666137695312, "logps/ref_chosen": -88.62652587890625, "logps/ref_rejected": -91.45091247558594, "logps/rejected": -357.0815124511719, "loss": 1.0751, "margin_dpo/margin_mean": 103.46046447753906, "margin_dpo/margin_std": 131.32321166992188, "step": 632 }, { "KL/chosen_KL_mean": -203.29052734375, "KL/mean": -233.33074951171875, "KL/rejected_KL_mean": -263.3710021972656, "KL/std": 108.14124298095703, "epoch": 0.9295154185022027, "fcm_dpo/beta": 0.00413405941799283, "fcm_dpo/delta": 0.02646186389029026, "fcm_dpo/margin": 60.0804557800293, "fcm_dpo/q_t": 0.4436579644680023, "grad_norm": 9.92850112915039, "learning_rate": 7.866980873399015e-09, "logits/chosen": -2.3172619342803955, "logits/rejected": -2.130617141723633, "logps/chosen": -284.6649475097656, "logps/ref_chosen": -81.37442016601562, "logps/ref_rejected": -98.62571716308594, "logps/rejected": -361.9967041015625, "loss": 1.2395, "margin_dpo/margin_mean": 60.08045196533203, "margin_dpo/margin_std": 144.52981567382812, "step": 633 }, { "KL/chosen_KL_mean": -188.1005401611328, "KL/mean": -227.23358154296875, "KL/rejected_KL_mean": -266.36663818359375, "KL/std": 97.98973083496094, "epoch": 0.9309838472834068, "fcm_dpo/beta": 0.004132546484470367, "fcm_dpo/delta": -0.03379357233643532, "fcm_dpo/margin": 78.2660903930664, "fcm_dpo/q_t": 0.42337095737457275, "grad_norm": 9.918196678161621, "learning_rate": 7.550765991247654e-09, "logits/chosen": -1.2049590349197388, "logits/rejected": -1.025159239768982, "logps/chosen": -284.223388671875, "logps/ref_chosen": -96.12284851074219, "logps/ref_rejected": -112.84780883789062, "logps/rejected": -379.2144470214844, "loss": 1.1338, "margin_dpo/margin_mean": 78.26608276367188, "margin_dpo/margin_std": 99.07841491699219, "step": 634 }, { "KL/chosen_KL_mean": -194.99090576171875, "KL/mean": -225.29229736328125, "KL/rejected_KL_mean": -255.59371948242188, "KL/std": 106.89424896240234, "epoch": 0.9324522760646109, "fcm_dpo/beta": 0.004132390022277832, "fcm_dpo/delta": 0.0330367237329483, "fcm_dpo/margin": 60.60279846191406, "fcm_dpo/q_t": 0.4419868588447571, "grad_norm": 11.799928665161133, "learning_rate": 7.240939871891699e-09, "logits/chosen": -1.8419935703277588, "logits/rejected": -1.3117855787277222, "logps/chosen": -293.6750183105469, "logps/ref_chosen": -98.68411254882812, "logps/ref_rejected": -89.89920043945312, "logps/rejected": -345.492919921875, "loss": 1.2194, "margin_dpo/margin_mean": 60.60279846191406, "margin_dpo/margin_std": 127.8775405883789, "step": 635 }, { "KL/chosen_KL_mean": -173.76181030273438, "KL/mean": -225.53909301757812, "KL/rejected_KL_mean": -277.31634521484375, "KL/std": 109.55888366699219, "epoch": 0.933920704845815, "fcm_dpo/beta": 0.004133112728595734, "fcm_dpo/delta": -0.02861359715461731, "fcm_dpo/margin": 103.55450439453125, "fcm_dpo/q_t": 0.399677574634552, "grad_norm": 8.947562217712402, "learning_rate": 6.937510679537628e-09, "logits/chosen": -1.4670990705490112, "logits/rejected": -0.9833395481109619, "logps/chosen": -264.1797790527344, "logps/ref_chosen": -90.41796112060547, "logps/ref_rejected": -87.70687866210938, "logps/rejected": -365.023193359375, "loss": 1.0516, "margin_dpo/margin_mean": 103.55450439453125, "margin_dpo/margin_std": 109.06664276123047, "step": 636 }, { "KL/chosen_KL_mean": -174.84136962890625, "KL/mean": -229.80905151367188, "KL/rejected_KL_mean": -284.7767333984375, "KL/std": 121.82740783691406, "epoch": 0.9353891336270191, "fcm_dpo/beta": 0.004113885574042797, "fcm_dpo/delta": -0.05345296114683151, "fcm_dpo/margin": 109.93533325195312, "fcm_dpo/q_t": 0.3940490186214447, "grad_norm": 8.195368766784668, "learning_rate": 6.640486409826785e-09, "logits/chosen": -1.9779016971588135, "logits/rejected": -1.8749200105667114, "logps/chosen": -257.29107666015625, "logps/ref_chosen": -82.44971466064453, "logps/ref_rejected": -104.02860260009766, "logps/rejected": -388.8053283691406, "loss": 1.0393, "margin_dpo/margin_mean": 109.93533325195312, "margin_dpo/margin_std": 116.01679992675781, "step": 637 }, { "KL/chosen_KL_mean": -163.5042724609375, "KL/mean": -217.22769165039062, "KL/rejected_KL_mean": -270.9510803222656, "KL/std": 108.30398559570312, "epoch": 0.9368575624082232, "fcm_dpo/beta": 0.004102465696632862, "fcm_dpo/delta": -0.041850265115499496, "fcm_dpo/margin": 107.4468002319336, "fcm_dpo/q_t": 0.3958735466003418, "grad_norm": 8.125937461853027, "learning_rate": 6.349874889624962e-09, "logits/chosen": -1.7744147777557373, "logits/rejected": -1.113417387008667, "logps/chosen": -255.42926025390625, "logps/ref_chosen": -91.92498779296875, "logps/ref_rejected": -86.28703308105469, "logps/rejected": -357.23809814453125, "loss": 1.0365, "margin_dpo/margin_mean": 107.44680786132812, "margin_dpo/margin_std": 100.37921142578125, "step": 638 }, { "KL/chosen_KL_mean": -217.85403442382812, "KL/mean": -243.15200805664062, "KL/rejected_KL_mean": -268.449951171875, "KL/std": 100.29324340820312, "epoch": 0.9383259911894273, "fcm_dpo/beta": 0.004102280829101801, "fcm_dpo/delta": 0.06845078617334366, "fcm_dpo/margin": 50.59590148925781, "fcm_dpo/q_t": 0.4509022831916809, "grad_norm": 9.168580055236816, "learning_rate": 6.065683776815933e-09, "logits/chosen": -1.3636586666107178, "logits/rejected": -0.5981815457344055, "logps/chosen": -322.381591796875, "logps/ref_chosen": -104.52755737304688, "logps/ref_rejected": -81.4803466796875, "logps/rejected": -349.9302978515625, "loss": 1.2535, "margin_dpo/margin_mean": 50.59590148925781, "margin_dpo/margin_std": 124.97648620605469, "step": 639 }, { "KL/chosen_KL_mean": -184.8734893798828, "KL/mean": -232.94715881347656, "KL/rejected_KL_mean": -281.0208435058594, "KL/std": 108.87355041503906, "epoch": 0.9397944199706314, "fcm_dpo/beta": 0.0041341278702020645, "fcm_dpo/delta": 0.0018553882837295532, "fcm_dpo/margin": 96.1473617553711, "fcm_dpo/q_t": 0.4065239727497101, "grad_norm": 11.459507942199707, "learning_rate": 5.7879205600998296e-09, "logits/chosen": -1.9653342962265015, "logits/rejected": -1.455316424369812, "logps/chosen": -282.7587585449219, "logps/ref_chosen": -97.88526916503906, "logps/ref_rejected": -112.70502471923828, "logps/rejected": -393.7258605957031, "loss": 1.0834, "margin_dpo/margin_mean": 96.14736938476562, "margin_dpo/margin_std": 108.8670425415039, "step": 640 }, { "KL/chosen_KL_mean": -169.15939331054688, "KL/mean": -219.54872131347656, "KL/rejected_KL_mean": -269.9380798339844, "KL/std": 125.58119201660156, "epoch": 0.9412628487518355, "fcm_dpo/beta": 0.004114219918847084, "fcm_dpo/delta": -0.014936191961169243, "fcm_dpo/margin": 100.77870178222656, "fcm_dpo/q_t": 0.4064510464668274, "grad_norm": 8.538479804992676, "learning_rate": 5.516592558795746e-09, "logits/chosen": -2.3514883518218994, "logits/rejected": -1.8164699077606201, "logps/chosen": -265.6050720214844, "logps/ref_chosen": -96.4456787109375, "logps/ref_rejected": -95.13568878173828, "logps/rejected": -365.0737609863281, "loss": 1.0873, "margin_dpo/margin_mean": 100.77869415283203, "margin_dpo/margin_std": 138.82662963867188, "step": 641 }, { "KL/chosen_KL_mean": -188.87020874023438, "KL/mean": -236.07574462890625, "KL/rejected_KL_mean": -283.28125, "KL/std": 108.20012664794922, "epoch": 0.9427312775330396, "fcm_dpo/beta": 0.004119611345231533, "fcm_dpo/delta": 0.011203395202755928, "fcm_dpo/margin": 94.4110336303711, "fcm_dpo/q_t": 0.411361962556839, "grad_norm": 10.90930461883545, "learning_rate": 5.251706922648868e-09, "logits/chosen": -1.379327416419983, "logits/rejected": -1.0352121591567993, "logps/chosen": -289.63006591796875, "logps/ref_chosen": -100.75984954833984, "logps/ref_rejected": -114.7076416015625, "logps/rejected": -397.9888916015625, "loss": 1.1051, "margin_dpo/margin_mean": 94.4110336303711, "margin_dpo/margin_std": 134.14910888671875, "step": 642 }, { "KL/chosen_KL_mean": -175.41177368164062, "KL/mean": -220.91351318359375, "KL/rejected_KL_mean": -266.41522216796875, "KL/std": 100.84690856933594, "epoch": 0.9441997063142438, "fcm_dpo/beta": 0.004110201261937618, "fcm_dpo/delta": 0.026108186691999435, "fcm_dpo/margin": 91.00345611572266, "fcm_dpo/q_t": 0.4119134545326233, "grad_norm": 11.415700912475586, "learning_rate": 4.993270631642038e-09, "logits/chosen": -2.206211805343628, "logits/rejected": -1.8301475048065186, "logps/chosen": -260.1554260253906, "logps/ref_chosen": -84.74365997314453, "logps/ref_rejected": -94.31842041015625, "logps/rejected": -360.733642578125, "loss": 1.106, "margin_dpo/margin_mean": 91.00346374511719, "margin_dpo/margin_std": 116.44977569580078, "step": 643 }, { "KL/chosen_KL_mean": -182.19558715820312, "KL/mean": -219.41091918945312, "KL/rejected_KL_mean": -256.6262512207031, "KL/std": 101.3681640625, "epoch": 0.9456681350954479, "fcm_dpo/beta": 0.00415519904345274, "fcm_dpo/delta": 0.09214293956756592, "fcm_dpo/margin": 74.4306640625, "fcm_dpo/q_t": 0.4271387457847595, "grad_norm": 10.91594123840332, "learning_rate": 4.741290495811873e-09, "logits/chosen": -1.9618909358978271, "logits/rejected": -1.6969542503356934, "logps/chosen": -267.518310546875, "logps/ref_chosen": -85.32274627685547, "logps/ref_rejected": -94.60861206054688, "logps/rejected": -351.23486328125, "loss": 1.1499, "margin_dpo/margin_mean": 74.43064880371094, "margin_dpo/margin_std": 108.91319274902344, "step": 644 }, { "KL/chosen_KL_mean": -194.284423828125, "KL/mean": -218.949462890625, "KL/rejected_KL_mean": -243.614501953125, "KL/std": 110.57415008544922, "epoch": 0.947136563876652, "fcm_dpo/beta": 0.0041809724643826485, "fcm_dpo/delta": 0.0787801519036293, "fcm_dpo/margin": 49.33007049560547, "fcm_dpo/q_t": 0.4519861340522766, "grad_norm": 9.718307495117188, "learning_rate": 4.495773155069299e-09, "logits/chosen": -1.2444889545440674, "logits/rejected": -1.070124626159668, "logps/chosen": -276.8746643066406, "logps/ref_chosen": -82.59024047851562, "logps/ref_rejected": -103.00375366210938, "logps/rejected": -346.6182556152344, "loss": 1.2577, "margin_dpo/margin_mean": 49.33007049560547, "margin_dpo/margin_std": 125.35694122314453, "step": 645 }, { "KL/chosen_KL_mean": -170.02645874023438, "KL/mean": -218.04541015625, "KL/rejected_KL_mean": -266.0643615722656, "KL/std": 96.34436798095703, "epoch": 0.9486049926578561, "fcm_dpo/beta": 0.0041885762475430965, "fcm_dpo/delta": -0.002464085817337036, "fcm_dpo/margin": 96.03790283203125, "fcm_dpo/q_t": 0.405120313167572, "grad_norm": 8.83475112915039, "learning_rate": 4.256725079024553e-09, "logits/chosen": -1.8784115314483643, "logits/rejected": -1.2778425216674805, "logps/chosen": -261.218505859375, "logps/ref_chosen": -91.1920394897461, "logps/ref_rejected": -83.77833557128906, "logps/rejected": -349.84271240234375, "loss": 1.0711, "margin_dpo/margin_mean": 96.03790283203125, "margin_dpo/margin_std": 104.44136047363281, "step": 646 }, { "KL/chosen_KL_mean": -185.0558319091797, "KL/mean": -227.13507080078125, "KL/rejected_KL_mean": -269.21429443359375, "KL/std": 116.10716247558594, "epoch": 0.9500734214390602, "fcm_dpo/beta": 0.004210649058222771, "fcm_dpo/delta": 0.04646524041891098, "fcm_dpo/margin": 84.15845489501953, "fcm_dpo/q_t": 0.4185153841972351, "grad_norm": 9.914180755615234, "learning_rate": 4.024152566816791e-09, "logits/chosen": -1.6011396646499634, "logits/rejected": -1.4661681652069092, "logps/chosen": -273.9002685546875, "logps/ref_chosen": -88.8444595336914, "logps/ref_rejected": -99.49833679199219, "logps/rejected": -368.7126159667969, "loss": 1.1326, "margin_dpo/margin_mean": 84.158447265625, "margin_dpo/margin_std": 128.91297912597656, "step": 647 }, { "KL/chosen_KL_mean": -171.845458984375, "KL/mean": -230.323974609375, "KL/rejected_KL_mean": -288.802490234375, "KL/std": 118.83293151855469, "epoch": 0.9515418502202643, "fcm_dpo/beta": 0.004191982559859753, "fcm_dpo/delta": -0.09249797463417053, "fcm_dpo/margin": 116.95703887939453, "fcm_dpo/q_t": 0.38618072867393494, "grad_norm": 9.835695266723633, "learning_rate": 3.798061746947995e-09, "logits/chosen": -1.8092339038848877, "logits/rejected": -1.734454870223999, "logps/chosen": -259.69354248046875, "logps/ref_chosen": -87.84810638427734, "logps/ref_rejected": -104.67005920410156, "logps/rejected": -393.4725341796875, "loss": 1.0205, "margin_dpo/margin_mean": 116.95703125, "margin_dpo/margin_std": 125.89228820800781, "step": 648 }, { "KL/chosen_KL_mean": -158.12045288085938, "KL/mean": -207.042724609375, "KL/rejected_KL_mean": -255.96499633789062, "KL/std": 104.23516845703125, "epoch": 0.9530102790014684, "fcm_dpo/beta": 0.004164888057857752, "fcm_dpo/delta": -0.007908973842859268, "fcm_dpo/margin": 97.84452819824219, "fcm_dpo/q_t": 0.40392300486564636, "grad_norm": 10.439946174621582, "learning_rate": 3.5784585771215235e-09, "logits/chosen": -2.4867160320281982, "logits/rejected": -2.0892417430877686, "logps/chosen": -247.81295776367188, "logps/ref_chosen": -89.6925048828125, "logps/ref_rejected": -88.70658111572266, "logps/rejected": -344.67156982421875, "loss": 1.0642, "margin_dpo/margin_mean": 97.84452819824219, "margin_dpo/margin_std": 100.66397094726562, "step": 649 }, { "KL/chosen_KL_mean": -188.24354553222656, "KL/mean": -234.2677001953125, "KL/rejected_KL_mean": -280.2918701171875, "KL/std": 112.35136413574219, "epoch": 0.9544787077826725, "fcm_dpo/beta": 0.004180354066193104, "fcm_dpo/delta": 0.015490900725126266, "fcm_dpo/margin": 92.04832458496094, "fcm_dpo/q_t": 0.41071081161499023, "grad_norm": 9.75493049621582, "learning_rate": 3.3653488440851253e-09, "logits/chosen": -1.4437450170516968, "logits/rejected": -1.2499384880065918, "logps/chosen": -278.17413330078125, "logps/ref_chosen": -89.93060302734375, "logps/ref_rejected": -102.61282348632812, "logps/rejected": -382.9046936035156, "loss": 1.11, "margin_dpo/margin_mean": 92.04832458496094, "margin_dpo/margin_std": 132.88858032226562, "step": 650 }, { "KL/chosen_KL_mean": -154.0522918701172, "KL/mean": -210.60397338867188, "KL/rejected_KL_mean": -267.1556701660156, "KL/std": 108.38768005371094, "epoch": 0.9559471365638766, "fcm_dpo/beta": 0.004157030023634434, "fcm_dpo/delta": -0.07188078761100769, "fcm_dpo/margin": 113.10337829589844, "fcm_dpo/q_t": 0.38892221450805664, "grad_norm": 9.0950288772583, "learning_rate": 3.158738163478475e-09, "logits/chosen": -2.1865296363830566, "logits/rejected": -2.012490749359131, "logps/chosen": -233.2396240234375, "logps/ref_chosen": -79.18731689453125, "logps/ref_rejected": -105.93333435058594, "logps/rejected": -373.0889892578125, "loss": 1.017, "margin_dpo/margin_mean": 113.10337829589844, "margin_dpo/margin_std": 106.13404846191406, "step": 651 }, { "KL/chosen_KL_mean": -163.78509521484375, "KL/mean": -210.56077575683594, "KL/rejected_KL_mean": -257.33642578125, "KL/std": 100.87310791015625, "epoch": 0.9574155653450808, "fcm_dpo/beta": 0.004158473573625088, "fcm_dpo/delta": 0.011123590171337128, "fcm_dpo/margin": 93.55135345458984, "fcm_dpo/q_t": 0.40789368748664856, "grad_norm": 9.080263137817383, "learning_rate": 2.9586319796851555e-09, "logits/chosen": -2.6430201530456543, "logits/rejected": -2.3525309562683105, "logps/chosen": -265.5753173828125, "logps/ref_chosen": -101.79022216796875, "logps/ref_rejected": -116.3245849609375, "logps/rejected": -373.6610107421875, "loss": 1.0726, "margin_dpo/margin_mean": 93.55135345458984, "margin_dpo/margin_std": 95.81214904785156, "step": 652 }, { "KL/chosen_KL_mean": -186.0380096435547, "KL/mean": -232.13299560546875, "KL/rejected_KL_mean": -278.22796630859375, "KL/std": 110.39732360839844, "epoch": 0.9588839941262849, "fcm_dpo/beta": 0.004162580240517855, "fcm_dpo/delta": 0.016545981168746948, "fcm_dpo/margin": 92.18995666503906, "fcm_dpo/q_t": 0.4085354208946228, "grad_norm": 9.65889835357666, "learning_rate": 2.7650355656892166e-09, "logits/chosen": -2.166877031326294, "logits/rejected": -1.9139078855514526, "logps/chosen": -279.3916015625, "logps/ref_chosen": -93.35359191894531, "logps/ref_rejected": -109.12324523925781, "logps/rejected": -387.3511962890625, "loss": 1.0877, "margin_dpo/margin_mean": 92.1899642944336, "margin_dpo/margin_std": 107.64662170410156, "step": 653 }, { "KL/chosen_KL_mean": -187.43255615234375, "KL/mean": -226.94622802734375, "KL/rejected_KL_mean": -266.4599304199219, "KL/std": 106.02590942382812, "epoch": 0.960352422907489, "fcm_dpo/beta": 0.0041835252195596695, "fcm_dpo/delta": 0.07055902481079102, "fcm_dpo/margin": 79.02738952636719, "fcm_dpo/q_t": 0.4247893691062927, "grad_norm": 11.835972785949707, "learning_rate": 2.577954022936174e-09, "logits/chosen": -2.300550937652588, "logits/rejected": -2.11796236038208, "logps/chosen": -276.548095703125, "logps/ref_chosen": -89.11553955078125, "logps/ref_rejected": -104.91995239257812, "logps/rejected": -371.3798828125, "loss": 1.1552, "margin_dpo/margin_mean": 79.02738952636719, "margin_dpo/margin_std": 133.13107299804688, "step": 654 }, { "KL/chosen_KL_mean": -171.2222137451172, "KL/mean": -218.49172973632812, "KL/rejected_KL_mean": -265.76123046875, "KL/std": 115.62846374511719, "epoch": 0.9618208516886931, "fcm_dpo/beta": 0.0041947197169065475, "fcm_dpo/delta": 0.003490721806883812, "fcm_dpo/margin": 94.5390396118164, "fcm_dpo/q_t": 0.40714073181152344, "grad_norm": 8.602540016174316, "learning_rate": 2.397392281198729e-09, "logits/chosen": -1.8841264247894287, "logits/rejected": -1.8867294788360596, "logps/chosen": -252.25831604003906, "logps/ref_chosen": -81.03610229492188, "logps/ref_rejected": -102.80233764648438, "logps/rejected": -368.5635681152344, "loss": 1.0937, "margin_dpo/margin_mean": 94.53904724121094, "margin_dpo/margin_std": 124.09255981445312, "step": 655 }, { "KL/chosen_KL_mean": -171.37582397460938, "KL/mean": -231.75680541992188, "KL/rejected_KL_mean": -292.1377868652344, "KL/std": 115.41425323486328, "epoch": 0.9632892804698973, "fcm_dpo/beta": 0.004157835617661476, "fcm_dpo/delta": -0.10480746626853943, "fcm_dpo/margin": 120.76197052001953, "fcm_dpo/q_t": 0.3836852014064789, "grad_norm": 9.658221244812012, "learning_rate": 2.223355098446622e-09, "logits/chosen": -1.3591506481170654, "logits/rejected": -1.4303390979766846, "logps/chosen": -256.701171875, "logps/ref_chosen": -85.32534790039062, "logps/ref_rejected": -118.33866119384766, "logps/rejected": -410.4764404296875, "loss": 1.0023, "margin_dpo/margin_mean": 120.761962890625, "margin_dpo/margin_std": 119.09832763671875, "step": 656 }, { "KL/chosen_KL_mean": -187.63662719726562, "KL/mean": -232.23681640625, "KL/rejected_KL_mean": -276.8370056152344, "KL/std": 106.36625671386719, "epoch": 0.9647577092511013, "fcm_dpo/beta": 0.004148798994719982, "fcm_dpo/delta": 0.030339453369379044, "fcm_dpo/margin": 89.20039367675781, "fcm_dpo/q_t": 0.41479969024658203, "grad_norm": 10.726730346679688, "learning_rate": 2.055847060721566e-09, "logits/chosen": -2.0343194007873535, "logits/rejected": -1.8686864376068115, "logps/chosen": -267.8343505859375, "logps/ref_chosen": -80.19772338867188, "logps/ref_rejected": -102.581298828125, "logps/rejected": -379.4183349609375, "loss": 1.1189, "margin_dpo/margin_mean": 89.20039367675781, "margin_dpo/margin_std": 127.93778228759766, "step": 657 }, { "KL/chosen_KL_mean": -184.05101013183594, "KL/mean": -225.86135864257812, "KL/rejected_KL_mean": -267.67169189453125, "KL/std": 100.55616760253906, "epoch": 0.9662261380323054, "fcm_dpo/beta": 0.0041833193972706795, "fcm_dpo/delta": 0.05093512684106827, "fcm_dpo/margin": 83.62065887451172, "fcm_dpo/q_t": 0.4176481068134308, "grad_norm": 8.537247657775879, "learning_rate": 1.8948725820160662e-09, "logits/chosen": -2.2053067684173584, "logits/rejected": -1.7498502731323242, "logps/chosen": -278.685546875, "logps/ref_chosen": -94.634521484375, "logps/ref_rejected": -101.6316146850586, "logps/rejected": -369.30328369140625, "loss": 1.1167, "margin_dpo/margin_mean": 83.62066650390625, "margin_dpo/margin_std": 107.93345642089844, "step": 658 }, { "KL/chosen_KL_mean": -167.4307861328125, "KL/mean": -217.44142150878906, "KL/rejected_KL_mean": -267.45208740234375, "KL/std": 108.06137084960938, "epoch": 0.9676945668135095, "fcm_dpo/beta": 0.004169822204858065, "fcm_dpo/delta": -0.01760134845972061, "fcm_dpo/margin": 100.02124786376953, "fcm_dpo/q_t": 0.40431806445121765, "grad_norm": 8.311103820800781, "learning_rate": 1.7404359041573723e-09, "logits/chosen": -2.1437501907348633, "logits/rejected": -1.179504156112671, "logps/chosen": -279.9866638183594, "logps/ref_chosen": -112.55587005615234, "logps/ref_rejected": -93.9216079711914, "logps/rejected": -361.3736572265625, "loss": 1.0961, "margin_dpo/margin_mean": 100.02125549316406, "margin_dpo/margin_std": 140.64071655273438, "step": 659 }, { "KL/chosen_KL_mean": -175.69651794433594, "KL/mean": -226.7316131591797, "KL/rejected_KL_mean": -277.7667236328125, "KL/std": 104.99344635009766, "epoch": 0.9691629955947136, "fcm_dpo/beta": 0.004167176783084869, "fcm_dpo/delta": -0.025897571817040443, "fcm_dpo/margin": 102.0701904296875, "fcm_dpo/q_t": 0.40059009194374084, "grad_norm": 11.31667423248291, "learning_rate": 1.592541096695571e-09, "logits/chosen": -1.7410497665405273, "logits/rejected": -1.1036680936813354, "logps/chosen": -269.0739440917969, "logps/ref_chosen": -93.37742614746094, "logps/ref_rejected": -81.39482116699219, "logps/rejected": -359.1615295410156, "loss": 1.0592, "margin_dpo/margin_mean": 102.0701904296875, "margin_dpo/margin_std": 115.01097106933594, "step": 660 }, { "KL/chosen_KL_mean": -175.06634521484375, "KL/mean": -232.54193115234375, "KL/rejected_KL_mean": -290.01751708984375, "KL/std": 117.2916259765625, "epoch": 0.9706314243759178, "fcm_dpo/beta": 0.004144606646150351, "fcm_dpo/delta": -0.07825462520122528, "fcm_dpo/margin": 114.951171875, "fcm_dpo/q_t": 0.39038658142089844, "grad_norm": 9.784979820251465, "learning_rate": 1.4511920567963908e-09, "logits/chosen": -2.0185415744781494, "logits/rejected": -1.5206613540649414, "logps/chosen": -262.9215087890625, "logps/ref_chosen": -87.85516357421875, "logps/ref_rejected": -92.40330505371094, "logps/rejected": -382.42083740234375, "loss": 1.0352, "margin_dpo/margin_mean": 114.951171875, "margin_dpo/margin_std": 130.60153198242188, "step": 661 }, { "KL/chosen_KL_mean": -171.02005004882812, "KL/mean": -223.704345703125, "KL/rejected_KL_mean": -276.3886413574219, "KL/std": 102.89617919921875, "epoch": 0.9720998531571219, "fcm_dpo/beta": 0.004124734550714493, "fcm_dpo/delta": -0.035384222865104675, "fcm_dpo/margin": 105.36859130859375, "fcm_dpo/q_t": 0.3979458212852478, "grad_norm": 10.346973419189453, "learning_rate": 1.3163925091384532e-09, "logits/chosen": -1.8384698629379272, "logits/rejected": -1.2771579027175903, "logps/chosen": -273.79986572265625, "logps/ref_chosen": -102.77980041503906, "logps/ref_rejected": -95.22531127929688, "logps/rejected": -371.61395263671875, "loss": 1.0469, "margin_dpo/margin_mean": 105.36859130859375, "margin_dpo/margin_std": 108.84596252441406, "step": 662 }, { "KL/chosen_KL_mean": -171.448486328125, "KL/mean": -219.08230590820312, "KL/rejected_KL_mean": -266.71612548828125, "KL/std": 105.92256164550781, "epoch": 0.973568281938326, "fcm_dpo/beta": 0.004123318940401077, "fcm_dpo/delta": 0.007292289286851883, "fcm_dpo/margin": 95.26766967773438, "fcm_dpo/q_t": 0.4084503650665283, "grad_norm": 10.68273639678955, "learning_rate": 1.1881460058152382e-09, "logits/chosen": -2.1778860092163086, "logits/rejected": -1.957489013671875, "logps/chosen": -267.7950744628906, "logps/ref_chosen": -96.34658813476562, "logps/ref_rejected": -120.52645874023438, "logps/rejected": -387.24261474609375, "loss": 1.0934, "margin_dpo/margin_mean": 95.26766967773438, "margin_dpo/margin_std": 123.21327209472656, "step": 663 }, { "KL/chosen_KL_mean": -191.30111694335938, "KL/mean": -234.4645538330078, "KL/rejected_KL_mean": -277.62799072265625, "KL/std": 109.58445739746094, "epoch": 0.9750367107195301, "fcm_dpo/beta": 0.004138381220400333, "fcm_dpo/delta": 0.0434366911649704, "fcm_dpo/margin": 86.32688903808594, "fcm_dpo/q_t": 0.4158845543861389, "grad_norm": 8.516119956970215, "learning_rate": 1.066455926241383e-09, "logits/chosen": -1.8140387535095215, "logits/rejected": -1.642183780670166, "logps/chosen": -283.1435241699219, "logps/ref_chosen": -91.84242248535156, "logps/ref_rejected": -111.8366928100586, "logps/rejected": -389.4646911621094, "loss": 1.1106, "margin_dpo/margin_mean": 86.3268814086914, "margin_dpo/margin_std": 108.4761734008789, "step": 664 }, { "KL/chosen_KL_mean": -162.95761108398438, "KL/mean": -210.8182373046875, "KL/rejected_KL_mean": -258.6788635253906, "KL/std": 106.0069351196289, "epoch": 0.9765051395007343, "fcm_dpo/beta": 0.004143881145864725, "fcm_dpo/delta": 0.0033657848834991455, "fcm_dpo/margin": 95.72123718261719, "fcm_dpo/q_t": 0.4060526192188263, "grad_norm": 8.06864070892334, "learning_rate": 9.513254770636137e-10, "logits/chosen": -2.3605947494506836, "logits/rejected": -1.926710844039917, "logps/chosen": -251.143798828125, "logps/ref_chosen": -88.18618774414062, "logps/ref_rejected": -91.9120101928711, "logps/rejected": -350.59088134765625, "loss": 1.077, "margin_dpo/margin_mean": 95.72123718261719, "margin_dpo/margin_std": 108.63481140136719, "step": 665 }, { "KL/chosen_KL_mean": -179.67604064941406, "KL/mean": -228.17626953125, "KL/rejected_KL_mean": -276.676513671875, "KL/std": 103.14291381835938, "epoch": 0.9779735682819384, "fcm_dpo/beta": 0.004134770482778549, "fcm_dpo/delta": -0.0011554453521966934, "fcm_dpo/margin": 97.00045776367188, "fcm_dpo/q_t": 0.40694648027420044, "grad_norm": 11.099916458129883, "learning_rate": 8.427576920763956e-10, "logits/chosen": -1.5606260299682617, "logits/rejected": -1.1774933338165283, "logps/chosen": -280.650634765625, "logps/ref_chosen": -100.97459411621094, "logps/ref_rejected": -101.24992370605469, "logps/rejected": -377.9264221191406, "loss": 1.087, "margin_dpo/margin_mean": 97.00045776367188, "margin_dpo/margin_std": 122.14839172363281, "step": 666 }, { "KL/chosen_KL_mean": -195.8563232421875, "KL/mean": -245.22705078125, "KL/rejected_KL_mean": -294.5977783203125, "KL/std": 107.78409576416016, "epoch": 0.9794419970631424, "fcm_dpo/beta": 0.004134554415941238, "fcm_dpo/delta": -0.008442575111985207, "fcm_dpo/margin": 98.74146270751953, "fcm_dpo/q_t": 0.4034131169319153, "grad_norm": 9.027989387512207, "learning_rate": 7.407554321417764e-10, "logits/chosen": -1.3676538467407227, "logits/rejected": -0.6978269815444946, "logps/chosen": -293.427490234375, "logps/ref_chosen": -97.5711669921875, "logps/ref_rejected": -93.58476257324219, "logps/rejected": -388.18255615234375, "loss": 1.0637, "margin_dpo/margin_mean": 98.74146270751953, "margin_dpo/margin_std": 104.31491088867188, "step": 667 }, { "KL/chosen_KL_mean": -173.09786987304688, "KL/mean": -230.94320678710938, "KL/rejected_KL_mean": -288.78851318359375, "KL/std": 114.91456604003906, "epoch": 0.9809104258443465, "fcm_dpo/beta": 0.004114450886845589, "fcm_dpo/delta": -0.0778254047036171, "fcm_dpo/margin": 115.6906509399414, "fcm_dpo/q_t": 0.39051228761672974, "grad_norm": 10.171438217163086, "learning_rate": 6.453213851142225e-10, "logits/chosen": -2.306455135345459, "logits/rejected": -1.7856180667877197, "logps/chosen": -275.67291259765625, "logps/ref_chosen": -102.5750503540039, "logps/ref_rejected": -108.81768798828125, "logps/rejected": -397.606201171875, "loss": 1.027, "margin_dpo/margin_mean": 115.6906509399414, "margin_dpo/margin_std": 125.12269592285156, "step": 668 }, { "KL/chosen_KL_mean": -157.6524658203125, "KL/mean": -213.17294311523438, "KL/rejected_KL_mean": -268.69342041015625, "KL/std": 111.88285827636719, "epoch": 0.9823788546255506, "fcm_dpo/beta": 0.0040878611616790295, "fcm_dpo/delta": -0.05516382306814194, "fcm_dpo/margin": 111.04096221923828, "fcm_dpo/q_t": 0.3941311240196228, "grad_norm": 9.402403831481934, "learning_rate": 5.564580657695939e-10, "logits/chosen": -1.7253546714782715, "logits/rejected": -1.329820156097412, "logps/chosen": -247.14723205566406, "logps/ref_chosen": -89.49478149414062, "logps/ref_rejected": -82.51950073242188, "logps/rejected": -351.21295166015625, "loss": 1.0386, "margin_dpo/margin_mean": 111.04096221923828, "margin_dpo/margin_std": 119.09214782714844, "step": 669 }, { "KL/chosen_KL_mean": -158.4515380859375, "KL/mean": -219.50779724121094, "KL/rejected_KL_mean": -280.56402587890625, "KL/std": 114.22224426269531, "epoch": 0.9838472834067548, "fcm_dpo/beta": 0.004054767079651356, "fcm_dpo/delta": -0.09750363230705261, "fcm_dpo/margin": 122.11248779296875, "fcm_dpo/q_t": 0.3833516240119934, "grad_norm": 7.476661682128906, "learning_rate": 4.741678157389739e-10, "logits/chosen": -1.7914434671401978, "logits/rejected": -1.2572290897369385, "logps/chosen": -253.90614318847656, "logps/ref_chosen": -95.45459747314453, "logps/ref_rejected": -101.53292846679688, "logps/rejected": -382.09698486328125, "loss": 1.0072, "margin_dpo/margin_mean": 122.11248779296875, "margin_dpo/margin_std": 119.4871826171875, "step": 670 }, { "KL/chosen_KL_mean": -193.36373901367188, "KL/mean": -235.04254150390625, "KL/rejected_KL_mean": -276.72137451171875, "KL/std": 105.78631591796875, "epoch": 0.9853157121879589, "fcm_dpo/beta": 0.004070744849741459, "fcm_dpo/delta": 0.06130343675613403, "fcm_dpo/margin": 83.35762023925781, "fcm_dpo/q_t": 0.42097437381744385, "grad_norm": 9.48980712890625, "learning_rate": 3.9845280344705245e-10, "logits/chosen": -1.7025372982025146, "logits/rejected": -1.4800764322280884, "logps/chosen": -275.48687744140625, "logps/ref_chosen": -82.12312316894531, "logps/ref_rejected": -90.21969604492188, "logps/rejected": -366.9410400390625, "loss": 1.1325, "margin_dpo/margin_mean": 83.35761260986328, "margin_dpo/margin_std": 117.642333984375, "step": 671 }, { "KL/chosen_KL_mean": -193.25042724609375, "KL/mean": -229.13946533203125, "KL/rejected_KL_mean": -265.02850341796875, "KL/std": 101.83818817138672, "epoch": 0.986784140969163, "fcm_dpo/beta": 0.0040968493558466434, "fcm_dpo/delta": 0.10757053643465042, "fcm_dpo/margin": 71.77810668945312, "fcm_dpo/q_t": 0.4308933913707733, "grad_norm": 12.626511573791504, "learning_rate": 3.293150240547549e-10, "logits/chosen": -2.1679110527038574, "logits/rejected": -1.8443044424057007, "logps/chosen": -283.31231689453125, "logps/ref_chosen": -90.0619125366211, "logps/ref_rejected": -100.45323181152344, "logps/rejected": -365.48175048828125, "loss": 1.1638, "margin_dpo/margin_mean": 71.77810668945312, "margin_dpo/margin_std": 108.26654052734375, "step": 672 }, { "KL/chosen_KL_mean": -172.74874877929688, "KL/mean": -213.12277221679688, "KL/rejected_KL_mean": -253.496826171875, "KL/std": 105.1376953125, "epoch": 0.9882525697503671, "fcm_dpo/beta": 0.004135543014854193, "fcm_dpo/delta": 0.06707511842250824, "fcm_dpo/margin": 80.74808502197266, "fcm_dpo/q_t": 0.42147284746170044, "grad_norm": 10.8684663772583, "learning_rate": 2.6675629940689504e-10, "logits/chosen": -1.8008537292480469, "logits/rejected": -1.4259765148162842, "logps/chosen": -252.01190185546875, "logps/ref_chosen": -79.26315307617188, "logps/ref_rejected": -91.34925079345703, "logps/rejected": -344.8460693359375, "loss": 1.1267, "margin_dpo/margin_mean": 80.74807739257812, "margin_dpo/margin_std": 106.74629211425781, "step": 673 }, { "KL/chosen_KL_mean": -161.00936889648438, "KL/mean": -218.79251098632812, "KL/rejected_KL_mean": -276.5756530761719, "KL/std": 105.09916687011719, "epoch": 0.9897209985315712, "fcm_dpo/beta": 0.004112974740564823, "fcm_dpo/delta": -0.07719054818153381, "fcm_dpo/margin": 115.56629943847656, "fcm_dpo/q_t": 0.38953882455825806, "grad_norm": 8.388043403625488, "learning_rate": 2.1077827798404725e-10, "logits/chosen": -1.7251453399658203, "logits/rejected": -1.3302440643310547, "logps/chosen": -236.46768188476562, "logps/ref_chosen": -75.45831298828125, "logps/ref_rejected": -76.20362854003906, "logps/rejected": -352.779296875, "loss": 1.018, "margin_dpo/margin_mean": 115.5662841796875, "margin_dpo/margin_std": 115.20941162109375, "step": 674 }, { "KL/chosen_KL_mean": -178.07435607910156, "KL/mean": -229.0012664794922, "KL/rejected_KL_mean": -279.92816162109375, "KL/std": 113.45880889892578, "epoch": 0.9911894273127754, "fcm_dpo/beta": 0.004089445807039738, "fcm_dpo/delta": -0.01726212352514267, "fcm_dpo/margin": 101.85381317138672, "fcm_dpo/q_t": 0.4032495319843292, "grad_norm": 10.7861328125, "learning_rate": 1.6138243485910863e-10, "logits/chosen": -1.3645881414413452, "logits/rejected": -1.017865777015686, "logps/chosen": -257.98388671875, "logps/ref_chosen": -79.90953063964844, "logps/ref_rejected": -81.21824645996094, "logps/rejected": -361.14642333984375, "loss": 1.0917, "margin_dpo/margin_mean": 101.85380554199219, "margin_dpo/margin_std": 139.26797485351562, "step": 675 }, { "KL/chosen_KL_mean": -192.17776489257812, "KL/mean": -241.98626708984375, "KL/rejected_KL_mean": -291.79473876953125, "KL/std": 108.77763366699219, "epoch": 0.9926578560939795, "fcm_dpo/beta": 0.004092554096132517, "fcm_dpo/delta": -0.007904548197984695, "fcm_dpo/margin": 99.61698150634766, "fcm_dpo/q_t": 0.40432265400886536, "grad_norm": 8.63825511932373, "learning_rate": 1.1857007165852472e-10, "logits/chosen": -1.8934168815612793, "logits/rejected": -1.300978660583496, "logps/chosen": -290.348876953125, "logps/ref_chosen": -98.17111206054688, "logps/ref_rejected": -95.024658203125, "logps/rejected": -386.81939697265625, "loss": 1.0806, "margin_dpo/margin_mean": 99.61698150634766, "margin_dpo/margin_std": 121.85092163085938, "step": 676 }, { "KL/chosen_KL_mean": -171.11822509765625, "KL/mean": -222.21356201171875, "KL/rejected_KL_mean": -273.30889892578125, "KL/std": 107.71270751953125, "epoch": 0.9941262848751835, "fcm_dpo/beta": 0.004086182452738285, "fcm_dpo/delta": -0.018013406544923782, "fcm_dpo/margin": 102.19068145751953, "fcm_dpo/q_t": 0.40157467126846313, "grad_norm": 8.241106033325195, "learning_rate": 8.23423165278725e-11, "logits/chosen": -1.760502815246582, "logits/rejected": -1.0893501043319702, "logps/chosen": -262.49749755859375, "logps/ref_chosen": -91.37928771972656, "logps/ref_rejected": -82.87776947021484, "logps/rejected": -356.1866760253906, "loss": 1.057, "margin_dpo/margin_mean": 102.190673828125, "margin_dpo/margin_std": 105.21155548095703, "step": 677 }, { "KL/chosen_KL_mean": -161.4202880859375, "KL/mean": -221.08929443359375, "KL/rejected_KL_mean": -280.75830078125, "KL/std": 114.70274353027344, "epoch": 0.9955947136563876, "fcm_dpo/beta": 0.004071141593158245, "fcm_dpo/delta": -0.08798402547836304, "fcm_dpo/margin": 119.33800506591797, "fcm_dpo/q_t": 0.3863942325115204, "grad_norm": 9.588170051574707, "learning_rate": 5.270012410216185e-11, "logits/chosen": -1.7387727499008179, "logits/rejected": -1.4145240783691406, "logps/chosen": -237.066162109375, "logps/ref_chosen": -75.6458740234375, "logps/ref_rejected": -86.96611022949219, "logps/rejected": -367.7243957519531, "loss": 1.0133, "margin_dpo/margin_mean": 119.33800506591797, "margin_dpo/margin_std": 116.71041870117188, "step": 678 }, { "KL/chosen_KL_mean": -186.90036010742188, "KL/mean": -236.03787231445312, "KL/rejected_KL_mean": -285.1754150390625, "KL/std": 96.0952377319336, "epoch": 0.9970631424375918, "fcm_dpo/beta": 0.004052022006362677, "fcm_dpo/delta": 0.0018185428343713284, "fcm_dpo/margin": 98.27503967285156, "fcm_dpo/q_t": 0.40641626715660095, "grad_norm": 8.639100074768066, "learning_rate": 2.9644275480772416e-11, "logits/chosen": -2.2520229816436768, "logits/rejected": -1.7381494045257568, "logps/chosen": -267.673828125, "logps/ref_chosen": -80.77344512939453, "logps/ref_rejected": -82.87850189208984, "logps/rejected": -368.05389404296875, "loss": 1.0676, "margin_dpo/margin_mean": 98.27503967285156, "margin_dpo/margin_std": 102.73526763916016, "step": 679 }, { "KL/chosen_KL_mean": -187.94369506835938, "KL/mean": -241.76229858398438, "KL/rejected_KL_mean": -295.5809326171875, "KL/std": 113.64466857910156, "epoch": 0.9985315712187959, "fcm_dpo/beta": 0.004055606666952372, "fcm_dpo/delta": -0.03765740990638733, "fcm_dpo/margin": 107.63724517822266, "fcm_dpo/q_t": 0.3989691734313965, "grad_norm": 7.507920742034912, "learning_rate": 1.31753782067201e-11, "logits/chosen": -1.957519292831421, "logits/rejected": -1.628006935119629, "logps/chosen": -295.6266174316406, "logps/ref_chosen": -107.68292999267578, "logps/ref_rejected": -116.0948715209961, "logps/rejected": -411.67578125, "loss": 1.0592, "margin_dpo/margin_mean": 107.63723754882812, "margin_dpo/margin_std": 123.32096862792969, "step": 680 }, { "KL/chosen_KL_mean": -170.96510314941406, "KL/mean": -217.71829223632812, "KL/rejected_KL_mean": -264.4714660644531, "KL/std": 107.1221923828125, "epoch": 1.0, "fcm_dpo/beta": 0.004048110917210579, "fcm_dpo/delta": 0.02185189723968506, "fcm_dpo/margin": 93.50635528564453, "fcm_dpo/q_t": 0.4116027057170868, "grad_norm": 10.250300407409668, "learning_rate": 3.2938662507808745e-12, "logits/chosen": -2.3380775451660156, "logits/rejected": -2.0294878482818604, "logps/chosen": -263.9761657714844, "logps/ref_chosen": -93.01106262207031, "logps/ref_rejected": -94.8221664428711, "logps/rejected": -359.29364013671875, "loss": 1.0974, "margin_dpo/margin_mean": 93.50634765625, "margin_dpo/margin_std": 117.61985778808594, "step": 681 }, { "epoch": 1.0, "step": 681, "total_flos": 0.0, "train_loss": 1.089209524370325, "train_runtime": 1972.735, "train_samples_per_second": 22.1, "train_steps_per_second": 0.345 } ], "logging_steps": 1, "max_steps": 681, "num_input_tokens_seen": 0, "num_train_epochs": 1, "save_steps": 50, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": false, "should_training_stop": false }, "attributes": {} } }, "total_flos": 0.0, "train_batch_size": 8, "trial_name": null, "trial_params": null }