{ "best_global_step": null, "best_metric": null, "best_model_checkpoint": null, "epoch": 1.0, "eval_steps": 100, "global_step": 681, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "KL/chosen_KL_mean": 0.05419158935546875, "KL/mean": 0.023573115468025208, "KL/rejected_KL_mean": -0.007045745849609375, "KL/std": 0.33097243309020996, "epoch": 0.0014684287812041115, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.06123492121696472, "fcm_dpo/q_t": 0.4984707534313202, "grad_norm": 38.819522857666016, "learning_rate": 0.0, "logits/chosen": 0.2677389979362488, "logits/rejected": 0.6191319227218628, "logps/chosen": -85.37664031982422, "logps/ref_chosen": -85.43083190917969, "logps/ref_rejected": -79.90458679199219, "logps/rejected": -79.91163635253906, "loss": 1.3807, "margin_dpo/margin_mean": 0.061235010623931885, "margin_dpo/margin_std": 0.4424324035644531, "step": 1 }, { "KL/chosen_KL_mean": -0.044902801513671875, "KL/mean": 0.017303764820098877, "KL/rejected_KL_mean": 0.07951736450195312, "KL/std": 0.2933448553085327, "epoch": 0.002936857562408223, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.12442219257354736, "fcm_dpo/q_t": 0.5031089186668396, "grad_norm": 29.759681701660156, "learning_rate": 7.246376811594203e-09, "logits/chosen": 0.47771453857421875, "logits/rejected": 0.5908545851707458, "logps/chosen": -82.11383056640625, "logps/ref_chosen": -82.06893157958984, "logps/ref_rejected": -81.65457916259766, "logps/rejected": -81.57505798339844, "loss": 1.3992, "margin_dpo/margin_mean": -0.12442246079444885, "margin_dpo/margin_std": 0.41358208656311035, "step": 2 }, { "KL/chosen_KL_mean": 0.01801300048828125, "KL/mean": 0.009846903383731842, "KL/rejected_KL_mean": 0.001682281494140625, "KL/std": 0.29592806100845337, "epoch": 0.004405286343612335, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.01633407175540924, "fcm_dpo/q_t": 0.4995919466018677, "grad_norm": 26.0485897064209, "learning_rate": 1.4492753623188406e-08, "logits/chosen": 0.42689090967178345, "logits/rejected": 0.6502213478088379, "logps/chosen": -93.79296875, "logps/ref_chosen": -93.81098937988281, "logps/ref_rejected": -74.22950744628906, "logps/rejected": -74.22782135009766, "loss": 1.3852, "margin_dpo/margin_mean": 0.016334399580955505, "margin_dpo/margin_std": 0.43817389011383057, "step": 3 }, { "KL/chosen_KL_mean": -0.032138824462890625, "KL/mean": -0.030376777052879333, "KL/rejected_KL_mean": -0.02861785888671875, "KL/std": 0.3369988203048706, "epoch": 0.005873715124816446, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.0035226643085479736, "fcm_dpo/q_t": 0.5000864267349243, "grad_norm": 32.7692756652832, "learning_rate": 2.1739130434782606e-08, "logits/chosen": 0.6462423801422119, "logits/rejected": 0.7780641913414001, "logps/chosen": -87.3246078491211, "logps/ref_chosen": -87.29246520996094, "logps/ref_rejected": -93.82425689697266, "logps/rejected": -93.85287475585938, "loss": 1.3872, "margin_dpo/margin_mean": -0.0035224556922912598, "margin_dpo/margin_std": 0.4537009596824646, "step": 4 }, { "KL/chosen_KL_mean": -0.055454254150390625, "KL/mean": -0.045780032873153687, "KL/rejected_KL_mean": -0.03610992431640625, "KL/std": 0.3053707480430603, "epoch": 0.007342143906020558, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.019347012042999268, "fcm_dpo/q_t": 0.5004826188087463, "grad_norm": 35.43703842163086, "learning_rate": 2.898550724637681e-08, "logits/chosen": 0.38495126366615295, "logits/rejected": 0.5912345051765442, "logps/chosen": -89.3922119140625, "logps/ref_chosen": -89.33676147460938, "logps/ref_rejected": -88.74783325195312, "logps/rejected": -88.78394317626953, "loss": 1.3888, "margin_dpo/margin_mean": -0.01934686303138733, "margin_dpo/margin_std": 0.4653624892234802, "step": 5 }, { "KL/chosen_KL_mean": -0.012523651123046875, "KL/mean": -0.03238530457019806, "KL/rejected_KL_mean": -0.05224609375, "KL/std": 0.31671661138534546, "epoch": 0.00881057268722467, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.03972157835960388, "fcm_dpo/q_t": 0.49900656938552856, "grad_norm": 36.194889068603516, "learning_rate": 3.6231884057971014e-08, "logits/chosen": 0.4861461818218231, "logits/rejected": 0.8017364740371704, "logps/chosen": -97.33399963378906, "logps/ref_chosen": -97.32147216796875, "logps/ref_rejected": -97.88345336914062, "logps/rejected": -97.93569946289062, "loss": 1.3829, "margin_dpo/margin_mean": 0.039721399545669556, "margin_dpo/margin_std": 0.4631843566894531, "step": 6 }, { "KL/chosen_KL_mean": -0.01750946044921875, "KL/mean": -0.010317608714103699, "KL/rejected_KL_mean": -0.00313568115234375, "KL/std": 0.3258499503135681, "epoch": 0.010279001468428781, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.014376074075698853, "fcm_dpo/q_t": 0.5003604888916016, "grad_norm": 38.085540771484375, "learning_rate": 4.347826086956521e-08, "logits/chosen": 0.4962475597858429, "logits/rejected": 0.5440775156021118, "logps/chosen": -86.66604614257812, "logps/ref_chosen": -86.64852905273438, "logps/ref_rejected": -109.61618041992188, "logps/rejected": -109.61931610107422, "loss": 1.3883, "margin_dpo/margin_mean": -0.014376431703567505, "margin_dpo/margin_std": 0.4693182706832886, "step": 7 }, { "KL/chosen_KL_mean": -0.0348968505859375, "KL/mean": 0.02407728135585785, "KL/rejected_KL_mean": 0.0830535888671875, "KL/std": 0.28958797454833984, "epoch": 0.011747430249632892, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.11795011162757874, "fcm_dpo/q_t": 0.5029476284980774, "grad_norm": 32.91734313964844, "learning_rate": 5.0724637681159424e-08, "logits/chosen": 0.4092610478401184, "logits/rejected": 0.7200788855552673, "logps/chosen": -89.95858001708984, "logps/ref_chosen": -89.9236831665039, "logps/ref_rejected": -86.22804260253906, "logps/rejected": -86.14498901367188, "loss": 1.3985, "margin_dpo/margin_mean": -0.11794936656951904, "margin_dpo/margin_std": 0.38808369636535645, "step": 8 }, { "KL/chosen_KL_mean": 0.07097244262695312, "KL/mean": 0.015693560242652893, "KL/rejected_KL_mean": -0.039577484130859375, "KL/std": 0.3280068635940552, "epoch": 0.013215859030837005, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.11054441332817078, "fcm_dpo/q_t": 0.4972390830516815, "grad_norm": 35.82514953613281, "learning_rate": 5.797101449275362e-08, "logits/chosen": 0.6291981935501099, "logits/rejected": 0.8066290616989136, "logps/chosen": -103.78616333007812, "logps/ref_chosen": -103.85713195800781, "logps/ref_rejected": -104.31932067871094, "logps/rejected": -104.35890197753906, "loss": 1.3758, "margin_dpo/margin_mean": 0.11054429411888123, "margin_dpo/margin_std": 0.46005770564079285, "step": 9 }, { "KL/chosen_KL_mean": 0.02231597900390625, "KL/mean": 0.07833842188119888, "KL/rejected_KL_mean": 0.13436126708984375, "KL/std": 0.2511771321296692, "epoch": 0.014684287812041116, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.11205075681209564, "fcm_dpo/q_t": 0.5027999877929688, "grad_norm": 33.62577438354492, "learning_rate": 6.521739130434782e-08, "logits/chosen": 0.5185714960098267, "logits/rejected": 0.7311661839485168, "logps/chosen": -76.18205261230469, "logps/ref_chosen": -76.20436096191406, "logps/ref_rejected": -87.152099609375, "logps/rejected": -87.01774597167969, "loss": 1.3978, "margin_dpo/margin_mean": -0.11205106973648071, "margin_dpo/margin_std": 0.335724413394928, "step": 10 }, { "KL/chosen_KL_mean": 0.06981277465820312, "KL/mean": 0.04370792955160141, "KL/rejected_KL_mean": 0.017604827880859375, "KL/std": 0.3104771375656128, "epoch": 0.016152716593245228, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.05220632255077362, "fcm_dpo/q_t": 0.49869662523269653, "grad_norm": 36.76396942138672, "learning_rate": 7.246376811594203e-08, "logits/chosen": 0.3010936677455902, "logits/rejected": 0.5179265141487122, "logps/chosen": -82.29668426513672, "logps/ref_chosen": -82.36650085449219, "logps/ref_rejected": -94.26461791992188, "logps/rejected": -94.24701690673828, "loss": 1.3817, "margin_dpo/margin_mean": 0.052206918597221375, "margin_dpo/margin_std": 0.4737238585948944, "step": 11 }, { "KL/chosen_KL_mean": -0.011959075927734375, "KL/mean": -0.013267628848552704, "KL/rejected_KL_mean": -0.0145721435546875, "KL/std": 0.34847086668014526, "epoch": 0.01762114537444934, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.0026122182607650757, "fcm_dpo/q_t": 0.4999332129955292, "grad_norm": 40.034034729003906, "learning_rate": 7.971014492753623e-08, "logits/chosen": 0.3836405873298645, "logits/rejected": 0.511702299118042, "logps/chosen": -99.11746215820312, "logps/ref_chosen": -99.10549926757812, "logps/ref_rejected": -110.27140808105469, "logps/rejected": -110.28598022460938, "loss": 1.3867, "margin_dpo/margin_mean": 0.002612382173538208, "margin_dpo/margin_std": 0.518186628818512, "step": 12 }, { "KL/chosen_KL_mean": -0.0435791015625, "KL/mean": -0.024171896278858185, "KL/rejected_KL_mean": -0.004764556884765625, "KL/std": 0.2932068705558777, "epoch": 0.01908957415565345, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.038815781474113464, "fcm_dpo/q_t": 0.500969648361206, "grad_norm": 42.037208557128906, "learning_rate": 8.695652173913042e-08, "logits/chosen": 0.37952056527137756, "logits/rejected": 0.6270250082015991, "logps/chosen": -90.60330200195312, "logps/ref_chosen": -90.55972290039062, "logps/ref_rejected": -93.69110107421875, "logps/rejected": -93.69586181640625, "loss": 1.3907, "margin_dpo/margin_mean": -0.03881540894508362, "margin_dpo/margin_std": 0.4327077269554138, "step": 13 }, { "KL/chosen_KL_mean": -0.10559463500976562, "KL/mean": -0.06970568001270294, "KL/rejected_KL_mean": -0.0338134765625, "KL/std": 0.3560220003128052, "epoch": 0.020558002936857563, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.07178828120231628, "fcm_dpo/q_t": 0.5017932653427124, "grad_norm": 35.77608871459961, "learning_rate": 9.420289855072464e-08, "logits/chosen": 0.9327787160873413, "logits/rejected": 1.0230083465576172, "logps/chosen": -99.93276977539062, "logps/ref_chosen": -99.82717895507812, "logps/ref_rejected": -108.94200134277344, "logps/rejected": -108.97581481933594, "loss": 1.394, "margin_dpo/margin_mean": -0.07178857922554016, "margin_dpo/margin_std": 0.43152570724487305, "step": 14 }, { "KL/chosen_KL_mean": -0.029644012451171875, "KL/mean": -0.030726686120033264, "KL/rejected_KL_mean": -0.03180694580078125, "KL/std": 0.33586567640304565, "epoch": 0.022026431718061675, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.0021653473377227783, "fcm_dpo/q_t": 0.4999483525753021, "grad_norm": 31.695486068725586, "learning_rate": 1.0144927536231885e-07, "logits/chosen": 0.4294240176677704, "logits/rejected": 0.5991781949996948, "logps/chosen": -78.93961334228516, "logps/ref_chosen": -78.90997314453125, "logps/ref_rejected": -90.06234741210938, "logps/rejected": -90.09414672851562, "loss": 1.3867, "margin_dpo/margin_mean": 0.0021652579307556152, "margin_dpo/margin_std": 0.49874815344810486, "step": 15 }, { "KL/chosen_KL_mean": 0.01975250244140625, "KL/mean": -0.0009732842445373535, "KL/rejected_KL_mean": -0.021701812744140625, "KL/std": 0.30277085304260254, "epoch": 0.023494860499265784, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.0414544939994812, "fcm_dpo/q_t": 0.49896475672721863, "grad_norm": 34.276187896728516, "learning_rate": 1.0869565217391303e-07, "logits/chosen": 0.6241447925567627, "logits/rejected": 0.8891364336013794, "logps/chosen": -97.40352630615234, "logps/ref_chosen": -97.42327880859375, "logps/ref_rejected": -90.59945678710938, "logps/rejected": -90.62116241455078, "loss": 1.3826, "margin_dpo/margin_mean": 0.04145437479019165, "margin_dpo/margin_std": 0.41995885968208313, "step": 16 }, { "KL/chosen_KL_mean": 0.0464630126953125, "KL/mean": 0.013510197401046753, "KL/rejected_KL_mean": -0.019443511962890625, "KL/std": 0.34516268968582153, "epoch": 0.024963289280469897, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.06591099500656128, "fcm_dpo/q_t": 0.49835169315338135, "grad_norm": 35.494384765625, "learning_rate": 1.1594202898550725e-07, "logits/chosen": 0.5926298499107361, "logits/rejected": 0.8359245657920837, "logps/chosen": -104.31784057617188, "logps/ref_chosen": -104.36430358886719, "logps/ref_rejected": -90.46772766113281, "logps/rejected": -90.48716735839844, "loss": 1.3804, "margin_dpo/margin_mean": 0.06591001152992249, "margin_dpo/margin_std": 0.5128095746040344, "step": 17 }, { "KL/chosen_KL_mean": 0.056804656982421875, "KL/mean": 0.0426379069685936, "KL/rejected_KL_mean": 0.028469085693359375, "KL/std": 0.32054784893989563, "epoch": 0.02643171806167401, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.028332993388175964, "fcm_dpo/q_t": 0.4992940127849579, "grad_norm": 41.7449951171875, "learning_rate": 1.2318840579710146e-07, "logits/chosen": 0.31673505902290344, "logits/rejected": 0.6540764570236206, "logps/chosen": -87.03515625, "logps/ref_chosen": -87.09195709228516, "logps/ref_rejected": -81.85072326660156, "logps/rejected": -81.82225036621094, "loss": 1.384, "margin_dpo/margin_mean": 0.02833305299282074, "margin_dpo/margin_std": 0.48277661204338074, "step": 18 }, { "KL/chosen_KL_mean": 0.02515411376953125, "KL/mean": -0.008823007345199585, "KL/rejected_KL_mean": -0.042804718017578125, "KL/std": 0.317953884601593, "epoch": 0.027900146842878122, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.06795555353164673, "fcm_dpo/q_t": 0.4983009099960327, "grad_norm": 31.350309371948242, "learning_rate": 1.3043478260869563e-07, "logits/chosen": 0.4514174461364746, "logits/rejected": 0.7146160006523132, "logps/chosen": -105.848388671875, "logps/ref_chosen": -105.87354278564453, "logps/ref_rejected": -96.93023681640625, "logps/rejected": -96.97303771972656, "loss": 1.38, "margin_dpo/margin_mean": 0.06795570254325867, "margin_dpo/margin_std": 0.44763243198394775, "step": 19 }, { "KL/chosen_KL_mean": 0.013187408447265625, "KL/mean": 0.021351009607315063, "KL/rejected_KL_mean": 0.029510498046875, "KL/std": 0.31396663188934326, "epoch": 0.02936857562408223, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.01632392406463623, "fcm_dpo/q_t": 0.5004087090492249, "grad_norm": 32.772891998291016, "learning_rate": 1.3768115942028986e-07, "logits/chosen": 0.21086394786834717, "logits/rejected": 0.4835556149482727, "logps/chosen": -90.74493408203125, "logps/ref_chosen": -90.75811767578125, "logps/ref_rejected": -85.91232299804688, "logps/rejected": -85.8828125, "loss": 1.3885, "margin_dpo/margin_mean": -0.016323953866958618, "margin_dpo/margin_std": 0.47240692377090454, "step": 20 }, { "KL/chosen_KL_mean": 0.0029144287109375, "KL/mean": -0.015282675623893738, "KL/rejected_KL_mean": -0.033477783203125, "KL/std": 0.26727402210235596, "epoch": 0.030837004405286344, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.03639879822731018, "fcm_dpo/q_t": 0.499089777469635, "grad_norm": 31.454870223999023, "learning_rate": 1.4492753623188405e-07, "logits/chosen": 0.5349259376525879, "logits/rejected": 0.7128177881240845, "logps/chosen": -80.3305435180664, "logps/ref_chosen": -80.33345794677734, "logps/ref_rejected": -83.93373107910156, "logps/rejected": -83.96720886230469, "loss": 1.3831, "margin_dpo/margin_mean": 0.03639945387840271, "margin_dpo/margin_std": 0.3977454900741577, "step": 21 }, { "KL/chosen_KL_mean": 0.028522491455078125, "KL/mean": 0.04464656114578247, "KL/rejected_KL_mean": 0.06076812744140625, "KL/std": 0.3194689154624939, "epoch": 0.032305433186490456, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.03225874900817871, "fcm_dpo/q_t": 0.5008060336112976, "grad_norm": 42.9687385559082, "learning_rate": 1.5217391304347825e-07, "logits/chosen": 0.49087780714035034, "logits/rejected": 0.6040288805961609, "logps/chosen": -95.36677551269531, "logps/ref_chosen": -95.39530181884766, "logps/ref_rejected": -103.47351837158203, "logps/rejected": -103.41275024414062, "loss": 1.3899, "margin_dpo/margin_mean": -0.03225898742675781, "margin_dpo/margin_std": 0.3678981065750122, "step": 22 }, { "KL/chosen_KL_mean": -0.023744583129882812, "KL/mean": 0.01652967929840088, "KL/rejected_KL_mean": 0.056812286376953125, "KL/std": 0.32836759090423584, "epoch": 0.033773861967694566, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.08055168390274048, "fcm_dpo/q_t": 0.5020133256912231, "grad_norm": 31.551267623901367, "learning_rate": 1.5942028985507245e-07, "logits/chosen": 0.7624632120132446, "logits/rejected": 0.943315327167511, "logps/chosen": -90.6612548828125, "logps/ref_chosen": -90.63751220703125, "logps/ref_rejected": -86.59425354003906, "logps/rejected": -86.53744506835938, "loss": 1.3949, "margin_dpo/margin_mean": -0.08055201172828674, "margin_dpo/margin_std": 0.47492456436157227, "step": 23 }, { "KL/chosen_KL_mean": 0.03722953796386719, "KL/mean": -0.0006896853446960449, "KL/rejected_KL_mean": -0.038608551025390625, "KL/std": 0.2770400047302246, "epoch": 0.03524229074889868, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.0758410096168518, "fcm_dpo/q_t": 0.498104989528656, "grad_norm": 45.75295639038086, "learning_rate": 1.6666666666666665e-07, "logits/chosen": 0.3441356420516968, "logits/rejected": 0.37922707200050354, "logps/chosen": -69.88006591796875, "logps/ref_chosen": -69.91728973388672, "logps/ref_rejected": -106.60990142822266, "logps/rejected": -106.64851379394531, "loss": 1.3791, "margin_dpo/margin_mean": 0.0758412629365921, "margin_dpo/margin_std": 0.3807234764099121, "step": 24 }, { "KL/chosen_KL_mean": 0.03569793701171875, "KL/mean": -0.004800528287887573, "KL/rejected_KL_mean": -0.045299530029296875, "KL/std": 0.39410075545310974, "epoch": 0.03671071953010279, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.0809900164604187, "fcm_dpo/q_t": 0.49797898530960083, "grad_norm": 37.050968170166016, "learning_rate": 1.7391304347826085e-07, "logits/chosen": 0.5247936248779297, "logits/rejected": 0.692723274230957, "logps/chosen": -80.78978729248047, "logps/ref_chosen": -80.82548522949219, "logps/ref_rejected": -95.95710754394531, "logps/rejected": -96.00241088867188, "loss": 1.3791, "margin_dpo/margin_mean": 0.08098980784416199, "margin_dpo/margin_std": 0.5588604211807251, "step": 25 }, { "KL/chosen_KL_mean": 0.010463714599609375, "KL/mean": -0.022343426942825317, "KL/rejected_KL_mean": -0.055149078369140625, "KL/std": 0.33425793051719666, "epoch": 0.0381791483113069, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.06561839580535889, "fcm_dpo/q_t": 0.49836117029190063, "grad_norm": 40.846405029296875, "learning_rate": 1.8115942028985507e-07, "logits/chosen": 0.31885746121406555, "logits/rejected": 0.5274312496185303, "logps/chosen": -88.89070892333984, "logps/ref_chosen": -88.90116882324219, "logps/ref_rejected": -109.82819366455078, "logps/rejected": -109.88334655761719, "loss": 1.3801, "margin_dpo/margin_mean": 0.0656188428401947, "margin_dpo/margin_std": 0.3876963257789612, "step": 26 }, { "KL/chosen_KL_mean": 0.0208587646484375, "KL/mean": 0.00705236941576004, "KL/rejected_KL_mean": -0.006763458251953125, "KL/std": 0.30177831649780273, "epoch": 0.039647577092511016, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.02762104570865631, "fcm_dpo/q_t": 0.4993098974227905, "grad_norm": 44.29174041748047, "learning_rate": 1.8840579710144927e-07, "logits/chosen": 0.055542007088661194, "logits/rejected": 0.2412649542093277, "logps/chosen": -77.57514953613281, "logps/ref_chosen": -77.59600830078125, "logps/ref_rejected": -103.93850708007812, "logps/rejected": -103.94526672363281, "loss": 1.384, "margin_dpo/margin_mean": 0.02762100100517273, "margin_dpo/margin_std": 0.43413975834846497, "step": 27 }, { "KL/chosen_KL_mean": 0.05838775634765625, "KL/mean": -0.018888160586357117, "KL/rejected_KL_mean": -0.096160888671875, "KL/std": 0.3182182013988495, "epoch": 0.041116005873715125, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.15454670786857605, "fcm_dpo/q_t": 0.49613893032073975, "grad_norm": 36.18867874145508, "learning_rate": 1.9565217391304347e-07, "logits/chosen": 0.5341115593910217, "logits/rejected": 0.74845290184021, "logps/chosen": -102.17018127441406, "logps/ref_chosen": -102.22856903076172, "logps/ref_rejected": -96.9594955444336, "logps/rejected": -97.05565643310547, "loss": 1.3714, "margin_dpo/margin_mean": 0.1545475423336029, "margin_dpo/margin_std": 0.46393883228302, "step": 28 }, { "KL/chosen_KL_mean": 0.029796600341796875, "KL/mean": -0.04521440714597702, "KL/rejected_KL_mean": -0.1202239990234375, "KL/std": 0.2746419906616211, "epoch": 0.042584434654919234, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.1500234752893448, "fcm_dpo/q_t": 0.4962507486343384, "grad_norm": 42.48250198364258, "learning_rate": 2.028985507246377e-07, "logits/chosen": 0.298464834690094, "logits/rejected": 0.5537115931510925, "logps/chosen": -88.61725616455078, "logps/ref_chosen": -88.64704895019531, "logps/ref_rejected": -102.96011352539062, "logps/rejected": -103.08033752441406, "loss": 1.3717, "margin_dpo/margin_mean": 0.15002469718456268, "margin_dpo/margin_std": 0.352841854095459, "step": 29 }, { "KL/chosen_KL_mean": 0.009586334228515625, "KL/mean": -0.06154375523328781, "KL/rejected_KL_mean": -0.13267898559570312, "KL/std": 0.3309122920036316, "epoch": 0.04405286343612335, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.14225663244724274, "fcm_dpo/q_t": 0.49644559621810913, "grad_norm": 38.739402770996094, "learning_rate": 2.1014492753623187e-07, "logits/chosen": 0.5708379149436951, "logits/rejected": 0.6858254671096802, "logps/chosen": -88.37879943847656, "logps/ref_chosen": -88.38838195800781, "logps/ref_rejected": -102.31889343261719, "logps/rejected": -102.45156860351562, "loss": 1.3726, "margin_dpo/margin_mean": 0.14225709438323975, "margin_dpo/margin_std": 0.4141353368759155, "step": 30 }, { "KL/chosen_KL_mean": -0.035480499267578125, "KL/mean": -0.09532057493925095, "KL/rejected_KL_mean": -0.15515899658203125, "KL/std": 0.28104445338249207, "epoch": 0.04552129221732746, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.11968068778514862, "fcm_dpo/q_t": 0.4970095455646515, "grad_norm": 31.303373336791992, "learning_rate": 2.1739130434782607e-07, "logits/chosen": 0.9836581349372864, "logits/rejected": 1.2598530054092407, "logps/chosen": -101.16114044189453, "logps/ref_chosen": -101.12565612792969, "logps/ref_rejected": -79.8584213256836, "logps/rejected": -80.01358032226562, "loss": 1.3747, "margin_dpo/margin_mean": 0.11968028545379639, "margin_dpo/margin_std": 0.3348178267478943, "step": 31 }, { "KL/chosen_KL_mean": -0.061847686767578125, "KL/mean": -0.08950134366750717, "KL/rejected_KL_mean": -0.11715316772460938, "KL/std": 0.35548287630081177, "epoch": 0.04698972099853157, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.05530880391597748, "fcm_dpo/q_t": 0.4986189603805542, "grad_norm": 35.21878433227539, "learning_rate": 2.2463768115942027e-07, "logits/chosen": 0.42202627658843994, "logits/rejected": 0.5617946982383728, "logps/chosen": -96.67887878417969, "logps/ref_chosen": -96.61702728271484, "logps/ref_rejected": -96.224365234375, "logps/rejected": -96.34152221679688, "loss": 1.3815, "margin_dpo/margin_mean": 0.05530978739261627, "margin_dpo/margin_std": 0.5153543949127197, "step": 32 }, { "KL/chosen_KL_mean": 0.001628875732421875, "KL/mean": -0.04754679650068283, "KL/rejected_KL_mean": -0.09672164916992188, "KL/std": 0.31931179761886597, "epoch": 0.048458149779735685, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.09835340082645416, "fcm_dpo/q_t": 0.4975426197052002, "grad_norm": 33.70878982543945, "learning_rate": 2.318840579710145e-07, "logits/chosen": 0.6354793906211853, "logits/rejected": 0.8171831369400024, "logps/chosen": -81.5193862915039, "logps/ref_chosen": -81.52101135253906, "logps/ref_rejected": -93.80595397949219, "logps/rejected": -93.90267944335938, "loss": 1.3771, "margin_dpo/margin_mean": 0.09835365414619446, "margin_dpo/margin_std": 0.47823020815849304, "step": 33 }, { "KL/chosen_KL_mean": 0.0430450439453125, "KL/mean": -0.07708846032619476, "KL/rejected_KL_mean": -0.19721603393554688, "KL/std": 0.3642929792404175, "epoch": 0.049926578560939794, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.24027082324028015, "fcm_dpo/q_t": 0.4940028190612793, "grad_norm": 40.969215393066406, "learning_rate": 2.391304347826087e-07, "logits/chosen": 0.5381206274032593, "logits/rejected": 0.7103798389434814, "logps/chosen": -77.15899658203125, "logps/ref_chosen": -77.20204162597656, "logps/ref_rejected": -106.71876525878906, "logps/rejected": -106.91598510742188, "loss": 1.3631, "margin_dpo/margin_mean": 0.24027130007743835, "margin_dpo/margin_std": 0.4968305826187134, "step": 34 }, { "KL/chosen_KL_mean": 0.01062774658203125, "KL/mean": -0.10152420401573181, "KL/rejected_KL_mean": -0.21367645263671875, "KL/std": 0.3811684250831604, "epoch": 0.0513950073421439, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.22430533170700073, "fcm_dpo/q_t": 0.49439704418182373, "grad_norm": 42.71918487548828, "learning_rate": 2.463768115942029e-07, "logits/chosen": 0.335090696811676, "logits/rejected": 0.4909539818763733, "logps/chosen": -77.55972290039062, "logps/ref_chosen": -77.57035064697266, "logps/ref_rejected": -112.18855285644531, "logps/rejected": -112.40222930908203, "loss": 1.3647, "margin_dpo/margin_mean": 0.22430622577667236, "margin_dpo/margin_std": 0.5345590114593506, "step": 35 }, { "KL/chosen_KL_mean": -0.01268768310546875, "KL/mean": -0.09679705649614334, "KL/rejected_KL_mean": -0.180908203125, "KL/std": 0.38911956548690796, "epoch": 0.05286343612334802, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.16821442544460297, "fcm_dpo/q_t": 0.49579930305480957, "grad_norm": 32.6117057800293, "learning_rate": 2.536231884057971e-07, "logits/chosen": 0.4062168002128601, "logits/rejected": 0.5958766937255859, "logps/chosen": -83.30093383789062, "logps/ref_chosen": -83.28823852539062, "logps/ref_rejected": -91.75741577148438, "logps/rejected": -91.93832397460938, "loss": 1.3702, "margin_dpo/margin_mean": 0.1682145893573761, "margin_dpo/margin_std": 0.5040839314460754, "step": 36 }, { "KL/chosen_KL_mean": -0.00870513916015625, "KL/mean": -0.15830036997795105, "KL/rejected_KL_mean": -0.30789947509765625, "KL/std": 0.47221916913986206, "epoch": 0.05433186490455213, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.2991906404495239, "fcm_dpo/q_t": 0.4925394058227539, "grad_norm": 35.66999816894531, "learning_rate": 2.6086956521739126e-07, "logits/chosen": 0.9145070314407349, "logits/rejected": 1.0948659181594849, "logps/chosen": -94.77980041503906, "logps/ref_chosen": -94.77108764648438, "logps/ref_rejected": -85.71726989746094, "logps/rejected": -86.02516174316406, "loss": 1.3578, "margin_dpo/margin_mean": 0.29919081926345825, "margin_dpo/margin_std": 0.6725693941116333, "step": 37 }, { "KL/chosen_KL_mean": 0.07264328002929688, "KL/mean": -0.05678700655698776, "KL/rejected_KL_mean": -0.18621444702148438, "KL/std": 0.4086042642593384, "epoch": 0.055800293685756244, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.25885826349258423, "fcm_dpo/q_t": 0.4935372769832611, "grad_norm": 37.20484161376953, "learning_rate": 2.681159420289855e-07, "logits/chosen": 0.5482184886932373, "logits/rejected": 0.6875032186508179, "logps/chosen": -75.9058609008789, "logps/ref_chosen": -75.97850036621094, "logps/ref_rejected": -104.10401916503906, "logps/rejected": -104.29023742675781, "loss": 1.3613, "margin_dpo/margin_mean": 0.2588585615158081, "margin_dpo/margin_std": 0.5284547805786133, "step": 38 }, { "KL/chosen_KL_mean": -0.048725128173828125, "KL/mean": -0.19109302759170532, "KL/rejected_KL_mean": -0.3334617614746094, "KL/std": 0.41185247898101807, "epoch": 0.05726872246696035, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.2847316265106201, "fcm_dpo/q_t": 0.4928908348083496, "grad_norm": 36.25815200805664, "learning_rate": 2.753623188405797e-07, "logits/chosen": 0.3941457271575928, "logits/rejected": 0.5761860013008118, "logps/chosen": -81.23450469970703, "logps/ref_chosen": -81.18577575683594, "logps/ref_rejected": -84.1959228515625, "logps/rejected": -84.52938079833984, "loss": 1.3589, "margin_dpo/margin_mean": 0.28473174571990967, "margin_dpo/margin_std": 0.5670312643051147, "step": 39 }, { "KL/chosen_KL_mean": 0.04741668701171875, "KL/mean": -0.0829850435256958, "KL/rejected_KL_mean": -0.21338653564453125, "KL/std": 0.35915321111679077, "epoch": 0.05873715124816446, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.26080721616744995, "fcm_dpo/q_t": 0.4934840798377991, "grad_norm": 28.99793243408203, "learning_rate": 2.8260869565217386e-07, "logits/chosen": 0.5965007543563843, "logits/rejected": 0.7917885184288025, "logps/chosen": -83.28515625, "logps/ref_chosen": -83.33257293701172, "logps/ref_rejected": -80.25591278076172, "logps/rejected": -80.46929931640625, "loss": 1.3609, "margin_dpo/margin_mean": 0.2608068287372589, "margin_dpo/margin_std": 0.4518221616744995, "step": 40 }, { "KL/chosen_KL_mean": -0.01788330078125, "KL/mean": -0.14398349821567535, "KL/rejected_KL_mean": -0.27008819580078125, "KL/std": 0.43517160415649414, "epoch": 0.06020558002936858, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.2522120177745819, "fcm_dpo/q_t": 0.49370720982551575, "grad_norm": 33.40452575683594, "learning_rate": 2.898550724637681e-07, "logits/chosen": 0.6236477494239807, "logits/rejected": 0.8108532428741455, "logps/chosen": -93.16654968261719, "logps/ref_chosen": -93.14866638183594, "logps/ref_rejected": -102.07920837402344, "logps/rejected": -102.34929656982422, "loss": 1.3623, "margin_dpo/margin_mean": 0.2522123456001282, "margin_dpo/margin_std": 0.6358495950698853, "step": 41 }, { "KL/chosen_KL_mean": 0.034381866455078125, "KL/mean": -0.19486773014068604, "KL/rejected_KL_mean": -0.4241180419921875, "KL/std": 0.5133959054946899, "epoch": 0.06167400881057269, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.4584972858428955, "fcm_dpo/q_t": 0.48855137825012207, "grad_norm": 40.651329040527344, "learning_rate": 2.971014492753623e-07, "logits/chosen": 0.39264267683029175, "logits/rejected": 0.5342455506324768, "logps/chosen": -90.667236328125, "logps/ref_chosen": -90.70162200927734, "logps/ref_rejected": -113.8646469116211, "logps/rejected": -114.28876495361328, "loss": 1.3418, "margin_dpo/margin_mean": 0.45849716663360596, "margin_dpo/margin_std": 0.584077000617981, "step": 42 }, { "KL/chosen_KL_mean": 0.021514892578125, "KL/mean": -0.19794650375843048, "KL/rejected_KL_mean": -0.41741180419921875, "KL/std": 0.5176049470901489, "epoch": 0.0631424375917768, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.4389236271381378, "fcm_dpo/q_t": 0.4890602231025696, "grad_norm": 37.85871124267578, "learning_rate": 3.043478260869565e-07, "logits/chosen": 0.6470640897750854, "logits/rejected": 0.8401391506195068, "logps/chosen": -89.62251281738281, "logps/ref_chosen": -89.64402770996094, "logps/ref_rejected": -103.95185852050781, "logps/rejected": -104.3692626953125, "loss": 1.3443, "margin_dpo/margin_mean": 0.43892422318458557, "margin_dpo/margin_std": 0.7477631568908691, "step": 43 }, { "KL/chosen_KL_mean": -0.026058197021484375, "KL/mean": -0.2544378638267517, "KL/rejected_KL_mean": -0.4828224182128906, "KL/std": 0.5010477304458618, "epoch": 0.06461086637298091, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.456767737865448, "fcm_dpo/q_t": 0.48859721422195435, "grad_norm": 34.155967712402344, "learning_rate": 3.115942028985507e-07, "logits/chosen": 0.8367877006530762, "logits/rejected": 0.9573251605033875, "logps/chosen": -81.90437316894531, "logps/ref_chosen": -81.8783187866211, "logps/ref_rejected": -113.421630859375, "logps/rejected": -113.90444946289062, "loss": 1.342, "margin_dpo/margin_mean": 0.45676785707473755, "margin_dpo/margin_std": 0.5923409461975098, "step": 44 }, { "KL/chosen_KL_mean": -0.11049652099609375, "KL/mean": -0.22296538949012756, "KL/rejected_KL_mean": -0.3354301452636719, "KL/std": 0.4330664277076721, "epoch": 0.06607929515418502, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.22493408620357513, "fcm_dpo/q_t": 0.49438217282295227, "grad_norm": 24.40647315979004, "learning_rate": 3.188405797101449e-07, "logits/chosen": 0.48652714490890503, "logits/rejected": 0.5948389172554016, "logps/chosen": -77.45509338378906, "logps/ref_chosen": -77.34459686279297, "logps/ref_rejected": -84.24774169921875, "logps/rejected": -84.58317565917969, "loss": 1.3647, "margin_dpo/margin_mean": 0.224934920668602, "margin_dpo/margin_std": 0.5600230097770691, "step": 45 }, { "KL/chosen_KL_mean": 0.03125762939453125, "KL/mean": -0.21650341153144836, "KL/rejected_KL_mean": -0.4642601013183594, "KL/std": 0.5851463079452515, "epoch": 0.06754772393538913, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.49552083015441895, "fcm_dpo/q_t": 0.4876311123371124, "grad_norm": 32.96702575683594, "learning_rate": 3.260869565217391e-07, "logits/chosen": 0.4339357316493988, "logits/rejected": 0.6726645231246948, "logps/chosen": -90.3095703125, "logps/ref_chosen": -90.3408203125, "logps/ref_rejected": -100.676513671875, "logps/rejected": -101.14077758789062, "loss": 1.3386, "margin_dpo/margin_mean": 0.49552121758461, "margin_dpo/margin_std": 0.7012072205543518, "step": 46 }, { "KL/chosen_KL_mean": -0.03223419189453125, "KL/mean": -0.2075231373310089, "KL/rejected_KL_mean": -0.3828163146972656, "KL/std": 0.5406893491744995, "epoch": 0.06901615271659324, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.3505786061286926, "fcm_dpo/q_t": 0.49125438928604126, "grad_norm": 34.141780853271484, "learning_rate": 3.333333333333333e-07, "logits/chosen": 0.40441077947616577, "logits/rejected": 0.5829853415489197, "logps/chosen": -104.44353485107422, "logps/ref_chosen": -104.41130065917969, "logps/ref_rejected": -101.25489807128906, "logps/rejected": -101.63771057128906, "loss": 1.353, "margin_dpo/margin_mean": 0.35057854652404785, "margin_dpo/margin_std": 0.7464984655380249, "step": 47 }, { "KL/chosen_KL_mean": -0.028476715087890625, "KL/mean": -0.26390624046325684, "KL/rejected_KL_mean": -0.49933624267578125, "KL/std": 0.634148120880127, "epoch": 0.07048458149779736, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.47085991501808167, "fcm_dpo/q_t": 0.4882550835609436, "grad_norm": 38.913291931152344, "learning_rate": 3.4057971014492755e-07, "logits/chosen": 0.15821820497512817, "logits/rejected": 0.35422125458717346, "logps/chosen": -91.12875366210938, "logps/ref_chosen": -91.10027313232422, "logps/ref_rejected": -95.08058166503906, "logps/rejected": -95.57991027832031, "loss": 1.3414, "margin_dpo/margin_mean": 0.47086048126220703, "margin_dpo/margin_std": 0.8205849528312683, "step": 48 }, { "KL/chosen_KL_mean": 0.06631088256835938, "KL/mean": -0.3106277883052826, "KL/rejected_KL_mean": -0.6875686645507812, "KL/std": 0.7828932404518127, "epoch": 0.07195301027900147, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.7538716197013855, "fcm_dpo/q_t": 0.48122939467430115, "grad_norm": 39.63159942626953, "learning_rate": 3.478260869565217e-07, "logits/chosen": 0.054642464965581894, "logits/rejected": 0.5203364491462708, "logps/chosen": -92.93737030029297, "logps/ref_chosen": -93.00367736816406, "logps/ref_rejected": -91.74899291992188, "logps/rejected": -92.43656921386719, "loss": 1.3149, "margin_dpo/margin_mean": 0.7538715600967407, "margin_dpo/margin_std": 0.9957276582717896, "step": 49 }, { "KL/chosen_KL_mean": -0.1480712890625, "KL/mean": -0.4775815010070801, "KL/rejected_KL_mean": -0.8070831298828125, "KL/std": 0.775669515132904, "epoch": 0.07342143906020558, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.6590114235877991, "fcm_dpo/q_t": 0.4835768938064575, "grad_norm": 33.30721664428711, "learning_rate": 3.5507246376811595e-07, "logits/chosen": 0.5360842943191528, "logits/rejected": 0.7067970037460327, "logps/chosen": -94.77488708496094, "logps/ref_chosen": -94.62681579589844, "logps/ref_rejected": -103.57435607910156, "logps/rejected": -104.38143920898438, "loss": 1.3236, "margin_dpo/margin_mean": 0.6590113639831543, "margin_dpo/margin_std": 0.9176972508430481, "step": 50 }, { "KL/chosen_KL_mean": -0.054714202880859375, "KL/mean": -0.3807927966117859, "KL/rejected_KL_mean": -0.7068710327148438, "KL/std": 0.8505513072013855, "epoch": 0.07488986784140969, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.6521493792533875, "fcm_dpo/q_t": 0.4837867021560669, "grad_norm": 29.927831649780273, "learning_rate": 3.6231884057971015e-07, "logits/chosen": 0.7329905033111572, "logits/rejected": 0.8798360824584961, "logps/chosen": -87.56199645996094, "logps/ref_chosen": -87.50727844238281, "logps/ref_rejected": -83.47235107421875, "logps/rejected": -84.17922973632812, "loss": 1.3253, "margin_dpo/margin_mean": 0.6521490812301636, "margin_dpo/margin_std": 1.0980899333953857, "step": 51 }, { "KL/chosen_KL_mean": 0.0141448974609375, "KL/mean": -0.411986768245697, "KL/rejected_KL_mean": -0.8381233215332031, "KL/std": 0.778254508972168, "epoch": 0.0763582966226138, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.8522641658782959, "fcm_dpo/q_t": 0.47876590490341187, "grad_norm": 31.903409957885742, "learning_rate": 3.695652173913043e-07, "logits/chosen": 0.2494804561138153, "logits/rejected": 0.449777215719223, "logps/chosen": -90.61611938476562, "logps/ref_chosen": -90.63026428222656, "logps/ref_rejected": -87.0390625, "logps/rejected": -87.87718200683594, "loss": 1.305, "margin_dpo/margin_mean": 0.852264404296875, "margin_dpo/margin_std": 0.916897714138031, "step": 52 }, { "KL/chosen_KL_mean": -0.0498809814453125, "KL/mean": -0.5524184107780457, "KL/rejected_KL_mean": -1.0549507141113281, "KL/std": 1.1000334024429321, "epoch": 0.07782672540381791, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.0050727128982544, "fcm_dpo/q_t": 0.4751381278038025, "grad_norm": 39.47797393798828, "learning_rate": 3.7681159420289855e-07, "logits/chosen": 0.08327862620353699, "logits/rejected": 0.19521111249923706, "logps/chosen": -81.6329574584961, "logps/ref_chosen": -81.58307647705078, "logps/ref_rejected": -95.66152954101562, "logps/rejected": -96.71647644042969, "loss": 1.2935, "margin_dpo/margin_mean": 1.0050725936889648, "margin_dpo/margin_std": 1.3506872653961182, "step": 53 }, { "KL/chosen_KL_mean": -0.038482666015625, "KL/mean": -0.5898535251617432, "KL/rejected_KL_mean": -1.1412277221679688, "KL/std": 1.2504780292510986, "epoch": 0.07929515418502203, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.1027414798736572, "fcm_dpo/q_t": 0.47257959842681885, "grad_norm": 40.124717712402344, "learning_rate": 3.8405797101449274e-07, "logits/chosen": 0.2811519503593445, "logits/rejected": 0.45574668049812317, "logps/chosen": -88.94865417480469, "logps/ref_chosen": -88.91017150878906, "logps/ref_rejected": -99.1175537109375, "logps/rejected": -100.25878143310547, "loss": 1.2826, "margin_dpo/margin_mean": 1.1027413606643677, "margin_dpo/margin_std": 1.1984407901763916, "step": 54 }, { "KL/chosen_KL_mean": 0.14495849609375, "KL/mean": -0.44793444871902466, "KL/rejected_KL_mean": -1.0408248901367188, "KL/std": 1.178361415863037, "epoch": 0.08076358296622614, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.1857811212539673, "fcm_dpo/q_t": 0.4706498384475708, "grad_norm": 35.26246643066406, "learning_rate": 3.9130434782608694e-07, "logits/chosen": 0.018801812082529068, "logits/rejected": 0.3150080442428589, "logps/chosen": -92.31095886230469, "logps/ref_chosen": -92.45591735839844, "logps/ref_rejected": -92.97093200683594, "logps/rejected": -94.01176452636719, "loss": 1.2772, "margin_dpo/margin_mean": 1.1857811212539673, "margin_dpo/margin_std": 1.4991127252578735, "step": 55 }, { "KL/chosen_KL_mean": -0.08123016357421875, "KL/mean": -0.6413096189498901, "KL/rejected_KL_mean": -1.201385498046875, "KL/std": 1.1786679029464722, "epoch": 0.08223201174743025, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.1201488971710205, "fcm_dpo/q_t": 0.47230932116508484, "grad_norm": 33.10283279418945, "learning_rate": 3.9855072463768114e-07, "logits/chosen": 0.1805080771446228, "logits/rejected": 0.5188891887664795, "logps/chosen": -87.31788635253906, "logps/ref_chosen": -87.23665618896484, "logps/ref_rejected": -100.64553833007812, "logps/rejected": -101.846923828125, "loss": 1.2837, "margin_dpo/margin_mean": 1.1201484203338623, "margin_dpo/margin_std": 1.5858896970748901, "step": 56 }, { "KL/chosen_KL_mean": -0.0514068603515625, "KL/mean": -0.6366605758666992, "KL/rejected_KL_mean": -1.2219200134277344, "KL/std": 1.2971317768096924, "epoch": 0.08370044052863436, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.1705279350280762, "fcm_dpo/q_t": 0.4710506796836853, "grad_norm": 33.408851623535156, "learning_rate": 4.057971014492754e-07, "logits/chosen": 0.5278745889663696, "logits/rejected": 0.7557357549667358, "logps/chosen": -98.2021484375, "logps/ref_chosen": -98.15074157714844, "logps/ref_rejected": -101.05284118652344, "logps/rejected": -102.2747573852539, "loss": 1.2793, "margin_dpo/margin_mean": 1.1705279350280762, "margin_dpo/margin_std": 1.6327052116394043, "step": 57 }, { "KL/chosen_KL_mean": 0.21571731567382812, "KL/mean": -0.5199761986732483, "KL/rejected_KL_mean": -1.2556610107421875, "KL/std": 1.4040195941925049, "epoch": 0.08516886930983847, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.4713754653930664, "fcm_dpo/q_t": 0.46378639340400696, "grad_norm": 38.12102508544922, "learning_rate": 4.1304347826086954e-07, "logits/chosen": 0.4386252462863922, "logits/rejected": 0.7860119938850403, "logps/chosen": -99.36524963378906, "logps/ref_chosen": -99.58097076416016, "logps/ref_rejected": -91.22227478027344, "logps/rejected": -92.47793579101562, "loss": 1.2535, "margin_dpo/margin_mean": 1.4713757038116455, "margin_dpo/margin_std": 1.8455278873443604, "step": 58 }, { "KL/chosen_KL_mean": 0.03876495361328125, "KL/mean": -0.6094352006912231, "KL/rejected_KL_mean": -1.2576370239257812, "KL/std": 1.2375140190124512, "epoch": 0.08663729809104258, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.2963955402374268, "fcm_dpo/q_t": 0.46789902448654175, "grad_norm": 34.11338424682617, "learning_rate": 4.2028985507246374e-07, "logits/chosen": 0.09331036359071732, "logits/rejected": 0.28541117906570435, "logps/chosen": -89.76356506347656, "logps/ref_chosen": -89.80232238769531, "logps/ref_rejected": -94.27667236328125, "logps/rejected": -95.53431701660156, "loss": 1.2667, "margin_dpo/margin_mean": 1.2963950634002686, "margin_dpo/margin_std": 1.5107533931732178, "step": 59 }, { "KL/chosen_KL_mean": 0.030788421630859375, "KL/mean": -0.5591611862182617, "KL/rejected_KL_mean": -1.1491165161132812, "KL/std": 1.2311577796936035, "epoch": 0.0881057268722467, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.1799085140228271, "fcm_dpo/q_t": 0.4707183837890625, "grad_norm": 28.915138244628906, "learning_rate": 4.2753623188405794e-07, "logits/chosen": 0.17282041907310486, "logits/rejected": 0.42514222860336304, "logps/chosen": -95.12492370605469, "logps/ref_chosen": -95.15571594238281, "logps/ref_rejected": -91.3724365234375, "logps/rejected": -92.52155303955078, "loss": 1.2764, "margin_dpo/margin_mean": 1.1799083948135376, "margin_dpo/margin_std": 1.3647682666778564, "step": 60 }, { "KL/chosen_KL_mean": -0.2078704833984375, "KL/mean": -0.7680006623268127, "KL/rejected_KL_mean": -1.3281364440917969, "KL/std": 1.3514353036880493, "epoch": 0.08957415565345081, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.1202651262283325, "fcm_dpo/q_t": 0.472463458776474, "grad_norm": 28.457597732543945, "learning_rate": 4.3478260869565214e-07, "logits/chosen": 0.1602925956249237, "logits/rejected": 0.30045902729034424, "logps/chosen": -85.94017791748047, "logps/ref_chosen": -85.73230743408203, "logps/ref_rejected": -97.96575927734375, "logps/rejected": -99.29389953613281, "loss": 1.2858, "margin_dpo/margin_mean": 1.1202644109725952, "margin_dpo/margin_std": 1.7892072200775146, "step": 61 }, { "KL/chosen_KL_mean": -0.09750747680664062, "KL/mean": -0.731864333152771, "KL/rejected_KL_mean": -1.3662223815917969, "KL/std": 1.4225494861602783, "epoch": 0.09104258443465492, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.2687207460403442, "fcm_dpo/q_t": 0.46867483854293823, "grad_norm": 28.60158348083496, "learning_rate": 4.420289855072464e-07, "logits/chosen": 0.15727725625038147, "logits/rejected": 0.3492550551891327, "logps/chosen": -81.73289489746094, "logps/ref_chosen": -81.63538360595703, "logps/ref_rejected": -84.03831481933594, "logps/rejected": -85.40453338623047, "loss": 1.2729, "margin_dpo/margin_mean": 1.2687196731567383, "margin_dpo/margin_std": 1.855647325515747, "step": 62 }, { "KL/chosen_KL_mean": -0.033748626708984375, "KL/mean": -0.7986141443252563, "KL/rejected_KL_mean": -1.5634803771972656, "KL/std": 1.3721587657928467, "epoch": 0.09251101321585903, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.5297342538833618, "fcm_dpo/q_t": 0.46211713552474976, "grad_norm": 35.054710388183594, "learning_rate": 4.4927536231884053e-07, "logits/chosen": 0.5110536217689514, "logits/rejected": 0.7312678098678589, "logps/chosen": -103.65780639648438, "logps/ref_chosen": -103.62405395507812, "logps/ref_rejected": -103.40303039550781, "logps/rejected": -104.96650695800781, "loss": 1.2454, "margin_dpo/margin_mean": 1.5297338962554932, "margin_dpo/margin_std": 1.5587282180786133, "step": 63 }, { "KL/chosen_KL_mean": -0.10355758666992188, "KL/mean": -0.9556412100791931, "KL/rejected_KL_mean": -1.8077392578125, "KL/std": 1.78635835647583, "epoch": 0.09397944199706314, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.7041810750961304, "fcm_dpo/q_t": 0.4582006335258484, "grad_norm": 35.068538665771484, "learning_rate": 4.5652173913043473e-07, "logits/chosen": 0.6043516993522644, "logits/rejected": 0.8259673118591309, "logps/chosen": -87.10514831542969, "logps/ref_chosen": -87.0015869140625, "logps/ref_rejected": -100.5854721069336, "logps/rejected": -102.3932113647461, "loss": 1.2341, "margin_dpo/margin_mean": 1.7041810750961304, "margin_dpo/margin_std": 2.0286264419555664, "step": 64 }, { "KL/chosen_KL_mean": -0.0607757568359375, "KL/mean": -0.9677102565765381, "KL/rejected_KL_mean": -1.8746452331542969, "KL/std": 1.6561270952224731, "epoch": 0.09544787077826726, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.8138678073883057, "fcm_dpo/q_t": 0.45543304085731506, "grad_norm": 38.30107116699219, "learning_rate": 4.63768115942029e-07, "logits/chosen": -0.06661674380302429, "logits/rejected": 0.07723595947027206, "logps/chosen": -91.2826919555664, "logps/ref_chosen": -91.22191619873047, "logps/ref_rejected": -115.33553314208984, "logps/rejected": -117.21017456054688, "loss": 1.2233, "margin_dpo/margin_mean": 1.8138679265975952, "margin_dpo/margin_std": 2.0140886306762695, "step": 65 }, { "KL/chosen_KL_mean": -0.33434295654296875, "KL/mean": -0.9711505770683289, "KL/rejected_KL_mean": -1.6079597473144531, "KL/std": 1.5632513761520386, "epoch": 0.09691629955947137, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.2736104726791382, "fcm_dpo/q_t": 0.46861565113067627, "grad_norm": 25.61758041381836, "learning_rate": 4.7101449275362313e-07, "logits/chosen": 0.30933254957199097, "logits/rejected": 0.5222980380058289, "logps/chosen": -84.11856842041016, "logps/ref_chosen": -83.78422546386719, "logps/ref_rejected": -82.7520980834961, "logps/rejected": -84.36006164550781, "loss": 1.2718, "margin_dpo/margin_mean": 1.2736105918884277, "margin_dpo/margin_std": 1.8205499649047852, "step": 66 }, { "KL/chosen_KL_mean": -0.8533248901367188, "KL/mean": -1.4257209300994873, "KL/rejected_KL_mean": -1.9981117248535156, "KL/std": 2.0455029010772705, "epoch": 0.09838472834067548, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.1447845697402954, "fcm_dpo/q_t": 0.4723505675792694, "grad_norm": 24.28444480895996, "learning_rate": 4.782608695652174e-07, "logits/chosen": 0.4081868529319763, "logits/rejected": 0.6900854110717773, "logps/chosen": -88.52629089355469, "logps/ref_chosen": -87.67295837402344, "logps/ref_rejected": -79.08674621582031, "logps/rejected": -81.0848617553711, "loss": 1.2942, "margin_dpo/margin_mean": 1.144784927368164, "margin_dpo/margin_std": 2.788841962814331, "step": 67 }, { "KL/chosen_KL_mean": -0.24430465698242188, "KL/mean": -1.2685749530792236, "KL/rejected_KL_mean": -2.2928390502929688, "KL/std": 2.1893463134765625, "epoch": 0.09985315712187959, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 2.048532247543335, "fcm_dpo/q_t": 0.4505225419998169, "grad_norm": 28.59839630126953, "learning_rate": 4.855072463768116e-07, "logits/chosen": 0.24755343794822693, "logits/rejected": 0.5789214968681335, "logps/chosen": -97.32315063476562, "logps/ref_chosen": -97.07884216308594, "logps/ref_rejected": -84.11872863769531, "logps/rejected": -86.41156768798828, "loss": 1.2099, "margin_dpo/margin_mean": 2.048532009124756, "margin_dpo/margin_std": 2.707021713256836, "step": 68 }, { "KL/chosen_KL_mean": -0.33158111572265625, "KL/mean": -1.3999078273773193, "KL/rejected_KL_mean": -2.4682350158691406, "KL/std": 2.3814938068389893, "epoch": 0.1013215859030837, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 2.1366519927978516, "fcm_dpo/q_t": 0.4485386610031128, "grad_norm": 30.916728973388672, "learning_rate": 4.927536231884058e-07, "logits/chosen": 0.06354180723428726, "logits/rejected": 0.17590491473674774, "logps/chosen": -86.0512924194336, "logps/ref_chosen": -85.71971130371094, "logps/ref_rejected": -109.48020935058594, "logps/rejected": -111.94844055175781, "loss": 1.2055, "margin_dpo/margin_mean": 2.1366517543792725, "margin_dpo/margin_std": 2.973220109939575, "step": 69 }, { "KL/chosen_KL_mean": -0.6108512878417969, "KL/mean": -1.9039943218231201, "KL/rejected_KL_mean": -3.1971397399902344, "KL/std": 2.7748351097106934, "epoch": 0.1027900146842878, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 2.586289405822754, "fcm_dpo/q_t": 0.4384194016456604, "grad_norm": 29.341352462768555, "learning_rate": 5e-07, "logits/chosen": 0.17052993178367615, "logits/rejected": 0.343351811170578, "logps/chosen": -95.62080383300781, "logps/ref_chosen": -95.00994873046875, "logps/ref_rejected": -96.21272277832031, "logps/rejected": -99.40986633300781, "loss": 1.1722, "margin_dpo/margin_mean": 2.5862884521484375, "margin_dpo/margin_std": 3.324324131011963, "step": 70 }, { "KL/chosen_KL_mean": -0.036632537841796875, "KL/mean": -1.8997746706008911, "KL/rejected_KL_mean": -3.7629127502441406, "KL/std": 3.59867525100708, "epoch": 0.10425844346549193, "fcm_dpo/beta": 0.10017916560173035, "fcm_dpo/delta": 0.027692411094903946, "fcm_dpo/margin": 3.726283073425293, "fcm_dpo/q_t": 0.4132327437400818, "grad_norm": 34.35282897949219, "learning_rate": 4.999967061337492e-07, "logits/chosen": 0.06169853359460831, "logits/rejected": 0.4402628540992737, "logps/chosen": -91.01398468017578, "logps/ref_chosen": -90.97735595703125, "logps/ref_rejected": -102.59103393554688, "logps/rejected": -106.35394287109375, "loss": 1.0866, "margin_dpo/margin_mean": 3.726283073425293, "margin_dpo/margin_std": 4.136852741241455, "step": 71 }, { "KL/chosen_KL_mean": -0.5575103759765625, "KL/mean": -2.022162437438965, "KL/rejected_KL_mean": -3.48681640625, "KL/std": 2.940340518951416, "epoch": 0.10572687224669604, "fcm_dpo/beta": 0.10143917053937912, "fcm_dpo/delta": 0.04375329613685608, "fcm_dpo/margin": 2.929307460784912, "fcm_dpo/q_t": 0.4291730523109436, "grad_norm": 29.753665924072266, "learning_rate": 4.999868246217933e-07, "logits/chosen": -0.42643800377845764, "logits/rejected": -0.12373970448970795, "logps/chosen": -98.44903564453125, "logps/ref_chosen": -97.89152526855469, "logps/ref_rejected": -100.19171142578125, "logps/rejected": -103.67852783203125, "loss": 1.1387, "margin_dpo/margin_mean": 2.929306983947754, "margin_dpo/margin_std": 3.3551158905029297, "step": 72 }, { "KL/chosen_KL_mean": -0.7645721435546875, "KL/mean": -2.3280248641967773, "KL/rejected_KL_mean": -3.8914794921875, "KL/std": 3.8170700073242188, "epoch": 0.10719530102790015, "fcm_dpo/beta": 0.10270863771438599, "fcm_dpo/delta": 0.08143049478530884, "fcm_dpo/margin": 3.12691330909729, "fcm_dpo/q_t": 0.42775601148605347, "grad_norm": 28.466285705566406, "learning_rate": 4.999703557245192e-07, "logits/chosen": -0.38065868616104126, "logits/rejected": -0.07968415319919586, "logps/chosen": -96.5335922241211, "logps/ref_chosen": -95.7690200805664, "logps/ref_rejected": -95.93243408203125, "logps/rejected": -99.82391357421875, "loss": 1.1475, "margin_dpo/margin_mean": 3.126913547515869, "margin_dpo/margin_std": 4.862720489501953, "step": 73 }, { "KL/chosen_KL_mean": -0.9901924133300781, "KL/mean": -2.7506957054138184, "KL/rejected_KL_mean": -4.511203765869141, "KL/std": 3.799988269805908, "epoch": 0.10866372980910426, "fcm_dpo/beta": 0.10353071987628937, "fcm_dpo/delta": 0.0368211604654789, "fcm_dpo/margin": 3.5210113525390625, "fcm_dpo/q_t": 0.41603541374206543, "grad_norm": 31.4624080657959, "learning_rate": 4.999472998758977e-07, "logits/chosen": -0.16242395341396332, "logits/rejected": -0.018897918984293938, "logps/chosen": -79.798583984375, "logps/ref_chosen": -78.80839538574219, "logps/ref_rejected": -101.64677429199219, "logps/rejected": -106.15797424316406, "loss": 1.106, "margin_dpo/margin_mean": 3.5210108757019043, "margin_dpo/margin_std": 4.538544654846191, "step": 74 }, { "KL/chosen_KL_mean": -1.0562019348144531, "KL/mean": -3.3269500732421875, "KL/rejected_KL_mean": -5.597698211669922, "KL/std": 4.597766876220703, "epoch": 0.11013215859030837, "fcm_dpo/beta": 0.10317438840866089, "fcm_dpo/delta": -0.07197670638561249, "fcm_dpo/margin": 4.541502952575684, "fcm_dpo/q_t": 0.3949120342731476, "grad_norm": 32.846946716308594, "learning_rate": 4.999176576834721e-07, "logits/chosen": -0.3875320553779602, "logits/rejected": -0.18362751603126526, "logps/chosen": -79.33805847167969, "logps/ref_chosen": -78.28185272216797, "logps/ref_rejected": -115.40311431884766, "logps/rejected": -121.00080871582031, "loss": 1.0407, "margin_dpo/margin_mean": 4.541502952575684, "margin_dpo/margin_std": 5.405215263366699, "step": 75 }, { "KL/chosen_KL_mean": -1.438232421875, "KL/mean": -2.984178066253662, "KL/rejected_KL_mean": -4.530128479003906, "KL/std": 3.786229133605957, "epoch": 0.11160058737151249, "fcm_dpo/beta": 0.10365289449691772, "fcm_dpo/delta": 0.08214224129915237, "fcm_dpo/margin": 3.0919015407562256, "fcm_dpo/q_t": 0.4267314672470093, "grad_norm": 23.542400360107422, "learning_rate": 4.998814299283415e-07, "logits/chosen": -0.6625537276268005, "logits/rejected": -0.2867332696914673, "logps/chosen": -89.31536865234375, "logps/ref_chosen": -87.87713623046875, "logps/ref_rejected": -85.71968078613281, "logps/rejected": -90.24980926513672, "loss": 1.1433, "margin_dpo/margin_mean": 3.0919017791748047, "margin_dpo/margin_std": 4.564672470092773, "step": 76 }, { "KL/chosen_KL_mean": -1.6675910949707031, "KL/mean": -3.354740619659424, "KL/rejected_KL_mean": -5.041893005371094, "KL/std": 4.480142593383789, "epoch": 0.1130690161527166, "fcm_dpo/beta": 0.10463721305131912, "fcm_dpo/delta": 0.048658907413482666, "fcm_dpo/margin": 3.374298095703125, "fcm_dpo/q_t": 0.4195900559425354, "grad_norm": 27.999977111816406, "learning_rate": 4.998386175651409e-07, "logits/chosen": -0.21670196950435638, "logits/rejected": 0.1504397988319397, "logps/chosen": -101.36793518066406, "logps/ref_chosen": -99.70034790039062, "logps/ref_rejected": -98.20576477050781, "logps/rejected": -103.24765014648438, "loss": 1.1312, "margin_dpo/margin_mean": 3.374298334121704, "margin_dpo/margin_std": 5.108901500701904, "step": 77 }, { "KL/chosen_KL_mean": -0.7116546630859375, "KL/mean": -2.511979103088379, "KL/rejected_KL_mean": -4.312297821044922, "KL/std": 3.678145170211792, "epoch": 0.1145374449339207, "fcm_dpo/beta": 0.10538917779922485, "fcm_dpo/delta": 0.021346315741539, "fcm_dpo/margin": 3.600642681121826, "fcm_dpo/q_t": 0.41206055879592896, "grad_norm": 26.094635009765625, "learning_rate": 4.997892217220159e-07, "logits/chosen": 0.1686553955078125, "logits/rejected": 0.4075925946235657, "logps/chosen": -91.00836181640625, "logps/ref_chosen": -90.29670715332031, "logps/ref_rejected": -91.13772583007812, "logps/rejected": -95.45002746582031, "loss": 1.0902, "margin_dpo/margin_mean": 3.600642442703247, "margin_dpo/margin_std": 4.339991092681885, "step": 78 }, { "KL/chosen_KL_mean": -0.8355979919433594, "KL/mean": -3.2423648834228516, "KL/rejected_KL_mean": -5.649135589599609, "KL/std": 5.327904224395752, "epoch": 0.11600587371512482, "fcm_dpo/beta": 0.10404185205698013, "fcm_dpo/delta": -0.10601618885993958, "fcm_dpo/margin": 4.813536643981934, "fcm_dpo/q_t": 0.3878142833709717, "grad_norm": 27.339481353759766, "learning_rate": 4.997332437005931e-07, "logits/chosen": -0.5722988843917847, "logits/rejected": -0.1865392029285431, "logps/chosen": -87.21392822265625, "logps/ref_chosen": -86.37832641601562, "logps/ref_rejected": -94.10777282714844, "logps/rejected": -99.75690460205078, "loss": 1.0278, "margin_dpo/margin_mean": 4.813536643981934, "margin_dpo/margin_std": 5.867785453796387, "step": 79 }, { "KL/chosen_KL_mean": -1.5975379943847656, "KL/mean": -3.4530553817749023, "KL/rejected_KL_mean": -5.308574676513672, "KL/std": 4.671290874481201, "epoch": 0.11747430249632893, "fcm_dpo/beta": 0.10355690121650696, "fcm_dpo/delta": 0.016326699405908585, "fcm_dpo/margin": 3.7110366821289062, "fcm_dpo/q_t": 0.41360679268836975, "grad_norm": 26.259410858154297, "learning_rate": 4.996706849759452e-07, "logits/chosen": -0.7483689188957214, "logits/rejected": -0.39665502309799194, "logps/chosen": -95.56785583496094, "logps/ref_chosen": -93.97032165527344, "logps/ref_rejected": -92.57441711425781, "logps/rejected": -97.88299560546875, "loss": 1.1026, "margin_dpo/margin_mean": 3.7110366821289062, "margin_dpo/margin_std": 5.134080410003662, "step": 80 }, { "KL/chosen_KL_mean": -1.1007575988769531, "KL/mean": -3.7640862464904785, "KL/rejected_KL_mean": -6.427410125732422, "KL/std": 5.826144218444824, "epoch": 0.11894273127753303, "fcm_dpo/beta": 0.10087832808494568, "fcm_dpo/delta": -0.1465744972229004, "fcm_dpo/margin": 5.326655387878418, "fcm_dpo/q_t": 0.38472476601600647, "grad_norm": 27.61595344543457, "learning_rate": 4.996015471965529e-07, "logits/chosen": -0.569584846496582, "logits/rejected": -0.14743685722351074, "logps/chosen": -100.93087768554688, "logps/ref_chosen": -99.83012390136719, "logps/ref_rejected": -133.67247009277344, "logps/rejected": -140.09988403320312, "loss": 1.0215, "margin_dpo/margin_mean": 5.326655387878418, "margin_dpo/margin_std": 6.914580821990967, "step": 81 }, { "KL/chosen_KL_mean": -2.0198593139648438, "KL/mean": -3.8523612022399902, "KL/rejected_KL_mean": -5.6848602294921875, "KL/std": 4.556976318359375, "epoch": 0.12041116005873716, "fcm_dpo/beta": 0.10029402375221252, "fcm_dpo/delta": -0.037920787930488586, "fcm_dpo/margin": 3.665005922317505, "fcm_dpo/q_t": 0.4152594804763794, "grad_norm": 26.13265609741211, "learning_rate": 4.995258321842611e-07, "logits/chosen": -0.048558514565229416, "logits/rejected": 0.17264960706233978, "logps/chosen": -85.06584167480469, "logps/ref_chosen": -83.04598236083984, "logps/ref_rejected": -94.52595520019531, "logps/rejected": -100.2108154296875, "loss": 1.1115, "margin_dpo/margin_mean": 3.665005922317505, "margin_dpo/margin_std": 4.93629789352417, "step": 82 }, { "KL/chosen_KL_mean": -2.360248565673828, "KL/mean": -4.392603397369385, "KL/rejected_KL_mean": -6.424961090087891, "KL/std": 5.152520179748535, "epoch": 0.12187958883994127, "fcm_dpo/beta": 0.09982265532016754, "fcm_dpo/delta": -0.005994098260998726, "fcm_dpo/margin": 4.064716339111328, "fcm_dpo/q_t": 0.410980761051178, "grad_norm": 27.43876075744629, "learning_rate": 4.994435419342304e-07, "logits/chosen": -0.32917851209640503, "logits/rejected": 0.10240879654884338, "logps/chosen": -94.53646850585938, "logps/ref_chosen": -92.17621612548828, "logps/ref_rejected": -107.74464416503906, "logps/rejected": -114.16960144042969, "loss": 1.1153, "margin_dpo/margin_mean": 4.064716339111328, "margin_dpo/margin_std": 6.495458602905273, "step": 83 }, { "KL/chosen_KL_mean": -2.9121971130371094, "KL/mean": -4.227639198303223, "KL/rejected_KL_mean": -5.543083190917969, "KL/std": 4.596038818359375, "epoch": 0.12334801762114538, "fcm_dpo/beta": 0.09968481957912445, "fcm_dpo/delta": -0.005473073106259108, "fcm_dpo/margin": 2.630887031555176, "fcm_dpo/q_t": 0.4405984878540039, "grad_norm": 28.258586883544922, "learning_rate": 4.993546786148857e-07, "logits/chosen": -0.20731906592845917, "logits/rejected": 0.20152854919433594, "logps/chosen": -104.43869018554688, "logps/ref_chosen": -101.5264892578125, "logps/ref_rejected": -92.42608642578125, "logps/rejected": -97.96916961669922, "loss": 1.2033, "margin_dpo/margin_mean": 2.630887031555176, "margin_dpo/margin_std": 4.9484453201293945, "step": 84 }, { "KL/chosen_KL_mean": -2.679340362548828, "KL/mean": -4.38173246383667, "KL/rejected_KL_mean": -6.0841217041015625, "KL/std": 5.083308696746826, "epoch": 0.12481644640234948, "fcm_dpo/beta": 0.09959776699542999, "fcm_dpo/delta": -0.008740188553929329, "fcm_dpo/margin": 3.4047844409942627, "fcm_dpo/q_t": 0.4238542914390564, "grad_norm": 26.950847625732422, "learning_rate": 4.992592445678582e-07, "logits/chosen": -0.49466782808303833, "logits/rejected": -0.1743880957365036, "logps/chosen": -98.80671691894531, "logps/ref_chosen": -96.12738037109375, "logps/ref_rejected": -85.05519104003906, "logps/rejected": -91.13931274414062, "loss": 1.1636, "margin_dpo/margin_mean": 3.40478515625, "margin_dpo/margin_std": 6.143662929534912, "step": 85 }, { "KL/chosen_KL_mean": -2.0633888244628906, "KL/mean": -4.257645606994629, "KL/rejected_KL_mean": -6.451904296875, "KL/std": 5.867281913757324, "epoch": 0.1262848751835536, "fcm_dpo/beta": 0.09884131699800491, "fcm_dpo/delta": -0.035395264625549316, "fcm_dpo/margin": 4.388519287109375, "fcm_dpo/q_t": 0.4091922342777252, "grad_norm": 22.061201095581055, "learning_rate": 4.991572423079235e-07, "logits/chosen": -0.56908118724823, "logits/rejected": -0.3259069323539734, "logps/chosen": -83.76765441894531, "logps/ref_chosen": -81.70426177978516, "logps/ref_rejected": -93.65550231933594, "logps/rejected": -100.10740661621094, "loss": 1.1072, "margin_dpo/margin_mean": 4.388518810272217, "margin_dpo/margin_std": 7.296498775482178, "step": 86 }, { "KL/chosen_KL_mean": -2.423919677734375, "KL/mean": -4.786927223205566, "KL/rejected_KL_mean": -7.149936676025391, "KL/std": 6.46231746673584, "epoch": 0.1277533039647577, "fcm_dpo/beta": 0.09758473932743073, "fcm_dpo/delta": -0.06449822336435318, "fcm_dpo/margin": 4.72601318359375, "fcm_dpo/q_t": 0.39699968695640564, "grad_norm": 26.262548446655273, "learning_rate": 4.990486745229364e-07, "logits/chosen": -0.9582849740982056, "logits/rejected": -0.5924580097198486, "logps/chosen": -95.1098861694336, "logps/ref_chosen": -92.68596649169922, "logps/ref_rejected": -102.91818237304688, "logps/rejected": -110.068115234375, "loss": 1.1015, "margin_dpo/margin_mean": 4.726013660430908, "margin_dpo/margin_std": 7.665117263793945, "step": 87 }, { "KL/chosen_KL_mean": -2.949321746826172, "KL/mean": -4.880241394042969, "KL/rejected_KL_mean": -6.811153411865234, "KL/std": 6.328596115112305, "epoch": 0.12922173274596183, "fcm_dpo/beta": 0.09653878211975098, "fcm_dpo/delta": -0.051709841936826706, "fcm_dpo/margin": 3.861827850341797, "fcm_dpo/q_t": 0.41755035519599915, "grad_norm": 23.8017578125, "learning_rate": 4.989335440737586e-07, "logits/chosen": -0.3058089017868042, "logits/rejected": -0.1184551939368248, "logps/chosen": -103.7123031616211, "logps/ref_chosen": -100.76298522949219, "logps/ref_rejected": -113.15037536621094, "logps/rejected": -119.9615249633789, "loss": 1.1418, "margin_dpo/margin_mean": 3.861827850341797, "margin_dpo/margin_std": 6.563701629638672, "step": 88 }, { "KL/chosen_KL_mean": -2.4566688537597656, "KL/mean": -4.524205207824707, "KL/rejected_KL_mean": -6.591743469238281, "KL/std": 5.3812031745910645, "epoch": 0.13069016152716592, "fcm_dpo/beta": 0.09631407260894775, "fcm_dpo/delta": 0.0016747862100601196, "fcm_dpo/margin": 4.135067939758301, "fcm_dpo/q_t": 0.41548407077789307, "grad_norm": 23.24942970275879, "learning_rate": 4.988118539941847e-07, "logits/chosen": -0.7394917607307434, "logits/rejected": -0.4035339951515198, "logps/chosen": -92.14775085449219, "logps/ref_chosen": -89.69108581542969, "logps/ref_rejected": -88.56832885742188, "logps/rejected": -95.16007995605469, "loss": 1.1251, "margin_dpo/margin_mean": 4.135067939758301, "margin_dpo/margin_std": 7.14996337890625, "step": 89 }, { "KL/chosen_KL_mean": -2.1671791076660156, "KL/mean": -4.8994526863098145, "KL/rejected_KL_mean": -7.631721496582031, "KL/std": 6.325523376464844, "epoch": 0.13215859030837004, "fcm_dpo/beta": 0.09513455629348755, "fcm_dpo/delta": -0.1262548714876175, "fcm_dpo/margin": 5.464543342590332, "fcm_dpo/q_t": 0.3883175551891327, "grad_norm": 25.596607208251953, "learning_rate": 4.986836074908615e-07, "logits/chosen": -0.47311776876449585, "logits/rejected": -0.3494108319282532, "logps/chosen": -83.54973602294922, "logps/ref_chosen": -81.38255310058594, "logps/ref_rejected": -117.77714538574219, "logps/rejected": -125.40885925292969, "loss": 1.0441, "margin_dpo/margin_mean": 5.464543342590332, "margin_dpo/margin_std": 7.588117599487305, "step": 90 }, { "KL/chosen_KL_mean": -2.4866104125976562, "KL/mean": -4.783910274505615, "KL/rejected_KL_mean": -7.081211090087891, "KL/std": 6.15728759765625, "epoch": 0.13362701908957417, "fcm_dpo/beta": 0.09375274181365967, "fcm_dpo/delta": -0.032131608575582504, "fcm_dpo/margin": 4.594605445861816, "fcm_dpo/q_t": 0.4077467620372772, "grad_norm": 25.023216247558594, "learning_rate": 4.985488079432037e-07, "logits/chosen": -0.33563512563705444, "logits/rejected": 0.08655708283185959, "logps/chosen": -99.70849609375, "logps/ref_chosen": -97.22187805175781, "logps/ref_rejected": -92.97674560546875, "logps/rejected": -100.05795288085938, "loss": 1.0999, "margin_dpo/margin_mean": 4.594604969024658, "margin_dpo/margin_std": 7.223147869110107, "step": 91 }, { "KL/chosen_KL_mean": -2.7732505798339844, "KL/mean": -4.78037166595459, "KL/rejected_KL_mean": -6.787494659423828, "KL/std": 6.127893924713135, "epoch": 0.13509544787077826, "fcm_dpo/beta": 0.09404444694519043, "fcm_dpo/delta": 0.023266423493623734, "fcm_dpo/margin": 4.014244079589844, "fcm_dpo/q_t": 0.4171380400657654, "grad_norm": 22.582233428955078, "learning_rate": 4.984074589033043e-07, "logits/chosen": -0.7812787294387817, "logits/rejected": -0.4103330969810486, "logps/chosen": -87.30352783203125, "logps/ref_chosen": -84.5302734375, "logps/ref_rejected": -84.5013198852539, "logps/rejected": -91.288818359375, "loss": 1.1295, "margin_dpo/margin_mean": 4.0142436027526855, "margin_dpo/margin_std": 6.596063613891602, "step": 92 }, { "KL/chosen_KL_mean": -2.9890403747558594, "KL/mean": -4.910215377807617, "KL/rejected_KL_mean": -6.831398010253906, "KL/std": 5.791092395782471, "epoch": 0.13656387665198239, "fcm_dpo/beta": 0.09467165172100067, "fcm_dpo/delta": 0.03749629482626915, "fcm_dpo/margin": 3.842360734939575, "fcm_dpo/q_t": 0.41992542147636414, "grad_norm": 25.639062881469727, "learning_rate": 4.982595640958425e-07, "logits/chosen": -0.9924514293670654, "logits/rejected": -0.42984145879745483, "logps/chosen": -93.23947143554688, "logps/ref_chosen": -90.25043487548828, "logps/ref_rejected": -84.0942153930664, "logps/rejected": -90.92561340332031, "loss": 1.1562, "margin_dpo/margin_mean": 3.842360734939575, "margin_dpo/margin_std": 6.998875617980957, "step": 93 }, { "KL/chosen_KL_mean": -3.112682342529297, "KL/mean": -6.050202369689941, "KL/rejected_KL_mean": -8.987720489501953, "KL/std": 6.811246871948242, "epoch": 0.13803230543318648, "fcm_dpo/beta": 0.09186561405658722, "fcm_dpo/delta": -0.1502506583929062, "fcm_dpo/margin": 5.875044345855713, "fcm_dpo/q_t": 0.38054198026657104, "grad_norm": 25.209552764892578, "learning_rate": 4.98105127417984e-07, "logits/chosen": -0.6663119792938232, "logits/rejected": -0.36196452379226685, "logps/chosen": -95.56690216064453, "logps/ref_chosen": -92.4542236328125, "logps/ref_rejected": -105.2472915649414, "logps/rejected": -114.23501586914062, "loss": 1.0342, "margin_dpo/margin_mean": 5.875043869018555, "margin_dpo/margin_std": 7.796442031860352, "step": 94 }, { "KL/chosen_KL_mean": -2.5027084350585938, "KL/mean": -4.828878402709961, "KL/rejected_KL_mean": -7.1550445556640625, "KL/std": 5.726435661315918, "epoch": 0.1395007342143906, "fcm_dpo/beta": 0.0918988585472107, "fcm_dpo/delta": -0.029026592150330544, "fcm_dpo/margin": 4.652338981628418, "fcm_dpo/q_t": 0.4076543152332306, "grad_norm": 21.351715087890625, "learning_rate": 4.979441529392784e-07, "logits/chosen": -0.8635698556900024, "logits/rejected": -0.5238780975341797, "logps/chosen": -81.37641906738281, "logps/ref_chosen": -78.87370300292969, "logps/ref_rejected": -83.59121704101562, "logps/rejected": -90.74626159667969, "loss": 1.0999, "margin_dpo/margin_mean": 4.652338981628418, "margin_dpo/margin_std": 7.098349571228027, "step": 95 }, { "KL/chosen_KL_mean": -2.2016448974609375, "KL/mean": -5.082788467407227, "KL/rejected_KL_mean": -7.963932037353516, "KL/std": 6.861926078796387, "epoch": 0.14096916299559473, "fcm_dpo/beta": 0.08951471745967865, "fcm_dpo/delta": -0.12249662727117538, "fcm_dpo/margin": 5.7622880935668945, "fcm_dpo/q_t": 0.3874228298664093, "grad_norm": 21.74850082397461, "learning_rate": 4.977766449015534e-07, "logits/chosen": -0.7713649272918701, "logits/rejected": -0.3251499533653259, "logps/chosen": -108.79380798339844, "logps/ref_chosen": -106.5921630859375, "logps/ref_rejected": -101.76802825927734, "logps/rejected": -109.73196411132812, "loss": 1.0383, "margin_dpo/margin_mean": 5.7622880935668945, "margin_dpo/margin_std": 7.617171287536621, "step": 96 }, { "KL/chosen_KL_mean": -3.3008499145507812, "KL/mean": -5.859622955322266, "KL/rejected_KL_mean": -8.418392181396484, "KL/std": 5.913722038269043, "epoch": 0.14243759177679882, "fcm_dpo/beta": 0.08901587128639221, "fcm_dpo/delta": -0.058812178671360016, "fcm_dpo/margin": 5.117546081542969, "fcm_dpo/q_t": 0.3970525860786438, "grad_norm": 24.633037567138672, "learning_rate": 4.976026077188012e-07, "logits/chosen": -0.4443942904472351, "logits/rejected": -0.035453300923109055, "logps/chosen": -91.98072814941406, "logps/ref_chosen": -88.67988586425781, "logps/ref_rejected": -84.81229400634766, "logps/rejected": -93.23068237304688, "loss": 1.0577, "margin_dpo/margin_mean": 5.117546081542969, "margin_dpo/margin_std": 6.3041229248046875, "step": 97 }, { "KL/chosen_KL_mean": -2.5518341064453125, "KL/mean": -5.429494857788086, "KL/rejected_KL_mean": -8.30715560913086, "KL/std": 5.7163214683532715, "epoch": 0.14390602055800295, "fcm_dpo/beta": 0.08634430170059204, "fcm_dpo/delta": -0.1030280590057373, "fcm_dpo/margin": 5.755323886871338, "fcm_dpo/q_t": 0.38794711232185364, "grad_norm": 22.770769119262695, "learning_rate": 4.974220459770639e-07, "logits/chosen": -0.6936355829238892, "logits/rejected": -0.4268031120300293, "logps/chosen": -94.79432678222656, "logps/ref_chosen": -92.24249267578125, "logps/ref_rejected": -101.51948547363281, "logps/rejected": -109.82664489746094, "loss": 1.022, "margin_dpo/margin_mean": 5.755324363708496, "margin_dpo/margin_std": 6.475984573364258, "step": 98 }, { "KL/chosen_KL_mean": -1.8986015319824219, "KL/mean": -5.120244979858398, "KL/rejected_KL_mean": -8.341896057128906, "KL/std": 6.887099742889404, "epoch": 0.14537444933920704, "fcm_dpo/beta": 0.08430320769548416, "fcm_dpo/delta": -0.1516081690788269, "fcm_dpo/margin": 6.443300724029541, "fcm_dpo/q_t": 0.38580048084259033, "grad_norm": 22.23921012878418, "learning_rate": 4.972349644343108e-07, "logits/chosen": -0.638316810131073, "logits/rejected": -0.4255262613296509, "logps/chosen": -74.083251953125, "logps/ref_chosen": -72.18464660644531, "logps/ref_rejected": -91.88131713867188, "logps/rejected": -100.22322082519531, "loss": 1.0236, "margin_dpo/margin_mean": 6.443300247192383, "margin_dpo/margin_std": 8.78098201751709, "step": 99 }, { "KL/chosen_KL_mean": -2.280132293701172, "KL/mean": -4.591710090637207, "KL/rejected_KL_mean": -6.903285980224609, "KL/std": 5.230842590332031, "epoch": 0.14684287812041116, "fcm_dpo/beta": 0.08361945301294327, "fcm_dpo/delta": 0.013753345236182213, "fcm_dpo/margin": 4.623149871826172, "fcm_dpo/q_t": 0.4108346700668335, "grad_norm": 19.484966278076172, "learning_rate": 4.970413680203148e-07, "logits/chosen": -0.6460772752761841, "logits/rejected": -0.1551763117313385, "logps/chosen": -91.79396057128906, "logps/ref_chosen": -89.51382446289062, "logps/ref_rejected": -81.21713256835938, "logps/rejected": -88.12041473388672, "loss": 1.0865, "margin_dpo/margin_mean": 4.6231489181518555, "margin_dpo/margin_std": 5.541533946990967, "step": 100 }, { "KL/chosen_KL_mean": -3.5491943359375, "KL/mean": -5.64581298828125, "KL/rejected_KL_mean": -7.742439270019531, "KL/std": 6.811181545257568, "epoch": 0.14831130690161526, "fcm_dpo/beta": 0.08362682163715363, "fcm_dpo/delta": -0.020485172048211098, "fcm_dpo/margin": 4.193233013153076, "fcm_dpo/q_t": 0.4199220538139343, "grad_norm": 23.749713897705078, "learning_rate": 4.968412618365215e-07, "logits/chosen": -0.8543105125427246, "logits/rejected": -0.5881494283676147, "logps/chosen": -97.952392578125, "logps/ref_chosen": -94.4031982421875, "logps/ref_rejected": -89.91001892089844, "logps/rejected": -97.65245056152344, "loss": 1.1539, "margin_dpo/margin_mean": 4.193233013153076, "margin_dpo/margin_std": 7.304439544677734, "step": 101 }, { "KL/chosen_KL_mean": -4.014743804931641, "KL/mean": -5.838558197021484, "KL/rejected_KL_mean": -7.662372589111328, "KL/std": 6.486747741699219, "epoch": 0.14977973568281938, "fcm_dpo/beta": 0.08330987393856049, "fcm_dpo/delta": -0.03804425895214081, "fcm_dpo/margin": 3.6476352214813232, "fcm_dpo/q_t": 0.43119144439697266, "grad_norm": 28.040851593017578, "learning_rate": 4.966346511559149e-07, "logits/chosen": -0.9621187448501587, "logits/rejected": -0.6033914685249329, "logps/chosen": -95.6241683959961, "logps/ref_chosen": -91.60942077636719, "logps/ref_rejected": -75.7689208984375, "logps/rejected": -83.43128967285156, "loss": 1.196, "margin_dpo/margin_mean": 3.6476354598999023, "margin_dpo/margin_std": 7.342247009277344, "step": 102 }, { "KL/chosen_KL_mean": -3.1960601806640625, "KL/mean": -6.231755256652832, "KL/rejected_KL_mean": -9.267452239990234, "KL/std": 6.851804733276367, "epoch": 0.1512481644640235, "fcm_dpo/beta": 0.08159910887479782, "fcm_dpo/delta": -0.1006719097495079, "fcm_dpo/margin": 6.071394920349121, "fcm_dpo/q_t": 0.3881691098213196, "grad_norm": 22.51178550720215, "learning_rate": 4.964215414228785e-07, "logits/chosen": -0.8078954219818115, "logits/rejected": -0.41563475131988525, "logps/chosen": -100.59844970703125, "logps/ref_chosen": -97.40238952636719, "logps/ref_rejected": -98.981689453125, "logps/rejected": -108.24913787841797, "loss": 1.0431, "margin_dpo/margin_mean": 6.0713958740234375, "margin_dpo/margin_std": 7.733498573303223, "step": 103 }, { "KL/chosen_KL_mean": -2.4227676391601562, "KL/mean": -5.7053046226501465, "KL/rejected_KL_mean": -8.987834930419922, "KL/std": 7.468371391296387, "epoch": 0.1527165932452276, "fcm_dpo/beta": 0.07966464757919312, "fcm_dpo/delta": -0.13002897799015045, "fcm_dpo/margin": 6.565065860748291, "fcm_dpo/q_t": 0.3874834179878235, "grad_norm": 18.731647491455078, "learning_rate": 4.96201938253052e-07, "logits/chosen": -0.9786155223846436, "logits/rejected": -0.6273901462554932, "logps/chosen": -97.27912902832031, "logps/ref_chosen": -94.85636138916016, "logps/ref_rejected": -97.02669525146484, "logps/rejected": -106.0145263671875, "loss": 1.0295, "margin_dpo/margin_mean": 6.565066814422607, "margin_dpo/margin_std": 8.832313537597656, "step": 104 }, { "KL/chosen_KL_mean": -3.2622146606445312, "KL/mean": -6.512721061706543, "KL/rejected_KL_mean": -9.763225555419922, "KL/std": 6.803992748260498, "epoch": 0.15418502202643172, "fcm_dpo/beta": 0.07827430963516235, "fcm_dpo/delta": -0.11460494995117188, "fcm_dpo/margin": 6.501008033752441, "fcm_dpo/q_t": 0.38681888580322266, "grad_norm": 21.944747924804688, "learning_rate": 4.959758474331832e-07, "logits/chosen": -1.006087303161621, "logits/rejected": -0.531781017780304, "logps/chosen": -100.02011108398438, "logps/ref_chosen": -96.75790405273438, "logps/ref_rejected": -101.5774917602539, "logps/rejected": -111.34071350097656, "loss": 1.0414, "margin_dpo/margin_mean": 6.501007080078125, "margin_dpo/margin_std": 8.554824829101562, "step": 105 }, { "KL/chosen_KL_mean": -3.2737464904785156, "KL/mean": -6.3534722328186035, "KL/rejected_KL_mean": -9.433197021484375, "KL/std": 7.004992961883545, "epoch": 0.15565345080763582, "fcm_dpo/beta": 0.07681487500667572, "fcm_dpo/delta": -0.07672860473394394, "fcm_dpo/margin": 6.159451007843018, "fcm_dpo/q_t": 0.39662793278694153, "grad_norm": 19.68122100830078, "learning_rate": 4.957432749209755e-07, "logits/chosen": -0.6537899374961853, "logits/rejected": -0.21296614408493042, "logps/chosen": -95.71165466308594, "logps/ref_chosen": -92.43791198730469, "logps/ref_rejected": -91.00970458984375, "logps/rejected": -100.44290161132812, "loss": 1.0606, "margin_dpo/margin_mean": 6.159450531005859, "margin_dpo/margin_std": 8.471151351928711, "step": 106 }, { "KL/chosen_KL_mean": -2.328693389892578, "KL/mean": -5.9570417404174805, "KL/rejected_KL_mean": -9.585391998291016, "KL/std": 8.865304946899414, "epoch": 0.15712187958883994, "fcm_dpo/beta": 0.07456070184707642, "fcm_dpo/delta": -0.14922761917114258, "fcm_dpo/margin": 7.256699562072754, "fcm_dpo/q_t": 0.3894772529602051, "grad_norm": 19.823720932006836, "learning_rate": 4.955042268449307e-07, "logits/chosen": -0.8180956244468689, "logits/rejected": -0.30201244354248047, "logps/chosen": -107.45370483398438, "logps/ref_chosen": -105.12500762939453, "logps/ref_rejected": -101.1824951171875, "logps/rejected": -110.76789093017578, "loss": 1.0426, "margin_dpo/margin_mean": 7.256699562072754, "margin_dpo/margin_std": 11.042577743530273, "step": 107 }, { "KL/chosen_KL_mean": -3.454601287841797, "KL/mean": -6.853583335876465, "KL/rejected_KL_mean": -10.252567291259766, "KL/std": 8.144183158874512, "epoch": 0.15859030837004406, "fcm_dpo/beta": 0.07261009514331818, "fcm_dpo/delta": -0.09935794025659561, "fcm_dpo/margin": 6.7979631423950195, "fcm_dpo/q_t": 0.39164412021636963, "grad_norm": 19.87030601501465, "learning_rate": 4.952587095041881e-07, "logits/chosen": -0.9941326975822449, "logits/rejected": -0.4892861545085907, "logps/chosen": -92.6639633178711, "logps/ref_chosen": -89.20936584472656, "logps/ref_rejected": -101.83784484863281, "logps/rejected": -112.09041595458984, "loss": 1.0476, "margin_dpo/margin_mean": 6.797963619232178, "margin_dpo/margin_std": 8.928738594055176, "step": 108 }, { "KL/chosen_KL_mean": -2.7665672302246094, "KL/mean": -6.983913421630859, "KL/rejected_KL_mean": -11.20126724243164, "KL/std": 8.303434371948242, "epoch": 0.16005873715124816, "fcm_dpo/beta": 0.07019108533859253, "fcm_dpo/delta": -0.20492924749851227, "fcm_dpo/margin": 8.434696197509766, "fcm_dpo/q_t": 0.37066853046417236, "grad_norm": 18.741079330444336, "learning_rate": 4.95006729368358e-07, "logits/chosen": -0.590620219707489, "logits/rejected": -0.35671481490135193, "logps/chosen": -102.95365905761719, "logps/ref_chosen": -100.18708801269531, "logps/ref_rejected": -103.53739929199219, "logps/rejected": -114.73866271972656, "loss": 0.9878, "margin_dpo/margin_mean": 8.43469524383545, "margin_dpo/margin_std": 10.254924774169922, "step": 109 }, { "KL/chosen_KL_mean": -3.585895538330078, "KL/mean": -6.090257167816162, "KL/rejected_KL_mean": -8.594623565673828, "KL/std": 6.798268795013428, "epoch": 0.16152716593245228, "fcm_dpo/beta": 0.06924136728048325, "fcm_dpo/delta": -0.030235530808568, "fcm_dpo/margin": 5.008722305297852, "fcm_dpo/q_t": 0.42062196135520935, "grad_norm": 20.035903930664062, "learning_rate": 4.947482930773511e-07, "logits/chosen": -0.281746506690979, "logits/rejected": 0.13460886478424072, "logps/chosen": -110.48161315917969, "logps/ref_chosen": -106.89572143554688, "logps/ref_rejected": -85.81120300292969, "logps/rejected": -94.40582275390625, "loss": 1.1373, "margin_dpo/margin_mean": 5.008722305297852, "margin_dpo/margin_std": 7.700854301452637, "step": 110 }, { "KL/chosen_KL_mean": -3.2048492431640625, "KL/mean": -8.13703727722168, "KL/rejected_KL_mean": -13.069229125976562, "KL/std": 9.644274711608887, "epoch": 0.16299559471365638, "fcm_dpo/beta": 0.06614179909229279, "fcm_dpo/delta": -0.27217304706573486, "fcm_dpo/margin": 9.864376068115234, "fcm_dpo/q_t": 0.3561099171638489, "grad_norm": 17.213592529296875, "learning_rate": 4.944834074412042e-07, "logits/chosen": -1.0692082643508911, "logits/rejected": -0.7875548601150513, "logps/chosen": -96.53421783447266, "logps/ref_chosen": -93.3293685913086, "logps/ref_rejected": -104.177490234375, "logps/rejected": -117.24671936035156, "loss": 0.9458, "margin_dpo/margin_mean": 9.864374160766602, "margin_dpo/margin_std": 10.657848358154297, "step": 111 }, { "KL/chosen_KL_mean": -3.6781959533691406, "KL/mean": -6.1637067794799805, "KL/rejected_KL_mean": -8.649219512939453, "KL/std": 5.5510945320129395, "epoch": 0.1644640234948605, "fcm_dpo/beta": 0.06587863713502884, "fcm_dpo/delta": 0.023251384496688843, "fcm_dpo/margin": 4.97102165222168, "fcm_dpo/q_t": 0.4243234395980835, "grad_norm": 17.179916381835938, "learning_rate": 4.942120794399002e-07, "logits/chosen": -1.283064365386963, "logits/rejected": -0.7845422625541687, "logps/chosen": -86.42523956298828, "logps/ref_chosen": -82.74703979492188, "logps/ref_rejected": -71.5584487915039, "logps/rejected": -80.20767211914062, "loss": 1.1436, "margin_dpo/margin_mean": 4.971022605895996, "margin_dpo/margin_std": 7.52631950378418, "step": 112 }, { "KL/chosen_KL_mean": -4.587512969970703, "KL/mean": -7.3055219650268555, "KL/rejected_KL_mean": -10.02353286743164, "KL/std": 7.405384063720703, "epoch": 0.16593245227606462, "fcm_dpo/beta": 0.066332146525383, "fcm_dpo/delta": 0.0408482551574707, "fcm_dpo/margin": 5.436017036437988, "fcm_dpo/q_t": 0.4170852601528168, "grad_norm": 20.547657012939453, "learning_rate": 4.939343162231841e-07, "logits/chosen": -0.9216668605804443, "logits/rejected": -0.3851248025894165, "logps/chosen": -102.48700714111328, "logps/ref_chosen": -97.89949035644531, "logps/ref_rejected": -84.41960144042969, "logps/rejected": -94.44313049316406, "loss": 1.1148, "margin_dpo/margin_mean": 5.436017990112305, "margin_dpo/margin_std": 7.101157188415527, "step": 113 }, { "KL/chosen_KL_mean": -4.290248870849609, "KL/mean": -8.092846870422363, "KL/rejected_KL_mean": -11.89544677734375, "KL/std": 8.676340103149414, "epoch": 0.16740088105726872, "fcm_dpo/beta": 0.06523959338665009, "fcm_dpo/delta": -0.10162682086229324, "fcm_dpo/margin": 7.605199337005615, "fcm_dpo/q_t": 0.393574059009552, "grad_norm": 20.153409957885742, "learning_rate": 4.936501251103751e-07, "logits/chosen": -1.0858216285705566, "logits/rejected": -0.5068268775939941, "logps/chosen": -100.78536987304688, "logps/ref_chosen": -96.4951171875, "logps/ref_rejected": -92.01536560058594, "logps/rejected": -103.91081237792969, "loss": 1.0683, "margin_dpo/margin_mean": 7.605199337005615, "margin_dpo/margin_std": 11.257364273071289, "step": 114 }, { "KL/chosen_KL_mean": -4.4661102294921875, "KL/mean": -8.248708724975586, "KL/rejected_KL_mean": -12.031307220458984, "KL/std": 9.088540077209473, "epoch": 0.16886930983847284, "fcm_dpo/beta": 0.06425650417804718, "fcm_dpo/delta": -0.09046189486980438, "fcm_dpo/margin": 7.565197944641113, "fcm_dpo/q_t": 0.3924819827079773, "grad_norm": 17.9512996673584, "learning_rate": 4.933595135901732e-07, "logits/chosen": -0.9558216333389282, "logits/rejected": -0.5450321435928345, "logps/chosen": -107.646240234375, "logps/ref_chosen": -103.18013000488281, "logps/ref_rejected": -105.71595764160156, "logps/rejected": -117.74726867675781, "loss": 1.0515, "margin_dpo/margin_mean": 7.565196990966797, "margin_dpo/margin_std": 9.938637733459473, "step": 115 }, { "KL/chosen_KL_mean": -3.9439430236816406, "KL/mean": -7.166526794433594, "KL/rejected_KL_mean": -10.389106750488281, "KL/std": 7.188760757446289, "epoch": 0.17033773861967694, "fcm_dpo/beta": 0.06402775645256042, "fcm_dpo/delta": -0.013450777158141136, "fcm_dpo/margin": 6.445163249969482, "fcm_dpo/q_t": 0.4059520661830902, "grad_norm": 17.014379501342773, "learning_rate": 4.930624893204624e-07, "logits/chosen": -1.0909994840621948, "logits/rejected": -0.6614570617675781, "logps/chosen": -85.1023178100586, "logps/ref_chosen": -81.15837097167969, "logps/ref_rejected": -86.30586242675781, "logps/rejected": -96.69496154785156, "loss": 1.0807, "margin_dpo/margin_mean": 6.445164203643799, "margin_dpo/margin_std": 8.192975997924805, "step": 116 }, { "KL/chosen_KL_mean": -4.727897644042969, "KL/mean": -7.359216690063477, "KL/rejected_KL_mean": -9.99053955078125, "KL/std": 7.311716556549072, "epoch": 0.17180616740088106, "fcm_dpo/beta": 0.06343423575162888, "fcm_dpo/delta": -0.024360468611121178, "fcm_dpo/margin": 5.262639045715332, "fcm_dpo/q_t": 0.4233490824699402, "grad_norm": 24.08353614807129, "learning_rate": 4.927590601281083e-07, "logits/chosen": -0.6956891417503357, "logits/rejected": -0.27032843232154846, "logps/chosen": -101.80885314941406, "logps/ref_chosen": -97.0809555053711, "logps/ref_rejected": -75.1399154663086, "logps/rejected": -85.13045501708984, "loss": 1.1571, "margin_dpo/margin_mean": 5.262638568878174, "margin_dpo/margin_std": 8.948646545410156, "step": 117 }, { "KL/chosen_KL_mean": -3.4797935485839844, "KL/mean": -7.198921203613281, "KL/rejected_KL_mean": -10.918048858642578, "KL/std": 8.009775161743164, "epoch": 0.17327459618208516, "fcm_dpo/beta": 0.062702976167202, "fcm_dpo/delta": -0.06976436078548431, "fcm_dpo/margin": 7.438255786895752, "fcm_dpo/q_t": 0.39276593923568726, "grad_norm": 16.052621841430664, "learning_rate": 4.924492340087524e-07, "logits/chosen": -1.125187635421753, "logits/rejected": -0.7884189486503601, "logps/chosen": -90.01602935791016, "logps/ref_chosen": -86.53623962402344, "logps/ref_rejected": -82.93563842773438, "logps/rejected": -93.85369110107422, "loss": 1.0382, "margin_dpo/margin_mean": 7.438255310058594, "margin_dpo/margin_std": 8.531219482421875, "step": 118 }, { "KL/chosen_KL_mean": -5.211345672607422, "KL/mean": -9.058032989501953, "KL/rejected_KL_mean": -12.904712677001953, "KL/std": 9.489660263061523, "epoch": 0.17474302496328928, "fcm_dpo/beta": 0.0616828054189682, "fcm_dpo/delta": -0.07856158912181854, "fcm_dpo/margin": 7.693366527557373, "fcm_dpo/q_t": 0.39377036690711975, "grad_norm": 18.52414894104004, "learning_rate": 4.92133019126601e-07, "logits/chosen": -1.3846263885498047, "logits/rejected": -1.0728895664215088, "logps/chosen": -109.82042694091797, "logps/ref_chosen": -104.60908508300781, "logps/ref_rejected": -109.15780639648438, "logps/rejected": -122.0625228881836, "loss": 1.0869, "margin_dpo/margin_mean": 7.6933674812316895, "margin_dpo/margin_std": 11.942338943481445, "step": 119 }, { "KL/chosen_KL_mean": -4.1529083251953125, "KL/mean": -9.279523849487305, "KL/rejected_KL_mean": -14.406135559082031, "KL/std": 11.35304069519043, "epoch": 0.1762114537444934, "fcm_dpo/beta": 0.05949506163597107, "fcm_dpo/delta": -0.22397297620773315, "fcm_dpo/margin": 10.253229141235352, "fcm_dpo/q_t": 0.37103092670440674, "grad_norm": 21.065217971801758, "learning_rate": 4.918104238142103e-07, "logits/chosen": -1.2521820068359375, "logits/rejected": -0.696804404258728, "logps/chosen": -118.84033203125, "logps/ref_chosen": -114.68742370605469, "logps/ref_rejected": -111.18586730957031, "logps/rejected": -125.59200286865234, "loss": 1.0193, "margin_dpo/margin_mean": 10.253229141235352, "margin_dpo/margin_std": 14.381040573120117, "step": 120 }, { "KL/chosen_KL_mean": -3.7394447326660156, "KL/mean": -8.40185832977295, "KL/rejected_KL_mean": -13.064281463623047, "KL/std": 8.866586685180664, "epoch": 0.1776798825256975, "fcm_dpo/beta": 0.057432860136032104, "fcm_dpo/delta": -0.14398802816867828, "fcm_dpo/margin": 9.324836730957031, "fcm_dpo/q_t": 0.3780592381954193, "grad_norm": 18.062376022338867, "learning_rate": 4.91481456572267e-07, "logits/chosen": -0.9575957655906677, "logits/rejected": -0.6242144107818604, "logps/chosen": -95.85610961914062, "logps/ref_chosen": -92.11666107177734, "logps/ref_rejected": -114.972900390625, "logps/rejected": -128.0371856689453, "loss": 0.9918, "margin_dpo/margin_mean": 9.324836730957031, "margin_dpo/margin_std": 9.832696914672852, "step": 121 }, { "KL/chosen_KL_mean": -4.112579345703125, "KL/mean": -9.427934646606445, "KL/rejected_KL_mean": -14.7432861328125, "KL/std": 9.578864097595215, "epoch": 0.17914831130690162, "fcm_dpo/beta": 0.0557304285466671, "fcm_dpo/delta": -0.20408757030963898, "fcm_dpo/margin": 10.630702018737793, "fcm_dpo/q_t": 0.36557281017303467, "grad_norm": 18.262664794921875, "learning_rate": 4.911461260693638e-07, "logits/chosen": -0.9353420734405518, "logits/rejected": -0.8377484083175659, "logps/chosen": -88.99015045166016, "logps/ref_chosen": -84.87757110595703, "logps/ref_rejected": -111.0829849243164, "logps/rejected": -125.8262710571289, "loss": 0.9693, "margin_dpo/margin_mean": 10.630702018737793, "margin_dpo/margin_std": 11.238935470581055, "step": 122 }, { "KL/chosen_KL_mean": -4.574275970458984, "KL/mean": -8.434359550476074, "KL/rejected_KL_mean": -12.294441223144531, "KL/std": 8.846650123596191, "epoch": 0.18061674008810572, "fcm_dpo/beta": 0.05413203313946724, "fcm_dpo/delta": -0.07251356542110443, "fcm_dpo/margin": 7.720160007476807, "fcm_dpo/q_t": 0.40424859523773193, "grad_norm": 15.763967514038086, "learning_rate": 4.908044411417711e-07, "logits/chosen": -1.1376043558120728, "logits/rejected": -0.8168488144874573, "logps/chosen": -103.05302429199219, "logps/ref_chosen": -98.47874450683594, "logps/ref_rejected": -93.85417175292969, "logps/rejected": -106.14862060546875, "loss": 1.084, "margin_dpo/margin_mean": 7.720160007476807, "margin_dpo/margin_std": 10.183891296386719, "step": 123 }, { "KL/chosen_KL_mean": -4.424888610839844, "KL/mean": -9.160396575927734, "KL/rejected_KL_mean": -13.89590072631836, "KL/std": 11.099146842956543, "epoch": 0.18208516886930984, "fcm_dpo/beta": 0.05327959358692169, "fcm_dpo/delta": -0.11012198030948639, "fcm_dpo/margin": 9.471003532409668, "fcm_dpo/q_t": 0.3945465683937073, "grad_norm": 16.024564743041992, "learning_rate": 4.904564107932048e-07, "logits/chosen": -1.1801691055297852, "logits/rejected": -0.91695237159729, "logps/chosen": -107.18778991699219, "logps/ref_chosen": -102.76290130615234, "logps/ref_rejected": -123.005615234375, "logps/rejected": -136.90151977539062, "loss": 1.066, "margin_dpo/margin_mean": 9.471002578735352, "margin_dpo/margin_std": 14.401384353637695, "step": 124 }, { "KL/chosen_KL_mean": -5.886226654052734, "KL/mean": -9.330833435058594, "KL/rejected_KL_mean": -12.775447845458984, "KL/std": 9.071002960205078, "epoch": 0.18355359765051396, "fcm_dpo/beta": 0.05318472534418106, "fcm_dpo/delta": 0.0348917655646801, "fcm_dpo/margin": 6.889215469360352, "fcm_dpo/q_t": 0.41773951053619385, "grad_norm": 16.47319984436035, "learning_rate": 4.90102044194588e-07, "logits/chosen": -0.8794206380844116, "logits/rejected": -0.6602913737297058, "logps/chosen": -96.3125, "logps/ref_chosen": -90.42627716064453, "logps/ref_rejected": -89.24901580810547, "logps/rejected": -102.02445983886719, "loss": 1.1348, "margin_dpo/margin_mean": 6.889216423034668, "margin_dpo/margin_std": 11.119590759277344, "step": 125 }, { "KL/chosen_KL_mean": -4.795345306396484, "KL/mean": -9.781375885009766, "KL/rejected_KL_mean": -14.767406463623047, "KL/std": 9.724905014038086, "epoch": 0.18502202643171806, "fcm_dpo/beta": 0.0523233599960804, "fcm_dpo/delta": -0.1284581869840622, "fcm_dpo/margin": 9.972065925598145, "fcm_dpo/q_t": 0.37919655442237854, "grad_norm": 15.988823890686035, "learning_rate": 4.897413506838102e-07, "logits/chosen": -0.8898293972015381, "logits/rejected": -0.6062628626823425, "logps/chosen": -99.21823120117188, "logps/ref_chosen": -94.42288208007812, "logps/ref_rejected": -103.88156127929688, "logps/rejected": -118.64896392822266, "loss": 0.9958, "margin_dpo/margin_mean": 9.972064971923828, "margin_dpo/margin_std": 10.190126419067383, "step": 126 }, { "KL/chosen_KL_mean": -4.822856903076172, "KL/mean": -8.551148414611816, "KL/rejected_KL_mean": -12.279441833496094, "KL/std": 8.983542442321777, "epoch": 0.18649045521292218, "fcm_dpo/beta": 0.05218739062547684, "fcm_dpo/delta": 0.011145832017064095, "fcm_dpo/margin": 7.4565839767456055, "fcm_dpo/q_t": 0.41303130984306335, "grad_norm": 16.119571685791016, "learning_rate": 4.89374339765481e-07, "logits/chosen": -0.7476365566253662, "logits/rejected": -0.39096903800964355, "logps/chosen": -89.57391357421875, "logps/ref_chosen": -84.75105285644531, "logps/ref_rejected": -83.54523468017578, "logps/rejected": -95.82467651367188, "loss": 1.1131, "margin_dpo/margin_mean": 7.4565839767456055, "margin_dpo/margin_std": 11.260820388793945, "step": 127 }, { "KL/chosen_KL_mean": -4.778011322021484, "KL/mean": -8.66218376159668, "KL/rejected_KL_mean": -12.546350479125977, "KL/std": 11.10682201385498, "epoch": 0.18795888399412627, "fcm_dpo/beta": 0.05228809267282486, "fcm_dpo/delta": -0.006903614848852158, "fcm_dpo/margin": 7.768337249755859, "fcm_dpo/q_t": 0.40844595432281494, "grad_norm": 18.162736892700195, "learning_rate": 4.890010211106795e-07, "logits/chosen": -0.6107750535011292, "logits/rejected": -0.11944863945245743, "logps/chosen": -95.39765930175781, "logps/ref_chosen": -90.61964416503906, "logps/ref_rejected": -83.10347747802734, "logps/rejected": -95.64982604980469, "loss": 1.1287, "margin_dpo/margin_mean": 7.768337249755859, "margin_dpo/margin_std": 13.658068656921387, "step": 128 }, { "KL/chosen_KL_mean": -5.3299407958984375, "KL/mean": -9.333892822265625, "KL/rejected_KL_mean": -13.337848663330078, "KL/std": 8.991896629333496, "epoch": 0.1894273127753304, "fcm_dpo/beta": 0.05203121155500412, "fcm_dpo/delta": -0.017507098615169525, "fcm_dpo/margin": 8.007903099060059, "fcm_dpo/q_t": 0.4092479348182678, "grad_norm": 14.906427383422852, "learning_rate": 4.88621404556699e-07, "logits/chosen": -0.797419548034668, "logits/rejected": -0.48800790309906006, "logps/chosen": -104.24864959716797, "logps/ref_chosen": -98.91870880126953, "logps/ref_rejected": -103.28419494628906, "logps/rejected": -116.62203979492188, "loss": 1.1071, "margin_dpo/margin_mean": 8.007903099060059, "margin_dpo/margin_std": 12.250895500183105, "step": 129 }, { "KL/chosen_KL_mean": -4.206462860107422, "KL/mean": -9.440500259399414, "KL/rejected_KL_mean": -14.67453384399414, "KL/std": 10.511327743530273, "epoch": 0.19089574155653452, "fcm_dpo/beta": 0.05113845691084862, "fcm_dpo/delta": -0.14308412373065948, "fcm_dpo/margin": 10.468074798583984, "fcm_dpo/q_t": 0.38214167952537537, "grad_norm": 14.121175765991211, "learning_rate": 4.882355001067891e-07, "logits/chosen": -1.132953405380249, "logits/rejected": -0.8869795203208923, "logps/chosen": -77.39015197753906, "logps/ref_chosen": -73.18368530273438, "logps/ref_rejected": -88.02154541015625, "logps/rejected": -102.69607543945312, "loss": 1.0169, "margin_dpo/margin_mean": 10.468074798583984, "margin_dpo/margin_std": 12.477448463439941, "step": 130 }, { "KL/chosen_KL_mean": -3.6156005859375, "KL/mean": -8.39569091796875, "KL/rejected_KL_mean": -13.175788879394531, "KL/std": 10.556526184082031, "epoch": 0.19236417033773862, "fcm_dpo/beta": 0.050069913268089294, "fcm_dpo/delta": -0.08292128890752792, "fcm_dpo/margin": 9.560185432434082, "fcm_dpo/q_t": 0.3923855423927307, "grad_norm": 16.485183715820312, "learning_rate": 4.878433179298909e-07, "logits/chosen": -1.0437589883804321, "logits/rejected": -0.8303720951080322, "logps/chosen": -77.5720443725586, "logps/ref_chosen": -73.9564437866211, "logps/ref_rejected": -94.45582580566406, "logps/rejected": -107.6316146850586, "loss": 1.0502, "margin_dpo/margin_mean": 9.560186386108398, "margin_dpo/margin_std": 12.052691459655762, "step": 131 }, { "KL/chosen_KL_mean": -5.328037261962891, "KL/mean": -10.026786804199219, "KL/rejected_KL_mean": -14.725532531738281, "KL/std": 10.929555892944336, "epoch": 0.19383259911894274, "fcm_dpo/beta": 0.04888708144426346, "fcm_dpo/delta": -0.0628490224480629, "fcm_dpo/margin": 9.39749526977539, "fcm_dpo/q_t": 0.3952639698982239, "grad_norm": 16.64833641052246, "learning_rate": 4.874448683603694e-07, "logits/chosen": -1.3084355592727661, "logits/rejected": -0.8912124633789062, "logps/chosen": -97.76751708984375, "logps/ref_chosen": -92.43948364257812, "logps/ref_rejected": -94.13252258300781, "logps/rejected": -108.85804748535156, "loss": 1.0639, "margin_dpo/margin_mean": 9.39749526977539, "margin_dpo/margin_std": 12.493152618408203, "step": 132 }, { "KL/chosen_KL_mean": -4.420093536376953, "KL/mean": -9.39143180847168, "KL/rejected_KL_mean": -14.362777709960938, "KL/std": 11.067865371704102, "epoch": 0.19530102790014683, "fcm_dpo/beta": 0.04835614562034607, "fcm_dpo/delta": -0.08480934053659439, "fcm_dpo/margin": 9.942688941955566, "fcm_dpo/q_t": 0.3915287256240845, "grad_norm": 14.990748405456543, "learning_rate": 4.870401618977415e-07, "logits/chosen": -1.5841182470321655, "logits/rejected": -1.1946488618850708, "logps/chosen": -97.53675842285156, "logps/ref_chosen": -93.11666107177734, "logps/ref_rejected": -102.09490966796875, "logps/rejected": -116.45768737792969, "loss": 1.0338, "margin_dpo/margin_mean": 9.94268798828125, "margin_dpo/margin_std": 11.722248077392578, "step": 133 }, { "KL/chosen_KL_mean": -4.661293029785156, "KL/mean": -10.001508712768555, "KL/rejected_KL_mean": -15.341720581054688, "KL/std": 10.434558868408203, "epoch": 0.19676945668135096, "fcm_dpo/beta": 0.04762697592377663, "fcm_dpo/delta": -0.11471561342477798, "fcm_dpo/margin": 10.680419921875, "fcm_dpo/q_t": 0.3836822509765625, "grad_norm": 14.525565147399902, "learning_rate": 4.866292092063986e-07, "logits/chosen": -1.5968610048294067, "logits/rejected": -1.2609632015228271, "logps/chosen": -88.21141815185547, "logps/ref_chosen": -83.55012512207031, "logps/ref_rejected": -93.8815689086914, "logps/rejected": -109.2232894897461, "loss": 1.0126, "margin_dpo/margin_mean": 10.680419921875, "margin_dpo/margin_std": 11.334150314331055, "step": 134 }, { "KL/chosen_KL_mean": -4.313234329223633, "KL/mean": -9.659897804260254, "KL/rejected_KL_mean": -15.006568908691406, "KL/std": 11.448062896728516, "epoch": 0.19823788546255505, "fcm_dpo/beta": 0.04640686511993408, "fcm_dpo/delta": -0.10114714503288269, "fcm_dpo/margin": 10.693326950073242, "fcm_dpo/q_t": 0.3898237943649292, "grad_norm": 18.776870727539062, "learning_rate": 4.862120211153265e-07, "logits/chosen": -1.171066403388977, "logits/rejected": -1.0541443824768066, "logps/chosen": -81.89585876464844, "logps/ref_chosen": -77.58262634277344, "logps/ref_rejected": -121.05648040771484, "logps/rejected": -136.06304931640625, "loss": 1.0323, "margin_dpo/margin_mean": 10.693326950073242, "margin_dpo/margin_std": 12.919585227966309, "step": 135 }, { "KL/chosen_KL_mean": -5.020286560058594, "KL/mean": -10.215295791625977, "KL/rejected_KL_mean": -15.410301208496094, "KL/std": 11.731208801269531, "epoch": 0.19970631424375918, "fcm_dpo/beta": 0.04551202431321144, "fcm_dpo/delta": -0.0765582025051117, "fcm_dpo/margin": 10.390012741088867, "fcm_dpo/q_t": 0.39236894249916077, "grad_norm": 14.227690696716309, "learning_rate": 4.857886086178193e-07, "logits/chosen": -1.4899359941482544, "logits/rejected": -1.057348608970642, "logps/chosen": -99.19302368164062, "logps/ref_chosen": -94.1727294921875, "logps/ref_rejected": -103.26428985595703, "logps/rejected": -118.67459106445312, "loss": 1.0329, "margin_dpo/margin_mean": 10.390012741088867, "margin_dpo/margin_std": 11.876028060913086, "step": 136 }, { "KL/chosen_KL_mean": -3.9241867065429688, "KL/mean": -12.21324348449707, "KL/rejected_KL_mean": -20.502304077148438, "KL/std": 15.162540435791016, "epoch": 0.2011747430249633, "fcm_dpo/beta": 0.04288076236844063, "fcm_dpo/delta": -0.33769649267196655, "fcm_dpo/margin": 16.578121185302734, "fcm_dpo/q_t": 0.34447985887527466, "grad_norm": 16.599565505981445, "learning_rate": 4.853589828711902e-07, "logits/chosen": -1.4613754749298096, "logits/rejected": -1.244441270828247, "logps/chosen": -82.04292297363281, "logps/ref_chosen": -78.11874389648438, "logps/ref_rejected": -117.92237091064453, "logps/rejected": -138.4246826171875, "loss": 0.8989, "margin_dpo/margin_mean": 16.5781192779541, "margin_dpo/margin_std": 16.622516632080078, "step": 137 }, { "KL/chosen_KL_mean": -6.602939605712891, "KL/mean": -11.714762687683105, "KL/rejected_KL_mean": -16.826583862304688, "KL/std": 11.099853515625, "epoch": 0.2026431718061674, "fcm_dpo/beta": 0.04212722182273865, "fcm_dpo/delta": -0.03219844400882721, "fcm_dpo/margin": 10.223644256591797, "fcm_dpo/q_t": 0.40117987990379333, "grad_norm": 13.062910079956055, "learning_rate": 4.849231551964771e-07, "logits/chosen": -1.0434367656707764, "logits/rejected": -0.7851938605308533, "logps/chosen": -99.4328842163086, "logps/ref_chosen": -92.82994079589844, "logps/ref_rejected": -98.64840698242188, "logps/rejected": -115.47499084472656, "loss": 1.0624, "margin_dpo/margin_mean": 10.223644256591797, "margin_dpo/margin_std": 12.262227058410645, "step": 138 }, { "KL/chosen_KL_mean": -4.611412048339844, "KL/mean": -10.478314399719238, "KL/rejected_KL_mean": -16.345218658447266, "KL/std": 12.116909980773926, "epoch": 0.20411160058737152, "fcm_dpo/beta": 0.04147449880838394, "fcm_dpo/delta": -0.09138133376836777, "fcm_dpo/margin": 11.733804702758789, "fcm_dpo/q_t": 0.3868618905544281, "grad_norm": 13.359282493591309, "learning_rate": 4.844811370781446e-07, "logits/chosen": -1.4438704252243042, "logits/rejected": -1.124290108680725, "logps/chosen": -83.61442565917969, "logps/ref_chosen": -79.00301361083984, "logps/ref_rejected": -85.82733154296875, "logps/rejected": -102.17254638671875, "loss": 1.0373, "margin_dpo/margin_mean": 11.733804702758789, "margin_dpo/margin_std": 14.385666847229004, "step": 139 }, { "KL/chosen_KL_mean": -5.039573669433594, "KL/mean": -10.38720703125, "KL/rejected_KL_mean": -15.734840393066406, "KL/std": 11.379945755004883, "epoch": 0.2055800293685756, "fcm_dpo/beta": 0.041134219616651535, "fcm_dpo/delta": -0.04176412522792816, "fcm_dpo/margin": 10.695271492004395, "fcm_dpo/q_t": 0.39817529916763306, "grad_norm": 12.633871078491211, "learning_rate": 4.840329401637809e-07, "logits/chosen": -1.629274845123291, "logits/rejected": -1.2231906652450562, "logps/chosen": -93.39202117919922, "logps/ref_chosen": -88.35244750976562, "logps/ref_rejected": -90.57476806640625, "logps/rejected": -106.30961608886719, "loss": 1.0519, "margin_dpo/margin_mean": 10.695271492004395, "margin_dpo/margin_std": 12.148259162902832, "step": 140 }, { "KL/chosen_KL_mean": -6.828273773193359, "KL/mean": -12.909975051879883, "KL/rejected_KL_mean": -18.991676330566406, "KL/std": 14.93606948852539, "epoch": 0.20704845814977973, "fcm_dpo/beta": 0.04045747220516205, "fcm_dpo/delta": -0.09682976454496384, "fcm_dpo/margin": 12.163406372070312, "fcm_dpo/q_t": 0.38734182715415955, "grad_norm": 15.835349082946777, "learning_rate": 4.83578576263792e-07, "logits/chosen": -1.2295082807540894, "logits/rejected": -0.8325952887535095, "logps/chosen": -114.04843139648438, "logps/ref_chosen": -107.22016143798828, "logps/ref_rejected": -104.71223449707031, "logps/rejected": -123.70390319824219, "loss": 1.0373, "margin_dpo/margin_mean": 12.163406372070312, "margin_dpo/margin_std": 15.071775436401367, "step": 141 }, { "KL/chosen_KL_mean": -6.979434967041016, "KL/mean": -12.973379135131836, "KL/rejected_KL_mean": -18.967323303222656, "KL/std": 13.900266647338867, "epoch": 0.20851688693098386, "fcm_dpo/beta": 0.039698414504528046, "fcm_dpo/delta": -0.07988056540489197, "fcm_dpo/margin": 11.98788833618164, "fcm_dpo/q_t": 0.39354777336120605, "grad_norm": 12.661876678466797, "learning_rate": 4.83118057351089e-07, "logits/chosen": -1.1199415922164917, "logits/rejected": -0.8576666116714478, "logps/chosen": -95.70838165283203, "logps/ref_chosen": -88.72894287109375, "logps/ref_rejected": -99.73038482666016, "logps/rejected": -118.69770812988281, "loss": 1.0381, "margin_dpo/margin_mean": 11.98788833618164, "margin_dpo/margin_std": 14.585189819335938, "step": 142 }, { "KL/chosen_KL_mean": -5.2153167724609375, "KL/mean": -10.112159729003906, "KL/rejected_KL_mean": -15.00899887084961, "KL/std": 11.589612007141113, "epoch": 0.20998531571218795, "fcm_dpo/beta": 0.039522238075733185, "fcm_dpo/delta": 0.013181895017623901, "fcm_dpo/margin": 9.793678283691406, "fcm_dpo/q_t": 0.4106484651565552, "grad_norm": 12.86412239074707, "learning_rate": 4.826513955607734e-07, "logits/chosen": -1.5898847579956055, "logits/rejected": -1.1357622146606445, "logps/chosen": -90.71989440917969, "logps/ref_chosen": -85.50457763671875, "logps/ref_rejected": -85.8056640625, "logps/rejected": -100.81465911865234, "loss": 1.0973, "margin_dpo/margin_mean": 9.793678283691406, "margin_dpo/margin_std": 12.777687072753906, "step": 143 }, { "KL/chosen_KL_mean": -6.343677520751953, "KL/mean": -11.225152969360352, "KL/rejected_KL_mean": -16.10663604736328, "KL/std": 12.24761962890625, "epoch": 0.21145374449339208, "fcm_dpo/beta": 0.03950557857751846, "fcm_dpo/delta": -0.050757840275764465, "fcm_dpo/margin": 9.762959480285645, "fcm_dpo/q_t": 0.4123876690864563, "grad_norm": 16.281400680541992, "learning_rate": 4.821786031898176e-07, "logits/chosen": -1.4441888332366943, "logits/rejected": -0.898901104927063, "logps/chosen": -111.03076934814453, "logps/ref_chosen": -104.68708801269531, "logps/ref_rejected": -87.71908569335938, "logps/rejected": -103.82571411132812, "loss": 1.1221, "margin_dpo/margin_mean": 9.762960433959961, "margin_dpo/margin_std": 15.188072204589844, "step": 144 }, { "KL/chosen_KL_mean": -5.382896423339844, "KL/mean": -11.173028945922852, "KL/rejected_KL_mean": -16.96316146850586, "KL/std": 11.726531982421875, "epoch": 0.21292217327459617, "fcm_dpo/beta": 0.0389803871512413, "fcm_dpo/delta": -0.05385002866387367, "fcm_dpo/margin": 11.5802640914917, "fcm_dpo/q_t": 0.3956826329231262, "grad_norm": 13.102893829345703, "learning_rate": 4.816996926967401e-07, "logits/chosen": -1.0844286680221558, "logits/rejected": -0.49777746200561523, "logps/chosen": -93.90278625488281, "logps/ref_chosen": -88.51988983154297, "logps/ref_rejected": -83.76513671875, "logps/rejected": -100.72830200195312, "loss": 1.0462, "margin_dpo/margin_mean": 11.580265045166016, "margin_dpo/margin_std": 13.240138053894043, "step": 145 }, { "KL/chosen_KL_mean": -5.799015045166016, "KL/mean": -12.270423889160156, "KL/rejected_KL_mean": -18.741836547851562, "KL/std": 12.897621154785156, "epoch": 0.2143906020558003, "fcm_dpo/beta": 0.038127027451992035, "fcm_dpo/delta": -0.09929933398962021, "fcm_dpo/margin": 12.942817687988281, "fcm_dpo/q_t": 0.3859749734401703, "grad_norm": 12.930309295654297, "learning_rate": 4.812146767012779e-07, "logits/chosen": -1.420979380607605, "logits/rejected": -0.751975953578949, "logps/chosen": -108.70487976074219, "logps/ref_chosen": -102.90586853027344, "logps/ref_rejected": -88.9493637084961, "logps/rejected": -107.69120025634766, "loss": 1.0203, "margin_dpo/margin_mean": 12.942819595336914, "margin_dpo/margin_std": 14.13513469696045, "step": 146 }, { "KL/chosen_KL_mean": -5.618473052978516, "KL/mean": -12.281608581542969, "KL/rejected_KL_mean": -18.944747924804688, "KL/std": 12.288924217224121, "epoch": 0.21585903083700442, "fcm_dpo/beta": 0.0375375859439373, "fcm_dpo/delta": -0.10548540949821472, "fcm_dpo/margin": 13.326274871826172, "fcm_dpo/q_t": 0.3839772939682007, "grad_norm": 12.124649047851562, "learning_rate": 4.807235679840536e-07, "logits/chosen": -1.4902106523513794, "logits/rejected": -0.9062970876693726, "logps/chosen": -94.88066101074219, "logps/ref_chosen": -89.26219177246094, "logps/ref_rejected": -76.50011444091797, "logps/rejected": -95.44486236572266, "loss": 1.003, "margin_dpo/margin_mean": 13.326274871826172, "margin_dpo/margin_std": 13.225658416748047, "step": 147 }, { "KL/chosen_KL_mean": -4.025173187255859, "KL/mean": -10.219099044799805, "KL/rejected_KL_mean": -16.41303253173828, "KL/std": 12.871241569519043, "epoch": 0.2173274596182085, "fcm_dpo/beta": 0.037258125841617584, "fcm_dpo/delta": -0.06528796255588531, "fcm_dpo/margin": 12.387852668762207, "fcm_dpo/q_t": 0.39699316024780273, "grad_norm": 12.853314399719238, "learning_rate": 4.802263794862384e-07, "logits/chosen": -1.6103897094726562, "logits/rejected": -1.215408444404602, "logps/chosen": -95.78805541992188, "logps/ref_chosen": -91.76287841796875, "logps/ref_rejected": -110.38552856445312, "logps/rejected": -126.7985610961914, "loss": 1.0535, "margin_dpo/margin_mean": 12.387852668762207, "margin_dpo/margin_std": 14.648200035095215, "step": 148 }, { "KL/chosen_KL_mean": -7.165077209472656, "KL/mean": -13.094527244567871, "KL/rejected_KL_mean": -19.023963928222656, "KL/std": 13.681352615356445, "epoch": 0.21879588839941264, "fcm_dpo/beta": 0.03651132062077522, "fcm_dpo/delta": -0.03489532321691513, "fcm_dpo/margin": 11.85888671875, "fcm_dpo/q_t": 0.4054684638977051, "grad_norm": 12.661170959472656, "learning_rate": 4.797231243092118e-07, "logits/chosen": -1.0527235269546509, "logits/rejected": -0.7100017666816711, "logps/chosen": -100.64909362792969, "logps/ref_chosen": -93.48401641845703, "logps/ref_rejected": -105.6436996459961, "logps/rejected": -124.66766357421875, "loss": 1.0861, "margin_dpo/margin_mean": 11.858888626098633, "margin_dpo/margin_std": 17.029895782470703, "step": 149 }, { "KL/chosen_KL_mean": -5.740791320800781, "KL/mean": -11.940508842468262, "KL/rejected_KL_mean": -18.140228271484375, "KL/std": 14.490161895751953, "epoch": 0.22026431718061673, "fcm_dpo/beta": 0.03621049225330353, "fcm_dpo/delta": -0.05155959725379944, "fcm_dpo/margin": 12.399433135986328, "fcm_dpo/q_t": 0.3988327085971832, "grad_norm": 11.37544059753418, "learning_rate": 4.792138157142157e-07, "logits/chosen": -1.078065276145935, "logits/rejected": -0.8071381449699402, "logps/chosen": -81.51947784423828, "logps/ref_chosen": -75.7786865234375, "logps/ref_rejected": -90.02676391601562, "logps/rejected": -108.1669921875, "loss": 1.0623, "margin_dpo/margin_mean": 12.399433135986328, "margin_dpo/margin_std": 16.125131607055664, "step": 150 }, { "KL/chosen_KL_mean": -7.623424530029297, "KL/mean": -13.627882957458496, "KL/rejected_KL_mean": -19.632335662841797, "KL/std": 14.161249160766602, "epoch": 0.22173274596182085, "fcm_dpo/beta": 0.0360143817961216, "fcm_dpo/delta": -0.033974550664424896, "fcm_dpo/margin": 12.008913040161133, "fcm_dpo/q_t": 0.40218406915664673, "grad_norm": 13.809005737304688, "learning_rate": 4.786984671220053e-07, "logits/chosen": -2.040083646774292, "logits/rejected": -1.446272373199463, "logps/chosen": -110.35765075683594, "logps/ref_chosen": -102.73422241210938, "logps/ref_rejected": -106.03781127929688, "logps/rejected": -125.67015075683594, "loss": 1.0737, "margin_dpo/margin_mean": 12.008913040161133, "margin_dpo/margin_std": 16.029272079467773, "step": 151 }, { "KL/chosen_KL_mean": -4.768917083740234, "KL/mean": -12.093599319458008, "KL/rejected_KL_mean": -19.41828155517578, "KL/std": 14.003814697265625, "epoch": 0.22320117474302498, "fcm_dpo/beta": 0.035251080989837646, "fcm_dpo/delta": -0.12301421165466309, "fcm_dpo/margin": 14.649358749389648, "fcm_dpo/q_t": 0.3814336359500885, "grad_norm": 12.385575294494629, "learning_rate": 4.78177092112495e-07, "logits/chosen": -1.3468904495239258, "logits/rejected": -0.9493122100830078, "logps/chosen": -102.35966491699219, "logps/ref_chosen": -97.59074401855469, "logps/ref_rejected": -110.49913024902344, "logps/rejected": -129.91741943359375, "loss": 1.0088, "margin_dpo/margin_mean": 14.649358749389648, "margin_dpo/margin_std": 16.299846649169922, "step": 152 }, { "KL/chosen_KL_mean": -7.291164398193359, "KL/mean": -12.519810676574707, "KL/rejected_KL_mean": -17.748458862304688, "KL/std": 13.610466003417969, "epoch": 0.22466960352422907, "fcm_dpo/beta": 0.03525884449481964, "fcm_dpo/delta": 0.0324854850769043, "fcm_dpo/margin": 10.457305908203125, "fcm_dpo/q_t": 0.41564375162124634, "grad_norm": 12.339768409729004, "learning_rate": 4.776497044244016e-07, "logits/chosen": -1.7729458808898926, "logits/rejected": -1.4528738260269165, "logps/chosen": -94.31676483154297, "logps/ref_chosen": -87.02560424804688, "logps/ref_rejected": -91.06854248046875, "logps/rejected": -108.81700134277344, "loss": 1.1235, "margin_dpo/margin_mean": 10.457305908203125, "margin_dpo/margin_std": 15.588835716247559, "step": 153 }, { "KL/chosen_KL_mean": -5.521923065185547, "KL/mean": -14.028284072875977, "KL/rejected_KL_mean": -22.534645080566406, "KL/std": 14.900197982788086, "epoch": 0.2261380323054332, "fcm_dpo/beta": 0.034528039395809174, "fcm_dpo/delta": -0.19871015846729279, "fcm_dpo/margin": 17.012727737426758, "fcm_dpo/q_t": 0.36560943722724915, "grad_norm": 12.075468063354492, "learning_rate": 4.771163179548808e-07, "logits/chosen": -1.2888035774230957, "logits/rejected": -0.959321141242981, "logps/chosen": -96.62759399414062, "logps/ref_chosen": -91.10566711425781, "logps/ref_rejected": -111.37238311767578, "logps/rejected": -133.9070281982422, "loss": 0.9544, "margin_dpo/margin_mean": 17.01272964477539, "margin_dpo/margin_std": 15.812162399291992, "step": 154 }, { "KL/chosen_KL_mean": -7.637653350830078, "KL/mean": -13.939414978027344, "KL/rejected_KL_mean": -20.241172790527344, "KL/std": 15.43907356262207, "epoch": 0.2276064610866373, "fcm_dpo/beta": 0.03364599868655205, "fcm_dpo/delta": -0.025743480771780014, "fcm_dpo/margin": 12.603522300720215, "fcm_dpo/q_t": 0.40080979466438293, "grad_norm": 14.663453102111816, "learning_rate": 4.7657694675916247e-07, "logits/chosen": -1.446264386177063, "logits/rejected": -1.0660717487335205, "logps/chosen": -102.19862365722656, "logps/ref_chosen": -94.56097412109375, "logps/ref_rejected": -84.77359008789062, "logps/rejected": -105.01476287841797, "loss": 1.1017, "margin_dpo/margin_mean": 12.603523254394531, "margin_dpo/margin_std": 19.157184600830078, "step": 155 }, { "KL/chosen_KL_mean": -9.37765121459961, "KL/mean": -15.051112174987793, "KL/rejected_KL_mean": -20.724571228027344, "KL/std": 13.716203689575195, "epoch": 0.2290748898678414, "fcm_dpo/beta": 0.03350888937711716, "fcm_dpo/delta": -0.038376789540052414, "fcm_dpo/margin": 11.34692096710205, "fcm_dpo/q_t": 0.4143773913383484, "grad_norm": 13.021503448486328, "learning_rate": 4.7603160505017893e-07, "logits/chosen": -1.6705093383789062, "logits/rejected": -1.1727981567382812, "logps/chosen": -104.03303527832031, "logps/ref_chosen": -94.65538787841797, "logps/ref_rejected": -84.41934204101562, "logps/rejected": -105.14391326904297, "loss": 1.1178, "margin_dpo/margin_mean": 11.34692096710205, "margin_dpo/margin_std": 16.84894371032715, "step": 156 }, { "KL/chosen_KL_mean": -8.810501098632812, "KL/mean": -16.74010467529297, "KL/rejected_KL_mean": -24.66971206665039, "KL/std": 14.787250518798828, "epoch": 0.2305433186490455, "fcm_dpo/beta": 0.03291456401348114, "fcm_dpo/delta": -0.1285938024520874, "fcm_dpo/margin": 15.85920524597168, "fcm_dpo/q_t": 0.3789578080177307, "grad_norm": 13.065104484558105, "learning_rate": 4.7548030719819154e-07, "logits/chosen": -1.9881575107574463, "logits/rejected": -1.5299711227416992, "logps/chosen": -105.82289123535156, "logps/ref_chosen": -97.01239013671875, "logps/ref_rejected": -111.70133972167969, "logps/rejected": -136.37106323242188, "loss": 0.9921, "margin_dpo/margin_mean": 15.859204292297363, "margin_dpo/margin_std": 15.631433486938477, "step": 157 }, { "KL/chosen_KL_mean": -7.910472869873047, "KL/mean": -16.244686126708984, "KL/rejected_KL_mean": -24.578895568847656, "KL/std": 15.939573287963867, "epoch": 0.23201174743024963, "fcm_dpo/beta": 0.03203149139881134, "fcm_dpo/delta": -0.14129498600959778, "fcm_dpo/margin": 16.668426513671875, "fcm_dpo/q_t": 0.3779933750629425, "grad_norm": 12.011423110961914, "learning_rate": 4.7492306773041136e-07, "logits/chosen": -1.4900768995285034, "logits/rejected": -1.2238035202026367, "logps/chosen": -102.6234359741211, "logps/ref_chosen": -94.71296691894531, "logps/ref_rejected": -120.93733978271484, "logps/rejected": -145.5162353515625, "loss": 0.9892, "margin_dpo/margin_mean": 16.668426513671875, "margin_dpo/margin_std": 17.173080444335938, "step": 158 }, { "KL/chosen_KL_mean": -7.4810791015625, "KL/mean": -14.905092239379883, "KL/rejected_KL_mean": -22.329105377197266, "KL/std": 15.53576946258545, "epoch": 0.23348017621145375, "fcm_dpo/beta": 0.03140883520245552, "fcm_dpo/delta": -0.06963557749986649, "fcm_dpo/margin": 14.848034858703613, "fcm_dpo/q_t": 0.39328068494796753, "grad_norm": 11.008720397949219, "learning_rate": 4.743599013306165e-07, "logits/chosen": -1.860887050628662, "logits/rejected": -1.351900577545166, "logps/chosen": -115.6993179321289, "logps/ref_chosen": -108.2182388305664, "logps/ref_rejected": -94.83901977539062, "logps/rejected": -117.16812896728516, "loss": 1.0437, "margin_dpo/margin_mean": 14.84803581237793, "margin_dpo/margin_std": 17.740581512451172, "step": 159 }, { "KL/chosen_KL_mean": -6.347347259521484, "KL/mean": -13.680153846740723, "KL/rejected_KL_mean": -21.012958526611328, "KL/std": 14.236058235168457, "epoch": 0.23494860499265785, "fcm_dpo/beta": 0.030983250588178635, "fcm_dpo/delta": -0.05716680362820625, "fcm_dpo/margin": 14.665613174438477, "fcm_dpo/q_t": 0.39353251457214355, "grad_norm": 11.312978744506836, "learning_rate": 4.737908228387656e-07, "logits/chosen": -1.736264705657959, "logits/rejected": -1.1935105323791504, "logps/chosen": -104.51075744628906, "logps/ref_chosen": -98.16340637207031, "logps/ref_rejected": -102.28097534179688, "logps/rejected": -123.29393768310547, "loss": 1.0346, "margin_dpo/margin_mean": 14.665614128112793, "margin_dpo/margin_std": 15.253125190734863, "step": 160 }, { "KL/chosen_KL_mean": -8.444732666015625, "KL/mean": -15.035680770874023, "KL/rejected_KL_mean": -21.626625061035156, "KL/std": 13.177711486816406, "epoch": 0.23641703377386197, "fcm_dpo/beta": 0.030808940529823303, "fcm_dpo/delta": -0.006680246442556381, "fcm_dpo/margin": 13.18189811706543, "fcm_dpo/q_t": 0.40630465745925903, "grad_norm": 11.289678573608398, "learning_rate": 4.7321584725060594e-07, "logits/chosen": -1.7891913652420044, "logits/rejected": -1.293304443359375, "logps/chosen": -95.03660583496094, "logps/ref_chosen": -86.59187316894531, "logps/ref_rejected": -89.01643371582031, "logps/rejected": -110.64305877685547, "loss": 1.0872, "margin_dpo/margin_mean": 13.18189811706543, "margin_dpo/margin_std": 17.073577880859375, "step": 161 }, { "KL/chosen_KL_mean": -7.480491638183594, "KL/mean": -14.86019515991211, "KL/rejected_KL_mean": -22.23989486694336, "KL/std": 15.275270462036133, "epoch": 0.23788546255506607, "fcm_dpo/beta": 0.030585046857595444, "fcm_dpo/delta": -0.05412214249372482, "fcm_dpo/margin": 14.759407997131348, "fcm_dpo/q_t": 0.3945804834365845, "grad_norm": 10.866828918457031, "learning_rate": 4.7263498971727905e-07, "logits/chosen": -1.2355608940124512, "logits/rejected": -0.8285146951675415, "logps/chosen": -104.88337707519531, "logps/ref_chosen": -97.40288543701172, "logps/ref_rejected": -95.86923217773438, "logps/rejected": -118.109130859375, "loss": 1.0426, "margin_dpo/margin_mean": 14.759408950805664, "margin_dpo/margin_std": 16.299522399902344, "step": 162 }, { "KL/chosen_KL_mean": -7.325176239013672, "KL/mean": -16.010160446166992, "KL/rejected_KL_mean": -24.695148468017578, "KL/std": 17.343318939208984, "epoch": 0.2393538913362702, "fcm_dpo/beta": 0.02993404120206833, "fcm_dpo/delta": -0.12696653604507446, "fcm_dpo/margin": 17.36997413635254, "fcm_dpo/q_t": 0.3817980885505676, "grad_norm": 9.801036834716797, "learning_rate": 4.720482655449212e-07, "logits/chosen": -1.6288502216339111, "logits/rejected": -1.2274677753448486, "logps/chosen": -89.50056457519531, "logps/ref_chosen": -82.1753921508789, "logps/ref_rejected": -82.86473846435547, "logps/rejected": -107.55989074707031, "loss": 1.0006, "margin_dpo/margin_mean": 17.36997413635254, "margin_dpo/margin_std": 18.650039672851562, "step": 163 }, { "KL/chosen_KL_mean": -11.081611633300781, "KL/mean": -17.008907318115234, "KL/rejected_KL_mean": -22.936203002929688, "KL/std": 13.699360847473145, "epoch": 0.24082232011747431, "fcm_dpo/beta": 0.02955673262476921, "fcm_dpo/delta": -0.0430353507399559, "fcm_dpo/margin": 11.854592323303223, "fcm_dpo/q_t": 0.41718682646751404, "grad_norm": 11.487072944641113, "learning_rate": 4.714556901942599e-07, "logits/chosen": -1.407117486000061, "logits/rejected": -0.9362297058105469, "logps/chosen": -101.55587768554688, "logps/ref_chosen": -90.47427368164062, "logps/ref_rejected": -87.02809143066406, "logps/rejected": -109.96429443359375, "loss": 1.1175, "margin_dpo/margin_mean": 11.854593276977539, "margin_dpo/margin_std": 15.119787216186523, "step": 164 }, { "KL/chosen_KL_mean": -7.5335235595703125, "KL/mean": -16.14388656616211, "KL/rejected_KL_mean": -24.754249572753906, "KL/std": 16.31938362121582, "epoch": 0.2422907488986784, "fcm_dpo/beta": 0.029155373573303223, "fcm_dpo/delta": -0.10733498632907867, "fcm_dpo/margin": 17.220731735229492, "fcm_dpo/q_t": 0.386141836643219, "grad_norm": 9.381265640258789, "learning_rate": 4.708572792802069e-07, "logits/chosen": -1.7909035682678223, "logits/rejected": -1.3680275678634644, "logps/chosen": -94.84675598144531, "logps/ref_chosen": -87.313232421875, "logps/ref_rejected": -81.902587890625, "logps/rejected": -106.65682983398438, "loss": 1.0092, "margin_dpo/margin_mean": 17.22072982788086, "margin_dpo/margin_std": 18.515459060668945, "step": 165 }, { "KL/chosen_KL_mean": -12.25210952758789, "KL/mean": -20.740781784057617, "KL/rejected_KL_mean": -29.229454040527344, "KL/std": 17.86658477783203, "epoch": 0.24375917767988253, "fcm_dpo/beta": 0.028545517474412918, "fcm_dpo/delta": -0.08894811570644379, "fcm_dpo/margin": 16.97733497619629, "fcm_dpo/q_t": 0.3932517170906067, "grad_norm": 10.706637382507324, "learning_rate": 4.702530485714461e-07, "logits/chosen": -1.5442438125610352, "logits/rejected": -1.3437598943710327, "logps/chosen": -93.83818054199219, "logps/ref_chosen": -81.58606719970703, "logps/ref_rejected": -103.16517639160156, "logps/rejected": -132.39462280273438, "loss": 1.0388, "margin_dpo/margin_mean": 16.97733497619629, "margin_dpo/margin_std": 21.5998592376709, "step": 166 }, { "KL/chosen_KL_mean": -10.866371154785156, "KL/mean": -20.000240325927734, "KL/rejected_KL_mean": -29.134113311767578, "KL/std": 16.47540855407715, "epoch": 0.24522760646108663, "fcm_dpo/beta": 0.027795948088169098, "fcm_dpo/delta": -0.11465759575366974, "fcm_dpo/margin": 18.26773452758789, "fcm_dpo/q_t": 0.3816662132740021, "grad_norm": 9.923656463623047, "learning_rate": 4.6964301399001877e-07, "logits/chosen": -1.185351848602295, "logits/rejected": -1.0142467021942139, "logps/chosen": -94.42803955078125, "logps/ref_chosen": -83.56167602539062, "logps/ref_rejected": -99.05305480957031, "logps/rejected": -128.18716430664062, "loss": 0.9928, "margin_dpo/margin_mean": 18.26773452758789, "margin_dpo/margin_std": 16.833255767822266, "step": 167 }, { "KL/chosen_KL_mean": -10.428199768066406, "KL/mean": -18.22240447998047, "KL/rejected_KL_mean": -26.01660919189453, "KL/std": 16.315227508544922, "epoch": 0.24669603524229075, "fcm_dpo/beta": 0.027679800987243652, "fcm_dpo/delta": -0.03289950639009476, "fcm_dpo/margin": 15.588400840759277, "fcm_dpo/q_t": 0.39915919303894043, "grad_norm": 10.774555206298828, "learning_rate": 4.690271916109034e-07, "logits/chosen": -1.5796266794204712, "logits/rejected": -1.2353956699371338, "logps/chosen": -90.35943603515625, "logps/ref_chosen": -79.93123626708984, "logps/ref_rejected": -80.10216522216797, "logps/rejected": -106.1187744140625, "loss": 1.0489, "margin_dpo/margin_mean": 15.588400840759277, "margin_dpo/margin_std": 16.35304832458496, "step": 168 }, { "KL/chosen_KL_mean": -9.043933868408203, "KL/mean": -18.00758171081543, "KL/rejected_KL_mean": -26.97122573852539, "KL/std": 17.62850570678711, "epoch": 0.24816446402349487, "fcm_dpo/beta": 0.0271998792886734, "fcm_dpo/delta": -0.09217746555805206, "fcm_dpo/margin": 17.927284240722656, "fcm_dpo/q_t": 0.3879433572292328, "grad_norm": 10.464804649353027, "learning_rate": 4.6840559766159235e-07, "logits/chosen": -1.6549845933914185, "logits/rejected": -1.4354002475738525, "logps/chosen": -91.94763946533203, "logps/ref_chosen": -82.90370178222656, "logps/ref_rejected": -90.12191009521484, "logps/rejected": -117.0931396484375, "loss": 1.0164, "margin_dpo/margin_mean": 17.92728614807129, "margin_dpo/margin_std": 18.866226196289062, "step": 169 }, { "KL/chosen_KL_mean": -10.63247299194336, "KL/mean": -18.46246337890625, "KL/rejected_KL_mean": -26.292449951171875, "KL/std": 14.761564254760742, "epoch": 0.24963289280469897, "fcm_dpo/beta": 0.026878267526626587, "fcm_dpo/delta": -0.022340044379234314, "fcm_dpo/margin": 15.659981727600098, "fcm_dpo/q_t": 0.40010008215904236, "grad_norm": 10.76467227935791, "learning_rate": 4.6777824852166437e-07, "logits/chosen": -1.5600578784942627, "logits/rejected": -1.3475031852722168, "logps/chosen": -85.61444091796875, "logps/ref_chosen": -74.98197174072266, "logps/ref_rejected": -82.16560363769531, "logps/rejected": -108.45805358886719, "loss": 1.0478, "margin_dpo/margin_mean": 15.659982681274414, "margin_dpo/margin_std": 14.7039794921875, "step": 170 }, { "KL/chosen_KL_mean": -13.5615234375, "KL/mean": -22.261917114257812, "KL/rejected_KL_mean": -30.96231460571289, "KL/std": 21.234651565551758, "epoch": 0.2511013215859031, "fcm_dpo/beta": 0.02671811357140541, "fcm_dpo/delta": -0.06803969293832779, "fcm_dpo/margin": 17.400794982910156, "fcm_dpo/q_t": 0.39949217438697815, "grad_norm": 12.291194915771484, "learning_rate": 4.6714516072235273e-07, "logits/chosen": -1.721679925918579, "logits/rejected": -1.1742870807647705, "logps/chosen": -132.54005432128906, "logps/ref_chosen": -118.97853088378906, "logps/ref_rejected": -115.76930236816406, "logps/rejected": -146.73162841796875, "loss": 1.0711, "margin_dpo/margin_mean": 17.400794982910156, "margin_dpo/margin_std": 25.43389320373535, "step": 171 }, { "KL/chosen_KL_mean": -14.527019500732422, "KL/mean": -22.23581314086914, "KL/rejected_KL_mean": -29.944602966308594, "KL/std": 15.254207611083984, "epoch": 0.2525697503671072, "fcm_dpo/beta": 0.026501990854740143, "fcm_dpo/delta": -0.009108584374189377, "fcm_dpo/margin": 15.417583465576172, "fcm_dpo/q_t": 0.4019724130630493, "grad_norm": 10.160428047180176, "learning_rate": 4.6650635094610966e-07, "logits/chosen": -2.3624086380004883, "logits/rejected": -1.6074013710021973, "logps/chosen": -115.6808090209961, "logps/ref_chosen": -101.15379333496094, "logps/ref_rejected": -91.00568389892578, "logps/rejected": -120.95028686523438, "loss": 1.046, "margin_dpo/margin_mean": 15.417583465576172, "margin_dpo/margin_std": 12.492011070251465, "step": 172 }, { "KL/chosen_KL_mean": -13.005489349365234, "KL/mean": -19.82510757446289, "KL/rejected_KL_mean": -26.64472198486328, "KL/std": 14.4640474319458, "epoch": 0.2540381791483113, "fcm_dpo/beta": 0.026738755404949188, "fcm_dpo/delta": 0.036543648689985275, "fcm_dpo/margin": 13.63923454284668, "fcm_dpo/q_t": 0.4138786494731903, "grad_norm": 9.789052963256836, "learning_rate": 4.6586183602616687e-07, "logits/chosen": -1.6370983123779297, "logits/rejected": -1.1626412868499756, "logps/chosen": -112.06217956542969, "logps/ref_chosen": -99.05668640136719, "logps/ref_rejected": -83.8507308959961, "logps/rejected": -110.49545288085938, "loss": 1.0944, "margin_dpo/margin_mean": 13.63923454284668, "margin_dpo/margin_std": 15.29142951965332, "step": 173 }, { "KL/chosen_KL_mean": -12.65925407409668, "KL/mean": -22.08783721923828, "KL/rejected_KL_mean": -31.51641845703125, "KL/std": 19.73651885986328, "epoch": 0.2555066079295154, "fcm_dpo/beta": 0.026379603892564774, "fcm_dpo/delta": -0.10244762897491455, "fcm_dpo/margin": 18.857162475585938, "fcm_dpo/q_t": 0.3849840462207794, "grad_norm": 11.376117706298828, "learning_rate": 4.652116329460919e-07, "logits/chosen": -1.8383495807647705, "logits/rejected": -1.7962310314178467, "logps/chosen": -93.06816101074219, "logps/ref_chosen": -80.40890502929688, "logps/ref_rejected": -108.20171356201172, "logps/rejected": -139.7181396484375, "loss": 1.0113, "margin_dpo/margin_mean": 18.857162475585938, "margin_dpo/margin_std": 19.820348739624023, "step": 174 }, { "KL/chosen_KL_mean": -16.196422576904297, "KL/mean": -25.205842971801758, "KL/rejected_KL_mean": -34.21526336669922, "KL/std": 17.551250457763672, "epoch": 0.25697503671071953, "fcm_dpo/beta": 0.02599494531750679, "fcm_dpo/delta": -0.07170958817005157, "fcm_dpo/margin": 18.018842697143555, "fcm_dpo/q_t": 0.38958460092544556, "grad_norm": 11.104044914245605, "learning_rate": 4.645557588393406e-07, "logits/chosen": -1.010321855545044, "logits/rejected": -0.7336099743843079, "logps/chosen": -98.91291809082031, "logps/ref_chosen": -82.71649169921875, "logps/ref_rejected": -96.66712951660156, "logps/rejected": -130.88238525390625, "loss": 1.0182, "margin_dpo/margin_mean": 18.018844604492188, "margin_dpo/margin_std": 16.789112091064453, "step": 175 }, { "KL/chosen_KL_mean": -13.433433532714844, "KL/mean": -22.715320587158203, "KL/rejected_KL_mean": -31.997203826904297, "KL/std": 19.08074951171875, "epoch": 0.25844346549192365, "fcm_dpo/beta": 0.025528497993946075, "fcm_dpo/delta": -0.07761147618293762, "fcm_dpo/margin": 18.563766479492188, "fcm_dpo/q_t": 0.39184433221817017, "grad_norm": 10.83478832244873, "learning_rate": 4.638942309888058e-07, "logits/chosen": -1.50587797164917, "logits/rejected": -1.3449994325637817, "logps/chosen": -93.06273651123047, "logps/ref_chosen": -79.62930297851562, "logps/ref_rejected": -102.6885986328125, "logps/rejected": -134.685791015625, "loss": 1.0394, "margin_dpo/margin_mean": 18.563766479492188, "margin_dpo/margin_std": 21.974594116210938, "step": 176 }, { "KL/chosen_KL_mean": -15.007614135742188, "KL/mean": -25.763689041137695, "KL/rejected_KL_mean": -36.5197639465332, "KL/std": 21.865760803222656, "epoch": 0.2599118942731278, "fcm_dpo/beta": 0.024997854605317116, "fcm_dpo/delta": -0.14533759653568268, "fcm_dpo/margin": 21.512149810791016, "fcm_dpo/q_t": 0.3792595863342285, "grad_norm": 9.867096900939941, "learning_rate": 4.6322706682636137e-07, "logits/chosen": -1.5718889236450195, "logits/rejected": -1.228896975517273, "logps/chosen": -107.56871032714844, "logps/ref_chosen": -92.56109619140625, "logps/ref_rejected": -103.33839416503906, "logps/rejected": -139.858154296875, "loss": 0.9895, "margin_dpo/margin_mean": 21.512149810791016, "margin_dpo/margin_std": 22.594806671142578, "step": 177 }, { "KL/chosen_KL_mean": -13.884201049804688, "KL/mean": -26.73963737487793, "KL/rejected_KL_mean": -39.595062255859375, "KL/std": 21.40091323852539, "epoch": 0.26138032305433184, "fcm_dpo/beta": 0.02386431209743023, "fcm_dpo/delta": -0.22777138650417328, "fcm_dpo/margin": 25.71085548400879, "fcm_dpo/q_t": 0.3595629036426544, "grad_norm": 10.852680206298828, "learning_rate": 4.6255428393240354e-07, "logits/chosen": -1.8012843132019043, "logits/rejected": -1.5873453617095947, "logps/chosen": -98.14999389648438, "logps/ref_chosen": -84.26579284667969, "logps/ref_rejected": -110.79730987548828, "logps/rejected": -150.39236450195312, "loss": 0.9296, "margin_dpo/margin_mean": 25.710853576660156, "margin_dpo/margin_std": 23.14470672607422, "step": 178 }, { "KL/chosen_KL_mean": -17.639938354492188, "KL/mean": -25.848388671875, "KL/rejected_KL_mean": -34.05683517456055, "KL/std": 20.54976463317871, "epoch": 0.26284875183553597, "fcm_dpo/beta": 0.023576252162456512, "fcm_dpo/delta": 0.01329508051276207, "fcm_dpo/margin": 16.416900634765625, "fcm_dpo/q_t": 0.41209518909454346, "grad_norm": 10.426217079162598, "learning_rate": 4.6187590003538724e-07, "logits/chosen": -1.7180531024932861, "logits/rejected": -1.2977184057235718, "logps/chosen": -111.0879135131836, "logps/ref_chosen": -93.4479751586914, "logps/ref_rejected": -94.25541687011719, "logps/rejected": -128.312255859375, "loss": 1.1112, "margin_dpo/margin_mean": 16.416898727416992, "margin_dpo/margin_std": 23.931289672851562, "step": 179 }, { "KL/chosen_KL_mean": -14.733882904052734, "KL/mean": -25.410295486450195, "KL/rejected_KL_mean": -36.086708068847656, "KL/std": 19.89807891845703, "epoch": 0.2643171806167401, "fcm_dpo/beta": 0.02324799634516239, "fcm_dpo/delta": -0.10189928114414215, "fcm_dpo/margin": 21.352828979492188, "fcm_dpo/q_t": 0.38459378480911255, "grad_norm": 12.427611351013184, "learning_rate": 4.611919330113591e-07, "logits/chosen": -1.5371224880218506, "logits/rejected": -1.3066015243530273, "logps/chosen": -103.56942749023438, "logps/ref_chosen": -88.83554077148438, "logps/ref_rejected": -103.90696716308594, "logps/rejected": -139.99368286132812, "loss": 1.0001, "margin_dpo/margin_mean": 21.352828979492188, "margin_dpo/margin_std": 20.412736892700195, "step": 180 }, { "KL/chosen_KL_mean": -14.737850189208984, "KL/mean": -22.852394104003906, "KL/rejected_KL_mean": -30.966934204101562, "KL/std": 18.975448608398438, "epoch": 0.2657856093979442, "fcm_dpo/beta": 0.023292385041713715, "fcm_dpo/delta": 0.022812366485595703, "fcm_dpo/margin": 16.229084014892578, "fcm_dpo/q_t": 0.4129781126976013, "grad_norm": 9.517890930175781, "learning_rate": 4.605024008834863e-07, "logits/chosen": -1.6671667098999023, "logits/rejected": -1.2102280855178833, "logps/chosen": -93.97181701660156, "logps/ref_chosen": -79.23396301269531, "logps/ref_rejected": -69.17442321777344, "logps/rejected": -100.141357421875, "loss": 1.0944, "margin_dpo/margin_mean": 16.229084014892578, "margin_dpo/margin_std": 20.26224136352539, "step": 181 }, { "KL/chosen_KL_mean": -18.547344207763672, "KL/mean": -29.14194107055664, "KL/rejected_KL_mean": -39.736534118652344, "KL/std": 21.912866592407227, "epoch": 0.26725403817914833, "fcm_dpo/beta": 0.02291329950094223, "fcm_dpo/delta": -0.09038232266902924, "fcm_dpo/margin": 21.189189910888672, "fcm_dpo/q_t": 0.3889124393463135, "grad_norm": 10.108099937438965, "learning_rate": 4.598073218215817e-07, "logits/chosen": -1.3922648429870605, "logits/rejected": -1.1766209602355957, "logps/chosen": -94.62953186035156, "logps/ref_chosen": -76.08218383789062, "logps/ref_rejected": -94.9686050415039, "logps/rejected": -134.70513916015625, "loss": 1.0278, "margin_dpo/margin_mean": 21.189189910888672, "margin_dpo/margin_std": 24.110326766967773, "step": 182 }, { "KL/chosen_KL_mean": -19.209930419921875, "KL/mean": -29.572113037109375, "KL/rejected_KL_mean": -39.934295654296875, "KL/std": 20.031349182128906, "epoch": 0.2687224669603524, "fcm_dpo/beta": 0.02255667932331562, "fcm_dpo/delta": -0.07131802290678024, "fcm_dpo/margin": 20.724355697631836, "fcm_dpo/q_t": 0.3906700611114502, "grad_norm": 10.490107536315918, "learning_rate": 4.5910671414162484e-07, "logits/chosen": -1.5747482776641846, "logits/rejected": -0.9683206081390381, "logps/chosen": -113.11783599853516, "logps/ref_chosen": -93.90790557861328, "logps/ref_rejected": -83.56061553955078, "logps/rejected": -123.49491119384766, "loss": 1.0237, "margin_dpo/margin_mean": 20.724355697631836, "margin_dpo/margin_std": 20.56180191040039, "step": 183 }, { "KL/chosen_KL_mean": -17.145397186279297, "KL/mean": -24.17635726928711, "KL/rejected_KL_mean": -31.207317352294922, "KL/std": 20.696651458740234, "epoch": 0.2701908957415565, "fcm_dpo/beta": 0.022848699241876602, "fcm_dpo/delta": 0.08124689012765884, "fcm_dpo/margin": 14.061912536621094, "fcm_dpo/q_t": 0.42483752965927124, "grad_norm": 9.723278045654297, "learning_rate": 4.5840059630527985e-07, "logits/chosen": -1.792701005935669, "logits/rejected": -1.5211716890335083, "logps/chosen": -104.0133056640625, "logps/ref_chosen": -86.86790466308594, "logps/ref_rejected": -84.28456115722656, "logps/rejected": -115.49187469482422, "loss": 1.1526, "margin_dpo/margin_mean": 14.06191349029541, "margin_dpo/margin_std": 22.382007598876953, "step": 184 }, { "KL/chosen_KL_mean": -18.211639404296875, "KL/mean": -26.628517150878906, "KL/rejected_KL_mean": -35.04539489746094, "KL/std": 20.271533966064453, "epoch": 0.27165932452276065, "fcm_dpo/beta": 0.023033609613776207, "fcm_dpo/delta": 0.012558795511722565, "fcm_dpo/margin": 16.833759307861328, "fcm_dpo/q_t": 0.41145211458206177, "grad_norm": 10.948394775390625, "learning_rate": 4.5768898691940836e-07, "logits/chosen": -2.1860392093658447, "logits/rejected": -1.571969747543335, "logps/chosen": -112.23306274414062, "logps/ref_chosen": -94.02142333984375, "logps/ref_rejected": -81.45668029785156, "logps/rejected": -116.5020751953125, "loss": 1.1032, "margin_dpo/margin_mean": 16.833759307861328, "margin_dpo/margin_std": 23.41136932373047, "step": 185 }, { "KL/chosen_KL_mean": -22.441036224365234, "KL/mean": -31.718746185302734, "KL/rejected_KL_mean": -40.996456146240234, "KL/std": 19.75735092163086, "epoch": 0.27312775330396477, "fcm_dpo/beta": 0.022826526314020157, "fcm_dpo/delta": -0.024883225560188293, "fcm_dpo/margin": 18.555416107177734, "fcm_dpo/q_t": 0.4012763798236847, "grad_norm": 11.483257293701172, "learning_rate": 4.5697190473557947e-07, "logits/chosen": -2.2207369804382324, "logits/rejected": -1.296262502670288, "logps/chosen": -136.63272094726562, "logps/ref_chosen": -114.19168090820312, "logps/ref_rejected": -93.33912658691406, "logps/rejected": -134.33558654785156, "loss": 1.0633, "margin_dpo/margin_mean": 18.555418014526367, "margin_dpo/margin_std": 21.36487579345703, "step": 186 }, { "KL/chosen_KL_mean": -19.209117889404297, "KL/mean": -28.090518951416016, "KL/rejected_KL_mean": -36.971920013427734, "KL/std": 22.092451095581055, "epoch": 0.2745961820851689, "fcm_dpo/beta": 0.02291002869606018, "fcm_dpo/delta": -0.007384965196251869, "fcm_dpo/margin": 17.76280975341797, "fcm_dpo/q_t": 0.4070413112640381, "grad_norm": 10.197686195373535, "learning_rate": 4.5624936864957555e-07, "logits/chosen": -1.8901567459106445, "logits/rejected": -1.6576085090637207, "logps/chosen": -96.07866668701172, "logps/ref_chosen": -76.86955261230469, "logps/ref_rejected": -88.16494750976562, "logps/rejected": -125.1368637084961, "loss": 1.0915, "margin_dpo/margin_mean": 17.762807846069336, "margin_dpo/margin_std": 24.168514251708984, "step": 187 }, { "KL/chosen_KL_mean": -14.895027160644531, "KL/mean": -24.94672966003418, "KL/rejected_KL_mean": -34.998435974121094, "KL/std": 21.68739128112793, "epoch": 0.27606461086637296, "fcm_dpo/beta": 0.02256125584244728, "fcm_dpo/delta": -0.05659133940935135, "fcm_dpo/margin": 20.103404998779297, "fcm_dpo/q_t": 0.3929678797721863, "grad_norm": 10.544901847839355, "learning_rate": 4.5552139770089454e-07, "logits/chosen": -2.15828800201416, "logits/rejected": -1.919528603553772, "logps/chosen": -94.08982849121094, "logps/ref_chosen": -79.1948013305664, "logps/ref_rejected": -96.82889556884766, "logps/rejected": -131.82733154296875, "loss": 1.0365, "margin_dpo/margin_mean": 20.103404998779297, "margin_dpo/margin_std": 20.449508666992188, "step": 188 }, { "KL/chosen_KL_mean": -19.69805145263672, "KL/mean": -30.020755767822266, "KL/rejected_KL_mean": -40.34346389770508, "KL/std": 21.775554656982422, "epoch": 0.2775330396475771, "fcm_dpo/beta": 0.022475745528936386, "fcm_dpo/delta": -0.06736817210912704, "fcm_dpo/margin": 20.645416259765625, "fcm_dpo/q_t": 0.39483535289764404, "grad_norm": 9.479969024658203, "learning_rate": 4.5478801107224794e-07, "logits/chosen": -1.806870698928833, "logits/rejected": -1.166749119758606, "logps/chosen": -103.59085083007812, "logps/ref_chosen": -83.8927993774414, "logps/ref_rejected": -79.46589660644531, "logps/rejected": -119.80935668945312, "loss": 1.0431, "margin_dpo/margin_mean": 20.645416259765625, "margin_dpo/margin_std": 24.183279037475586, "step": 189 }, { "KL/chosen_KL_mean": -15.783485412597656, "KL/mean": -28.141357421875, "KL/rejected_KL_mean": -40.499237060546875, "KL/std": 24.050844192504883, "epoch": 0.2790014684287812, "fcm_dpo/beta": 0.021788340061903, "fcm_dpo/delta": -0.14646783471107483, "fcm_dpo/margin": 24.71575355529785, "fcm_dpo/q_t": 0.3746356666088104, "grad_norm": 10.152409553527832, "learning_rate": 4.5404922808905543e-07, "logits/chosen": -1.8220771551132202, "logits/rejected": -1.2215911149978638, "logps/chosen": -119.2249526977539, "logps/ref_chosen": -103.44146728515625, "logps/ref_rejected": -109.21502685546875, "logps/rejected": -149.71426391601562, "loss": 0.9751, "margin_dpo/margin_mean": 24.71575355529785, "margin_dpo/margin_std": 22.809528350830078, "step": 190 }, { "KL/chosen_KL_mean": -15.049739837646484, "KL/mean": -29.086788177490234, "KL/rejected_KL_mean": -43.12383270263672, "KL/std": 26.152400970458984, "epoch": 0.28046989720998533, "fcm_dpo/beta": 0.021027402952313423, "fcm_dpo/delta": -0.20201078057289124, "fcm_dpo/margin": 28.074092864990234, "fcm_dpo/q_t": 0.3653144836425781, "grad_norm": 11.1878080368042, "learning_rate": 4.5330506821893565e-07, "logits/chosen": -1.4565821886062622, "logits/rejected": -1.1412721872329712, "logps/chosen": -114.83525085449219, "logps/ref_chosen": -99.78551483154297, "logps/ref_rejected": -116.00028228759766, "logps/rejected": -159.12411499023438, "loss": 0.9504, "margin_dpo/margin_mean": 28.074092864990234, "margin_dpo/margin_std": 26.740522384643555, "step": 191 }, { "KL/chosen_KL_mean": -24.489482879638672, "KL/mean": -34.255035400390625, "KL/rejected_KL_mean": -44.020572662353516, "KL/std": 20.901046752929688, "epoch": 0.28193832599118945, "fcm_dpo/beta": 0.02073926106095314, "fcm_dpo/delta": -0.005302004981786013, "fcm_dpo/margin": 19.53109359741211, "fcm_dpo/q_t": 0.40668633580207825, "grad_norm": 11.313491821289062, "learning_rate": 4.5255555107119336e-07, "logits/chosen": -1.6908667087554932, "logits/rejected": -1.4428727626800537, "logps/chosen": -125.34980773925781, "logps/ref_chosen": -100.86032104492188, "logps/ref_rejected": -109.44500732421875, "logps/rejected": -153.465576171875, "loss": 1.081, "margin_dpo/margin_mean": 19.53109359741211, "margin_dpo/margin_std": 24.421367645263672, "step": 192 }, { "KL/chosen_KL_mean": -20.182708740234375, "KL/mean": -28.216341018676758, "KL/rejected_KL_mean": -36.249977111816406, "KL/std": 22.246002197265625, "epoch": 0.2834067547723935, "fcm_dpo/beta": 0.020767875015735626, "fcm_dpo/delta": 0.006026466842740774, "fcm_dpo/margin": 16.067264556884766, "fcm_dpo/q_t": 0.42281368374824524, "grad_norm": 10.591568946838379, "learning_rate": 4.5180069639630236e-07, "logits/chosen": -2.098865270614624, "logits/rejected": -1.5169203281402588, "logps/chosen": -113.12102508544922, "logps/ref_chosen": -92.93831634521484, "logps/ref_rejected": -83.10917663574219, "logps/rejected": -119.35916137695312, "loss": 1.1554, "margin_dpo/margin_mean": 16.067264556884766, "margin_dpo/margin_std": 26.86675262451172, "step": 193 }, { "KL/chosen_KL_mean": -20.637863159179688, "KL/mean": -29.951847076416016, "KL/rejected_KL_mean": -39.26583480834961, "KL/std": 22.802433013916016, "epoch": 0.28487518355359764, "fcm_dpo/beta": 0.020880235359072685, "fcm_dpo/delta": 0.011257613077759743, "fcm_dpo/margin": 18.627979278564453, "fcm_dpo/q_t": 0.4104408025741577, "grad_norm": 9.059990882873535, "learning_rate": 4.510405240853854e-07, "logits/chosen": -2.1468214988708496, "logits/rejected": -1.750035285949707, "logps/chosen": -84.42943572998047, "logps/ref_chosen": -63.79157257080078, "logps/ref_rejected": -68.72665405273438, "logps/rejected": -107.99249267578125, "loss": 1.0945, "margin_dpo/margin_mean": 18.627979278564453, "margin_dpo/margin_std": 24.174999237060547, "step": 194 }, { "KL/chosen_KL_mean": -19.76596450805664, "KL/mean": -32.27701187133789, "KL/rejected_KL_mean": -44.78805160522461, "KL/std": 23.19590950012207, "epoch": 0.28634361233480177, "fcm_dpo/beta": 0.020515337586402893, "fcm_dpo/delta": -0.11929795145988464, "fcm_dpo/margin": 25.0220947265625, "fcm_dpo/q_t": 0.38101837038993835, "grad_norm": 9.40182113647461, "learning_rate": 4.5027505416968985e-07, "logits/chosen": -1.3792896270751953, "logits/rejected": -1.2623875141143799, "logps/chosen": -103.65142822265625, "logps/ref_chosen": -83.88545989990234, "logps/ref_rejected": -104.00125122070312, "logps/rejected": -148.789306640625, "loss": 1.0065, "margin_dpo/margin_mean": 25.022092819213867, "margin_dpo/margin_std": 26.55062484741211, "step": 195 }, { "KL/chosen_KL_mean": -23.175148010253906, "KL/mean": -34.83287811279297, "KL/rejected_KL_mean": -46.4906120300293, "KL/std": 24.843109130859375, "epoch": 0.2878120411160059, "fcm_dpo/beta": 0.020115695893764496, "fcm_dpo/delta": -0.07241245359182358, "fcm_dpo/margin": 23.31546401977539, "fcm_dpo/q_t": 0.38920408487319946, "grad_norm": 9.05499267578125, "learning_rate": 4.495043068200599e-07, "logits/chosen": -1.4280736446380615, "logits/rejected": -0.9839767217636108, "logps/chosen": -112.57650756835938, "logps/ref_chosen": -89.4013671875, "logps/ref_rejected": -83.34735107421875, "logps/rejected": -129.83795166015625, "loss": 1.0194, "margin_dpo/margin_mean": 23.31546401977539, "margin_dpo/margin_std": 22.447128295898438, "step": 196 }, { "KL/chosen_KL_mean": -23.49209976196289, "KL/mean": -30.68783187866211, "KL/rejected_KL_mean": -37.8835563659668, "KL/std": 20.021530151367188, "epoch": 0.28928046989721, "fcm_dpo/beta": 0.020043786615133286, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 14.391450881958008, "fcm_dpo/q_t": 0.4311813712120056, "grad_norm": 9.273676872253418, "learning_rate": 4.4872830234640493e-07, "logits/chosen": -1.269303798675537, "logits/rejected": -0.9314581751823425, "logps/chosen": -110.33280944824219, "logps/ref_chosen": -86.84070587158203, "logps/ref_rejected": -82.79289245605469, "logps/rejected": -120.67645263671875, "loss": 1.1617, "margin_dpo/margin_mean": 14.391451835632324, "margin_dpo/margin_std": 21.266876220703125, "step": 197 }, { "KL/chosen_KL_mean": -27.17208480834961, "KL/mean": -38.82708740234375, "KL/rejected_KL_mean": -50.48210144042969, "KL/std": 23.719057083129883, "epoch": 0.2907488986784141, "fcm_dpo/beta": 0.019858654588460922, "fcm_dpo/delta": -0.06592979282140732, "fcm_dpo/margin": 23.310012817382812, "fcm_dpo/q_t": 0.39266514778137207, "grad_norm": 10.997335433959961, "learning_rate": 4.479470611971645e-07, "logits/chosen": -1.7973957061767578, "logits/rejected": -1.6265045404434204, "logps/chosen": -112.22032928466797, "logps/ref_chosen": -85.04824829101562, "logps/ref_rejected": -104.10365295410156, "logps/rejected": -154.58575439453125, "loss": 1.03, "margin_dpo/margin_mean": 23.31001091003418, "margin_dpo/margin_std": 24.526268005371094, "step": 198 }, { "KL/chosen_KL_mean": -26.518173217773438, "KL/mean": -36.141231536865234, "KL/rejected_KL_mean": -45.764286041259766, "KL/std": 23.040512084960938, "epoch": 0.2922173274596182, "fcm_dpo/beta": 0.019856173545122147, "fcm_dpo/delta": 0.018514636904001236, "fcm_dpo/margin": 19.246116638183594, "fcm_dpo/q_t": 0.41099148988723755, "grad_norm": 9.613306045532227, "learning_rate": 4.471606039587695e-07, "logits/chosen": -1.7797781229019165, "logits/rejected": -1.525411605834961, "logps/chosen": -108.30487060546875, "logps/ref_chosen": -81.78669738769531, "logps/ref_rejected": -90.790771484375, "logps/rejected": -136.5550537109375, "loss": 1.0949, "margin_dpo/margin_mean": 19.246116638183594, "margin_dpo/margin_std": 24.207881927490234, "step": 199 }, { "KL/chosen_KL_mean": -25.89590835571289, "KL/mean": -37.32862854003906, "KL/rejected_KL_mean": -48.76133346557617, "KL/std": 24.73147964477539, "epoch": 0.2936857562408223, "fcm_dpo/beta": 0.019621796905994415, "fcm_dpo/delta": -0.051508162170648575, "fcm_dpo/margin": 22.86542510986328, "fcm_dpo/q_t": 0.3969246447086334, "grad_norm": 9.600115776062012, "learning_rate": 4.4636895135509966e-07, "logits/chosen": -1.6544878482818604, "logits/rejected": -1.111642599105835, "logps/chosen": -110.16400146484375, "logps/ref_chosen": -84.26809692382812, "logps/ref_rejected": -86.96786499023438, "logps/rejected": -135.7292022705078, "loss": 1.0611, "margin_dpo/margin_mean": 22.86542510986328, "margin_dpo/margin_std": 28.457225799560547, "step": 200 }, { "KL/chosen_KL_mean": -27.968212127685547, "KL/mean": -39.04167175292969, "KL/rejected_KL_mean": -50.11512756347656, "KL/std": 26.870845794677734, "epoch": 0.29515418502202645, "fcm_dpo/beta": 0.01959862932562828, "fcm_dpo/delta": -0.03564136475324631, "fcm_dpo/margin": 22.14691925048828, "fcm_dpo/q_t": 0.4006343185901642, "grad_norm": 9.832130432128906, "learning_rate": 4.455721242469372e-07, "logits/chosen": -1.5167807340621948, "logits/rejected": -1.1362334489822388, "logps/chosen": -135.74070739746094, "logps/ref_chosen": -107.77249145507812, "logps/ref_rejected": -119.79248046875, "logps/rejected": -169.90760803222656, "loss": 1.075, "margin_dpo/margin_mean": 22.14691925048828, "margin_dpo/margin_std": 29.343774795532227, "step": 201 }, { "KL/chosen_KL_mean": -26.233989715576172, "KL/mean": -36.52886962890625, "KL/rejected_KL_mean": -46.82374572753906, "KL/std": 22.212574005126953, "epoch": 0.2966226138032305, "fcm_dpo/beta": 0.019527671858668327, "fcm_dpo/delta": -0.0021812673658132553, "fcm_dpo/margin": 20.589757919311523, "fcm_dpo/q_t": 0.40716424584388733, "grad_norm": 10.090387344360352, "learning_rate": 4.4477014363141755e-07, "logits/chosen": -1.749312162399292, "logits/rejected": -1.6240307092666626, "logps/chosen": -102.2064437866211, "logps/ref_chosen": -75.97245788574219, "logps/ref_rejected": -94.4599838256836, "logps/rejected": -141.28372192382812, "loss": 1.0825, "margin_dpo/margin_mean": 20.58975601196289, "margin_dpo/margin_std": 25.11485481262207, "step": 202 }, { "KL/chosen_KL_mean": -23.40564727783203, "KL/mean": -35.04082489013672, "KL/rejected_KL_mean": -46.67601013183594, "KL/std": 23.13131332397461, "epoch": 0.29809104258443464, "fcm_dpo/beta": 0.019376806914806366, "fcm_dpo/delta": -0.05327478423714638, "fcm_dpo/margin": 23.270366668701172, "fcm_dpo/q_t": 0.39427393674850464, "grad_norm": 9.077618598937988, "learning_rate": 4.439630306414758e-07, "logits/chosen": -1.965078353881836, "logits/rejected": -1.5162969827651978, "logps/chosen": -118.372802734375, "logps/ref_chosen": -94.96715545654297, "logps/ref_rejected": -92.8876724243164, "logps/rejected": -139.56369018554688, "loss": 1.0361, "margin_dpo/margin_mean": 23.270366668701172, "margin_dpo/margin_std": 23.98265838623047, "step": 203 }, { "KL/chosen_KL_mean": -29.340232849121094, "KL/mean": -39.35368347167969, "KL/rejected_KL_mean": -49.36712646484375, "KL/std": 23.16506576538086, "epoch": 0.29955947136563876, "fcm_dpo/beta": 0.019382819533348083, "fcm_dpo/delta": 0.0121227465569973, "fcm_dpo/margin": 20.026893615722656, "fcm_dpo/q_t": 0.4106913208961487, "grad_norm": 9.94970417022705, "learning_rate": 4.431508065452897e-07, "logits/chosen": -1.9286906719207764, "logits/rejected": -1.274007797241211, "logps/chosen": -145.69741821289062, "logps/ref_chosen": -116.35719299316406, "logps/ref_rejected": -93.39759063720703, "logps/rejected": -142.76470947265625, "loss": 1.1013, "margin_dpo/margin_mean": 20.026893615722656, "margin_dpo/margin_std": 26.898170471191406, "step": 204 }, { "KL/chosen_KL_mean": -27.196041107177734, "KL/mean": -40.15739440917969, "KL/rejected_KL_mean": -53.118751525878906, "KL/std": 25.728809356689453, "epoch": 0.3010279001468429, "fcm_dpo/beta": 0.019023960456252098, "fcm_dpo/delta": -0.09826859086751938, "fcm_dpo/margin": 25.922714233398438, "fcm_dpo/q_t": 0.3828265070915222, "grad_norm": 8.933135032653809, "learning_rate": 4.4233349274571974e-07, "logits/chosen": -2.452038288116455, "logits/rejected": -2.041184425354004, "logps/chosen": -116.05538940429688, "logps/ref_chosen": -88.85934448242188, "logps/ref_rejected": -91.8544921875, "logps/rejected": -144.97323608398438, "loss": 0.9938, "margin_dpo/margin_mean": 25.922714233398438, "margin_dpo/margin_std": 21.94232940673828, "step": 205 }, { "KL/chosen_KL_mean": -27.82909393310547, "KL/mean": -39.71849060058594, "KL/rejected_KL_mean": -51.60789108276367, "KL/std": 24.90050506591797, "epoch": 0.302496328928047, "fcm_dpo/beta": 0.01876676455140114, "fcm_dpo/delta": -0.048801764845848083, "fcm_dpo/margin": 23.77880096435547, "fcm_dpo/q_t": 0.3938303589820862, "grad_norm": 10.346532821655273, "learning_rate": 4.415111107797445e-07, "logits/chosen": -1.6061272621154785, "logits/rejected": -1.459729552268982, "logps/chosen": -104.37544250488281, "logps/ref_chosen": -76.54634857177734, "logps/ref_rejected": -102.95314025878906, "logps/rejected": -154.56103515625, "loss": 1.0278, "margin_dpo/margin_mean": 23.77880096435547, "margin_dpo/margin_std": 21.266902923583984, "step": 206 }, { "KL/chosen_KL_mean": -29.162586212158203, "KL/mean": -42.509910583496094, "KL/rejected_KL_mean": -55.85722732543945, "KL/std": 30.608707427978516, "epoch": 0.3039647577092511, "fcm_dpo/beta": 0.018516261130571365, "fcm_dpo/delta": -0.09914683550596237, "fcm_dpo/margin": 26.694644927978516, "fcm_dpo/q_t": 0.3901873230934143, "grad_norm": 10.37959098815918, "learning_rate": 4.4068368231789365e-07, "logits/chosen": -1.9148609638214111, "logits/rejected": -1.4796404838562012, "logps/chosen": -115.39423370361328, "logps/ref_chosen": -86.23164367675781, "logps/ref_rejected": -90.65512084960938, "logps/rejected": -146.51234436035156, "loss": 1.0456, "margin_dpo/margin_mean": 26.69464683532715, "margin_dpo/margin_std": 35.2462272644043, "step": 207 }, { "KL/chosen_KL_mean": -30.22491455078125, "KL/mean": -43.85370635986328, "KL/rejected_KL_mean": -57.48249816894531, "KL/std": 27.858169555664062, "epoch": 0.3054331864904552, "fcm_dpo/beta": 0.018213478848338127, "fcm_dpo/delta": -0.1014234870672226, "fcm_dpo/margin": 27.25757598876953, "fcm_dpo/q_t": 0.38294517993927, "grad_norm": 10.183534622192383, "learning_rate": 4.398512291636768e-07, "logits/chosen": -1.8578312397003174, "logits/rejected": -1.5064001083374023, "logps/chosen": -124.38446044921875, "logps/ref_chosen": -94.1595458984375, "logps/ref_rejected": -100.96233367919922, "logps/rejected": -158.44482421875, "loss": 1.0027, "margin_dpo/margin_mean": 27.257577896118164, "margin_dpo/margin_std": 25.58877944946289, "step": 208 }, { "KL/chosen_KL_mean": -25.645763397216797, "KL/mean": -37.72901916503906, "KL/rejected_KL_mean": -49.8122673034668, "KL/std": 24.893207550048828, "epoch": 0.3069016152716593, "fcm_dpo/beta": 0.01788686215877533, "fcm_dpo/delta": -0.034120529890060425, "fcm_dpo/margin": 24.16650390625, "fcm_dpo/q_t": 0.3998543918132782, "grad_norm": 9.04115104675293, "learning_rate": 4.3901377325300857e-07, "logits/chosen": -2.3702800273895264, "logits/rejected": -1.9165239334106445, "logps/chosen": -109.81632995605469, "logps/ref_chosen": -84.17056274414062, "logps/ref_rejected": -87.61955261230469, "logps/rejected": -137.43182373046875, "loss": 1.0607, "margin_dpo/margin_mean": 24.16650390625, "margin_dpo/margin_std": 28.215757369995117, "step": 209 }, { "KL/chosen_KL_mean": -29.096385955810547, "KL/mean": -40.607025146484375, "KL/rejected_KL_mean": -52.1176643371582, "KL/std": 26.404951095581055, "epoch": 0.30837004405286345, "fcm_dpo/beta": 0.01786329597234726, "fcm_dpo/delta": -0.011780261993408203, "fcm_dpo/margin": 23.02127456665039, "fcm_dpo/q_t": 0.4044468402862549, "grad_norm": 8.486321449279785, "learning_rate": 4.381713366536311e-07, "logits/chosen": -1.9202561378479004, "logits/rejected": -1.53126060962677, "logps/chosen": -110.26756286621094, "logps/ref_chosen": -81.17117309570312, "logps/ref_rejected": -84.17478942871094, "logps/rejected": -136.29244995117188, "loss": 1.0696, "margin_dpo/margin_mean": 23.02127456665039, "margin_dpo/margin_std": 26.58807373046875, "step": 210 }, { "KL/chosen_KL_mean": -32.55935287475586, "KL/mean": -43.85276794433594, "KL/rejected_KL_mean": -55.146183013916016, "KL/std": 27.905582427978516, "epoch": 0.30983847283406757, "fcm_dpo/beta": 0.01790366694331169, "fcm_dpo/delta": -0.004649447277188301, "fcm_dpo/margin": 22.586833953857422, "fcm_dpo/q_t": 0.4066464304924011, "grad_norm": 10.353544235229492, "learning_rate": 4.373239415645323e-07, "logits/chosen": -2.105949640274048, "logits/rejected": -1.484708547592163, "logps/chosen": -141.27206420898438, "logps/ref_chosen": -108.71270751953125, "logps/ref_rejected": -93.55564880371094, "logps/rejected": -148.70184326171875, "loss": 1.081, "margin_dpo/margin_mean": 22.58683204650879, "margin_dpo/margin_std": 28.160480499267578, "step": 211 }, { "KL/chosen_KL_mean": -22.877002716064453, "KL/mean": -39.357173919677734, "KL/rejected_KL_mean": -55.83734130859375, "KL/std": 31.732728958129883, "epoch": 0.31130690161527164, "fcm_dpo/beta": 0.01734536699950695, "fcm_dpo/delta": -0.1826426386833191, "fcm_dpo/margin": 32.9603385925293, "fcm_dpo/q_t": 0.3686683177947998, "grad_norm": 9.021686553955078, "learning_rate": 4.3647161031536086e-07, "logits/chosen": -1.8028687238693237, "logits/rejected": -1.3849356174468994, "logps/chosen": -121.23895263671875, "logps/ref_chosen": -98.36195373535156, "logps/ref_rejected": -109.88999938964844, "logps/rejected": -165.7273406982422, "loss": 0.9615, "margin_dpo/margin_mean": 32.96033477783203, "margin_dpo/margin_std": 31.690784454345703, "step": 212 }, { "KL/chosen_KL_mean": -26.760753631591797, "KL/mean": -39.19908905029297, "KL/rejected_KL_mean": -51.637420654296875, "KL/std": 28.971054077148438, "epoch": 0.31277533039647576, "fcm_dpo/beta": 0.01711571030318737, "fcm_dpo/delta": -0.027080245316028595, "fcm_dpo/margin": 24.876659393310547, "fcm_dpo/q_t": 0.4012739062309265, "grad_norm": 11.205018043518066, "learning_rate": 4.3561436536583774e-07, "logits/chosen": -1.945738434791565, "logits/rejected": -1.1565263271331787, "logps/chosen": -134.81607055664062, "logps/ref_chosen": -108.05531311035156, "logps/ref_rejected": -100.14414978027344, "logps/rejected": -151.7815704345703, "loss": 1.0737, "margin_dpo/margin_mean": 24.87665557861328, "margin_dpo/margin_std": 31.63521957397461, "step": 213 }, { "KL/chosen_KL_mean": -28.489566802978516, "KL/mean": -40.16156005859375, "KL/rejected_KL_mean": -51.83355712890625, "KL/std": 30.93045425415039, "epoch": 0.3142437591776799, "fcm_dpo/beta": 0.01714509166777134, "fcm_dpo/delta": -0.0002644965425133705, "fcm_dpo/margin": 23.34398651123047, "fcm_dpo/q_t": 0.40931522846221924, "grad_norm": 8.833428382873535, "learning_rate": 4.3475222930516473e-07, "logits/chosen": -2.026113748550415, "logits/rejected": -1.9088587760925293, "logps/chosen": -106.29430389404297, "logps/ref_chosen": -77.80473327636719, "logps/ref_rejected": -84.27578735351562, "logps/rejected": -136.10934448242188, "loss": 1.1037, "margin_dpo/margin_mean": 23.3439884185791, "margin_dpo/margin_std": 33.94078063964844, "step": 214 }, { "KL/chosen_KL_mean": -24.645503997802734, "KL/mean": -40.430931091308594, "KL/rejected_KL_mean": -56.21635055541992, "KL/std": 30.444000244140625, "epoch": 0.315712187958884, "fcm_dpo/beta": 0.016785655170679092, "fcm_dpo/delta": -0.13718369603157043, "fcm_dpo/margin": 31.570846557617188, "fcm_dpo/q_t": 0.37727463245391846, "grad_norm": 8.714178085327148, "learning_rate": 4.3388522485142885e-07, "logits/chosen": -1.6730263233184814, "logits/rejected": -1.193305492401123, "logps/chosen": -109.7593994140625, "logps/ref_chosen": -85.1138916015625, "logps/ref_rejected": -96.86151123046875, "logps/rejected": -153.07786560058594, "loss": 0.9818, "margin_dpo/margin_mean": 31.570846557617188, "margin_dpo/margin_std": 29.977745056152344, "step": 215 }, { "KL/chosen_KL_mean": -26.72441864013672, "KL/mean": -40.45320129394531, "KL/rejected_KL_mean": -54.18199157714844, "KL/std": 27.406570434570312, "epoch": 0.31718061674008813, "fcm_dpo/beta": 0.016590487211942673, "fcm_dpo/delta": -0.05830288678407669, "fcm_dpo/margin": 27.45757293701172, "fcm_dpo/q_t": 0.39406317472457886, "grad_norm": 8.250274658203125, "learning_rate": 4.330133748510036e-07, "logits/chosen": -1.9427597522735596, "logits/rejected": -1.5463614463806152, "logps/chosen": -107.31672668457031, "logps/ref_chosen": -80.59230041503906, "logps/ref_rejected": -81.41983795166016, "logps/rejected": -135.60183715820312, "loss": 1.0346, "margin_dpo/margin_mean": 27.45757293701172, "margin_dpo/margin_std": 28.32803726196289, "step": 216 }, { "KL/chosen_KL_mean": -32.52296829223633, "KL/mean": -43.812591552734375, "KL/rejected_KL_mean": -55.10221481323242, "KL/std": 28.377426147460938, "epoch": 0.3186490455212922, "fcm_dpo/beta": 0.016561094671487808, "fcm_dpo/delta": 0.026995552703738213, "fcm_dpo/margin": 22.579242706298828, "fcm_dpo/q_t": 0.4147772490978241, "grad_norm": 10.1908597946167, "learning_rate": 4.3213670227794757e-07, "logits/chosen": -2.124011516571045, "logits/rejected": -1.8209376335144043, "logps/chosen": -125.99555206298828, "logps/ref_chosen": -93.47257995605469, "logps/ref_rejected": -103.488525390625, "logps/rejected": -158.59072875976562, "loss": 1.1129, "margin_dpo/margin_mean": 22.579242706298828, "margin_dpo/margin_std": 32.31659698486328, "step": 217 }, { "KL/chosen_KL_mean": -32.00503921508789, "KL/mean": -45.61492919921875, "KL/rejected_KL_mean": -59.224830627441406, "KL/std": 28.565357208251953, "epoch": 0.3201174743024963, "fcm_dpo/beta": 0.01639879308640957, "fcm_dpo/delta": -0.04879341274499893, "fcm_dpo/margin": 27.21978759765625, "fcm_dpo/q_t": 0.39750951528549194, "grad_norm": 8.804474830627441, "learning_rate": 4.3125523023339815e-07, "logits/chosen": -1.317617416381836, "logits/rejected": -0.9291328191757202, "logps/chosen": -121.06387329101562, "logps/ref_chosen": -89.05883026123047, "logps/ref_rejected": -94.30680847167969, "logps/rejected": -153.53164672851562, "loss": 1.0623, "margin_dpo/margin_mean": 27.21978759765625, "margin_dpo/margin_std": 34.204444885253906, "step": 218 }, { "KL/chosen_KL_mean": -28.777660369873047, "KL/mean": -40.968997955322266, "KL/rejected_KL_mean": -53.16034698486328, "KL/std": 33.21924591064453, "epoch": 0.32158590308370044, "fcm_dpo/beta": 0.01631225273013115, "fcm_dpo/delta": -0.053448427468538284, "fcm_dpo/margin": 24.382694244384766, "fcm_dpo/q_t": 0.4085707664489746, "grad_norm": 9.802894592285156, "learning_rate": 4.303689819449636e-07, "logits/chosen": -1.6217575073242188, "logits/rejected": -1.154639720916748, "logps/chosen": -129.78500366210938, "logps/ref_chosen": -101.00733947753906, "logps/ref_rejected": -92.46794128417969, "logps/rejected": -145.6282958984375, "loss": 1.1063, "margin_dpo/margin_mean": 24.382692337036133, "margin_dpo/margin_std": 35.364593505859375, "step": 219 }, { "KL/chosen_KL_mean": -29.587989807128906, "KL/mean": -41.061065673828125, "KL/rejected_KL_mean": -52.534149169921875, "KL/std": 27.626510620117188, "epoch": 0.32305433186490456, "fcm_dpo/beta": 0.016299601644277573, "fcm_dpo/delta": 0.02697490155696869, "fcm_dpo/margin": 22.946155548095703, "fcm_dpo/q_t": 0.41226431727409363, "grad_norm": 10.13264274597168, "learning_rate": 4.2947798076611047e-07, "logits/chosen": -1.6190385818481445, "logits/rejected": -1.276505708694458, "logps/chosen": -125.12520599365234, "logps/ref_chosen": -95.53721618652344, "logps/ref_rejected": -94.30703735351562, "logps/rejected": -146.8411865234375, "loss": 1.0947, "margin_dpo/margin_mean": 22.946155548095703, "margin_dpo/margin_std": 27.265180587768555, "step": 220 }, { "KL/chosen_KL_mean": -27.93502426147461, "KL/mean": -45.54465866088867, "KL/rejected_KL_mean": -63.154293060302734, "KL/std": 34.48823165893555, "epoch": 0.3245227606461087, "fcm_dpo/beta": 0.016002081334590912, "fcm_dpo/delta": -0.17308306694030762, "fcm_dpo/margin": 35.219268798828125, "fcm_dpo/q_t": 0.3696337938308716, "grad_norm": 9.928122520446777, "learning_rate": 4.285822501755485e-07, "logits/chosen": -1.3340513706207275, "logits/rejected": -1.1993815898895264, "logps/chosen": -110.77989196777344, "logps/ref_chosen": -82.84486389160156, "logps/ref_rejected": -110.81179809570312, "logps/rejected": -173.96609497070312, "loss": 0.9595, "margin_dpo/margin_mean": 35.219268798828125, "margin_dpo/margin_std": 30.34347915649414, "step": 221 }, { "KL/chosen_KL_mean": -35.79014587402344, "KL/mean": -50.069541931152344, "KL/rejected_KL_mean": -64.34893798828125, "KL/std": 33.09998321533203, "epoch": 0.32599118942731276, "fcm_dpo/beta": 0.015705151483416557, "fcm_dpo/delta": -0.050947126001119614, "fcm_dpo/margin": 28.558786392211914, "fcm_dpo/q_t": 0.3975737690925598, "grad_norm": 9.994668006896973, "learning_rate": 4.276818137766118e-07, "logits/chosen": -2.027966022491455, "logits/rejected": -1.6006577014923096, "logps/chosen": -130.93212890625, "logps/ref_chosen": -95.14198303222656, "logps/ref_rejected": -106.80441284179688, "logps/rejected": -171.15335083007812, "loss": 1.075, "margin_dpo/margin_mean": 28.558786392211914, "margin_dpo/margin_std": 38.62836456298828, "step": 222 }, { "KL/chosen_KL_mean": -35.09429168701172, "KL/mean": -45.19826889038086, "KL/rejected_KL_mean": -55.302249908447266, "KL/std": 29.226844787597656, "epoch": 0.3274596182085169, "fcm_dpo/beta": 0.015683729201555252, "fcm_dpo/delta": 0.027387013658881187, "fcm_dpo/margin": 20.20795440673828, "fcm_dpo/q_t": 0.42626410722732544, "grad_norm": 9.692822456359863, "learning_rate": 4.2677669529663686e-07, "logits/chosen": -2.1440961360931396, "logits/rejected": -1.7515130043029785, "logps/chosen": -120.66940307617188, "logps/ref_chosen": -85.57511138916016, "logps/ref_rejected": -86.45237731933594, "logps/rejected": -141.754638671875, "loss": 1.1548, "margin_dpo/margin_mean": 20.20795440673828, "margin_dpo/margin_std": 32.085304260253906, "step": 223 }, { "KL/chosen_KL_mean": -32.0039176940918, "KL/mean": -46.522911071777344, "KL/rejected_KL_mean": -61.041908264160156, "KL/std": 32.32599639892578, "epoch": 0.328928046989721, "fcm_dpo/beta": 0.015548836439847946, "fcm_dpo/delta": -0.05398253723978996, "fcm_dpo/margin": 29.037982940673828, "fcm_dpo/q_t": 0.39580273628234863, "grad_norm": 9.209607124328613, "learning_rate": 4.2586691858633747e-07, "logits/chosen": -1.727626085281372, "logits/rejected": -1.264815330505371, "logps/chosen": -114.72772216796875, "logps/ref_chosen": -82.72380065917969, "logps/ref_rejected": -82.59538269042969, "logps/rejected": -143.63729858398438, "loss": 1.049, "margin_dpo/margin_mean": 29.037982940673828, "margin_dpo/margin_std": 33.81342315673828, "step": 224 }, { "KL/chosen_KL_mean": -35.956302642822266, "KL/mean": -50.97644805908203, "KL/rejected_KL_mean": -65.99658966064453, "KL/std": 33.929115295410156, "epoch": 0.3303964757709251, "fcm_dpo/beta": 0.015419520437717438, "fcm_dpo/delta": -0.06630764901638031, "fcm_dpo/margin": 30.0402889251709, "fcm_dpo/q_t": 0.39472413063049316, "grad_norm": 10.009520530700684, "learning_rate": 4.249525076191759e-07, "logits/chosen": -2.102365016937256, "logits/rejected": -1.6344006061553955, "logps/chosen": -131.6339874267578, "logps/ref_chosen": -95.67768859863281, "logps/ref_rejected": -105.09687805175781, "logps/rejected": -171.09347534179688, "loss": 1.0543, "margin_dpo/margin_mean": 30.0402889251709, "margin_dpo/margin_std": 38.083045959472656, "step": 225 }, { "KL/chosen_KL_mean": -29.665660858154297, "KL/mean": -44.15570831298828, "KL/rejected_KL_mean": -58.64575958251953, "KL/std": 33.27996826171875, "epoch": 0.33186490455212925, "fcm_dpo/beta": 0.015142346732318401, "fcm_dpo/delta": -0.04130810871720314, "fcm_dpo/margin": 28.98008918762207, "fcm_dpo/q_t": 0.3973340094089508, "grad_norm": 8.312088012695312, "learning_rate": 4.2403348649073167e-07, "logits/chosen": -1.6455552577972412, "logits/rejected": -1.0894885063171387, "logps/chosen": -123.1265869140625, "logps/ref_chosen": -93.46092224121094, "logps/ref_rejected": -86.7017593383789, "logps/rejected": -145.34751892089844, "loss": 1.0442, "margin_dpo/margin_mean": 28.98008918762207, "margin_dpo/margin_std": 29.74630355834961, "step": 226 }, { "KL/chosen_KL_mean": -40.30400466918945, "KL/mean": -55.16033935546875, "KL/rejected_KL_mean": -70.01666259765625, "KL/std": 32.16132354736328, "epoch": 0.3333333333333333, "fcm_dpo/beta": 0.015080028213560581, "fcm_dpo/delta": -0.05030110478401184, "fcm_dpo/margin": 29.712661743164062, "fcm_dpo/q_t": 0.3975616693496704, "grad_norm": 9.174317359924316, "learning_rate": 4.2310987941806615e-07, "logits/chosen": -1.498534917831421, "logits/rejected": -1.1503180265426636, "logps/chosen": -133.11827087402344, "logps/ref_chosen": -92.81427001953125, "logps/ref_rejected": -104.73693084716797, "logps/rejected": -174.75360107421875, "loss": 1.065, "margin_dpo/margin_mean": 29.712661743164062, "margin_dpo/margin_std": 38.15606689453125, "step": 227 }, { "KL/chosen_KL_mean": -37.99431610107422, "KL/mean": -49.339935302734375, "KL/rejected_KL_mean": -60.6855583190918, "KL/std": 30.187368392944336, "epoch": 0.33480176211453744, "fcm_dpo/beta": 0.01515631377696991, "fcm_dpo/delta": 0.058105990290641785, "fcm_dpo/margin": 22.69123077392578, "fcm_dpo/q_t": 0.4199024438858032, "grad_norm": 9.403865814208984, "learning_rate": 4.2218171073908463e-07, "logits/chosen": -1.9922826290130615, "logits/rejected": -1.5676965713500977, "logps/chosen": -132.03143310546875, "logps/ref_chosen": -94.03712463378906, "logps/ref_rejected": -96.02151489257812, "logps/rejected": -156.7070770263672, "loss": 1.1353, "margin_dpo/margin_mean": 22.691232681274414, "margin_dpo/margin_std": 34.02064514160156, "step": 228 }, { "KL/chosen_KL_mean": -33.82004928588867, "KL/mean": -52.36869812011719, "KL/rejected_KL_mean": -70.9173583984375, "KL/std": 36.59883499145508, "epoch": 0.33627019089574156, "fcm_dpo/beta": 0.014859197661280632, "fcm_dpo/delta": -0.1599859595298767, "fcm_dpo/margin": 37.09730529785156, "fcm_dpo/q_t": 0.37526413798332214, "grad_norm": 9.268818855285645, "learning_rate": 4.212490049118951e-07, "logits/chosen": -2.0356123447418213, "logits/rejected": -1.42457115650177, "logps/chosen": -129.39772033691406, "logps/ref_chosen": -95.57766723632812, "logps/ref_rejected": -89.17379760742188, "logps/rejected": -160.09115600585938, "loss": 0.9875, "margin_dpo/margin_mean": 37.09730529785156, "margin_dpo/margin_std": 40.07815170288086, "step": 229 }, { "KL/chosen_KL_mean": -35.235069274902344, "KL/mean": -54.34660339355469, "KL/rejected_KL_mean": -73.4581298828125, "KL/std": 36.1552734375, "epoch": 0.3377386196769457, "fcm_dpo/beta": 0.014383679255843163, "fcm_dpo/delta": -0.1585429459810257, "fcm_dpo/margin": 38.22306442260742, "fcm_dpo/q_t": 0.3721032738685608, "grad_norm": 10.260030746459961, "learning_rate": 4.203117865141635e-07, "logits/chosen": -1.8191139698028564, "logits/rejected": -1.758512258529663, "logps/chosen": -98.94869995117188, "logps/ref_chosen": -63.713626861572266, "logps/ref_rejected": -91.90870666503906, "logps/rejected": -165.36683654785156, "loss": 0.9715, "margin_dpo/margin_mean": 38.223060607910156, "margin_dpo/margin_std": 36.017547607421875, "step": 230 }, { "KL/chosen_KL_mean": -38.84626770019531, "KL/mean": -55.4265022277832, "KL/rejected_KL_mean": -72.00672912597656, "KL/std": 34.953025817871094, "epoch": 0.3392070484581498, "fcm_dpo/beta": 0.014057666063308716, "fcm_dpo/delta": -0.07013809680938721, "fcm_dpo/margin": 33.16046905517578, "fcm_dpo/q_t": 0.3900415301322937, "grad_norm": 8.214221000671387, "learning_rate": 4.1937008024246625e-07, "logits/chosen": -1.631027102470398, "logits/rejected": -1.0523672103881836, "logps/chosen": -134.30194091796875, "logps/ref_chosen": -95.45567321777344, "logps/ref_rejected": -80.95568084716797, "logps/rejected": -152.96240234375, "loss": 1.0173, "margin_dpo/margin_mean": 33.16046905517578, "margin_dpo/margin_std": 30.208650588989258, "step": 231 }, { "KL/chosen_KL_mean": -41.343650817871094, "KL/mean": -55.100006103515625, "KL/rejected_KL_mean": -68.85635375976562, "KL/std": 32.287376403808594, "epoch": 0.3406754772393539, "fcm_dpo/beta": 0.014084059745073318, "fcm_dpo/delta": 0.013005422428250313, "fcm_dpo/margin": 27.512710571289062, "fcm_dpo/q_t": 0.4092198610305786, "grad_norm": 9.009343147277832, "learning_rate": 4.1842391091163933e-07, "logits/chosen": -1.3948099613189697, "logits/rejected": -0.9005382061004639, "logps/chosen": -138.24090576171875, "logps/ref_chosen": -96.89726257324219, "logps/ref_rejected": -89.76461791992188, "logps/rejected": -158.6209716796875, "loss": 1.086, "margin_dpo/margin_mean": 27.512710571289062, "margin_dpo/margin_std": 32.424991607666016, "step": 232 }, { "KL/chosen_KL_mean": -42.68800354003906, "KL/mean": -62.00927734375, "KL/rejected_KL_mean": -81.33053588867188, "KL/std": 40.835723876953125, "epoch": 0.342143906020558, "fcm_dpo/beta": 0.013795394450426102, "fcm_dpo/delta": -0.1408371776342392, "fcm_dpo/margin": 38.64254379272461, "fcm_dpo/q_t": 0.3769005537033081, "grad_norm": 9.590836524963379, "learning_rate": 4.174733034541245e-07, "logits/chosen": -1.6353737115859985, "logits/rejected": -1.4302332401275635, "logps/chosen": -131.7383270263672, "logps/ref_chosen": -89.05032348632812, "logps/ref_rejected": -112.75917053222656, "logps/rejected": -194.08970642089844, "loss": 0.991, "margin_dpo/margin_mean": 38.64254379272461, "margin_dpo/margin_std": 39.588863372802734, "step": 233 }, { "KL/chosen_KL_mean": -41.83635711669922, "KL/mean": -62.46691131591797, "KL/rejected_KL_mean": -83.09745788574219, "KL/std": 39.073936462402344, "epoch": 0.3436123348017621, "fcm_dpo/beta": 0.013297352939844131, "fcm_dpo/delta": -0.15955108404159546, "fcm_dpo/margin": 41.261104583740234, "fcm_dpo/q_t": 0.3744843602180481, "grad_norm": 8.52673625946045, "learning_rate": 4.165182829193126e-07, "logits/chosen": -1.558506965637207, "logits/rejected": -1.4900007247924805, "logps/chosen": -116.15472412109375, "logps/ref_chosen": -74.318359375, "logps/ref_rejected": -106.38758850097656, "logps/rejected": -189.48504638671875, "loss": 0.9866, "margin_dpo/margin_mean": 41.261104583740234, "margin_dpo/margin_std": 42.67225646972656, "step": 234 }, { "KL/chosen_KL_mean": -50.23050308227539, "KL/mean": -65.73869323730469, "KL/rejected_KL_mean": -81.24687194824219, "KL/std": 35.055206298828125, "epoch": 0.34508076358296624, "fcm_dpo/beta": 0.013276444748044014, "fcm_dpo/delta": -0.01230689324438572, "fcm_dpo/margin": 31.016374588012695, "fcm_dpo/q_t": 0.40316057205200195, "grad_norm": 8.82662296295166, "learning_rate": 4.1555887447288255e-07, "logits/chosen": -1.70002019405365, "logits/rejected": -1.1506282091140747, "logps/chosen": -148.44754028320312, "logps/ref_chosen": -98.217041015625, "logps/ref_rejected": -97.24677276611328, "logps/rejected": -178.49365234375, "loss": 1.0618, "margin_dpo/margin_mean": 31.016376495361328, "margin_dpo/margin_std": 33.02741241455078, "step": 235 }, { "KL/chosen_KL_mean": -42.19235610961914, "KL/mean": -62.37560272216797, "KL/rejected_KL_mean": -82.55885314941406, "KL/std": 44.101383209228516, "epoch": 0.3465491923641703, "fcm_dpo/beta": 0.013020765967667103, "fcm_dpo/delta": -0.1323883831501007, "fcm_dpo/margin": 40.36650085449219, "fcm_dpo/q_t": 0.3842836618423462, "grad_norm": 9.429834365844727, "learning_rate": 4.1459510339613946e-07, "logits/chosen": -1.4334239959716797, "logits/rejected": -1.4379512071609497, "logps/chosen": -121.03009033203125, "logps/ref_chosen": -78.83773040771484, "logps/ref_rejected": -109.06343078613281, "logps/rejected": -191.62228393554688, "loss": 1.01, "margin_dpo/margin_mean": 40.36650085449219, "margin_dpo/margin_std": 47.81488037109375, "step": 236 }, { "KL/chosen_KL_mean": -50.26822280883789, "KL/mean": -64.86390686035156, "KL/rejected_KL_mean": -79.45960235595703, "KL/std": 38.93791198730469, "epoch": 0.34801762114537443, "fcm_dpo/beta": 0.012950649484992027, "fcm_dpo/delta": 0.022814584895968437, "fcm_dpo/margin": 29.191373825073242, "fcm_dpo/q_t": 0.416756808757782, "grad_norm": 8.735091209411621, "learning_rate": 4.136269950853473e-07, "logits/chosen": -1.8456820249557495, "logits/rejected": -1.507851243019104, "logps/chosen": -135.47952270507812, "logps/ref_chosen": -85.21128845214844, "logps/ref_rejected": -99.90999603271484, "logps/rejected": -179.36959838867188, "loss": 1.1409, "margin_dpo/margin_mean": 29.191375732421875, "margin_dpo/margin_std": 50.543724060058594, "step": 237 }, { "KL/chosen_KL_mean": -54.99638748168945, "KL/mean": -72.25209045410156, "KL/rejected_KL_mean": -89.50779724121094, "KL/std": 40.485992431640625, "epoch": 0.34948604992657856, "fcm_dpo/beta": 0.012862354516983032, "fcm_dpo/delta": -0.04601479694247246, "fcm_dpo/margin": 34.51141357421875, "fcm_dpo/q_t": 0.3984161913394928, "grad_norm": 8.139162063598633, "learning_rate": 4.126545750510605e-07, "logits/chosen": -1.2899951934814453, "logits/rejected": -1.141869068145752, "logps/chosen": -133.72763061523438, "logps/ref_chosen": -78.73123168945312, "logps/ref_rejected": -95.41840362548828, "logps/rejected": -184.92620849609375, "loss": 1.0573, "margin_dpo/margin_mean": 34.511409759521484, "margin_dpo/margin_std": 41.94647979736328, "step": 238 }, { "KL/chosen_KL_mean": -49.67461013793945, "KL/mean": -65.66973876953125, "KL/rejected_KL_mean": -81.66487884521484, "KL/std": 36.1599235534668, "epoch": 0.3509544787077827, "fcm_dpo/beta": 0.012679128907620907, "fcm_dpo/delta": -0.065593421459198, "fcm_dpo/margin": 31.990262985229492, "fcm_dpo/q_t": 0.4057391881942749, "grad_norm": 8.987502098083496, "learning_rate": 4.116778689174514e-07, "logits/chosen": -2.1470353603363037, "logits/rejected": -1.692811369895935, "logps/chosen": -142.27554321289062, "logps/ref_chosen": -92.60093688964844, "logps/ref_rejected": -100.51769256591797, "logps/rejected": -182.1825714111328, "loss": 1.0857, "margin_dpo/margin_mean": 31.990262985229492, "margin_dpo/margin_std": 40.13188171386719, "step": 239 }, { "KL/chosen_KL_mean": -51.856868743896484, "KL/mean": -66.01220703125, "KL/rejected_KL_mean": -80.16754150390625, "KL/std": 36.49290466308594, "epoch": 0.3524229074889868, "fcm_dpo/beta": 0.012731213122606277, "fcm_dpo/delta": 0.04104982316493988, "fcm_dpo/margin": 28.3106689453125, "fcm_dpo/q_t": 0.4163438677787781, "grad_norm": 10.008689880371094, "learning_rate": 4.106969024216348e-07, "logits/chosen": -1.9556775093078613, "logits/rejected": -1.5861685276031494, "logps/chosen": -138.01663208007812, "logps/ref_chosen": -86.15977478027344, "logps/ref_rejected": -80.45567321777344, "logps/rejected": -160.62319946289062, "loss": 1.118, "margin_dpo/margin_mean": 28.3106689453125, "margin_dpo/margin_std": 39.285972595214844, "step": 240 }, { "KL/chosen_KL_mean": -47.5243034362793, "KL/mean": -72.66444396972656, "KL/rejected_KL_mean": -97.80459594726562, "KL/std": 41.57490539550781, "epoch": 0.35389133627019087, "fcm_dpo/beta": 0.012402208521962166, "fcm_dpo/delta": -0.23778831958770752, "fcm_dpo/margin": 50.280296325683594, "fcm_dpo/q_t": 0.35778650641441345, "grad_norm": 8.693832397460938, "learning_rate": 4.097117014129903e-07, "logits/chosen": -1.7208023071289062, "logits/rejected": -1.036415934562683, "logps/chosen": -148.57025146484375, "logps/ref_chosen": -101.04594421386719, "logps/ref_rejected": -94.04934692382812, "logps/rejected": -191.85394287109375, "loss": 0.9329, "margin_dpo/margin_mean": 50.280296325683594, "margin_dpo/margin_std": 46.47297668457031, "step": 241 }, { "KL/chosen_KL_mean": -56.00395584106445, "KL/mean": -74.20791625976562, "KL/rejected_KL_mean": -92.41185760498047, "KL/std": 37.318153381347656, "epoch": 0.355359765051395, "fcm_dpo/beta": 0.012145690619945526, "fcm_dpo/delta": -0.04419766739010811, "fcm_dpo/margin": 36.407894134521484, "fcm_dpo/q_t": 0.3966217041015625, "grad_norm": 8.951445579528809, "learning_rate": 4.087222918524807e-07, "logits/chosen": -1.9276995658874512, "logits/rejected": -1.2692008018493652, "logps/chosen": -151.67662048339844, "logps/ref_chosen": -95.67266845703125, "logps/ref_rejected": -90.65454864501953, "logps/rejected": -183.06640625, "loss": 1.0525, "margin_dpo/margin_mean": 36.40789031982422, "margin_dpo/margin_std": 41.32374954223633, "step": 242 }, { "KL/chosen_KL_mean": -55.74171447753906, "KL/mean": -73.79441833496094, "KL/rejected_KL_mean": -91.84712219238281, "KL/std": 40.06411361694336, "epoch": 0.3568281938325991, "fcm_dpo/beta": 0.011970823630690575, "fcm_dpo/delta": -0.09341253340244293, "fcm_dpo/margin": 36.10541534423828, "fcm_dpo/q_t": 0.40096449851989746, "grad_norm": 8.581452369689941, "learning_rate": 4.07728699811968e-07, "logits/chosen": -1.9148585796356201, "logits/rejected": -1.0913387537002563, "logps/chosen": -153.7731170654297, "logps/ref_chosen": -98.03140258789062, "logps/ref_rejected": -83.18806457519531, "logps/rejected": -175.03518676757812, "loss": 1.081, "margin_dpo/margin_mean": 36.10541534423828, "margin_dpo/margin_std": 47.55536651611328, "step": 243 }, { "KL/chosen_KL_mean": -58.18250274658203, "KL/mean": -77.11045837402344, "KL/rejected_KL_mean": -96.03840637207031, "KL/std": 43.899818420410156, "epoch": 0.35829662261380324, "fcm_dpo/beta": 0.011755963787436485, "fcm_dpo/delta": -0.04726836830377579, "fcm_dpo/margin": 37.85589599609375, "fcm_dpo/q_t": 0.39697739481925964, "grad_norm": 8.456755638122559, "learning_rate": 4.067309514735267e-07, "logits/chosen": -2.235485315322876, "logits/rejected": -1.9078938961029053, "logps/chosen": -147.076416015625, "logps/ref_chosen": -88.8939208984375, "logps/ref_rejected": -102.57278442382812, "logps/rejected": -198.61117553710938, "loss": 1.0505, "margin_dpo/margin_mean": 37.85589599609375, "margin_dpo/margin_std": 43.53936767578125, "step": 244 }, { "KL/chosen_KL_mean": -49.47932815551758, "KL/mean": -69.46237182617188, "KL/rejected_KL_mean": -89.44541931152344, "KL/std": 40.532493591308594, "epoch": 0.35976505139500736, "fcm_dpo/beta": 0.0116469357162714, "fcm_dpo/delta": -0.06862208247184753, "fcm_dpo/margin": 39.96609878540039, "fcm_dpo/q_t": 0.39160656929016113, "grad_norm": 7.924368858337402, "learning_rate": 4.057290731287531e-07, "logits/chosen": -2.3287272453308105, "logits/rejected": -1.4995757341384888, "logps/chosen": -153.67332458496094, "logps/ref_chosen": -104.19400024414062, "logps/ref_rejected": -92.65645599365234, "logps/rejected": -182.10186767578125, "loss": 1.029, "margin_dpo/margin_mean": 39.96609878540039, "margin_dpo/margin_std": 41.65141296386719, "step": 245 }, { "KL/chosen_KL_mean": -64.39878845214844, "KL/mean": -80.34806823730469, "KL/rejected_KL_mean": -96.29736328125, "KL/std": 39.891204833984375, "epoch": 0.36123348017621143, "fcm_dpo/beta": 0.011543956585228443, "fcm_dpo/delta": -0.018736502155661583, "fcm_dpo/margin": 31.898571014404297, "fcm_dpo/q_t": 0.41454970836639404, "grad_norm": 8.051033020019531, "learning_rate": 4.047230911780736e-07, "logits/chosen": -1.9757585525512695, "logits/rejected": -1.4433469772338867, "logps/chosen": -167.61782836914062, "logps/ref_chosen": -103.21904754638672, "logps/ref_rejected": -90.99221801757812, "logps/rejected": -187.28958129882812, "loss": 1.1101, "margin_dpo/margin_mean": 31.898571014404297, "margin_dpo/margin_std": 43.04576873779297, "step": 246 }, { "KL/chosen_KL_mean": -58.211830139160156, "KL/mean": -84.24368286132812, "KL/rejected_KL_mean": -110.27554321289062, "KL/std": 44.67066192626953, "epoch": 0.36270190895741555, "fcm_dpo/beta": 0.01116771437227726, "fcm_dpo/delta": -0.19386975467205048, "fcm_dpo/margin": 52.06371307373047, "fcm_dpo/q_t": 0.3671821355819702, "grad_norm": 8.201760292053223, "learning_rate": 4.0371303213004814e-07, "logits/chosen": -2.1689987182617188, "logits/rejected": -1.9922220706939697, "logps/chosen": -145.20620727539062, "logps/ref_chosen": -86.99437713623047, "logps/ref_rejected": -111.33802795410156, "logps/rejected": -221.61355590820312, "loss": 0.9595, "margin_dpo/margin_mean": 52.06371307373047, "margin_dpo/margin_std": 50.39811706542969, "step": 247 }, { "KL/chosen_KL_mean": -67.36201477050781, "KL/mean": -84.65104675292969, "KL/rejected_KL_mean": -101.9400863647461, "KL/std": 39.40555953979492, "epoch": 0.3641703377386197, "fcm_dpo/beta": 0.011161180213093758, "fcm_dpo/delta": 0.014418689534068108, "fcm_dpo/margin": 34.578086853027344, "fcm_dpo/q_t": 0.40965962409973145, "grad_norm": 8.75762939453125, "learning_rate": 4.0269892260067197e-07, "logits/chosen": -1.566325068473816, "logits/rejected": -1.5425187349319458, "logps/chosen": -142.14752197265625, "logps/ref_chosen": -74.7855224609375, "logps/ref_rejected": -98.27689361572266, "logps/rejected": -200.21697998046875, "loss": 1.086, "margin_dpo/margin_mean": 34.578086853027344, "margin_dpo/margin_std": 40.4761848449707, "step": 248 }, { "KL/chosen_KL_mean": -74.44125366210938, "KL/mean": -90.29890441894531, "KL/rejected_KL_mean": -106.15655517578125, "KL/std": 42.263832092285156, "epoch": 0.3656387665198238, "fcm_dpo/beta": 0.011107325553894043, "fcm_dpo/delta": -0.013334457762539387, "fcm_dpo/margin": 31.715316772460938, "fcm_dpo/q_t": 0.4185119569301605, "grad_norm": 9.714157104492188, "learning_rate": 4.0168078931267426e-07, "logits/chosen": -2.2583506107330322, "logits/rejected": -1.7227230072021484, "logps/chosen": -170.14505004882812, "logps/ref_chosen": -95.70379638671875, "logps/ref_rejected": -87.14646911621094, "logps/rejected": -193.30303955078125, "loss": 1.1341, "margin_dpo/margin_mean": 31.715316772460938, "margin_dpo/margin_std": 48.34738540649414, "step": 249 }, { "KL/chosen_KL_mean": -66.18019104003906, "KL/mean": -84.00900268554688, "KL/rejected_KL_mean": -101.83782958984375, "KL/std": 42.873809814453125, "epoch": 0.3671071953010279, "fcm_dpo/beta": 0.011063864454627037, "fcm_dpo/delta": 0.005284234881401062, "fcm_dpo/margin": 35.65764617919922, "fcm_dpo/q_t": 0.40786194801330566, "grad_norm": 8.91695499420166, "learning_rate": 4.006586590948141e-07, "logits/chosen": -1.9831795692443848, "logits/rejected": -1.0479307174682617, "logps/chosen": -180.23239135742188, "logps/ref_chosen": -114.05221557617188, "logps/ref_rejected": -81.08767700195312, "logps/rejected": -182.92550659179688, "loss": 1.0959, "margin_dpo/margin_mean": 35.65764617919922, "margin_dpo/margin_std": 45.877777099609375, "step": 250 }, { "KL/chosen_KL_mean": -66.26416015625, "KL/mean": -82.0506591796875, "KL/rejected_KL_mean": -97.83717346191406, "KL/std": 40.05830383300781, "epoch": 0.368575624082232, "fcm_dpo/beta": 0.01105173397809267, "fcm_dpo/delta": -0.03037196956574917, "fcm_dpo/margin": 31.572999954223633, "fcm_dpo/q_t": 0.41971632838249207, "grad_norm": 9.536917686462402, "learning_rate": 3.9963255888117325e-07, "logits/chosen": -1.7843058109283447, "logits/rejected": -1.071089267730713, "logps/chosen": -163.97544860839844, "logps/ref_chosen": -97.71128845214844, "logps/ref_rejected": -83.52742004394531, "logps/rejected": -181.36459350585938, "loss": 1.1369, "margin_dpo/margin_mean": 31.573001861572266, "margin_dpo/margin_std": 48.03728103637695, "step": 251 }, { "KL/chosen_KL_mean": -66.95657348632812, "KL/mean": -82.80113220214844, "KL/rejected_KL_mean": -98.64569091796875, "KL/std": 42.470863342285156, "epoch": 0.3700440528634361, "fcm_dpo/beta": 0.011149127036333084, "fcm_dpo/delta": 0.04827743023633957, "fcm_dpo/margin": 31.68912696838379, "fcm_dpo/q_t": 0.4175231456756592, "grad_norm": 9.4552640914917, "learning_rate": 3.9860251571044666e-07, "logits/chosen": -1.9976928234100342, "logits/rejected": -1.4521489143371582, "logps/chosen": -175.94268798828125, "logps/ref_chosen": -108.9861068725586, "logps/ref_rejected": -91.56423950195312, "logps/rejected": -190.20993041992188, "loss": 1.1253, "margin_dpo/margin_mean": 31.68912696838379, "margin_dpo/margin_std": 45.009246826171875, "step": 252 }, { "KL/chosen_KL_mean": -73.47262573242188, "KL/mean": -88.30828857421875, "KL/rejected_KL_mean": -103.1439437866211, "KL/std": 46.262168884277344, "epoch": 0.37151248164464024, "fcm_dpo/beta": 0.011135118082165718, "fcm_dpo/delta": -0.021412089467048645, "fcm_dpo/margin": 29.67132568359375, "fcm_dpo/q_t": 0.42164188623428345, "grad_norm": 9.413656234741211, "learning_rate": 3.9756855672522986e-07, "logits/chosen": -2.139962911605835, "logits/rejected": -1.935544490814209, "logps/chosen": -173.68893432617188, "logps/ref_chosen": -100.21630096435547, "logps/ref_rejected": -105.67670440673828, "logps/rejected": -208.82064819335938, "loss": 1.1329, "margin_dpo/margin_mean": 29.671321868896484, "margin_dpo/margin_std": 39.909759521484375, "step": 253 }, { "KL/chosen_KL_mean": -70.78187561035156, "KL/mean": -86.1494140625, "KL/rejected_KL_mean": -101.5169448852539, "KL/std": 43.68431854248047, "epoch": 0.37298091042584436, "fcm_dpo/beta": 0.01108785904943943, "fcm_dpo/delta": -0.01054837554693222, "fcm_dpo/margin": 30.735069274902344, "fcm_dpo/q_t": 0.4197158217430115, "grad_norm": 9.133874893188477, "learning_rate": 3.965307091713037e-07, "logits/chosen": -2.117870330810547, "logits/rejected": -1.5947855710983276, "logps/chosen": -169.51705932617188, "logps/ref_chosen": -98.73518371582031, "logps/ref_rejected": -93.73825073242188, "logps/rejected": -195.25518798828125, "loss": 1.1394, "margin_dpo/margin_mean": 30.735069274902344, "margin_dpo/margin_std": 46.54502487182617, "step": 254 }, { "KL/chosen_KL_mean": -71.57162475585938, "KL/mean": -95.09077453613281, "KL/rejected_KL_mean": -118.60993957519531, "KL/std": 49.213096618652344, "epoch": 0.3744493392070485, "fcm_dpo/beta": 0.010904960334300995, "fcm_dpo/delta": -0.1189572736620903, "fcm_dpo/margin": 47.03831100463867, "fcm_dpo/q_t": 0.38124898076057434, "grad_norm": 9.016454696655273, "learning_rate": 3.954890003969163e-07, "logits/chosen": -1.624290943145752, "logits/rejected": -1.2691798210144043, "logps/chosen": -161.95419311523438, "logps/ref_chosen": -90.382568359375, "logps/ref_rejected": -97.07624816894531, "logps/rejected": -215.68618774414062, "loss": 1.0063, "margin_dpo/margin_mean": 47.03831481933594, "margin_dpo/margin_std": 49.886207580566406, "step": 255 }, { "KL/chosen_KL_mean": -70.11643981933594, "KL/mean": -91.02188873291016, "KL/rejected_KL_mean": -111.92735290527344, "KL/std": 51.18582534790039, "epoch": 0.37591776798825255, "fcm_dpo/beta": 0.01076935138553381, "fcm_dpo/delta": -0.052641574293375015, "fcm_dpo/margin": 41.8109016418457, "fcm_dpo/q_t": 0.39821550250053406, "grad_norm": 9.411343574523926, "learning_rate": 3.944434578520628e-07, "logits/chosen": -1.8807837963104248, "logits/rejected": -1.5417015552520752, "logps/chosen": -158.86924743652344, "logps/ref_chosen": -88.7528076171875, "logps/ref_rejected": -98.49383544921875, "logps/rejected": -210.42117309570312, "loss": 1.0817, "margin_dpo/margin_mean": 41.81090545654297, "margin_dpo/margin_std": 59.800994873046875, "step": 256 }, { "KL/chosen_KL_mean": -71.79951477050781, "KL/mean": -93.05375671386719, "KL/rejected_KL_mean": -114.30799865722656, "KL/std": 52.42218780517578, "epoch": 0.37738619676945667, "fcm_dpo/beta": 0.010478893294930458, "fcm_dpo/delta": -0.11083962023258209, "fcm_dpo/margin": 42.50849151611328, "fcm_dpo/q_t": 0.3968268036842346, "grad_norm": 8.39576244354248, "learning_rate": 3.933941090877615e-07, "logits/chosen": -1.7643895149230957, "logits/rejected": -1.3534268140792847, "logps/chosen": -154.60302734375, "logps/ref_chosen": -82.80352020263672, "logps/ref_rejected": -85.8677978515625, "logps/rejected": -200.17578125, "loss": 1.0616, "margin_dpo/margin_mean": 42.50849151611328, "margin_dpo/margin_std": 50.7281494140625, "step": 257 }, { "KL/chosen_KL_mean": -75.10973358154297, "KL/mean": -94.32515716552734, "KL/rejected_KL_mean": -113.54057312011719, "KL/std": 49.886898040771484, "epoch": 0.3788546255506608, "fcm_dpo/beta": 0.010417868383228779, "fcm_dpo/delta": -0.05857791751623154, "fcm_dpo/margin": 38.430850982666016, "fcm_dpo/q_t": 0.4060943126678467, "grad_norm": 9.19739818572998, "learning_rate": 3.923409817553284e-07, "logits/chosen": -2.39119553565979, "logits/rejected": -2.0479345321655273, "logps/chosen": -165.2974853515625, "logps/ref_chosen": -90.187744140625, "logps/ref_rejected": -103.47068786621094, "logps/rejected": -217.01126098632812, "loss": 1.0873, "margin_dpo/margin_mean": 38.43085479736328, "margin_dpo/margin_std": 47.67232131958008, "step": 258 }, { "KL/chosen_KL_mean": -81.25570678710938, "KL/mean": -97.72335815429688, "KL/rejected_KL_mean": -114.19100952148438, "KL/std": 50.4412727355957, "epoch": 0.3803230543318649, "fcm_dpo/beta": 0.010299797169864178, "fcm_dpo/delta": -0.027616608887910843, "fcm_dpo/margin": 32.935298919677734, "fcm_dpo/q_t": 0.420768141746521, "grad_norm": 8.87748908996582, "learning_rate": 3.9128410360564793e-07, "logits/chosen": -1.2344012260437012, "logits/rejected": -0.9772180318832397, "logps/chosen": -172.02825927734375, "logps/ref_chosen": -90.77254486083984, "logps/ref_rejected": -94.58816528320312, "logps/rejected": -208.7791748046875, "loss": 1.1585, "margin_dpo/margin_mean": 32.935298919677734, "margin_dpo/margin_std": 56.87708282470703, "step": 259 }, { "KL/chosen_KL_mean": -83.33119201660156, "KL/mean": -97.52334594726562, "KL/rejected_KL_mean": -111.71551513671875, "KL/std": 50.935203552246094, "epoch": 0.38179148311306904, "fcm_dpo/beta": 0.010396572761237621, "fcm_dpo/delta": 0.04675988852977753, "fcm_dpo/margin": 28.384326934814453, "fcm_dpo/q_t": 0.4314086437225342, "grad_norm": 10.557072639465332, "learning_rate": 3.9022350248844246e-07, "logits/chosen": -1.667799472808838, "logits/rejected": -1.6990429162979126, "logps/chosen": -158.92388916015625, "logps/ref_chosen": -75.59269714355469, "logps/ref_rejected": -100.84554290771484, "logps/rejected": -212.56106567382812, "loss": 1.1916, "margin_dpo/margin_mean": 28.384326934814453, "margin_dpo/margin_std": 55.32361602783203, "step": 260 }, { "KL/chosen_KL_mean": -64.40402221679688, "KL/mean": -88.78857421875, "KL/rejected_KL_mean": -113.17312622070312, "KL/std": 51.38774871826172, "epoch": 0.3832599118942731, "fcm_dpo/beta": 0.010184694081544876, "fcm_dpo/delta": -0.1028708964586258, "fcm_dpo/margin": 48.76910400390625, "fcm_dpo/q_t": 0.3848683834075928, "grad_norm": 8.344627380371094, "learning_rate": 3.891592063515376e-07, "logits/chosen": -2.0689573287963867, "logits/rejected": -1.705404281616211, "logps/chosen": -158.658935546875, "logps/ref_chosen": -94.25491333007812, "logps/ref_rejected": -93.65699005126953, "logps/rejected": -206.83010864257812, "loss": 1.0113, "margin_dpo/margin_mean": 48.769107818603516, "margin_dpo/margin_std": 50.05211639404297, "step": 261 }, { "KL/chosen_KL_mean": -69.95246887207031, "KL/mean": -93.08432006835938, "KL/rejected_KL_mean": -116.21617126464844, "KL/std": 54.20262908935547, "epoch": 0.38472834067547723, "fcm_dpo/beta": 0.010046085342764854, "fcm_dpo/delta": -0.06842421740293503, "fcm_dpo/margin": 46.263694763183594, "fcm_dpo/q_t": 0.3925879895687103, "grad_norm": 8.307077407836914, "learning_rate": 3.880912432401264e-07, "logits/chosen": -1.6251637935638428, "logits/rejected": -1.0782082080841064, "logps/chosen": -155.2197723388672, "logps/ref_chosen": -85.26730346679688, "logps/ref_rejected": -90.82609558105469, "logps/rejected": -207.04226684570312, "loss": 1.0439, "margin_dpo/margin_mean": 46.263694763183594, "margin_dpo/margin_std": 54.11823654174805, "step": 262 }, { "KL/chosen_KL_mean": -56.42060852050781, "KL/mean": -87.4281997680664, "KL/rejected_KL_mean": -118.435791015625, "KL/std": 58.96156311035156, "epoch": 0.38619676945668135, "fcm_dpo/beta": 0.009745280258357525, "fcm_dpo/delta": -0.21718664467334747, "fcm_dpo/margin": 62.01518249511719, "fcm_dpo/q_t": 0.36159005761146545, "grad_norm": 8.110187530517578, "learning_rate": 3.870196412960302e-07, "logits/chosen": -2.079312801361084, "logits/rejected": -1.5224335193634033, "logps/chosen": -152.1785125732422, "logps/ref_chosen": -95.75790405273438, "logps/ref_rejected": -101.83377075195312, "logps/rejected": -220.26956176757812, "loss": 0.9562, "margin_dpo/margin_mean": 62.01518249511719, "margin_dpo/margin_std": 62.88063049316406, "step": 263 }, { "KL/chosen_KL_mean": -65.93309020996094, "KL/mean": -92.15192413330078, "KL/rejected_KL_mean": -118.37074279785156, "KL/std": 55.61305236816406, "epoch": 0.3876651982378855, "fcm_dpo/beta": 0.009461034089326859, "fcm_dpo/delta": -0.10133536159992218, "fcm_dpo/margin": 52.43765640258789, "fcm_dpo/q_t": 0.384945809841156, "grad_norm": 8.082246780395508, "learning_rate": 3.8594442875695665e-07, "logits/chosen": -2.0536587238311768, "logits/rejected": -1.7670159339904785, "logps/chosen": -156.55575561523438, "logps/ref_chosen": -90.6226577758789, "logps/ref_rejected": -100.32554626464844, "logps/rejected": -218.6962890625, "loss": 1.0171, "margin_dpo/margin_mean": 52.437660217285156, "margin_dpo/margin_std": 56.434661865234375, "step": 264 }, { "KL/chosen_KL_mean": -69.6064224243164, "KL/mean": -92.77266693115234, "KL/rejected_KL_mean": -115.93890380859375, "KL/std": 56.830081939697266, "epoch": 0.3891336270190896, "fcm_dpo/beta": 0.009339086711406708, "fcm_dpo/delta": -0.03467705845832825, "fcm_dpo/margin": 46.33248519897461, "fcm_dpo/q_t": 0.4013739824295044, "grad_norm": 8.375724792480469, "learning_rate": 3.848656339557562e-07, "logits/chosen": -2.082062244415283, "logits/rejected": -1.6759120225906372, "logps/chosen": -161.978759765625, "logps/ref_chosen": -92.37232971191406, "logps/ref_rejected": -94.62757873535156, "logps/rejected": -210.5664825439453, "loss": 1.0774, "margin_dpo/margin_mean": 46.33248519897461, "margin_dpo/margin_std": 61.289188385009766, "step": 265 }, { "KL/chosen_KL_mean": -81.2215576171875, "KL/mean": -102.11510467529297, "KL/rejected_KL_mean": -123.00863647460938, "KL/std": 55.078590393066406, "epoch": 0.39060205580029367, "fcm_dpo/beta": 0.009357543662190437, "fcm_dpo/delta": 0.009292619302868843, "fcm_dpo/margin": 41.787078857421875, "fcm_dpo/q_t": 0.41033831238746643, "grad_norm": 10.080767631530762, "learning_rate": 3.8378328531967507e-07, "logits/chosen": -2.2015106678009033, "logits/rejected": -1.3638226985931396, "logps/chosen": -183.4215850830078, "logps/ref_chosen": -102.20002746582031, "logps/ref_rejected": -74.36642456054688, "logps/rejected": -197.37506103515625, "loss": 1.1048, "margin_dpo/margin_mean": 41.787078857421875, "margin_dpo/margin_std": 58.27301025390625, "step": 266 }, { "KL/chosen_KL_mean": -72.06327819824219, "KL/mean": -96.59632110595703, "KL/rejected_KL_mean": -121.12936401367188, "KL/std": 55.48495864868164, "epoch": 0.3920704845814978, "fcm_dpo/beta": 0.009264141321182251, "fcm_dpo/delta": -0.05764763057231903, "fcm_dpo/margin": 49.066078186035156, "fcm_dpo/q_t": 0.39444106817245483, "grad_norm": 8.82445240020752, "learning_rate": 3.8269741136960646e-07, "logits/chosen": -1.9128168821334839, "logits/rejected": -1.3005517721176147, "logps/chosen": -176.34927368164062, "logps/ref_chosen": -104.28599548339844, "logps/ref_rejected": -95.98719024658203, "logps/rejected": -217.11654663085938, "loss": 1.0404, "margin_dpo/margin_mean": 49.06608200073242, "margin_dpo/margin_std": 53.13525390625, "step": 267 }, { "KL/chosen_KL_mean": -71.71980285644531, "KL/mean": -100.52813720703125, "KL/rejected_KL_mean": -129.33645629882812, "KL/std": 55.80779266357422, "epoch": 0.3935389133627019, "fcm_dpo/beta": 0.009052696637809277, "fcm_dpo/delta": -0.12931329011917114, "fcm_dpo/margin": 57.616661071777344, "fcm_dpo/q_t": 0.3780568242073059, "grad_norm": 9.015339851379395, "learning_rate": 3.8160804071933894e-07, "logits/chosen": -1.6295409202575684, "logits/rejected": -1.4765753746032715, "logps/chosen": -158.416015625, "logps/ref_chosen": -86.69622039794922, "logps/ref_rejected": -109.19184112548828, "logps/rejected": -238.52828979492188, "loss": 0.9905, "margin_dpo/margin_mean": 57.616661071777344, "margin_dpo/margin_std": 55.16349411010742, "step": 268 }, { "KL/chosen_KL_mean": -76.3770751953125, "KL/mean": -104.35591125488281, "KL/rejected_KL_mean": -132.33473205566406, "KL/std": 59.76306915283203, "epoch": 0.39500734214390604, "fcm_dpo/beta": 0.008916812017560005, "fcm_dpo/delta": -0.10403021425008774, "fcm_dpo/margin": 55.957672119140625, "fcm_dpo/q_t": 0.3841118812561035, "grad_norm": 8.239960670471191, "learning_rate": 3.8051520207480204e-07, "logits/chosen": -1.8281971216201782, "logits/rejected": -1.3735320568084717, "logps/chosen": -181.348876953125, "logps/ref_chosen": -104.97181701660156, "logps/ref_rejected": -112.47647094726562, "logps/rejected": -244.8112030029297, "loss": 1.0093, "margin_dpo/margin_mean": 55.957672119140625, "margin_dpo/margin_std": 57.233211517333984, "step": 269 }, { "KL/chosen_KL_mean": -84.89106750488281, "KL/mean": -108.30947875976562, "KL/rejected_KL_mean": -131.72787475585938, "KL/std": 58.81729507446289, "epoch": 0.3964757709251101, "fcm_dpo/beta": 0.008856823667883873, "fcm_dpo/delta": -0.015634391456842422, "fcm_dpo/margin": 46.836814880371094, "fcm_dpo/q_t": 0.40438681840896606, "grad_norm": 8.963788986206055, "learning_rate": 3.794189242333106e-07, "logits/chosen": -2.467940330505371, "logits/rejected": -2.057849884033203, "logps/chosen": -185.96490478515625, "logps/ref_chosen": -101.07383728027344, "logps/ref_rejected": -117.75289154052734, "logps/rejected": -249.48077392578125, "loss": 1.0975, "margin_dpo/margin_mean": 46.83681106567383, "margin_dpo/margin_std": 66.76217651367188, "step": 270 }, { "KL/chosen_KL_mean": -70.38786315917969, "KL/mean": -99.63778686523438, "KL/rejected_KL_mean": -128.88772583007812, "KL/std": 55.40803527832031, "epoch": 0.39794419970631423, "fcm_dpo/beta": 0.008662492968142033, "fcm_dpo/delta": -0.11264573782682419, "fcm_dpo/margin": 58.49987030029297, "fcm_dpo/q_t": 0.38125231862068176, "grad_norm": 8.793933868408203, "learning_rate": 3.7831923608280514e-07, "logits/chosen": -1.5933396816253662, "logits/rejected": -1.145467758178711, "logps/chosen": -167.11245727539062, "logps/ref_chosen": -96.72459411621094, "logps/ref_rejected": -98.5244140625, "logps/rejected": -227.41213989257812, "loss": 1.0142, "margin_dpo/margin_mean": 58.49987030029297, "margin_dpo/margin_std": 63.44873809814453, "step": 271 }, { "KL/chosen_KL_mean": -60.55150604248047, "KL/mean": -88.66481018066406, "KL/rejected_KL_mean": -116.77812194824219, "KL/std": 61.343833923339844, "epoch": 0.39941262848751835, "fcm_dpo/beta": 0.008516614325344563, "fcm_dpo/delta": -0.08284822106361389, "fcm_dpo/margin": 56.226619720458984, "fcm_dpo/q_t": 0.3901873826980591, "grad_norm": 8.260068893432617, "learning_rate": 3.772161666010912e-07, "logits/chosen": -1.4849441051483154, "logits/rejected": -1.3740112781524658, "logps/chosen": -141.52871704101562, "logps/ref_chosen": -80.97721862792969, "logps/ref_rejected": -108.55535888671875, "logps/rejected": -225.33348083496094, "loss": 1.0387, "margin_dpo/margin_mean": 56.226619720458984, "margin_dpo/margin_std": 66.62297821044922, "step": 272 }, { "KL/chosen_KL_mean": -83.14810943603516, "KL/mean": -111.72622680664062, "KL/rejected_KL_mean": -140.30435180664062, "KL/std": 63.00804138183594, "epoch": 0.4008810572687225, "fcm_dpo/beta": 0.008320118300616741, "fcm_dpo/delta": -0.08099813759326935, "fcm_dpo/margin": 57.15624237060547, "fcm_dpo/q_t": 0.3903518319129944, "grad_norm": 8.189800262451172, "learning_rate": 3.761097448550755e-07, "logits/chosen": -1.490915298461914, "logits/rejected": -1.0723785161972046, "logps/chosen": -175.37271118164062, "logps/ref_chosen": -92.22460174560547, "logps/ref_rejected": -97.3630599975586, "logps/rejected": -237.66741943359375, "loss": 1.0532, "margin_dpo/margin_mean": 57.15624237060547, "margin_dpo/margin_std": 71.69188690185547, "step": 273 }, { "KL/chosen_KL_mean": -84.57870483398438, "KL/mean": -107.02603912353516, "KL/rejected_KL_mean": -129.473388671875, "KL/std": 61.506961822509766, "epoch": 0.4023494860499266, "fcm_dpo/beta": 0.00835760310292244, "fcm_dpo/delta": 0.025692790746688843, "fcm_dpo/margin": 44.89469528198242, "fcm_dpo/q_t": 0.4126904010772705, "grad_norm": 12.220967292785645, "learning_rate": 3.75e-07, "logits/chosen": -1.8258918523788452, "logits/rejected": -1.4234145879745483, "logps/chosen": -166.839599609375, "logps/ref_chosen": -82.26089477539062, "logps/ref_rejected": -83.87699890136719, "logps/rejected": -213.3503875732422, "loss": 1.0993, "margin_dpo/margin_mean": 44.894691467285156, "margin_dpo/margin_std": 56.543540954589844, "step": 274 }, { "KL/chosen_KL_mean": -71.63504028320312, "KL/mean": -96.9306640625, "KL/rejected_KL_mean": -122.22628784179688, "KL/std": 60.558570861816406, "epoch": 0.40381791483113066, "fcm_dpo/beta": 0.008400652557611465, "fcm_dpo/delta": -0.0265403613448143, "fcm_dpo/margin": 50.59123611450195, "fcm_dpo/q_t": 0.4022720158100128, "grad_norm": 9.108122825622559, "learning_rate": 3.738869612786737e-07, "logits/chosen": -2.0346293449401855, "logits/rejected": -1.709783911705017, "logps/chosen": -151.32200622558594, "logps/ref_chosen": -79.68696594238281, "logps/ref_rejected": -98.7509765625, "logps/rejected": -220.97726440429688, "loss": 1.0955, "margin_dpo/margin_mean": 50.59123611450195, "margin_dpo/margin_std": 72.06424713134766, "step": 275 }, { "KL/chosen_KL_mean": -85.46694946289062, "KL/mean": -112.68284606933594, "KL/rejected_KL_mean": -139.8987579345703, "KL/std": 61.15618133544922, "epoch": 0.4052863436123348, "fcm_dpo/beta": 0.008227425627410412, "fcm_dpo/delta": -0.05128801241517067, "fcm_dpo/margin": 54.43180465698242, "fcm_dpo/q_t": 0.3975059986114502, "grad_norm": 9.379951477050781, "learning_rate": 3.7277065802070204e-07, "logits/chosen": -2.101893663406372, "logits/rejected": -1.5263922214508057, "logps/chosen": -172.00665283203125, "logps/ref_chosen": -86.53970336914062, "logps/ref_rejected": -77.85394287109375, "logps/rejected": -217.75270080566406, "loss": 1.0668, "margin_dpo/margin_mean": 54.43180847167969, "margin_dpo/margin_std": 69.58363342285156, "step": 276 }, { "KL/chosen_KL_mean": -92.11480712890625, "KL/mean": -111.81443786621094, "KL/rejected_KL_mean": -131.51406860351562, "KL/std": 59.30174255371094, "epoch": 0.4067547723935389, "fcm_dpo/beta": 0.00826319307088852, "fcm_dpo/delta": 0.002128008520230651, "fcm_dpo/margin": 39.399261474609375, "fcm_dpo/q_t": 0.4241357445716858, "grad_norm": 9.081292152404785, "learning_rate": 3.71651119641714e-07, "logits/chosen": -2.0738203525543213, "logits/rejected": -1.658502459526062, "logps/chosen": -176.35891723632812, "logps/ref_chosen": -84.24411010742188, "logps/ref_rejected": -98.83421325683594, "logps/rejected": -230.34829711914062, "loss": 1.1479, "margin_dpo/margin_mean": 39.399261474609375, "margin_dpo/margin_std": 60.69454574584961, "step": 277 }, { "KL/chosen_KL_mean": -88.09430694580078, "KL/mean": -113.26759338378906, "KL/rejected_KL_mean": -138.44088745117188, "KL/std": 63.547183990478516, "epoch": 0.40822320117474303, "fcm_dpo/beta": 0.008229157887399197, "fcm_dpo/delta": -0.015029809437692165, "fcm_dpo/margin": 50.346580505371094, "fcm_dpo/q_t": 0.40309515595436096, "grad_norm": 10.935393333435059, "learning_rate": 3.705283756425872e-07, "logits/chosen": -2.0817856788635254, "logits/rejected": -1.8657386302947998, "logps/chosen": -170.52618408203125, "logps/ref_chosen": -82.431884765625, "logps/ref_rejected": -97.85691833496094, "logps/rejected": -236.2978057861328, "loss": 1.0819, "margin_dpo/margin_mean": 50.346580505371094, "margin_dpo/margin_std": 64.09684753417969, "step": 278 }, { "KL/chosen_KL_mean": -90.63104248046875, "KL/mean": -121.99627685546875, "KL/rejected_KL_mean": -153.3614959716797, "KL/std": 67.62579345703125, "epoch": 0.40969162995594716, "fcm_dpo/beta": 0.008074283599853516, "fcm_dpo/delta": -0.11279429495334625, "fcm_dpo/margin": 62.73046112060547, "fcm_dpo/q_t": 0.3830791115760803, "grad_norm": 10.03566837310791, "learning_rate": 3.6940245560867e-07, "logits/chosen": -1.8028552532196045, "logits/rejected": -1.5892622470855713, "logps/chosen": -175.79904174804688, "logps/ref_chosen": -85.16799926757812, "logps/ref_rejected": -94.12664794921875, "logps/rejected": -247.48814392089844, "loss": 1.0114, "margin_dpo/margin_mean": 62.73046112060547, "margin_dpo/margin_std": 67.60432434082031, "step": 279 }, { "KL/chosen_KL_mean": -93.82257080078125, "KL/mean": -124.91815948486328, "KL/rejected_KL_mean": -156.01376342773438, "KL/std": 61.90663146972656, "epoch": 0.4111600587371512, "fcm_dpo/beta": 0.007951401174068451, "fcm_dpo/delta": -0.09931014478206635, "fcm_dpo/margin": 62.19118881225586, "fcm_dpo/q_t": 0.3849431872367859, "grad_norm": 8.708633422851562, "learning_rate": 3.6827338920900253e-07, "logits/chosen": -1.8133509159088135, "logits/rejected": -1.6158638000488281, "logps/chosen": -179.67898559570312, "logps/ref_chosen": -85.85641479492188, "logps/ref_rejected": -104.11859130859375, "logps/rejected": -260.1323547363281, "loss": 1.0142, "margin_dpo/margin_mean": 62.191192626953125, "margin_dpo/margin_std": 64.64448547363281, "step": 280 }, { "KL/chosen_KL_mean": -100.50975036621094, "KL/mean": -131.24822998046875, "KL/rejected_KL_mean": -161.9866943359375, "KL/std": 71.71414184570312, "epoch": 0.41262848751835535, "fcm_dpo/beta": 0.0077834296971559525, "fcm_dpo/delta": -0.08270800113677979, "fcm_dpo/margin": 61.4769401550293, "fcm_dpo/q_t": 0.39019155502319336, "grad_norm": 7.265856742858887, "learning_rate": 3.6714120619553435e-07, "logits/chosen": -2.1806912422180176, "logits/rejected": -1.4496757984161377, "logps/chosen": -187.06057739257812, "logps/ref_chosen": -86.55082702636719, "logps/ref_rejected": -88.62866973876953, "logps/rejected": -250.6153564453125, "loss": 1.0364, "margin_dpo/margin_mean": 61.47693634033203, "margin_dpo/margin_std": 72.03846740722656, "step": 281 }, { "KL/chosen_KL_mean": -97.80078887939453, "KL/mean": -136.38632202148438, "KL/rejected_KL_mean": -174.97186279296875, "KL/std": 69.38382720947266, "epoch": 0.41409691629955947, "fcm_dpo/beta": 0.007540118880569935, "fcm_dpo/delta": -0.19331520795822144, "fcm_dpo/margin": 77.17108154296875, "fcm_dpo/q_t": 0.36470791697502136, "grad_norm": 8.835838317871094, "learning_rate": 3.660059364023408e-07, "logits/chosen": -1.7645771503448486, "logits/rejected": -1.3814326524734497, "logps/chosen": -202.90589904785156, "logps/ref_chosen": -105.10511016845703, "logps/ref_rejected": -102.85336303710938, "logps/rejected": -277.82525634765625, "loss": 0.9485, "margin_dpo/margin_mean": 77.17108154296875, "margin_dpo/margin_std": 69.40178680419922, "step": 282 }, { "KL/chosen_KL_mean": -112.97630310058594, "KL/mean": -140.03945922851562, "KL/rejected_KL_mean": -167.10260009765625, "KL/std": 69.02311706542969, "epoch": 0.4155653450807636, "fcm_dpo/beta": 0.007465697359293699, "fcm_dpo/delta": -0.004264989867806435, "fcm_dpo/margin": 54.126319885253906, "fcm_dpo/q_t": 0.4070722162723541, "grad_norm": 9.946036338806152, "learning_rate": 3.6486760974483685e-07, "logits/chosen": -2.327394962310791, "logits/rejected": -1.9982503652572632, "logps/chosen": -208.02890014648438, "logps/ref_chosen": -95.05259704589844, "logps/ref_rejected": -103.54454803466797, "logps/rejected": -270.64715576171875, "loss": 1.099, "margin_dpo/margin_mean": 54.126319885253906, "margin_dpo/margin_std": 76.44354248046875, "step": 283 }, { "KL/chosen_KL_mean": -102.30014038085938, "KL/mean": -136.99880981445312, "KL/rejected_KL_mean": -171.697509765625, "KL/std": 68.04427337646484, "epoch": 0.4170337738619677, "fcm_dpo/beta": 0.007310959044843912, "fcm_dpo/delta": -0.11388857662677765, "fcm_dpo/margin": 69.39735412597656, "fcm_dpo/q_t": 0.38020601868629456, "grad_norm": 9.636674880981445, "learning_rate": 3.6372625621898863e-07, "logits/chosen": -1.767768144607544, "logits/rejected": -1.4314764738082886, "logps/chosen": -189.96661376953125, "logps/ref_chosen": -87.66647338867188, "logps/ref_rejected": -98.75103759765625, "logps/rejected": -270.44854736328125, "loss": 0.9887, "margin_dpo/margin_mean": 69.39735412597656, "margin_dpo/margin_std": 61.032047271728516, "step": 284 }, { "KL/chosen_KL_mean": -114.56338500976562, "KL/mean": -138.91604614257812, "KL/rejected_KL_mean": -163.26869201660156, "KL/std": 62.78680419921875, "epoch": 0.4185022026431718, "fcm_dpo/beta": 0.007360531948506832, "fcm_dpo/delta": 0.042854174971580505, "fcm_dpo/margin": 48.705291748046875, "fcm_dpo/q_t": 0.4151710867881775, "grad_norm": 9.950057983398438, "learning_rate": 3.625819059005228e-07, "logits/chosen": -2.6420364379882812, "logits/rejected": -2.376955032348633, "logps/chosen": -208.99642944335938, "logps/ref_chosen": -94.43303680419922, "logps/ref_rejected": -104.07194519042969, "logps/rejected": -267.34063720703125, "loss": 1.1066, "margin_dpo/margin_mean": 48.705291748046875, "margin_dpo/margin_std": 59.84783935546875, "step": 285 }, { "KL/chosen_KL_mean": -118.46646118164062, "KL/mean": -151.51300048828125, "KL/rejected_KL_mean": -184.55950927734375, "KL/std": 66.89373779296875, "epoch": 0.4199706314243759, "fcm_dpo/beta": 0.00725883524864912, "fcm_dpo/delta": -0.08393802493810654, "fcm_dpo/margin": 66.09306335449219, "fcm_dpo/q_t": 0.38787949085235596, "grad_norm": 8.2915620803833, "learning_rate": 3.614345889441346e-07, "logits/chosen": -2.117314100265503, "logits/rejected": -1.6650042533874512, "logps/chosen": -222.18685913085938, "logps/ref_chosen": -103.72039794921875, "logps/ref_rejected": -96.25775909423828, "logps/rejected": -280.8172607421875, "loss": 1.024, "margin_dpo/margin_mean": 66.09307098388672, "margin_dpo/margin_std": 69.53059387207031, "step": 286 }, { "KL/chosen_KL_mean": -136.50839233398438, "KL/mean": -155.32199096679688, "KL/rejected_KL_mean": -174.13558959960938, "KL/std": 60.80683517456055, "epoch": 0.42143906020558003, "fcm_dpo/beta": 0.007252223789691925, "fcm_dpo/delta": 0.01974192075431347, "fcm_dpo/margin": 37.62720489501953, "fcm_dpo/q_t": 0.43733546137809753, "grad_norm": 8.115185737609863, "learning_rate": 3.6028433558269275e-07, "logits/chosen": -2.132850170135498, "logits/rejected": -1.5235352516174316, "logps/chosen": -230.39828491210938, "logps/ref_chosen": -93.88988494873047, "logps/ref_rejected": -83.33365631103516, "logps/rejected": -257.46923828125, "loss": 1.1979, "margin_dpo/margin_mean": 37.62720489501953, "margin_dpo/margin_std": 70.94268035888672, "step": 287 }, { "KL/chosen_KL_mean": -128.83718872070312, "KL/mean": -163.05276489257812, "KL/rejected_KL_mean": -197.26834106445312, "KL/std": 69.10311126708984, "epoch": 0.42290748898678415, "fcm_dpo/beta": 0.007152985781431198, "fcm_dpo/delta": -0.09431672096252441, "fcm_dpo/margin": 68.43115997314453, "fcm_dpo/q_t": 0.38811296224594116, "grad_norm": 7.692909240722656, "learning_rate": 3.5913117612644327e-07, "logits/chosen": -1.9543590545654297, "logits/rejected": -1.5476887226104736, "logps/chosen": -216.9932098388672, "logps/ref_chosen": -88.15602111816406, "logps/ref_rejected": -93.28195190429688, "logps/rejected": -290.55029296875, "loss": 1.0251, "margin_dpo/margin_mean": 68.43116760253906, "margin_dpo/margin_std": 77.25398254394531, "step": 288 }, { "KL/chosen_KL_mean": -125.1337890625, "KL/mean": -159.77838134765625, "KL/rejected_KL_mean": -194.4229736328125, "KL/std": 72.15738677978516, "epoch": 0.4243759177679883, "fcm_dpo/beta": 0.007046650163829327, "fcm_dpo/delta": -0.09268496930599213, "fcm_dpo/margin": 69.2891845703125, "fcm_dpo/q_t": 0.3900146484375, "grad_norm": 7.289619445800781, "learning_rate": 3.5797514096221024e-07, "logits/chosen": -2.028315544128418, "logits/rejected": -1.8274290561676025, "logps/chosen": -200.52671813964844, "logps/ref_chosen": -75.39292907714844, "logps/ref_rejected": -93.15428161621094, "logps/rejected": -287.5772705078125, "loss": 1.0345, "margin_dpo/margin_mean": 69.28919219970703, "margin_dpo/margin_std": 83.13552856445312, "step": 289 }, { "KL/chosen_KL_mean": -134.67037963867188, "KL/mean": -170.7037353515625, "KL/rejected_KL_mean": -206.73707580566406, "KL/std": 76.6993408203125, "epoch": 0.42584434654919234, "fcm_dpo/beta": 0.00693401787430048, "fcm_dpo/delta": -0.10508199781179428, "fcm_dpo/margin": 72.06668090820312, "fcm_dpo/q_t": 0.3863818049430847, "grad_norm": 8.005448341369629, "learning_rate": 3.568162605525952e-07, "logits/chosen": -1.3975692987442017, "logits/rejected": -1.442676067352295, "logps/chosen": -222.71237182617188, "logps/ref_chosen": -88.04197692871094, "logps/ref_rejected": -123.212158203125, "logps/rejected": -329.94921875, "loss": 1.0212, "margin_dpo/margin_mean": 72.06668090820312, "margin_dpo/margin_std": 80.52359008789062, "step": 290 }, { "KL/chosen_KL_mean": -126.65396881103516, "KL/mean": -157.83328247070312, "KL/rejected_KL_mean": -189.0126190185547, "KL/std": 72.67715454101562, "epoch": 0.42731277533039647, "fcm_dpo/beta": 0.006835309322923422, "fcm_dpo/delta": -0.027408163994550705, "fcm_dpo/margin": 62.35863494873047, "fcm_dpo/q_t": 0.40002545714378357, "grad_norm": 8.919699668884277, "learning_rate": 3.5565456543517485e-07, "logits/chosen": -2.2441418170928955, "logits/rejected": -1.7710058689117432, "logps/chosen": -220.74920654296875, "logps/ref_chosen": -94.09524536132812, "logps/ref_rejected": -96.05006408691406, "logps/rejected": -285.06268310546875, "loss": 1.0625, "margin_dpo/margin_mean": 62.35863494873047, "margin_dpo/margin_std": 72.18458557128906, "step": 291 }, { "KL/chosen_KL_mean": -131.94564819335938, "KL/mean": -163.62689208984375, "KL/rejected_KL_mean": -195.30810546875, "KL/std": 70.27583312988281, "epoch": 0.4287812041116006, "fcm_dpo/beta": 0.006783951073884964, "fcm_dpo/delta": -0.03119458258152008, "fcm_dpo/margin": 63.362449645996094, "fcm_dpo/q_t": 0.39915913343429565, "grad_norm": 8.372907638549805, "learning_rate": 3.5449008622169583e-07, "logits/chosen": -2.0467820167541504, "logits/rejected": -1.6140203475952148, "logps/chosen": -220.19607543945312, "logps/ref_chosen": -88.25041198730469, "logps/ref_rejected": -96.41764068603516, "logps/rejected": -291.72576904296875, "loss": 1.0483, "margin_dpo/margin_mean": 63.362449645996094, "margin_dpo/margin_std": 65.77003479003906, "step": 292 }, { "KL/chosen_KL_mean": -131.47671508789062, "KL/mean": -156.93624877929688, "KL/rejected_KL_mean": -182.39581298828125, "KL/std": 74.41465759277344, "epoch": 0.4302496328928047, "fcm_dpo/beta": 0.006768643390387297, "fcm_dpo/delta": -0.008427482098340988, "fcm_dpo/margin": 50.919090270996094, "fcm_dpo/q_t": 0.42108622193336487, "grad_norm": 7.748796463012695, "learning_rate": 3.5332285359726846e-07, "logits/chosen": -2.3342254161834717, "logits/rejected": -1.762291431427002, "logps/chosen": -218.85325622558594, "logps/ref_chosen": -87.37654113769531, "logps/ref_rejected": -85.75579833984375, "logps/rejected": -268.151611328125, "loss": 1.1374, "margin_dpo/margin_mean": 50.919090270996094, "margin_dpo/margin_std": 78.06587982177734, "step": 293 }, { "KL/chosen_KL_mean": -134.5666046142578, "KL/mean": -155.5863037109375, "KL/rejected_KL_mean": -176.6060333251953, "KL/std": 69.97149658203125, "epoch": 0.43171806167400884, "fcm_dpo/beta": 0.006784706376492977, "fcm_dpo/delta": 0.03208325430750847, "fcm_dpo/margin": 42.03943634033203, "fcm_dpo/q_t": 0.4344545304775238, "grad_norm": 8.094246864318848, "learning_rate": 3.5215289831955786e-07, "logits/chosen": -1.7406806945800781, "logits/rejected": -1.5746405124664307, "logps/chosen": -208.07455444335938, "logps/ref_chosen": -73.50794982910156, "logps/ref_rejected": -88.08877563476562, "logps/rejected": -264.69482421875, "loss": 1.1972, "margin_dpo/margin_mean": 42.03943634033203, "margin_dpo/margin_std": 81.99540710449219, "step": 294 }, { "KL/chosen_KL_mean": -127.7253646850586, "KL/mean": -160.01303100585938, "KL/rejected_KL_mean": -192.30072021484375, "KL/std": 71.88095092773438, "epoch": 0.4331864904552129, "fcm_dpo/beta": 0.006742920260876417, "fcm_dpo/delta": -0.037477537989616394, "fcm_dpo/margin": 64.57534790039062, "fcm_dpo/q_t": 0.39943987131118774, "grad_norm": 8.162626266479492, "learning_rate": 3.509802512179737e-07, "logits/chosen": -1.652780532836914, "logits/rejected": -1.49755859375, "logps/chosen": -205.4908447265625, "logps/ref_chosen": -77.76548767089844, "logps/ref_rejected": -94.24726867675781, "logps/rejected": -286.5479736328125, "loss": 1.0618, "margin_dpo/margin_mean": 64.57534790039062, "margin_dpo/margin_std": 77.56766510009766, "step": 295 }, { "KL/chosen_KL_mean": -137.2755126953125, "KL/mean": -171.8076171875, "KL/rejected_KL_mean": -206.33970642089844, "KL/std": 72.12962341308594, "epoch": 0.434654919236417, "fcm_dpo/beta": 0.006669612601399422, "fcm_dpo/delta": -0.06399206817150116, "fcm_dpo/margin": 69.06419372558594, "fcm_dpo/q_t": 0.3919883668422699, "grad_norm": 11.0608491897583, "learning_rate": 3.498049431928577e-07, "logits/chosen": -2.4091262817382812, "logits/rejected": -1.9138436317443848, "logps/chosen": -235.13192749023438, "logps/ref_chosen": -97.85641479492188, "logps/ref_rejected": -100.81631469726562, "logps/rejected": -307.156005859375, "loss": 1.0342, "margin_dpo/margin_mean": 69.06419372558594, "margin_dpo/margin_std": 72.87110900878906, "step": 296 }, { "KL/chosen_KL_mean": -139.03265380859375, "KL/mean": -168.15167236328125, "KL/rejected_KL_mean": -197.27072143554688, "KL/std": 72.32681274414062, "epoch": 0.43612334801762115, "fcm_dpo/beta": 0.006693677511066198, "fcm_dpo/delta": 0.010481396690011024, "fcm_dpo/margin": 58.23806381225586, "fcm_dpo/q_t": 0.4106457233428955, "grad_norm": 11.073972702026367, "learning_rate": 3.486270052146694e-07, "logits/chosen": -1.2457356452941895, "logits/rejected": -0.9378162622451782, "logps/chosen": -227.59848022460938, "logps/ref_chosen": -88.56583404541016, "logps/ref_rejected": -101.55656433105469, "logps/rejected": -298.8272705078125, "loss": 1.0969, "margin_dpo/margin_mean": 58.23806381225586, "margin_dpo/margin_std": 77.65412902832031, "step": 297 }, { "KL/chosen_KL_mean": -117.33425903320312, "KL/mean": -154.497314453125, "KL/rejected_KL_mean": -191.6603546142578, "KL/std": 78.61767578125, "epoch": 0.43759177679882527, "fcm_dpo/beta": 0.006604637950658798, "fcm_dpo/delta": -0.09547838568687439, "fcm_dpo/margin": 74.32611083984375, "fcm_dpo/q_t": 0.38755857944488525, "grad_norm": 11.828282356262207, "learning_rate": 3.474464683231698e-07, "logits/chosen": -2.0623486042022705, "logits/rejected": -2.0400123596191406, "logps/chosen": -212.21469116210938, "logps/ref_chosen": -94.88043975830078, "logps/ref_rejected": -122.31101989746094, "logps/rejected": -313.97137451171875, "loss": 1.0178, "margin_dpo/margin_mean": 74.32611083984375, "margin_dpo/margin_std": 79.70893096923828, "step": 298 }, { "KL/chosen_KL_mean": -122.30178833007812, "KL/mean": -159.39028930664062, "KL/rejected_KL_mean": -196.478759765625, "KL/std": 75.82727813720703, "epoch": 0.4390602055800294, "fcm_dpo/beta": 0.006472492590546608, "fcm_dpo/delta": -0.08419737219810486, "fcm_dpo/margin": 74.1769790649414, "fcm_dpo/q_t": 0.38858693838119507, "grad_norm": 9.64562702178955, "learning_rate": 3.462633636266041e-07, "logits/chosen": -1.1129204034805298, "logits/rejected": -0.8912756443023682, "logps/chosen": -202.71014404296875, "logps/ref_chosen": -80.40835571289062, "logps/ref_rejected": -89.53717041015625, "logps/rejected": -286.01593017578125, "loss": 1.0237, "margin_dpo/margin_mean": 74.1769790649414, "margin_dpo/margin_std": 79.17498779296875, "step": 299 }, { "KL/chosen_KL_mean": -129.1915740966797, "KL/mean": -159.88795471191406, "KL/rejected_KL_mean": -190.58432006835938, "KL/std": 79.55728912353516, "epoch": 0.44052863436123346, "fcm_dpo/beta": 0.006427213549613953, "fcm_dpo/delta": 0.005332227796316147, "fcm_dpo/margin": 61.39275360107422, "fcm_dpo/q_t": 0.4087502062320709, "grad_norm": 8.07475471496582, "learning_rate": 3.4507772230088147e-07, "logits/chosen": -1.635288119316101, "logits/rejected": -1.4246439933776855, "logps/chosen": -217.35047912597656, "logps/ref_chosen": -88.15890502929688, "logps/ref_rejected": -100.93919372558594, "logps/rejected": -291.52349853515625, "loss": 1.1024, "margin_dpo/margin_mean": 61.39275360107422, "margin_dpo/margin_std": 85.14608764648438, "step": 300 }, { "KL/chosen_KL_mean": -118.31253814697266, "KL/mean": -156.63668823242188, "KL/rejected_KL_mean": -194.9608154296875, "KL/std": 77.95761108398438, "epoch": 0.4419970631424376, "fcm_dpo/beta": 0.006336267106235027, "fcm_dpo/delta": -0.09099871665239334, "fcm_dpo/margin": 76.64825439453125, "fcm_dpo/q_t": 0.3882323205471039, "grad_norm": 9.345137596130371, "learning_rate": 3.4388957558875316e-07, "logits/chosen": -1.7852119207382202, "logits/rejected": -1.355362892150879, "logps/chosen": -203.81744384765625, "logps/ref_chosen": -85.50491333007812, "logps/ref_rejected": -98.91683959960938, "logps/rejected": -293.8776550292969, "loss": 1.0314, "margin_dpo/margin_mean": 76.64825439453125, "margin_dpo/margin_std": 87.937744140625, "step": 301 }, { "KL/chosen_KL_mean": -114.86396026611328, "KL/mean": -147.6676788330078, "KL/rejected_KL_mean": -180.47140502929688, "KL/std": 80.09053802490234, "epoch": 0.4434654919236417, "fcm_dpo/beta": 0.006313342601060867, "fcm_dpo/delta": -0.014902262017130852, "fcm_dpo/margin": 65.60742950439453, "fcm_dpo/q_t": 0.4030223488807678, "grad_norm": 8.341588020324707, "learning_rate": 3.426989547989902e-07, "logits/chosen": -1.2355989217758179, "logits/rejected": -1.059578776359558, "logps/chosen": -194.31436157226562, "logps/ref_chosen": -79.45040893554688, "logps/ref_rejected": -103.0591049194336, "logps/rejected": -283.530517578125, "loss": 1.0722, "margin_dpo/margin_mean": 65.60742950439453, "margin_dpo/margin_std": 78.26306915283203, "step": 302 }, { "KL/chosen_KL_mean": -107.10307312011719, "KL/mean": -142.83676147460938, "KL/rejected_KL_mean": -178.57046508789062, "KL/std": 74.27822875976562, "epoch": 0.44493392070484583, "fcm_dpo/beta": 0.006287604570388794, "fcm_dpo/delta": -0.051686663180589676, "fcm_dpo/margin": 71.46737670898438, "fcm_dpo/q_t": 0.3948175311088562, "grad_norm": 8.15832805633545, "learning_rate": 3.4150589130555773e-07, "logits/chosen": -1.8444539308547974, "logits/rejected": -1.384230375289917, "logps/chosen": -202.7013702392578, "logps/ref_chosen": -95.59829711914062, "logps/ref_rejected": -92.0645751953125, "logps/rejected": -270.6350402832031, "loss": 1.0464, "margin_dpo/margin_mean": 71.4673843383789, "margin_dpo/margin_std": 79.21310424804688, "step": 303 }, { "KL/chosen_KL_mean": -105.83399963378906, "KL/mean": -143.13702392578125, "KL/rejected_KL_mean": -180.44003295898438, "KL/std": 77.14070129394531, "epoch": 0.44640234948604995, "fcm_dpo/beta": 0.006196558475494385, "fcm_dpo/delta": -0.06529698520898819, "fcm_dpo/margin": 74.60604858398438, "fcm_dpo/q_t": 0.3918082118034363, "grad_norm": 8.952920913696289, "learning_rate": 3.403104165467883e-07, "logits/chosen": -2.0361664295196533, "logits/rejected": -1.6098146438598633, "logps/chosen": -205.28713989257812, "logps/ref_chosen": -99.4531478881836, "logps/ref_rejected": -98.08226013183594, "logps/rejected": -278.5223083496094, "loss": 1.0336, "margin_dpo/margin_mean": 74.60604858398438, "margin_dpo/margin_std": 79.76287841796875, "step": 304 }, { "KL/chosen_KL_mean": -115.79327392578125, "KL/mean": -151.1674346923828, "KL/rejected_KL_mean": -186.54159545898438, "KL/std": 80.5308837890625, "epoch": 0.447870778267254, "fcm_dpo/beta": 0.006108911242336035, "fcm_dpo/delta": -0.03442249447107315, "fcm_dpo/margin": 70.74832153320312, "fcm_dpo/q_t": 0.39917391538619995, "grad_norm": 7.5331597328186035, "learning_rate": 3.391125620245535e-07, "logits/chosen": -1.8691487312316895, "logits/rejected": -1.3629894256591797, "logps/chosen": -219.05667114257812, "logps/ref_chosen": -103.26339721679688, "logps/ref_rejected": -97.82897186279297, "logps/rejected": -284.37054443359375, "loss": 1.0702, "margin_dpo/margin_mean": 70.74832153320312, "margin_dpo/margin_std": 86.82716369628906, "step": 305 }, { "KL/chosen_KL_mean": -104.38932800292969, "KL/mean": -145.96383666992188, "KL/rejected_KL_mean": -187.538330078125, "KL/std": 76.4020004272461, "epoch": 0.44933920704845814, "fcm_dpo/beta": 0.006018854212015867, "fcm_dpo/delta": -0.1061825156211853, "fcm_dpo/margin": 83.14901733398438, "fcm_dpo/q_t": 0.3811030685901642, "grad_norm": 9.700840950012207, "learning_rate": 3.3791235930343417e-07, "logits/chosen": -2.1679553985595703, "logits/rejected": -1.6040107011795044, "logps/chosen": -204.6931610107422, "logps/ref_chosen": -100.3038330078125, "logps/ref_rejected": -92.23086547851562, "logps/rejected": -279.76922607421875, "loss": 0.9912, "margin_dpo/margin_mean": 83.14901733398438, "margin_dpo/margin_std": 70.9165267944336, "step": 306 }, { "KL/chosen_KL_mean": -109.17141723632812, "KL/mean": -140.42274475097656, "KL/rejected_KL_mean": -171.67405700683594, "KL/std": 77.14170837402344, "epoch": 0.45080763582966227, "fcm_dpo/beta": 0.006004738621413708, "fcm_dpo/delta": 0.02550465799868107, "fcm_dpo/margin": 62.50264358520508, "fcm_dpo/q_t": 0.4135735034942627, "grad_norm": 8.71324634552002, "learning_rate": 3.367098400098881e-07, "logits/chosen": -1.8640072345733643, "logits/rejected": -1.5111415386199951, "logps/chosen": -205.82118225097656, "logps/ref_chosen": -96.64976501464844, "logps/ref_rejected": -93.34860229492188, "logps/rejected": -265.02264404296875, "loss": 1.1178, "margin_dpo/margin_mean": 62.50264358520508, "margin_dpo/margin_std": 91.26786041259766, "step": 307 }, { "KL/chosen_KL_mean": -114.31552124023438, "KL/mean": -145.63380432128906, "KL/rejected_KL_mean": -176.95208740234375, "KL/std": 78.7996597290039, "epoch": 0.4522760646108664, "fcm_dpo/beta": 0.005973818711936474, "fcm_dpo/delta": -0.04912327229976654, "fcm_dpo/margin": 62.63658142089844, "fcm_dpo/q_t": 0.41376030445098877, "grad_norm": 10.472161293029785, "learning_rate": 3.355050358314172e-07, "logits/chosen": -1.5214544534683228, "logits/rejected": -1.2289180755615234, "logps/chosen": -200.04019165039062, "logps/ref_chosen": -85.72467041015625, "logps/ref_rejected": -86.42317962646484, "logps/rejected": -263.3752746582031, "loss": 1.1221, "margin_dpo/margin_mean": 62.63658142089844, "margin_dpo/margin_std": 91.94395446777344, "step": 308 }, { "KL/chosen_KL_mean": -128.42152404785156, "KL/mean": -156.42520141601562, "KL/rejected_KL_mean": -184.42886352539062, "KL/std": 83.79991149902344, "epoch": 0.45374449339207046, "fcm_dpo/beta": 0.006035213358700275, "fcm_dpo/delta": 0.06410080194473267, "fcm_dpo/margin": 56.00735855102539, "fcm_dpo/q_t": 0.4215248227119446, "grad_norm": 11.60377025604248, "learning_rate": 3.3429797851573183e-07, "logits/chosen": -1.9197332859039307, "logits/rejected": -1.5002105236053467, "logps/chosen": -211.57298278808594, "logps/ref_chosen": -83.15145874023438, "logps/ref_rejected": -86.58602142333984, "logps/rejected": -271.014892578125, "loss": 1.1401, "margin_dpo/margin_mean": 56.00735855102539, "margin_dpo/margin_std": 83.6112060546875, "step": 309 }, { "KL/chosen_KL_mean": -116.37451934814453, "KL/mean": -147.36968994140625, "KL/rejected_KL_mean": -178.3648681640625, "KL/std": 76.15359497070312, "epoch": 0.4552129221732746, "fcm_dpo/beta": 0.006068498827517033, "fcm_dpo/delta": 0.024749252945184708, "fcm_dpo/margin": 61.990352630615234, "fcm_dpo/q_t": 0.41219815611839294, "grad_norm": 8.85814380645752, "learning_rate": 3.3308869986991487e-07, "logits/chosen": -2.462876319885254, "logits/rejected": -2.066950559616089, "logps/chosen": -206.11251831054688, "logps/ref_chosen": -89.73799133300781, "logps/ref_rejected": -87.07083129882812, "logps/rejected": -265.4356994628906, "loss": 1.1014, "margin_dpo/margin_mean": 61.99034881591797, "margin_dpo/margin_std": 80.15214538574219, "step": 310 }, { "KL/chosen_KL_mean": -134.4075927734375, "KL/mean": -174.58982849121094, "KL/rejected_KL_mean": -214.77206420898438, "KL/std": 77.66455841064453, "epoch": 0.4566813509544787, "fcm_dpo/beta": 0.006006065756082535, "fcm_dpo/delta": -0.08682767301797867, "fcm_dpo/margin": 80.36447143554688, "fcm_dpo/q_t": 0.38752785325050354, "grad_norm": 9.148625373840332, "learning_rate": 3.3187723175958346e-07, "logits/chosen": -1.7456867694854736, "logits/rejected": -1.0247340202331543, "logps/chosen": -222.93402099609375, "logps/ref_chosen": -88.52644348144531, "logps/ref_rejected": -82.26608276367188, "logps/rejected": -297.03814697265625, "loss": 1.0124, "margin_dpo/margin_mean": 80.3644790649414, "margin_dpo/margin_std": 79.02332305908203, "step": 311 }, { "KL/chosen_KL_mean": -130.44659423828125, "KL/mean": -163.57403564453125, "KL/rejected_KL_mean": -196.7014923095703, "KL/std": 85.45756530761719, "epoch": 0.4581497797356828, "fcm_dpo/beta": 0.005996899679303169, "fcm_dpo/delta": 0.0025873929262161255, "fcm_dpo/margin": 66.2549057006836, "fcm_dpo/q_t": 0.40717554092407227, "grad_norm": 9.376420974731445, "learning_rate": 3.306636061080487e-07, "logits/chosen": -1.6214861869812012, "logits/rejected": -1.098444938659668, "logps/chosen": -210.8472900390625, "logps/ref_chosen": -80.40069580078125, "logps/ref_rejected": -82.4471435546875, "logps/rejected": -279.14862060546875, "loss": 1.0856, "margin_dpo/margin_mean": 66.2549057006836, "margin_dpo/margin_std": 81.75543975830078, "step": 312 }, { "KL/chosen_KL_mean": -124.4384765625, "KL/mean": -157.04403686523438, "KL/rejected_KL_mean": -189.6496124267578, "KL/std": 83.68032836914062, "epoch": 0.45961820851688695, "fcm_dpo/beta": 0.006004225462675095, "fcm_dpo/delta": 0.008609505370259285, "fcm_dpo/margin": 65.21114349365234, "fcm_dpo/q_t": 0.40929821133613586, "grad_norm": 9.61097526550293, "learning_rate": 3.2944785489547537e-07, "logits/chosen": -2.390892505645752, "logits/rejected": -1.9862465858459473, "logps/chosen": -208.58038330078125, "logps/ref_chosen": -84.14190673828125, "logps/ref_rejected": -93.42684173583984, "logps/rejected": -283.0764465332031, "loss": 1.1017, "margin_dpo/margin_mean": 65.21115112304688, "margin_dpo/margin_std": 89.83334350585938, "step": 313 }, { "KL/chosen_KL_mean": -126.9460678100586, "KL/mean": -158.96875, "KL/rejected_KL_mean": -190.99139404296875, "KL/std": 83.79634094238281, "epoch": 0.461086637298091, "fcm_dpo/beta": 0.005988355726003647, "fcm_dpo/delta": 0.017022665590047836, "fcm_dpo/margin": 64.04533386230469, "fcm_dpo/q_t": 0.4116547703742981, "grad_norm": 10.217790603637695, "learning_rate": 3.2823001015803857e-07, "logits/chosen": -2.0049421787261963, "logits/rejected": -1.7885639667510986, "logps/chosen": -215.75579833984375, "logps/ref_chosen": -88.80972290039062, "logps/ref_rejected": -104.15892028808594, "logps/rejected": -295.15032958984375, "loss": 1.1001, "margin_dpo/margin_mean": 64.04534149169922, "margin_dpo/margin_std": 84.52468872070312, "step": 314 }, { "KL/chosen_KL_mean": -122.24842071533203, "KL/mean": -155.42909240722656, "KL/rejected_KL_mean": -188.60977172851562, "KL/std": 88.0732192993164, "epoch": 0.46255506607929514, "fcm_dpo/beta": 0.006007509771734476, "fcm_dpo/delta": 0.001381636131554842, "fcm_dpo/margin": 66.36135864257812, "fcm_dpo/q_t": 0.4093374013900757, "grad_norm": 8.968510627746582, "learning_rate": 3.270101039870797e-07, "logits/chosen": -1.5781664848327637, "logits/rejected": -1.4049468040466309, "logps/chosen": -203.0421142578125, "logps/ref_chosen": -80.793701171875, "logps/ref_rejected": -92.13711547851562, "logps/rejected": -280.74688720703125, "loss": 1.1129, "margin_dpo/margin_mean": 66.36135864257812, "margin_dpo/margin_std": 100.99237060546875, "step": 315 }, { "KL/chosen_KL_mean": -124.93754577636719, "KL/mean": -160.77418518066406, "KL/rejected_KL_mean": -196.61085510253906, "KL/std": 86.6968002319336, "epoch": 0.46402349486049926, "fcm_dpo/beta": 0.005990965291857719, "fcm_dpo/delta": -0.030701998621225357, "fcm_dpo/margin": 71.67329406738281, "fcm_dpo/q_t": 0.40148815512657166, "grad_norm": 9.289080619812012, "learning_rate": 3.2578816852826086e-07, "logits/chosen": -1.7152100801467896, "logits/rejected": -1.716172695159912, "logps/chosen": -209.12179565429688, "logps/ref_chosen": -84.18424987792969, "logps/ref_rejected": -105.6908187866211, "logps/rejected": -302.3016662597656, "loss": 1.0704, "margin_dpo/margin_mean": 71.67329406738281, "margin_dpo/margin_std": 90.77346801757812, "step": 316 }, { "KL/chosen_KL_mean": -130.26666259765625, "KL/mean": -174.43026733398438, "KL/rejected_KL_mean": -218.5938720703125, "KL/std": 95.59025573730469, "epoch": 0.4654919236417034, "fcm_dpo/beta": 0.005883518606424332, "fcm_dpo/delta": -0.1260424256324768, "fcm_dpo/margin": 88.32722473144531, "fcm_dpo/q_t": 0.38176554441452026, "grad_norm": 10.334203720092773, "learning_rate": 3.2456423598071783e-07, "logits/chosen": -1.7716938257217407, "logits/rejected": -1.2840113639831543, "logps/chosen": -218.14015197753906, "logps/ref_chosen": -87.87348937988281, "logps/ref_rejected": -108.26519775390625, "logps/rejected": -326.85906982421875, "loss": 1.0172, "margin_dpo/margin_mean": 88.32722473144531, "margin_dpo/margin_std": 102.13755798339844, "step": 317 }, { "KL/chosen_KL_mean": -127.30432891845703, "KL/mean": -166.45045471191406, "KL/rejected_KL_mean": -205.59658813476562, "KL/std": 80.25211334228516, "epoch": 0.4669603524229075, "fcm_dpo/beta": 0.005772769451141357, "fcm_dpo/delta": -0.05450741946697235, "fcm_dpo/margin": 78.29225158691406, "fcm_dpo/q_t": 0.39487743377685547, "grad_norm": 10.423051834106445, "learning_rate": 3.233383385962115e-07, "logits/chosen": -2.327599287033081, "logits/rejected": -1.6279561519622803, "logps/chosen": -225.50985717773438, "logps/ref_chosen": -98.20553588867188, "logps/ref_rejected": -88.13629150390625, "logps/rejected": -293.7328796386719, "loss": 1.0458, "margin_dpo/margin_mean": 78.29225158691406, "margin_dpo/margin_std": 88.71526336669922, "step": 318 }, { "KL/chosen_KL_mean": -109.34317016601562, "KL/mean": -152.42117309570312, "KL/rejected_KL_mean": -195.49920654296875, "KL/std": 83.83075714111328, "epoch": 0.4684287812041116, "fcm_dpo/beta": 0.00567377544939518, "fcm_dpo/delta": -0.09361159056425095, "fcm_dpo/margin": 86.1560287475586, "fcm_dpo/q_t": 0.38663536310195923, "grad_norm": 10.118432998657227, "learning_rate": 3.2211050867827805e-07, "logits/chosen": -1.8486950397491455, "logits/rejected": -1.7804803848266602, "logps/chosen": -191.84654235839844, "logps/ref_chosen": -82.50337219238281, "logps/ref_rejected": -118.65068054199219, "logps/rejected": -314.14990234375, "loss": 1.0223, "margin_dpo/margin_mean": 86.15603637695312, "margin_dpo/margin_std": 94.27151489257812, "step": 319 }, { "KL/chosen_KL_mean": -111.82501220703125, "KL/mean": -146.86178588867188, "KL/rejected_KL_mean": -181.89852905273438, "KL/std": 84.91935729980469, "epoch": 0.4698972099853157, "fcm_dpo/beta": 0.005651119165122509, "fcm_dpo/delta": 0.00413152202963829, "fcm_dpo/margin": 70.07352447509766, "fcm_dpo/q_t": 0.40768054127693176, "grad_norm": 8.290468215942383, "learning_rate": 3.208807785813777e-07, "logits/chosen": -1.9670660495758057, "logits/rejected": -1.7768141031265259, "logps/chosen": -198.96224975585938, "logps/ref_chosen": -87.13722229003906, "logps/ref_rejected": -103.22412109375, "logps/rejected": -285.1226501464844, "loss": 1.0807, "margin_dpo/margin_mean": 70.07352447509766, "margin_dpo/margin_std": 82.98087310791016, "step": 320 }, { "KL/chosen_KL_mean": -125.65251922607422, "KL/mean": -166.46595764160156, "KL/rejected_KL_mean": -207.27938842773438, "KL/std": 87.3461685180664, "epoch": 0.4713656387665198, "fcm_dpo/beta": 0.005591265857219696, "fcm_dpo/delta": -0.05955302342772484, "fcm_dpo/margin": 81.62687683105469, "fcm_dpo/q_t": 0.3923654556274414, "grad_norm": 8.765765190124512, "learning_rate": 3.1964918071004217e-07, "logits/chosen": -1.909494400024414, "logits/rejected": -1.5170204639434814, "logps/chosen": -218.75718688964844, "logps/ref_chosen": -93.10466766357422, "logps/ref_rejected": -97.4681396484375, "logps/rejected": -304.7475280761719, "loss": 1.0382, "margin_dpo/margin_mean": 81.62687683105469, "margin_dpo/margin_std": 87.35755920410156, "step": 321 }, { "KL/chosen_KL_mean": -116.93401336669922, "KL/mean": -160.39947509765625, "KL/rejected_KL_mean": -203.8649139404297, "KL/std": 86.72179412841797, "epoch": 0.47283406754772395, "fcm_dpo/beta": 0.005490246694535017, "fcm_dpo/delta": -0.08251510560512543, "fcm_dpo/margin": 86.9309310913086, "fcm_dpo/q_t": 0.3887970447540283, "grad_norm": 13.106282234191895, "learning_rate": 3.184157475180207e-07, "logits/chosen": -1.781909704208374, "logits/rejected": -1.427588701248169, "logps/chosen": -209.12692260742188, "logps/ref_chosen": -92.19291687011719, "logps/ref_rejected": -100.89321899414062, "logps/rejected": -304.75811767578125, "loss": 1.0304, "margin_dpo/margin_mean": 86.9309310913086, "margin_dpo/margin_std": 93.81636810302734, "step": 322 }, { "KL/chosen_KL_mean": -134.53424072265625, "KL/mean": -171.5132293701172, "KL/rejected_KL_mean": -208.49220275878906, "KL/std": 87.30924987792969, "epoch": 0.47430249632892807, "fcm_dpo/beta": 0.005486944690346718, "fcm_dpo/delta": -0.006131318397819996, "fcm_dpo/margin": 73.95794677734375, "fcm_dpo/q_t": 0.4063902199268341, "grad_norm": 11.118532180786133, "learning_rate": 3.171805115074251e-07, "logits/chosen": -2.0952534675598145, "logits/rejected": -1.7223830223083496, "logps/chosen": -214.8063507080078, "logps/ref_chosen": -80.27210235595703, "logps/ref_rejected": -80.3033447265625, "logps/rejected": -288.7955322265625, "loss": 1.0806, "margin_dpo/margin_mean": 73.95794677734375, "margin_dpo/margin_std": 91.36995697021484, "step": 323 }, { "KL/chosen_KL_mean": -134.28904724121094, "KL/mean": -172.3663787841797, "KL/rejected_KL_mean": -210.44369506835938, "KL/std": 87.37994384765625, "epoch": 0.47577092511013214, "fcm_dpo/beta": 0.005488350056111813, "fcm_dpo/delta": -0.018764495849609375, "fcm_dpo/margin": 76.1546401977539, "fcm_dpo/q_t": 0.4014281630516052, "grad_norm": 8.76894474029541, "learning_rate": 3.1594350522787295e-07, "logits/chosen": -1.72676420211792, "logits/rejected": -1.0805652141571045, "logps/chosen": -230.01840209960938, "logps/ref_chosen": -95.72935485839844, "logps/ref_rejected": -94.16924285888672, "logps/rejected": -304.6129150390625, "loss": 1.0603, "margin_dpo/margin_mean": 76.1546401977539, "margin_dpo/margin_std": 82.4627685546875, "step": 324 }, { "KL/chosen_KL_mean": -123.01898193359375, "KL/mean": -164.42840576171875, "KL/rejected_KL_mean": -205.83782958984375, "KL/std": 83.34144592285156, "epoch": 0.47723935389133626, "fcm_dpo/beta": 0.00543711893260479, "fcm_dpo/delta": -0.05263667553663254, "fcm_dpo/margin": 82.81886291503906, "fcm_dpo/q_t": 0.394151896238327, "grad_norm": 7.848616123199463, "learning_rate": 3.147047612756302e-07, "logits/chosen": -1.790490746498108, "logits/rejected": -1.0584759712219238, "logps/chosen": -221.72584533691406, "logps/ref_chosen": -98.70686340332031, "logps/ref_rejected": -76.40809631347656, "logps/rejected": -282.2459411621094, "loss": 1.0391, "margin_dpo/margin_mean": 82.81885528564453, "margin_dpo/margin_std": 86.77519226074219, "step": 325 }, { "KL/chosen_KL_mean": -137.7166748046875, "KL/mean": -174.91812133789062, "KL/rejected_KL_mean": -212.11956787109375, "KL/std": 83.25706481933594, "epoch": 0.4787077826725404, "fcm_dpo/beta": 0.005412403494119644, "fcm_dpo/delta": -0.0028189420700073242, "fcm_dpo/margin": 74.40288543701172, "fcm_dpo/q_t": 0.4054105281829834, "grad_norm": 7.6784586906433105, "learning_rate": 3.134643122927519e-07, "logits/chosen": -2.5124287605285645, "logits/rejected": -1.8499565124511719, "logps/chosen": -242.37799072265625, "logps/ref_chosen": -104.66130828857422, "logps/ref_rejected": -86.82537841796875, "logps/rejected": -298.9449462890625, "loss": 1.08, "margin_dpo/margin_mean": 74.40289306640625, "margin_dpo/margin_std": 89.81488037109375, "step": 326 }, { "KL/chosen_KL_mean": -132.90896606445312, "KL/mean": -168.94619750976562, "KL/rejected_KL_mean": -204.98342895507812, "KL/std": 76.07303619384766, "epoch": 0.4801762114537445, "fcm_dpo/beta": 0.005408043973147869, "fcm_dpo/delta": 0.01050557941198349, "fcm_dpo/margin": 72.07447814941406, "fcm_dpo/q_t": 0.40942442417144775, "grad_norm": 8.35208797454834, "learning_rate": 3.1222219096622264e-07, "logits/chosen": -1.8414480686187744, "logits/rejected": -1.2603416442871094, "logps/chosen": -240.3491973876953, "logps/ref_chosen": -107.44023895263672, "logps/ref_rejected": -117.05323028564453, "logps/rejected": -322.03668212890625, "loss": 1.085, "margin_dpo/margin_mean": 72.0744857788086, "margin_dpo/margin_std": 86.0423583984375, "step": 327 }, { "KL/chosen_KL_mean": -129.3306884765625, "KL/mean": -176.6916046142578, "KL/rejected_KL_mean": -224.05252075195312, "KL/std": 90.3766098022461, "epoch": 0.48164464023494863, "fcm_dpo/beta": 0.0053399354219436646, "fcm_dpo/delta": -0.11138816177845001, "fcm_dpo/margin": 94.72184753417969, "fcm_dpo/q_t": 0.3810993432998657, "grad_norm": 9.81959342956543, "learning_rate": 3.1097843002709427e-07, "logits/chosen": -1.6090867519378662, "logits/rejected": -1.307708740234375, "logps/chosen": -215.94137573242188, "logps/ref_chosen": -86.6106948852539, "logps/ref_rejected": -97.40188598632812, "logps/rejected": -321.45440673828125, "loss": 0.9933, "margin_dpo/margin_mean": 94.72184753417969, "margin_dpo/margin_std": 86.73387145996094, "step": 328 }, { "KL/chosen_KL_mean": -135.27142333984375, "KL/mean": -172.5872802734375, "KL/rejected_KL_mean": -209.9031524658203, "KL/std": 87.32764434814453, "epoch": 0.4831130690161527, "fcm_dpo/beta": 0.005290657747536898, "fcm_dpo/delta": 0.0050959512591362, "fcm_dpo/margin": 74.63172912597656, "fcm_dpo/q_t": 0.40841954946517944, "grad_norm": 9.771174430847168, "learning_rate": 3.0973306224962437e-07, "logits/chosen": -1.3904471397399902, "logits/rejected": -0.8803360462188721, "logps/chosen": -235.88450622558594, "logps/ref_chosen": -100.61308288574219, "logps/ref_rejected": -104.95370483398438, "logps/rejected": -314.85687255859375, "loss": 1.0881, "margin_dpo/margin_mean": 74.63172912597656, "margin_dpo/margin_std": 93.00950622558594, "step": 329 }, { "KL/chosen_KL_mean": -114.90520477294922, "KL/mean": -154.475830078125, "KL/rejected_KL_mean": -194.0464630126953, "KL/std": 84.89775848388672, "epoch": 0.4845814977973568, "fcm_dpo/beta": 0.005290227476507425, "fcm_dpo/delta": -0.01954089291393757, "fcm_dpo/margin": 79.14125061035156, "fcm_dpo/q_t": 0.3999236524105072, "grad_norm": 8.447620391845703, "learning_rate": 3.084861204504122e-07, "logits/chosen": -1.7703384160995483, "logits/rejected": -1.6913669109344482, "logps/chosen": -191.8636474609375, "logps/ref_chosen": -76.95843505859375, "logps/ref_rejected": -92.81494140625, "logps/rejected": -286.86138916015625, "loss": 1.0504, "margin_dpo/margin_mean": 79.14125061035156, "margin_dpo/margin_std": 75.37956237792969, "step": 330 }, { "KL/chosen_KL_mean": -124.10118103027344, "KL/mean": -168.7129669189453, "KL/rejected_KL_mean": -213.32473754882812, "KL/std": 83.27133178710938, "epoch": 0.48604992657856094, "fcm_dpo/beta": 0.005252513103187084, "fcm_dpo/delta": -0.07201417535543442, "fcm_dpo/margin": 89.22357940673828, "fcm_dpo/q_t": 0.3895244896411896, "grad_norm": 9.674551010131836, "learning_rate": 3.072376374875335e-07, "logits/chosen": -1.8069652318954468, "logits/rejected": -1.5011857748031616, "logps/chosen": -207.1605987548828, "logps/ref_chosen": -83.05941772460938, "logps/ref_rejected": -94.13113403320312, "logps/rejected": -307.45587158203125, "loss": 1.0183, "margin_dpo/margin_mean": 89.22357940673828, "margin_dpo/margin_std": 83.74002075195312, "step": 331 }, { "KL/chosen_KL_mean": -132.55140686035156, "KL/mean": -163.17306518554688, "KL/rejected_KL_mean": -193.79473876953125, "KL/std": 91.64155578613281, "epoch": 0.48751835535976507, "fcm_dpo/beta": 0.00528405886143446, "fcm_dpo/delta": 0.07880137115716934, "fcm_dpo/margin": 61.24333953857422, "fcm_dpo/q_t": 0.4242613911628723, "grad_norm": 7.782529354095459, "learning_rate": 3.059876462596758e-07, "logits/chosen": -1.995274543762207, "logits/rejected": -1.5595709085464478, "logps/chosen": -214.4822998046875, "logps/ref_chosen": -81.93089294433594, "logps/ref_rejected": -82.84590148925781, "logps/rejected": -276.640625, "loss": 1.1426, "margin_dpo/margin_mean": 61.24333953857422, "margin_dpo/margin_std": 89.94986724853516, "step": 332 }, { "KL/chosen_KL_mean": -153.6798095703125, "KL/mean": -196.93731689453125, "KL/rejected_KL_mean": -240.19480895996094, "KL/std": 96.05758666992188, "epoch": 0.4889867841409692, "fcm_dpo/beta": 0.0052257683128118515, "fcm_dpo/delta": -0.055502861738204956, "fcm_dpo/margin": 86.51498413085938, "fcm_dpo/q_t": 0.3956648111343384, "grad_norm": 8.08462142944336, "learning_rate": 3.0473617970527015e-07, "logits/chosen": -1.3471949100494385, "logits/rejected": -0.9994245767593384, "logps/chosen": -242.21240234375, "logps/ref_chosen": -88.53257751464844, "logps/ref_rejected": -102.46601867675781, "logps/rejected": -342.66082763671875, "loss": 1.055, "margin_dpo/margin_mean": 86.51498413085938, "margin_dpo/margin_std": 103.09303283691406, "step": 333 }, { "KL/chosen_KL_mean": -139.8485107421875, "KL/mean": -177.95394897460938, "KL/rejected_KL_mean": -216.0594024658203, "KL/std": 90.49869537353516, "epoch": 0.49045521292217326, "fcm_dpo/beta": 0.005182007793337107, "fcm_dpo/delta": -0.055397916585206985, "fcm_dpo/margin": 76.21089935302734, "fcm_dpo/q_t": 0.40680527687072754, "grad_norm": 8.630681037902832, "learning_rate": 3.034832708016243e-07, "logits/chosen": -1.450178861618042, "logits/rejected": -1.1560436487197876, "logps/chosen": -237.60629272460938, "logps/ref_chosen": -97.75778198242188, "logps/ref_rejected": -100.0058364868164, "logps/rejected": -316.06524658203125, "loss": 1.0976, "margin_dpo/margin_mean": 76.21089935302734, "margin_dpo/margin_std": 100.74238586425781, "step": 334 }, { "KL/chosen_KL_mean": -138.5660400390625, "KL/mean": -171.50950622558594, "KL/rejected_KL_mean": -204.45297241210938, "KL/std": 90.08235168457031, "epoch": 0.4919236417033774, "fcm_dpo/beta": 0.0052277748472988605, "fcm_dpo/delta": 0.057526759803295135, "fcm_dpo/margin": 65.88694763183594, "fcm_dpo/q_t": 0.4200003743171692, "grad_norm": 12.09694766998291, "learning_rate": 3.022289525640531e-07, "logits/chosen": -2.28955078125, "logits/rejected": -1.8641899824142456, "logps/chosen": -234.94207763671875, "logps/ref_chosen": -96.37603759765625, "logps/ref_rejected": -94.92401123046875, "logps/rejected": -299.3769836425781, "loss": 1.1348, "margin_dpo/margin_mean": 65.8869400024414, "margin_dpo/margin_std": 98.77403259277344, "step": 335 }, { "KL/chosen_KL_mean": -142.24081420898438, "KL/mean": -184.5833282470703, "KL/rejected_KL_mean": -226.9258270263672, "KL/std": 92.70210266113281, "epoch": 0.4933920704845815, "fcm_dpo/beta": 0.005219377111643553, "fcm_dpo/delta": -0.043948303908109665, "fcm_dpo/margin": 84.68500518798828, "fcm_dpo/q_t": 0.3972393870353699, "grad_norm": 10.240348815917969, "learning_rate": 3.009732580450086e-07, "logits/chosen": -1.8144869804382324, "logits/rejected": -1.6664538383483887, "logps/chosen": -230.77035522460938, "logps/ref_chosen": -88.52952575683594, "logps/ref_rejected": -109.5557861328125, "logps/rejected": -336.48162841796875, "loss": 1.0528, "margin_dpo/margin_mean": 84.68501281738281, "margin_dpo/margin_std": 98.26302337646484, "step": 336 }, { "KL/chosen_KL_mean": -139.7945098876953, "KL/mean": -180.14407348632812, "KL/rejected_KL_mean": -220.4936065673828, "KL/std": 93.29644775390625, "epoch": 0.4948604992657856, "fcm_dpo/beta": 0.005189026705920696, "fcm_dpo/delta": -0.019588202238082886, "fcm_dpo/margin": 80.6991195678711, "fcm_dpo/q_t": 0.4038439095020294, "grad_norm": 8.789505004882812, "learning_rate": 2.9971622033320914e-07, "logits/chosen": -2.2602341175079346, "logits/rejected": -1.9733288288116455, "logps/chosen": -237.10690307617188, "logps/ref_chosen": -97.3123779296875, "logps/ref_rejected": -110.13134765625, "logps/rejected": -330.62493896484375, "loss": 1.0863, "margin_dpo/margin_mean": 80.69911193847656, "margin_dpo/margin_std": 109.38632202148438, "step": 337 }, { "KL/chosen_KL_mean": -145.0054473876953, "KL/mean": -181.9836883544922, "KL/rejected_KL_mean": -218.96194458007812, "KL/std": 94.61917114257812, "epoch": 0.49632892804698975, "fcm_dpo/beta": 0.005185029469430447, "fcm_dpo/delta": 0.01719367876648903, "fcm_dpo/margin": 73.95649719238281, "fcm_dpo/q_t": 0.40975597500801086, "grad_norm": 8.891013145446777, "learning_rate": 2.984578725527675e-07, "logits/chosen": -1.9223698377609253, "logits/rejected": -1.7721664905548096, "logps/chosen": -228.63555908203125, "logps/ref_chosen": -83.63011169433594, "logps/ref_rejected": -93.69235229492188, "logps/rejected": -312.654296875, "loss": 1.0946, "margin_dpo/margin_mean": 73.95649719238281, "margin_dpo/margin_std": 92.86002349853516, "step": 338 }, { "KL/chosen_KL_mean": -154.381591796875, "KL/mean": -193.88868713378906, "KL/rejected_KL_mean": -233.39578247070312, "KL/std": 93.60277557373047, "epoch": 0.4977973568281938, "fcm_dpo/beta": 0.005157374776899815, "fcm_dpo/delta": -0.07026338577270508, "fcm_dpo/margin": 79.01419830322266, "fcm_dpo/q_t": 0.40499579906463623, "grad_norm": 7.875832557678223, "learning_rate": 2.9719824786231796e-07, "logits/chosen": -2.87162709236145, "logits/rejected": -2.442060708999634, "logps/chosen": -254.2333984375, "logps/ref_chosen": -99.85182189941406, "logps/ref_rejected": -109.0743637084961, "logps/rejected": -342.47015380859375, "loss": 1.0918, "margin_dpo/margin_mean": 79.01419830322266, "margin_dpo/margin_std": 104.06044006347656, "step": 339 }, { "KL/chosen_KL_mean": -166.36679077148438, "KL/mean": -200.3663787841797, "KL/rejected_KL_mean": -234.365966796875, "KL/std": 85.84724426269531, "epoch": 0.49926578560939794, "fcm_dpo/beta": 0.0051616826094686985, "fcm_dpo/delta": 0.05077475309371948, "fcm_dpo/margin": 67.99917602539062, "fcm_dpo/q_t": 0.4177219569683075, "grad_norm": 9.743352890014648, "learning_rate": 2.959373794541426e-07, "logits/chosen": -1.3466472625732422, "logits/rejected": -0.8567553758621216, "logps/chosen": -251.34075927734375, "logps/ref_chosen": -84.97396850585938, "logps/ref_rejected": -78.80033874511719, "logps/rejected": -313.16632080078125, "loss": 1.1216, "margin_dpo/margin_mean": 67.99917602539062, "margin_dpo/margin_std": 94.52733612060547, "step": 340 }, { "KL/chosen_KL_mean": -173.53192138671875, "KL/mean": -206.74200439453125, "KL/rejected_KL_mean": -239.95208740234375, "KL/std": 89.88375854492188, "epoch": 0.5007342143906021, "fcm_dpo/beta": 0.005205621011555195, "fcm_dpo/delta": 0.05621008947491646, "fcm_dpo/margin": 66.42013549804688, "fcm_dpo/q_t": 0.4203876256942749, "grad_norm": 9.62618637084961, "learning_rate": 2.946753005532965e-07, "logits/chosen": -1.385262131690979, "logits/rejected": -1.1342041492462158, "logps/chosen": -258.7958984375, "logps/ref_chosen": -85.26399230957031, "logps/ref_rejected": -106.08595275878906, "logps/rejected": -346.03802490234375, "loss": 1.1403, "margin_dpo/margin_mean": 66.4201431274414, "margin_dpo/margin_std": 103.22789001464844, "step": 341 }, { "KL/chosen_KL_mean": -138.84603881835938, "KL/mean": -187.59945678710938, "KL/rejected_KL_mean": -236.3529052734375, "KL/std": 90.45417022705078, "epoch": 0.5022026431718062, "fcm_dpo/beta": 0.0051222434267401695, "fcm_dpo/delta": -0.10586446523666382, "fcm_dpo/margin": 97.50687408447266, "fcm_dpo/q_t": 0.3817700743675232, "grad_norm": 7.682787895202637, "learning_rate": 2.934120444167326e-07, "logits/chosen": -1.290515661239624, "logits/rejected": -0.5167274475097656, "logps/chosen": -239.88462829589844, "logps/ref_chosen": -101.03860473632812, "logps/ref_rejected": -98.89851379394531, "logps/rejected": -335.25140380859375, "loss": 1.0104, "margin_dpo/margin_mean": 97.50688171386719, "margin_dpo/margin_std": 96.39700317382812, "step": 342 }, { "KL/chosen_KL_mean": -151.55612182617188, "KL/mean": -191.76034545898438, "KL/rejected_KL_mean": -231.9645538330078, "KL/std": 94.27001953125, "epoch": 0.5036710719530103, "fcm_dpo/beta": 0.005087699741125107, "fcm_dpo/delta": -0.06798040866851807, "fcm_dpo/margin": 80.40845489501953, "fcm_dpo/q_t": 0.404390424489975, "grad_norm": 9.761009216308594, "learning_rate": 2.9214764433242476e-07, "logits/chosen": -2.1991000175476074, "logits/rejected": -2.0898547172546387, "logps/chosen": -235.30348205566406, "logps/ref_chosen": -83.74736022949219, "logps/ref_rejected": -116.57908630371094, "logps/rejected": -348.54364013671875, "loss": 1.0825, "margin_dpo/margin_mean": 80.408447265625, "margin_dpo/margin_std": 98.9737548828125, "step": 343 }, { "KL/chosen_KL_mean": -150.92111206054688, "KL/mean": -193.98861694335938, "KL/rejected_KL_mean": -237.05615234375, "KL/std": 99.92767333984375, "epoch": 0.5051395007342144, "fcm_dpo/beta": 0.005051136948168278, "fcm_dpo/delta": -0.03701796382665634, "fcm_dpo/margin": 86.13502502441406, "fcm_dpo/q_t": 0.39953723549842834, "grad_norm": 9.656281471252441, "learning_rate": 2.9088213361849126e-07, "logits/chosen": -1.4665610790252686, "logits/rejected": -1.212233304977417, "logps/chosen": -229.17507934570312, "logps/ref_chosen": -78.25396728515625, "logps/ref_rejected": -95.42870330810547, "logps/rejected": -332.48486328125, "loss": 1.0674, "margin_dpo/margin_mean": 86.13502502441406, "margin_dpo/margin_std": 105.87095642089844, "step": 344 }, { "KL/chosen_KL_mean": -155.3314208984375, "KL/mean": -200.91249084472656, "KL/rejected_KL_mean": -246.49356079101562, "KL/std": 100.1898193359375, "epoch": 0.5066079295154186, "fcm_dpo/beta": 0.004966835491359234, "fcm_dpo/delta": -0.055428341031074524, "fcm_dpo/margin": 91.16216278076172, "fcm_dpo/q_t": 0.39493459463119507, "grad_norm": 9.278047561645508, "learning_rate": 2.896155456223163e-07, "logits/chosen": -1.5918264389038086, "logits/rejected": -1.2987147569656372, "logps/chosen": -253.41090393066406, "logps/ref_chosen": -98.0794906616211, "logps/ref_rejected": -104.03477478027344, "logps/rejected": -350.5283203125, "loss": 1.0457, "margin_dpo/margin_mean": 91.16215515136719, "margin_dpo/margin_std": 103.647216796875, "step": 345 }, { "KL/chosen_KL_mean": -172.59291076660156, "KL/mean": -207.95062255859375, "KL/rejected_KL_mean": -243.30833435058594, "KL/std": 93.20742797851562, "epoch": 0.5080763582966226, "fcm_dpo/beta": 0.004995340947061777, "fcm_dpo/delta": 0.04847484827041626, "fcm_dpo/margin": 70.71542358398438, "fcm_dpo/q_t": 0.4186241626739502, "grad_norm": 7.907067775726318, "learning_rate": 2.883479137196714e-07, "logits/chosen": -1.9634160995483398, "logits/rejected": -1.6597533226013184, "logps/chosen": -253.27639770507812, "logps/ref_chosen": -80.68348693847656, "logps/ref_rejected": -85.89260864257812, "logps/rejected": -329.200927734375, "loss": 1.1313, "margin_dpo/margin_mean": 70.71542358398438, "margin_dpo/margin_std": 107.04525756835938, "step": 346 }, { "KL/chosen_KL_mean": -163.02511596679688, "KL/mean": -196.81716918945312, "KL/rejected_KL_mean": -230.60923767089844, "KL/std": 92.2974853515625, "epoch": 0.5095447870778267, "fcm_dpo/beta": 0.005008503794670105, "fcm_dpo/delta": -0.0007321417797356844, "fcm_dpo/margin": 67.58413696289062, "fcm_dpo/q_t": 0.42133665084838867, "grad_norm": 8.228876113891602, "learning_rate": 2.8707927131383614e-07, "logits/chosen": -1.9228236675262451, "logits/rejected": -1.6043643951416016, "logps/chosen": -256.7044677734375, "logps/ref_chosen": -93.67938232421875, "logps/ref_rejected": -97.54251098632812, "logps/rejected": -328.1517333984375, "loss": 1.1363, "margin_dpo/margin_mean": 67.58413696289062, "margin_dpo/margin_std": 99.87893676757812, "step": 347 }, { "KL/chosen_KL_mean": -140.70156860351562, "KL/mean": -179.17735290527344, "KL/rejected_KL_mean": -217.65313720703125, "KL/std": 86.80039978027344, "epoch": 0.5110132158590308, "fcm_dpo/beta": 0.004934805445373058, "fcm_dpo/delta": -0.07375401258468628, "fcm_dpo/margin": 76.95158386230469, "fcm_dpo/q_t": 0.40964022278785706, "grad_norm": 7.939823150634766, "learning_rate": 2.858096518347179e-07, "logits/chosen": -1.9222362041473389, "logits/rejected": -1.6479917764663696, "logps/chosen": -230.95379638671875, "logps/ref_chosen": -90.25224304199219, "logps/ref_rejected": -95.77027893066406, "logps/rejected": -313.42340087890625, "loss": 1.0937, "margin_dpo/margin_mean": 76.95158386230469, "margin_dpo/margin_std": 88.59011840820312, "step": 348 }, { "KL/chosen_KL_mean": -152.14627075195312, "KL/mean": -187.83560180664062, "KL/rejected_KL_mean": -223.52493286132812, "KL/std": 95.50184631347656, "epoch": 0.5124816446402349, "fcm_dpo/beta": 0.004928614944219589, "fcm_dpo/delta": -0.012560423463582993, "fcm_dpo/margin": 71.37866973876953, "fcm_dpo/q_t": 0.4180968403816223, "grad_norm": 9.119296073913574, "learning_rate": 2.845390887379706e-07, "logits/chosen": -2.0517802238464355, "logits/rejected": -1.826784372329712, "logps/chosen": -244.20436096191406, "logps/ref_chosen": -92.05809020996094, "logps/ref_rejected": -103.96527099609375, "logps/rejected": -327.4902038574219, "loss": 1.1233, "margin_dpo/margin_mean": 71.37867736816406, "margin_dpo/margin_std": 99.10256958007812, "step": 349 }, { "KL/chosen_KL_mean": -160.86972045898438, "KL/mean": -191.12863159179688, "KL/rejected_KL_mean": -221.38754272460938, "KL/std": 87.87226867675781, "epoch": 0.5139500734214391, "fcm_dpo/beta": 0.0049631791189312935, "fcm_dpo/delta": 0.041226498782634735, "fcm_dpo/margin": 60.51781463623047, "fcm_dpo/q_t": 0.4296115040779114, "grad_norm": 10.321072578430176, "learning_rate": 2.8326761550411346e-07, "logits/chosen": -1.853127121925354, "logits/rejected": -1.6269001960754395, "logps/chosen": -247.92391967773438, "logps/ref_chosen": -87.05419921875, "logps/ref_rejected": -95.72193145751953, "logps/rejected": -317.1094665527344, "loss": 1.1631, "margin_dpo/margin_mean": 60.51781463623047, "margin_dpo/margin_std": 95.85125732421875, "step": 350 }, { "KL/chosen_KL_mean": -160.72219848632812, "KL/mean": -203.95358276367188, "KL/rejected_KL_mean": -247.18499755859375, "KL/std": 98.07379913330078, "epoch": 0.5154185022026432, "fcm_dpo/beta": 0.004950874485075474, "fcm_dpo/delta": -0.02933877892792225, "fcm_dpo/margin": 86.46279907226562, "fcm_dpo/q_t": 0.40006253123283386, "grad_norm": 8.126477241516113, "learning_rate": 2.819952656376487e-07, "logits/chosen": -1.7150453329086304, "logits/rejected": -1.4142982959747314, "logps/chosen": -249.8912353515625, "logps/ref_chosen": -89.16903686523438, "logps/ref_rejected": -108.29801940917969, "logps/rejected": -355.4830322265625, "loss": 1.0666, "margin_dpo/margin_mean": 86.46279907226562, "margin_dpo/margin_std": 104.249755859375, "step": 351 }, { "KL/chosen_KL_mean": -147.03170776367188, "KL/mean": -187.869140625, "KL/rejected_KL_mean": -228.70657348632812, "KL/std": 96.88475036621094, "epoch": 0.5168869309838473, "fcm_dpo/beta": 0.004932219162583351, "fcm_dpo/delta": -0.0029572767671197653, "fcm_dpo/margin": 81.67485046386719, "fcm_dpo/q_t": 0.4076157212257385, "grad_norm": 8.16756820678711, "learning_rate": 2.8072207266617854e-07, "logits/chosen": -2.105515480041504, "logits/rejected": -1.5747778415679932, "logps/chosen": -243.84320068359375, "logps/ref_chosen": -96.81147766113281, "logps/ref_rejected": -84.40483093261719, "logps/rejected": -313.11138916015625, "loss": 1.0876, "margin_dpo/margin_mean": 81.67485046386719, "margin_dpo/margin_std": 104.96588897705078, "step": 352 }, { "KL/chosen_KL_mean": -168.7980194091797, "KL/mean": -195.0948944091797, "KL/rejected_KL_mean": -221.39175415039062, "KL/std": 85.62964630126953, "epoch": 0.5183553597650514, "fcm_dpo/beta": 0.004969088826328516, "fcm_dpo/delta": 0.03824581205844879, "fcm_dpo/margin": 52.59373092651367, "fcm_dpo/q_t": 0.43823564052581787, "grad_norm": 9.168176651000977, "learning_rate": 2.794480701395219e-07, "logits/chosen": -2.0197463035583496, "logits/rejected": -1.6257328987121582, "logps/chosen": -254.16050720214844, "logps/ref_chosen": -85.36248779296875, "logps/ref_rejected": -87.36732482910156, "logps/rejected": -308.75909423828125, "loss": 1.1997, "margin_dpo/margin_mean": 52.59373092651367, "margin_dpo/margin_std": 96.77818298339844, "step": 353 }, { "KL/chosen_KL_mean": -147.91342163085938, "KL/mean": -188.85592651367188, "KL/rejected_KL_mean": -229.7984619140625, "KL/std": 86.54969787597656, "epoch": 0.5198237885462555, "fcm_dpo/beta": 0.004973916336894035, "fcm_dpo/delta": -0.00767357274889946, "fcm_dpo/margin": 81.8850326538086, "fcm_dpo/q_t": 0.4037133455276489, "grad_norm": 7.6839118003845215, "learning_rate": 2.781732916288303e-07, "logits/chosen": -1.8105194568634033, "logits/rejected": -1.3358511924743652, "logps/chosen": -243.71458435058594, "logps/ref_chosen": -95.8011703491211, "logps/ref_rejected": -94.86614227294922, "logps/rejected": -324.66461181640625, "loss": 1.058, "margin_dpo/margin_mean": 81.8850326538086, "margin_dpo/margin_std": 80.45028686523438, "step": 354 }, { "KL/chosen_KL_mean": -146.56759643554688, "KL/mean": -184.26931762695312, "KL/rejected_KL_mean": -221.97100830078125, "KL/std": 88.38886260986328, "epoch": 0.5212922173274597, "fcm_dpo/beta": 0.004964661784470081, "fcm_dpo/delta": 0.026508353650569916, "fcm_dpo/margin": 75.40342712402344, "fcm_dpo/q_t": 0.4120803773403168, "grad_norm": 9.448277473449707, "learning_rate": 2.7689777072570284e-07, "logits/chosen": -2.308893918991089, "logits/rejected": -1.8136093616485596, "logps/chosen": -232.75314331054688, "logps/ref_chosen": -86.185546875, "logps/ref_rejected": -88.4615707397461, "logps/rejected": -310.4325866699219, "loss": 1.1017, "margin_dpo/margin_mean": 75.40342712402344, "margin_dpo/margin_std": 94.25363159179688, "step": 355 }, { "KL/chosen_KL_mean": -172.5458984375, "KL/mean": -202.02325439453125, "KL/rejected_KL_mean": -231.50059509277344, "KL/std": 86.2015609741211, "epoch": 0.5227606461086637, "fcm_dpo/beta": 0.005002472549676895, "fcm_dpo/delta": 0.029246121644973755, "fcm_dpo/margin": 58.95470428466797, "fcm_dpo/q_t": 0.43088269233703613, "grad_norm": 8.20483684539795, "learning_rate": 2.7562154104130176e-07, "logits/chosen": -1.788015604019165, "logits/rejected": -1.3173956871032715, "logps/chosen": -263.364013671875, "logps/ref_chosen": -90.818115234375, "logps/ref_rejected": -82.1273193359375, "logps/rejected": -313.6279296875, "loss": 1.1721, "margin_dpo/margin_mean": 58.95470428466797, "margin_dpo/margin_std": 98.95693969726562, "step": 356 }, { "KL/chosen_KL_mean": -145.1062774658203, "KL/mean": -191.907958984375, "KL/rejected_KL_mean": -238.70965576171875, "KL/std": 94.88236999511719, "epoch": 0.5242290748898678, "fcm_dpo/beta": 0.004938648547977209, "fcm_dpo/delta": -0.06639204919338226, "fcm_dpo/margin": 93.60336303710938, "fcm_dpo/q_t": 0.3908548951148987, "grad_norm": 7.912759780883789, "learning_rate": 2.7434463620546594e-07, "logits/chosen": -1.7317296266555786, "logits/rejected": -1.3505488634109497, "logps/chosen": -232.61102294921875, "logps/ref_chosen": -87.5047607421875, "logps/ref_rejected": -92.24937438964844, "logps/rejected": -330.9590148925781, "loss": 1.0244, "margin_dpo/margin_mean": 93.60336303710938, "margin_dpo/margin_std": 88.33078002929688, "step": 357 }, { "KL/chosen_KL_mean": -155.79161071777344, "KL/mean": -190.37872314453125, "KL/rejected_KL_mean": -224.96585083007812, "KL/std": 87.9743423461914, "epoch": 0.5256975036710719, "fcm_dpo/beta": 0.004991848021745682, "fcm_dpo/delta": 0.05666042119264603, "fcm_dpo/margin": 69.17424011230469, "fcm_dpo/q_t": 0.42042723298072815, "grad_norm": 8.212221145629883, "learning_rate": 2.730670898658255e-07, "logits/chosen": -1.6254724264144897, "logits/rejected": -1.1543527841567993, "logps/chosen": -250.84707641601562, "logps/ref_chosen": -95.05546569824219, "logps/ref_rejected": -96.47611999511719, "logps/rejected": -321.44195556640625, "loss": 1.1222, "margin_dpo/margin_mean": 69.17424011230469, "margin_dpo/margin_std": 94.25694274902344, "step": 358 }, { "KL/chosen_KL_mean": -175.09396362304688, "KL/mean": -207.33108520507812, "KL/rejected_KL_mean": -239.56820678710938, "KL/std": 82.54708862304688, "epoch": 0.527165932452276, "fcm_dpo/beta": 0.004995310213416815, "fcm_dpo/delta": -0.012033559381961823, "fcm_dpo/margin": 64.47423553466797, "fcm_dpo/q_t": 0.4248253107070923, "grad_norm": 9.227970123291016, "learning_rate": 2.717889356869146e-07, "logits/chosen": -1.5680854320526123, "logits/rejected": -0.9995710849761963, "logps/chosen": -262.46728515625, "logps/ref_chosen": -87.37332916259766, "logps/ref_rejected": -88.77188110351562, "logps/rejected": -328.340087890625, "loss": 1.1479, "margin_dpo/margin_mean": 64.47423553466797, "margin_dpo/margin_std": 96.67312622070312, "step": 359 }, { "KL/chosen_KL_mean": -159.5872802734375, "KL/mean": -200.59890747070312, "KL/rejected_KL_mean": -241.6105499267578, "KL/std": 83.31660461425781, "epoch": 0.5286343612334802, "fcm_dpo/beta": 0.004984111059457064, "fcm_dpo/delta": -0.009212283417582512, "fcm_dpo/margin": 82.02326965332031, "fcm_dpo/q_t": 0.4031431972980499, "grad_norm": 13.064961433410645, "learning_rate": 2.7051020734928443e-07, "logits/chosen": -1.4418290853500366, "logits/rejected": -1.0884792804718018, "logps/chosen": -234.81072998046875, "logps/ref_chosen": -75.22344970703125, "logps/ref_rejected": -76.09400177001953, "logps/rejected": -317.70452880859375, "loss": 1.0648, "margin_dpo/margin_mean": 82.02326202392578, "margin_dpo/margin_std": 88.16264343261719, "step": 360 }, { "KL/chosen_KL_mean": -162.84571838378906, "KL/mean": -199.18038940429688, "KL/rejected_KL_mean": -235.51507568359375, "KL/std": 95.06013488769531, "epoch": 0.5301027900146843, "fcm_dpo/beta": 0.005016217939555645, "fcm_dpo/delta": 0.0367913693189621, "fcm_dpo/margin": 72.66934967041016, "fcm_dpo/q_t": 0.4160331189632416, "grad_norm": 9.651906967163086, "learning_rate": 2.6923093854861593e-07, "logits/chosen": -1.5508899688720703, "logits/rejected": -1.30221688747406, "logps/chosen": -244.79904174804688, "logps/ref_chosen": -81.95333099365234, "logps/ref_rejected": -96.67919921875, "logps/rejected": -332.19427490234375, "loss": 1.1157, "margin_dpo/margin_mean": 72.66934204101562, "margin_dpo/margin_std": 101.4786376953125, "step": 361 }, { "KL/chosen_KL_mean": -167.842041015625, "KL/mean": -206.40011596679688, "KL/rejected_KL_mean": -244.95822143554688, "KL/std": 89.92999267578125, "epoch": 0.5315712187958884, "fcm_dpo/beta": 0.005022239871323109, "fcm_dpo/delta": 0.013153315521776676, "fcm_dpo/margin": 77.11618041992188, "fcm_dpo/q_t": 0.4091334342956543, "grad_norm": 8.851387023925781, "learning_rate": 2.679511629948319e-07, "logits/chosen": -1.782833218574524, "logits/rejected": -1.5787432193756104, "logps/chosen": -257.1817626953125, "logps/ref_chosen": -89.33971405029297, "logps/ref_rejected": -110.50196838378906, "logps/rejected": -355.460205078125, "loss": 1.0929, "margin_dpo/margin_mean": 77.11618041992188, "margin_dpo/margin_std": 96.0059814453125, "step": 362 }, { "KL/chosen_KL_mean": -157.31036376953125, "KL/mean": -206.67315673828125, "KL/rejected_KL_mean": -256.0359191894531, "KL/std": 94.60166931152344, "epoch": 0.5330396475770925, "fcm_dpo/beta": 0.004984427243471146, "fcm_dpo/delta": -0.09679950773715973, "fcm_dpo/margin": 98.72555541992188, "fcm_dpo/q_t": 0.3860166668891907, "grad_norm": 11.03801441192627, "learning_rate": 2.6667091441120816e-07, "logits/chosen": -1.9064676761627197, "logits/rejected": -1.480465292930603, "logps/chosen": -238.0853729248047, "logps/ref_chosen": -80.77499389648438, "logps/ref_rejected": -79.43267822265625, "logps/rejected": -335.4685974121094, "loss": 1.0159, "margin_dpo/margin_mean": 98.72555541992188, "margin_dpo/margin_std": 103.05851745605469, "step": 363 }, { "KL/chosen_KL_mean": -155.9902801513672, "KL/mean": -195.59634399414062, "KL/rejected_KL_mean": -235.202392578125, "KL/std": 87.65835571289062, "epoch": 0.5345080763582967, "fcm_dpo/beta": 0.004912791773676872, "fcm_dpo/delta": -0.05458861589431763, "fcm_dpo/margin": 79.21211242675781, "fcm_dpo/q_t": 0.40737155079841614, "grad_norm": 11.701790809631348, "learning_rate": 2.6539022653348575e-07, "logits/chosen": -2.0065109729766846, "logits/rejected": -1.864483118057251, "logps/chosen": -232.51988220214844, "logps/ref_chosen": -76.52960205078125, "logps/ref_rejected": -97.56173706054688, "logps/rejected": -332.7641296386719, "loss": 1.0787, "margin_dpo/margin_mean": 79.21211242675781, "margin_dpo/margin_std": 83.88143920898438, "step": 364 }, { "KL/chosen_KL_mean": -150.7875213623047, "KL/mean": -194.66790771484375, "KL/rejected_KL_mean": -238.54830932617188, "KL/std": 94.74967956542969, "epoch": 0.5359765051395007, "fcm_dpo/beta": 0.004881038796156645, "fcm_dpo/delta": -0.029769405722618103, "fcm_dpo/margin": 87.76078033447266, "fcm_dpo/q_t": 0.4015900790691376, "grad_norm": 8.28507137298584, "learning_rate": 2.641091331089811e-07, "logits/chosen": -1.6776100397109985, "logits/rejected": -1.5518178939819336, "logps/chosen": -231.33241271972656, "logps/ref_chosen": -80.54489135742188, "logps/ref_rejected": -99.32316589355469, "logps/rejected": -337.8714599609375, "loss": 1.0689, "margin_dpo/margin_mean": 87.76078033447266, "margin_dpo/margin_std": 108.048095703125, "step": 365 }, { "KL/chosen_KL_mean": -152.82333374023438, "KL/mean": -188.84579467773438, "KL/rejected_KL_mean": -224.86822509765625, "KL/std": 98.29963684082031, "epoch": 0.5374449339207048, "fcm_dpo/beta": 0.004892690107226372, "fcm_dpo/delta": 0.04924327880144119, "fcm_dpo/margin": 72.04490661621094, "fcm_dpo/q_t": 0.41934844851493835, "grad_norm": 8.162010192871094, "learning_rate": 2.6282766789569736e-07, "logits/chosen": -1.9759223461151123, "logits/rejected": -2.008493185043335, "logps/chosen": -225.0614013671875, "logps/ref_chosen": -72.23806762695312, "logps/ref_rejected": -91.67182922363281, "logps/rejected": -316.5400695800781, "loss": 1.1159, "margin_dpo/margin_mean": 72.04490661621094, "margin_dpo/margin_std": 96.59625244140625, "step": 366 }, { "KL/chosen_KL_mean": -138.40423583984375, "KL/mean": -181.97520446777344, "KL/rejected_KL_mean": -225.54617309570312, "KL/std": 86.47904968261719, "epoch": 0.5389133627019089, "fcm_dpo/beta": 0.0048882076516747475, "fcm_dpo/delta": -0.02711433917284012, "fcm_dpo/margin": 87.14193725585938, "fcm_dpo/q_t": 0.39972636103630066, "grad_norm": 7.999646186828613, "learning_rate": 2.615458646614349e-07, "logits/chosen": -1.9296679496765137, "logits/rejected": -1.590559482574463, "logps/chosen": -223.82623291015625, "logps/ref_chosen": -85.42201232910156, "logps/ref_rejected": -83.51779174804688, "logps/rejected": -309.06396484375, "loss": 1.0498, "margin_dpo/margin_mean": 87.14193725585938, "margin_dpo/margin_std": 89.44941711425781, "step": 367 }, { "KL/chosen_KL_mean": -129.32821655273438, "KL/mean": -177.3518829345703, "KL/rejected_KL_mean": -225.37554931640625, "KL/std": 95.51463317871094, "epoch": 0.540381791483113, "fcm_dpo/beta": 0.00484818872064352, "fcm_dpo/delta": -0.06891346722841263, "fcm_dpo/margin": 96.0473403930664, "fcm_dpo/q_t": 0.3902171850204468, "grad_norm": 9.052199363708496, "learning_rate": 2.6026375718290083e-07, "logits/chosen": -1.3902904987335205, "logits/rejected": -1.3585437536239624, "logps/chosen": -203.16934204101562, "logps/ref_chosen": -73.84112548828125, "logps/ref_rejected": -104.69528198242188, "logps/rejected": -330.0708312988281, "loss": 1.0287, "margin_dpo/margin_mean": 96.0473403930664, "margin_dpo/margin_std": 96.81156921386719, "step": 368 }, { "KL/chosen_KL_mean": -143.9194793701172, "KL/mean": -188.78811645507812, "KL/rejected_KL_mean": -233.65673828125, "KL/std": 88.20640563964844, "epoch": 0.5418502202643172, "fcm_dpo/beta": 0.0047968486323952675, "fcm_dpo/delta": -0.03182859346270561, "fcm_dpo/margin": 89.73724365234375, "fcm_dpo/q_t": 0.39866721630096436, "grad_norm": 8.820496559143066, "learning_rate": 2.589813792448196e-07, "logits/chosen": -1.7988767623901367, "logits/rejected": -1.4377846717834473, "logps/chosen": -242.38143920898438, "logps/ref_chosen": -98.46195983886719, "logps/ref_rejected": -99.15672302246094, "logps/rejected": -332.8134765625, "loss": 1.0444, "margin_dpo/margin_mean": 89.73724365234375, "margin_dpo/margin_std": 88.95266723632812, "step": 369 }, { "KL/chosen_KL_mean": -159.89451599121094, "KL/mean": -189.7515869140625, "KL/rejected_KL_mean": -219.608642578125, "KL/std": 84.88975524902344, "epoch": 0.5433186490455213, "fcm_dpo/beta": 0.004805641248822212, "fcm_dpo/delta": 0.05085427314043045, "fcm_dpo/margin": 59.71413040161133, "fcm_dpo/q_t": 0.43212318420410156, "grad_norm": 8.281012535095215, "learning_rate": 2.5769876463904263e-07, "logits/chosen": -1.9239224195480347, "logits/rejected": -1.7259197235107422, "logps/chosen": -256.6702575683594, "logps/ref_chosen": -96.77574157714844, "logps/ref_rejected": -102.74411010742188, "logps/rejected": -322.3527526855469, "loss": 1.1738, "margin_dpo/margin_mean": 59.71412658691406, "margin_dpo/margin_std": 98.31381225585938, "step": 370 }, { "KL/chosen_KL_mean": -152.7358856201172, "KL/mean": -185.91757202148438, "KL/rejected_KL_mean": -219.0992431640625, "KL/std": 85.54147338867188, "epoch": 0.5447870778267254, "fcm_dpo/beta": 0.004836600739508867, "fcm_dpo/delta": 0.013481990434229374, "fcm_dpo/margin": 66.36337280273438, "fcm_dpo/q_t": 0.4238986670970917, "grad_norm": 10.920252799987793, "learning_rate": 2.5641594716365744e-07, "logits/chosen": -1.62270987033844, "logits/rejected": -1.2429583072662354, "logps/chosen": -255.72608947753906, "logps/ref_chosen": -102.99020385742188, "logps/ref_rejected": -105.35062408447266, "logps/rejected": -324.44989013671875, "loss": 1.14, "margin_dpo/margin_mean": 66.36337280273438, "margin_dpo/margin_std": 92.62242126464844, "step": 371 }, { "KL/chosen_KL_mean": -129.4332275390625, "KL/mean": -179.0995330810547, "KL/rejected_KL_mean": -228.76585388183594, "KL/std": 99.67709350585938, "epoch": 0.5462555066079295, "fcm_dpo/beta": 0.004791527055203915, "fcm_dpo/delta": -0.07968680560588837, "fcm_dpo/margin": 99.33262634277344, "fcm_dpo/q_t": 0.3892465829849243, "grad_norm": 7.302544116973877, "learning_rate": 2.551329606220976e-07, "logits/chosen": -2.0579938888549805, "logits/rejected": -1.4430837631225586, "logps/chosen": -220.45945739746094, "logps/ref_chosen": -91.02623748779297, "logps/ref_rejected": -85.6102294921875, "logps/rejected": -314.3760986328125, "loss": 1.0223, "margin_dpo/margin_mean": 99.33262634277344, "margin_dpo/margin_std": 102.0501708984375, "step": 372 }, { "KL/chosen_KL_mean": -150.50437927246094, "KL/mean": -187.273193359375, "KL/rejected_KL_mean": -224.0419921875, "KL/std": 95.267578125, "epoch": 0.5477239353891337, "fcm_dpo/beta": 0.004796641878783703, "fcm_dpo/delta": 0.04902341216802597, "fcm_dpo/margin": 73.53761291503906, "fcm_dpo/q_t": 0.4177188277244568, "grad_norm": 9.591846466064453, "learning_rate": 2.538498388222517e-07, "logits/chosen": -2.039701461791992, "logits/rejected": -1.5097781419754028, "logps/chosen": -250.57745361328125, "logps/ref_chosen": -100.07307434082031, "logps/ref_rejected": -91.11634826660156, "logps/rejected": -315.1583251953125, "loss": 1.1219, "margin_dpo/margin_mean": 73.53761291503906, "margin_dpo/margin_std": 102.26766967773438, "step": 373 }, { "KL/chosen_KL_mean": -130.49325561523438, "KL/mean": -170.98703002929688, "KL/rejected_KL_mean": -211.48077392578125, "KL/std": 90.057373046875, "epoch": 0.5491923641703378, "fcm_dpo/beta": 0.004810908809304237, "fcm_dpo/delta": 0.01072636991739273, "fcm_dpo/margin": 80.98753356933594, "fcm_dpo/q_t": 0.40843844413757324, "grad_norm": 8.041906356811523, "learning_rate": 2.525666155755725e-07, "logits/chosen": -2.3030543327331543, "logits/rejected": -1.9203490018844604, "logps/chosen": -228.50559997558594, "logps/ref_chosen": -98.01234436035156, "logps/ref_rejected": -100.75224304199219, "logps/rejected": -312.2330322265625, "loss": 1.0759, "margin_dpo/margin_mean": 80.98753356933594, "margin_dpo/margin_std": 86.94792938232422, "step": 374 }, { "KL/chosen_KL_mean": -131.91326904296875, "KL/mean": -178.53237915039062, "KL/rejected_KL_mean": -225.1514892578125, "KL/std": 87.07174682617188, "epoch": 0.5506607929515418, "fcm_dpo/beta": 0.004762294702231884, "fcm_dpo/delta": -0.04704749584197998, "fcm_dpo/margin": 93.23819732666016, "fcm_dpo/q_t": 0.394857257604599, "grad_norm": 8.932513236999512, "learning_rate": 2.512833246961859e-07, "logits/chosen": -1.653382658958435, "logits/rejected": -1.4214141368865967, "logps/chosen": -216.30955505371094, "logps/ref_chosen": -84.39627838134766, "logps/ref_rejected": -95.7188491821289, "logps/rejected": -320.870361328125, "loss": 1.0396, "margin_dpo/margin_mean": 93.23819732666016, "margin_dpo/margin_std": 92.36711120605469, "step": 375 }, { "KL/chosen_KL_mean": -127.81919860839844, "KL/mean": -178.79754638671875, "KL/rejected_KL_mean": -229.77593994140625, "KL/std": 100.33113098144531, "epoch": 0.5521292217327459, "fcm_dpo/beta": 0.004714367911219597, "fcm_dpo/delta": -0.08483125269412994, "fcm_dpo/margin": 101.95670318603516, "fcm_dpo/q_t": 0.3879043757915497, "grad_norm": 8.880372047424316, "learning_rate": 2.5e-07, "logits/chosen": -1.8324880599975586, "logits/rejected": -1.524383783340454, "logps/chosen": -223.55690002441406, "logps/ref_chosen": -95.73770141601562, "logps/ref_rejected": -111.97331237792969, "logps/rejected": -341.7492370605469, "loss": 1.0218, "margin_dpo/margin_mean": 101.95670318603516, "margin_dpo/margin_std": 107.13682556152344, "step": 376 }, { "KL/chosen_KL_mean": -135.4739990234375, "KL/mean": -188.46055603027344, "KL/rejected_KL_mean": -241.44712829589844, "KL/std": 100.69512939453125, "epoch": 0.55359765051395, "fcm_dpo/beta": 0.00461684912443161, "fcm_dpo/delta": -0.09442128241062164, "fcm_dpo/margin": 105.97311401367188, "fcm_dpo/q_t": 0.3847930431365967, "grad_norm": 8.08694839477539, "learning_rate": 2.487166753038141e-07, "logits/chosen": -1.624734878540039, "logits/rejected": -1.510082721710205, "logps/chosen": -214.4009552001953, "logps/ref_chosen": -78.92694854736328, "logps/ref_rejected": -105.89892578125, "logps/rejected": -347.3460693359375, "loss": 1.0068, "margin_dpo/margin_mean": 105.97311401367188, "margin_dpo/margin_std": 100.46519470214844, "step": 377 }, { "KL/chosen_KL_mean": -131.78115844726562, "KL/mean": -173.58877563476562, "KL/rejected_KL_mean": -215.39642333984375, "KL/std": 95.47067260742188, "epoch": 0.5550660792951542, "fcm_dpo/beta": 0.004638203419744968, "fcm_dpo/delta": 0.012258481234312057, "fcm_dpo/margin": 83.61526489257812, "fcm_dpo/q_t": 0.4080488383769989, "grad_norm": 9.341957092285156, "learning_rate": 2.4743338442442754e-07, "logits/chosen": -2.2326016426086426, "logits/rejected": -1.9615634679794312, "logps/chosen": -207.153564453125, "logps/ref_chosen": -75.37240600585938, "logps/ref_rejected": -93.70409393310547, "logps/rejected": -309.10052490234375, "loss": 1.0875, "margin_dpo/margin_mean": 83.61526489257812, "margin_dpo/margin_std": 98.60076904296875, "step": 378 }, { "KL/chosen_KL_mean": -154.72335815429688, "KL/mean": -191.8759765625, "KL/rejected_KL_mean": -229.02859497070312, "KL/std": 98.63135528564453, "epoch": 0.5565345080763583, "fcm_dpo/beta": 0.004657992161810398, "fcm_dpo/delta": 0.0558420866727829, "fcm_dpo/margin": 74.30522155761719, "fcm_dpo/q_t": 0.4205613434314728, "grad_norm": 8.567704200744629, "learning_rate": 2.461501611777483e-07, "logits/chosen": -1.7665473222732544, "logits/rejected": -1.6926254034042358, "logps/chosen": -243.57904052734375, "logps/ref_chosen": -88.85568237304688, "logps/ref_rejected": -121.38114166259766, "logps/rejected": -350.40972900390625, "loss": 1.1353, "margin_dpo/margin_mean": 74.30522155761719, "margin_dpo/margin_std": 112.70030212402344, "step": 379 }, { "KL/chosen_KL_mean": -169.372802734375, "KL/mean": -214.88067626953125, "KL/rejected_KL_mean": -260.3885803222656, "KL/std": 101.7571792602539, "epoch": 0.5580029368575624, "fcm_dpo/beta": 0.004655890166759491, "fcm_dpo/delta": -0.024813536554574966, "fcm_dpo/margin": 91.0157470703125, "fcm_dpo/q_t": 0.4028235375881195, "grad_norm": 11.913869857788086, "learning_rate": 2.4486703937790243e-07, "logits/chosen": -1.3702871799468994, "logits/rejected": -1.3439240455627441, "logps/chosen": -249.20602416992188, "logps/ref_chosen": -79.83321380615234, "logps/ref_rejected": -110.7447509765625, "logps/rejected": -371.1333312988281, "loss": 1.0722, "margin_dpo/margin_mean": 91.0157470703125, "margin_dpo/margin_std": 114.99800109863281, "step": 380 }, { "KL/chosen_KL_mean": -149.1160125732422, "KL/mean": -188.42755126953125, "KL/rejected_KL_mean": -227.7390899658203, "KL/std": 97.11800384521484, "epoch": 0.5594713656387665, "fcm_dpo/beta": 0.0046757897362113, "fcm_dpo/delta": 0.03359299525618553, "fcm_dpo/margin": 78.62306213378906, "fcm_dpo/q_t": 0.41347819566726685, "grad_norm": 9.305055618286133, "learning_rate": 2.435840528363426e-07, "logits/chosen": -1.821177363395691, "logits/rejected": -1.2757813930511475, "logps/chosen": -235.68284606933594, "logps/ref_chosen": -86.56683349609375, "logps/ref_rejected": -84.416748046875, "logps/rejected": -312.15582275390625, "loss": 1.105, "margin_dpo/margin_mean": 78.62306213378906, "margin_dpo/margin_std": 100.54057312011719, "step": 381 }, { "KL/chosen_KL_mean": -146.64369201660156, "KL/mean": -191.88885498046875, "KL/rejected_KL_mean": -237.13404846191406, "KL/std": 95.49838256835938, "epoch": 0.5609397944199707, "fcm_dpo/beta": 0.0046416353434324265, "fcm_dpo/delta": -0.021616537123918533, "fcm_dpo/margin": 90.4903564453125, "fcm_dpo/q_t": 0.4013921916484833, "grad_norm": 8.945796966552734, "learning_rate": 2.4230123536095745e-07, "logits/chosen": -1.8326973915100098, "logits/rejected": -1.5663193464279175, "logps/chosen": -251.10549926757812, "logps/ref_chosen": -104.46180725097656, "logps/ref_rejected": -114.60223388671875, "logps/rejected": -351.73626708984375, "loss": 1.0611, "margin_dpo/margin_mean": 90.4903564453125, "margin_dpo/margin_std": 97.75071716308594, "step": 382 }, { "KL/chosen_KL_mean": -159.60577392578125, "KL/mean": -197.79856872558594, "KL/rejected_KL_mean": -235.9913787841797, "KL/std": 98.73554229736328, "epoch": 0.5624082232011748, "fcm_dpo/beta": 0.004684256389737129, "fcm_dpo/delta": 0.04377155005931854, "fcm_dpo/margin": 76.38562774658203, "fcm_dpo/q_t": 0.4160931408405304, "grad_norm": 8.794547080993652, "learning_rate": 2.4101862075518037e-07, "logits/chosen": -1.4984644651412964, "logits/rejected": -1.3735969066619873, "logps/chosen": -241.72183227539062, "logps/ref_chosen": -82.11607360839844, "logps/ref_rejected": -98.77716064453125, "logps/rejected": -334.7685546875, "loss": 1.1107, "margin_dpo/margin_mean": 76.38562774658203, "margin_dpo/margin_std": 96.79356384277344, "step": 383 }, { "KL/chosen_KL_mean": -170.21539306640625, "KL/mean": -208.12240600585938, "KL/rejected_KL_mean": -246.0294189453125, "KL/std": 102.38072204589844, "epoch": 0.5638766519823789, "fcm_dpo/beta": 0.004683772101998329, "fcm_dpo/delta": -0.040233880281448364, "fcm_dpo/margin": 75.81401062011719, "fcm_dpo/q_t": 0.4166222810745239, "grad_norm": 10.724540710449219, "learning_rate": 2.397362428170992e-07, "logits/chosen": -1.8102113008499146, "logits/rejected": -1.5873998403549194, "logps/chosen": -250.80064392089844, "logps/ref_chosen": -80.58525085449219, "logps/ref_rejected": -91.44789123535156, "logps/rejected": -337.477294921875, "loss": 1.1361, "margin_dpo/margin_mean": 75.81401062011719, "margin_dpo/margin_std": 113.7035140991211, "step": 384 }, { "KL/chosen_KL_mean": -142.04183959960938, "KL/mean": -191.25686645507812, "KL/rejected_KL_mean": -240.47186279296875, "KL/std": 92.16252136230469, "epoch": 0.5653450807635829, "fcm_dpo/beta": 0.004603315144777298, "fcm_dpo/delta": -0.056393664330244064, "fcm_dpo/margin": 98.43002319335938, "fcm_dpo/q_t": 0.3944080173969269, "grad_norm": 9.644135475158691, "learning_rate": 2.3845413533856514e-07, "logits/chosen": -2.1324715614318848, "logits/rejected": -1.5153433084487915, "logps/chosen": -234.80201721191406, "logps/ref_chosen": -92.76016235351562, "logps/ref_rejected": -84.45997619628906, "logps/rejected": -324.9318542480469, "loss": 1.0414, "margin_dpo/margin_mean": 98.43002319335938, "margin_dpo/margin_std": 105.903076171875, "step": 385 }, { "KL/chosen_KL_mean": -150.625732421875, "KL/mean": -197.98562622070312, "KL/rejected_KL_mean": -245.3455352783203, "KL/std": 94.32642364501953, "epoch": 0.566813509544787, "fcm_dpo/beta": 0.004587274976074696, "fcm_dpo/delta": -0.03607851266860962, "fcm_dpo/margin": 94.71981048583984, "fcm_dpo/q_t": 0.39677664637565613, "grad_norm": 7.512143135070801, "learning_rate": 2.3717233210430254e-07, "logits/chosen": -1.7768217325210571, "logits/rejected": -1.4961553812026978, "logps/chosen": -235.98468017578125, "logps/ref_chosen": -85.35896301269531, "logps/ref_rejected": -96.99065399169922, "logps/rejected": -342.336181640625, "loss": 1.0511, "margin_dpo/margin_mean": 94.71980285644531, "margin_dpo/margin_std": 102.27782440185547, "step": 386 }, { "KL/chosen_KL_mean": -175.24859619140625, "KL/mean": -217.31689453125, "KL/rejected_KL_mean": -259.38519287109375, "KL/std": 91.85528564453125, "epoch": 0.5682819383259912, "fcm_dpo/beta": 0.004596139770001173, "fcm_dpo/delta": 0.013757986016571522, "fcm_dpo/margin": 84.1365966796875, "fcm_dpo/q_t": 0.4088144898414612, "grad_norm": 10.681920051574707, "learning_rate": 2.3589086689101889e-07, "logits/chosen": -2.393043041229248, "logits/rejected": -1.7053593397140503, "logps/chosen": -277.9367370605469, "logps/ref_chosen": -102.68814086914062, "logps/ref_rejected": -99.42858123779297, "logps/rejected": -358.81378173828125, "loss": 1.0813, "margin_dpo/margin_mean": 84.13660430908203, "margin_dpo/margin_std": 94.17853546142578, "step": 387 }, { "KL/chosen_KL_mean": -177.23260498046875, "KL/mean": -223.73065185546875, "KL/rejected_KL_mean": -270.22869873046875, "KL/std": 101.81842041015625, "epoch": 0.5697503671071953, "fcm_dpo/beta": 0.004570071119815111, "fcm_dpo/delta": -0.026149000972509384, "fcm_dpo/margin": 92.99610900878906, "fcm_dpo/q_t": 0.4001759886741638, "grad_norm": 8.60345458984375, "learning_rate": 2.3460977346651428e-07, "logits/chosen": -1.6761045455932617, "logits/rejected": -1.5581592321395874, "logps/chosen": -260.8178405761719, "logps/ref_chosen": -83.58523559570312, "logps/ref_rejected": -110.10037231445312, "logps/rejected": -380.3291015625, "loss": 1.0607, "margin_dpo/margin_mean": 92.99610900878906, "margin_dpo/margin_std": 105.09429931640625, "step": 388 }, { "KL/chosen_KL_mean": -182.1714630126953, "KL/mean": -221.9415283203125, "KL/rejected_KL_mean": -261.7115783691406, "KL/std": 103.77534484863281, "epoch": 0.5712187958883994, "fcm_dpo/beta": 0.00459069199860096, "fcm_dpo/delta": 0.036189205944538116, "fcm_dpo/margin": 79.54012298583984, "fcm_dpo/q_t": 0.4151958227157593, "grad_norm": 10.999826431274414, "learning_rate": 2.3332908558879177e-07, "logits/chosen": -2.4839155673980713, "logits/rejected": -2.0433268547058105, "logps/chosen": -271.99371337890625, "logps/ref_chosen": -89.822265625, "logps/ref_rejected": -84.85376739501953, "logps/rejected": -346.5653381347656, "loss": 1.1166, "margin_dpo/margin_mean": 79.54012298583984, "margin_dpo/margin_std": 112.048828125, "step": 389 }, { "KL/chosen_KL_mean": -182.55450439453125, "KL/mean": -220.90225219726562, "KL/rejected_KL_mean": -259.25, "KL/std": 101.57498931884766, "epoch": 0.5726872246696035, "fcm_dpo/beta": 0.00457828538492322, "fcm_dpo/delta": -0.025065837427973747, "fcm_dpo/margin": 76.69551849365234, "fcm_dpo/q_t": 0.4198220372200012, "grad_norm": 8.391576766967773, "learning_rate": 2.320488370051681e-07, "logits/chosen": -1.4965027570724487, "logits/rejected": -1.2872850894927979, "logps/chosen": -260.0869445800781, "logps/ref_chosen": -77.53244018554688, "logps/ref_rejected": -92.58392333984375, "logps/rejected": -351.83392333984375, "loss": 1.1337, "margin_dpo/margin_mean": 76.69551849365234, "margin_dpo/margin_std": 116.51912689208984, "step": 390 }, { "KL/chosen_KL_mean": -164.91554260253906, "KL/mean": -206.34451293945312, "KL/rejected_KL_mean": -247.77349853515625, "KL/std": 97.03406524658203, "epoch": 0.5741556534508077, "fcm_dpo/beta": 0.004579827189445496, "fcm_dpo/delta": 0.021230699494481087, "fcm_dpo/margin": 82.85796356201172, "fcm_dpo/q_t": 0.4110923409461975, "grad_norm": 8.695920944213867, "learning_rate": 2.3076906145138405e-07, "logits/chosen": -2.0394270420074463, "logits/rejected": -1.5955464839935303, "logps/chosen": -253.14846801757812, "logps/ref_chosen": -88.2329330444336, "logps/ref_rejected": -93.00756072998047, "logps/rejected": -340.78106689453125, "loss": 1.0954, "margin_dpo/margin_mean": 82.85796356201172, "margin_dpo/margin_std": 101.58324432373047, "step": 391 }, { "KL/chosen_KL_mean": -167.07406616210938, "KL/mean": -217.9665069580078, "KL/rejected_KL_mean": -268.85894775390625, "KL/std": 109.32381439208984, "epoch": 0.5756240822320118, "fcm_dpo/beta": 0.004566463176161051, "fcm_dpo/delta": -0.06795135885477066, "fcm_dpo/margin": 101.78485870361328, "fcm_dpo/q_t": 0.3917716145515442, "grad_norm": 9.26330280303955, "learning_rate": 2.294897926507156e-07, "logits/chosen": -1.6375110149383545, "logits/rejected": -1.5211455821990967, "logps/chosen": -254.8184814453125, "logps/ref_chosen": -87.74442291259766, "logps/ref_rejected": -108.57071685791016, "logps/rejected": -377.4296569824219, "loss": 1.0424, "margin_dpo/margin_mean": 101.78485870361328, "margin_dpo/margin_std": 116.9224624633789, "step": 392 }, { "KL/chosen_KL_mean": -148.2543487548828, "KL/mean": -193.87881469726562, "KL/rejected_KL_mean": -239.5032958984375, "KL/std": 102.76055908203125, "epoch": 0.5770925110132159, "fcm_dpo/beta": 0.004524551331996918, "fcm_dpo/delta": -0.013424969278275967, "fcm_dpo/margin": 91.24895477294922, "fcm_dpo/q_t": 0.4031013548374176, "grad_norm": 8.333599090576172, "learning_rate": 2.2821106431308543e-07, "logits/chosen": -1.726250410079956, "logits/rejected": -1.40533447265625, "logps/chosen": -224.8181610107422, "logps/ref_chosen": -76.56381225585938, "logps/ref_rejected": -79.72618103027344, "logps/rejected": -319.2294921875, "loss": 1.0645, "margin_dpo/margin_mean": 91.24895477294922, "margin_dpo/margin_std": 100.66075134277344, "step": 393 }, { "KL/chosen_KL_mean": -157.91297912597656, "KL/mean": -207.65185546875, "KL/rejected_KL_mean": -257.39068603515625, "KL/std": 98.01484680175781, "epoch": 0.57856093979442, "fcm_dpo/beta": 0.004491634666919708, "fcm_dpo/delta": -0.04899781942367554, "fcm_dpo/margin": 99.47770690917969, "fcm_dpo/q_t": 0.39410465955734253, "grad_norm": 8.270844459533691, "learning_rate": 2.2693291013417452e-07, "logits/chosen": -1.4862959384918213, "logits/rejected": -1.2529828548431396, "logps/chosen": -242.6116485595703, "logps/ref_chosen": -84.69866180419922, "logps/ref_rejected": -97.457763671875, "logps/rejected": -354.84844970703125, "loss": 1.0328, "margin_dpo/margin_mean": 99.47771453857422, "margin_dpo/margin_std": 95.81720733642578, "step": 394 }, { "KL/chosen_KL_mean": -163.55706787109375, "KL/mean": -201.77471923828125, "KL/rejected_KL_mean": -239.99237060546875, "KL/std": 102.40203094482422, "epoch": 0.580029368575624, "fcm_dpo/beta": 0.004466316197067499, "fcm_dpo/delta": -0.014659581705927849, "fcm_dpo/margin": 76.43531036376953, "fcm_dpo/q_t": 0.4225883483886719, "grad_norm": 8.785511016845703, "learning_rate": 2.2565536379453404e-07, "logits/chosen": -2.625396490097046, "logits/rejected": -2.230534076690674, "logps/chosen": -252.4447021484375, "logps/ref_chosen": -88.88763427734375, "logps/ref_rejected": -91.0455322265625, "logps/rejected": -331.03790283203125, "loss": 1.1646, "margin_dpo/margin_mean": 76.43531036376953, "margin_dpo/margin_std": 138.156005859375, "step": 395 }, { "KL/chosen_KL_mean": -172.32266235351562, "KL/mean": -206.53025817871094, "KL/rejected_KL_mean": -240.73785400390625, "KL/std": 90.76689147949219, "epoch": 0.5814977973568282, "fcm_dpo/beta": 0.0045280614867806435, "fcm_dpo/delta": 0.09311157464981079, "fcm_dpo/margin": 68.41517639160156, "fcm_dpo/q_t": 0.42777177691459656, "grad_norm": 10.51520824432373, "learning_rate": 2.2437845895869825e-07, "logits/chosen": -1.9896714687347412, "logits/rejected": -1.5045146942138672, "logps/chosen": -274.79400634765625, "logps/ref_chosen": -102.47132873535156, "logps/ref_rejected": -93.23478698730469, "logps/rejected": -333.9726257324219, "loss": 1.156, "margin_dpo/margin_mean": 68.4151611328125, "margin_dpo/margin_std": 106.41094207763672, "step": 396 }, { "KL/chosen_KL_mean": -155.91302490234375, "KL/mean": -216.68316650390625, "KL/rejected_KL_mean": -277.45330810546875, "KL/std": 105.60340881347656, "epoch": 0.5829662261380323, "fcm_dpo/beta": 0.004443579353392124, "fcm_dpo/delta": -0.14838063716888428, "fcm_dpo/margin": 121.54026794433594, "fcm_dpo/q_t": 0.37464988231658936, "grad_norm": 8.144950866699219, "learning_rate": 2.2310222927429716e-07, "logits/chosen": -2.2715883255004883, "logits/rejected": -1.6745511293411255, "logps/chosen": -256.95745849609375, "logps/ref_chosen": -101.04441833496094, "logps/ref_rejected": -108.8511734008789, "logps/rejected": -386.3044738769531, "loss": 0.9905, "margin_dpo/margin_mean": 121.54027557373047, "margin_dpo/margin_std": 126.0580825805664, "step": 397 }, { "KL/chosen_KL_mean": -164.30319213867188, "KL/mean": -207.94981384277344, "KL/rejected_KL_mean": -251.59645080566406, "KL/std": 103.51817321777344, "epoch": 0.5844346549192364, "fcm_dpo/beta": 0.004421258345246315, "fcm_dpo/delta": 0.01460410375148058, "fcm_dpo/margin": 87.29324340820312, "fcm_dpo/q_t": 0.4108482599258423, "grad_norm": 7.92824125289917, "learning_rate": 2.2182670837116972e-07, "logits/chosen": -2.2637898921966553, "logits/rejected": -1.9915108680725098, "logps/chosen": -259.3524169921875, "logps/ref_chosen": -95.04922485351562, "logps/ref_rejected": -114.00563049316406, "logps/rejected": -365.6020812988281, "loss": 1.1065, "margin_dpo/margin_mean": 87.29324340820312, "margin_dpo/margin_std": 122.97125244140625, "step": 398 }, { "KL/chosen_KL_mean": -143.33665466308594, "KL/mean": -199.37667846679688, "KL/rejected_KL_mean": -255.41668701171875, "KL/std": 93.90657043457031, "epoch": 0.5859030837004405, "fcm_dpo/beta": 0.0043755630031228065, "fcm_dpo/delta": -0.09497334063053131, "fcm_dpo/margin": 112.08001708984375, "fcm_dpo/q_t": 0.3842849135398865, "grad_norm": 8.265376091003418, "learning_rate": 2.2055192986047804e-07, "logits/chosen": -2.267026901245117, "logits/rejected": -1.4912976026535034, "logps/chosen": -239.97520446777344, "logps/ref_chosen": -96.63854217529297, "logps/ref_rejected": -83.53645324707031, "logps/rejected": -338.953125, "loss": 1.0014, "margin_dpo/margin_mean": 112.08000946044922, "margin_dpo/margin_std": 101.56687927246094, "step": 399 }, { "KL/chosen_KL_mean": -150.24603271484375, "KL/mean": -207.30279541015625, "KL/rejected_KL_mean": -264.3595275878906, "KL/std": 103.20948791503906, "epoch": 0.5873715124816447, "fcm_dpo/beta": 0.004286617040634155, "fcm_dpo/delta": -0.093747079372406, "fcm_dpo/margin": 114.11351013183594, "fcm_dpo/q_t": 0.3858647346496582, "grad_norm": 12.003571510314941, "learning_rate": 2.192779273338215e-07, "logits/chosen": -2.319026470184326, "logits/rejected": -1.847064733505249, "logps/chosen": -255.54917907714844, "logps/ref_chosen": -105.30314636230469, "logps/ref_rejected": -108.6555404663086, "logps/rejected": -373.01507568359375, "loss": 1.0177, "margin_dpo/margin_mean": 114.11351013183594, "margin_dpo/margin_std": 118.26309967041016, "step": 400 }, { "KL/chosen_KL_mean": -165.57501220703125, "KL/mean": -200.97299194335938, "KL/rejected_KL_mean": -236.3709716796875, "KL/std": 105.03636932373047, "epoch": 0.5888399412628488, "fcm_dpo/beta": 0.004283515270799398, "fcm_dpo/delta": 0.02043536677956581, "fcm_dpo/margin": 70.79595184326172, "fcm_dpo/q_t": 0.4315440356731415, "grad_norm": 10.821425437927246, "learning_rate": 2.1800473436235136e-07, "logits/chosen": -1.5078041553497314, "logits/rejected": -1.2638230323791504, "logps/chosen": -249.8641815185547, "logps/ref_chosen": -84.28916931152344, "logps/ref_rejected": -90.943115234375, "logps/rejected": -327.3140869140625, "loss": 1.1767, "margin_dpo/margin_mean": 70.79595184326172, "margin_dpo/margin_std": 128.64962768554688, "step": 401 }, { "KL/chosen_KL_mean": -146.51083374023438, "KL/mean": -213.16024780273438, "KL/rejected_KL_mean": -279.8096923828125, "KL/std": 111.65316009521484, "epoch": 0.5903083700440529, "fcm_dpo/beta": 0.004186196718364954, "fcm_dpo/delta": -0.1671200692653656, "fcm_dpo/margin": 133.29884338378906, "fcm_dpo/q_t": 0.36797064542770386, "grad_norm": 7.594605922698975, "learning_rate": 2.1673238449588665e-07, "logits/chosen": -2.0720481872558594, "logits/rejected": -1.617252230644226, "logps/chosen": -230.1039581298828, "logps/ref_chosen": -83.59312438964844, "logps/ref_rejected": -87.81027221679688, "logps/rejected": -367.61993408203125, "loss": 0.9542, "margin_dpo/margin_mean": 133.29884338378906, "margin_dpo/margin_std": 109.2518081665039, "step": 402 }, { "KL/chosen_KL_mean": -149.97921752929688, "KL/mean": -199.19219970703125, "KL/rejected_KL_mean": -248.40518188476562, "KL/std": 107.48170471191406, "epoch": 0.591776798825257, "fcm_dpo/beta": 0.004130107816308737, "fcm_dpo/delta": -0.006883140653371811, "fcm_dpo/margin": 98.42596435546875, "fcm_dpo/q_t": 0.40466803312301636, "grad_norm": 8.713815689086914, "learning_rate": 2.154609112620295e-07, "logits/chosen": -2.3627402782440186, "logits/rejected": -1.9727840423583984, "logps/chosen": -223.7322998046875, "logps/ref_chosen": -73.75308227539062, "logps/ref_rejected": -83.92012786865234, "logps/rejected": -332.3253173828125, "loss": 1.0755, "margin_dpo/margin_mean": 98.42597198486328, "margin_dpo/margin_std": 115.85055541992188, "step": 403 }, { "KL/chosen_KL_mean": -178.7392120361328, "KL/mean": -220.9769287109375, "KL/rejected_KL_mean": -263.21466064453125, "KL/std": 103.2603988647461, "epoch": 0.593245227606461, "fcm_dpo/beta": 0.004159621428698301, "fcm_dpo/delta": 0.050401441752910614, "fcm_dpo/margin": 84.47544860839844, "fcm_dpo/q_t": 0.41857510805130005, "grad_norm": 11.859652519226074, "learning_rate": 2.1419034816528218e-07, "logits/chosen": -1.8403348922729492, "logits/rejected": -1.4935059547424316, "logps/chosen": -258.4154052734375, "logps/ref_chosen": -79.67617797851562, "logps/ref_rejected": -84.280517578125, "logps/rejected": -347.49517822265625, "loss": 1.1309, "margin_dpo/margin_mean": 84.47544860839844, "margin_dpo/margin_std": 126.14936828613281, "step": 404 }, { "KL/chosen_KL_mean": -182.48565673828125, "KL/mean": -225.05123901367188, "KL/rejected_KL_mean": -267.6167907714844, "KL/std": 97.38519287109375, "epoch": 0.5947136563876652, "fcm_dpo/beta": 0.004169545602053404, "fcm_dpo/delta": -0.022491540759801865, "fcm_dpo/margin": 85.13111877441406, "fcm_dpo/q_t": 0.4164101481437683, "grad_norm": 7.930446624755859, "learning_rate": 2.129207286861638e-07, "logits/chosen": -1.6639671325683594, "logits/rejected": -1.1961365938186646, "logps/chosen": -278.9476318359375, "logps/ref_chosen": -96.46195220947266, "logps/ref_rejected": -92.87071228027344, "logps/rejected": -360.48748779296875, "loss": 1.108, "margin_dpo/margin_mean": 85.13111877441406, "margin_dpo/margin_std": 103.06645965576172, "step": 405 }, { "KL/chosen_KL_mean": -173.97332763671875, "KL/mean": -212.91693115234375, "KL/rejected_KL_mean": -251.86053466796875, "KL/std": 104.26785278320312, "epoch": 0.5961820851688693, "fcm_dpo/beta": 0.004206824582070112, "fcm_dpo/delta": 0.07479364424943924, "fcm_dpo/margin": 77.88721466064453, "fcm_dpo/q_t": 0.4238826036453247, "grad_norm": 13.93974781036377, "learning_rate": 2.1165208628032861e-07, "logits/chosen": -1.925182819366455, "logits/rejected": -1.758012294769287, "logps/chosen": -252.10728454589844, "logps/ref_chosen": -78.13396453857422, "logps/ref_rejected": -98.28359985351562, "logps/rejected": -350.1441345214844, "loss": 1.1485, "margin_dpo/margin_mean": 77.88722229003906, "margin_dpo/margin_std": 122.2257308959961, "step": 406 }, { "KL/chosen_KL_mean": -180.55955505371094, "KL/mean": -214.61465454101562, "KL/rejected_KL_mean": -248.6697540283203, "KL/std": 102.51031494140625, "epoch": 0.5976505139500734, "fcm_dpo/beta": 0.004268460441380739, "fcm_dpo/delta": 0.05369499698281288, "fcm_dpo/margin": 68.11019134521484, "fcm_dpo/q_t": 0.4316493570804596, "grad_norm": 9.310810089111328, "learning_rate": 2.1038445437768375e-07, "logits/chosen": -2.3327717781066895, "logits/rejected": -1.8287901878356934, "logps/chosen": -264.5723876953125, "logps/ref_chosen": -84.01283264160156, "logps/ref_rejected": -82.78103637695312, "logps/rejected": -331.4508056640625, "loss": 1.1686, "margin_dpo/margin_mean": 68.11019134521484, "margin_dpo/margin_std": 109.58731079101562, "step": 407 }, { "KL/chosen_KL_mean": -172.81137084960938, "KL/mean": -207.1467742919922, "KL/rejected_KL_mean": -241.48214721679688, "KL/std": 97.80386352539062, "epoch": 0.5991189427312775, "fcm_dpo/beta": 0.004303076304495335, "fcm_dpo/delta": 0.04038481041789055, "fcm_dpo/margin": 68.6707763671875, "fcm_dpo/q_t": 0.4315441846847534, "grad_norm": 8.947710037231445, "learning_rate": 2.0911786638150872e-07, "logits/chosen": -2.3426196575164795, "logits/rejected": -1.7310137748718262, "logps/chosen": -277.27313232421875, "logps/ref_chosen": -104.46174621582031, "logps/ref_rejected": -96.37217712402344, "logps/rejected": -337.8543395996094, "loss": 1.1775, "margin_dpo/margin_mean": 68.67078399658203, "margin_dpo/margin_std": 121.33287048339844, "step": 408 }, { "KL/chosen_KL_mean": -184.01516723632812, "KL/mean": -219.9274139404297, "KL/rejected_KL_mean": -255.83966064453125, "KL/std": 100.07394409179688, "epoch": 0.6005873715124816, "fcm_dpo/beta": 0.0043605901300907135, "fcm_dpo/delta": 0.08961852639913559, "fcm_dpo/margin": 71.82449340820312, "fcm_dpo/q_t": 0.4256886839866638, "grad_norm": 8.713200569152832, "learning_rate": 2.0785235566757517e-07, "logits/chosen": -1.6637320518493652, "logits/rejected": -1.2354702949523926, "logps/chosen": -281.6834716796875, "logps/ref_chosen": -97.66830444335938, "logps/ref_rejected": -90.04583740234375, "logps/rejected": -345.885498046875, "loss": 1.155, "margin_dpo/margin_mean": 71.82449340820312, "margin_dpo/margin_std": 110.18793487548828, "step": 409 }, { "KL/chosen_KL_mean": -178.03671264648438, "KL/mean": -221.854248046875, "KL/rejected_KL_mean": -265.6717529296875, "KL/std": 106.2457275390625, "epoch": 0.6020558002936858, "fcm_dpo/beta": 0.004389758221805096, "fcm_dpo/delta": 0.015918675810098648, "fcm_dpo/margin": 87.63505554199219, "fcm_dpo/q_t": 0.4089387357234955, "grad_norm": 8.035231590270996, "learning_rate": 2.065879555832674e-07, "logits/chosen": -1.6794209480285645, "logits/rejected": -1.5484282970428467, "logps/chosen": -254.50595092773438, "logps/ref_chosen": -76.46923828125, "logps/ref_rejected": -88.64064025878906, "logps/rejected": -354.3124084472656, "loss": 1.0905, "margin_dpo/margin_mean": 87.63505554199219, "margin_dpo/margin_std": 106.79444885253906, "step": 410 }, { "KL/chosen_KL_mean": -191.19830322265625, "KL/mean": -231.706298828125, "KL/rejected_KL_mean": -272.21429443359375, "KL/std": 98.99314880371094, "epoch": 0.6035242290748899, "fcm_dpo/beta": 0.004386193584650755, "fcm_dpo/delta": -0.009879241697490215, "fcm_dpo/margin": 81.01599884033203, "fcm_dpo/q_t": 0.4174742102622986, "grad_norm": 8.319835662841797, "learning_rate": 2.0532469944670343e-07, "logits/chosen": -2.338230848312378, "logits/rejected": -1.934354543685913, "logps/chosen": -278.3645935058594, "logps/ref_chosen": -87.16630554199219, "logps/ref_rejected": -87.09603118896484, "logps/rejected": -359.3103332519531, "loss": 1.1154, "margin_dpo/margin_mean": 81.0159912109375, "margin_dpo/margin_std": 106.55110168457031, "step": 411 }, { "KL/chosen_KL_mean": -181.22537231445312, "KL/mean": -221.19668579101562, "KL/rejected_KL_mean": -261.1679992675781, "KL/std": 99.29476928710938, "epoch": 0.604992657856094, "fcm_dpo/beta": 0.004409436136484146, "fcm_dpo/delta": 0.049248237162828445, "fcm_dpo/margin": 79.94264221191406, "fcm_dpo/q_t": 0.4195641279220581, "grad_norm": 9.73497486114502, "learning_rate": 2.0406262054585738e-07, "logits/chosen": -2.3409171104431152, "logits/rejected": -2.1250534057617188, "logps/chosen": -260.1727294921875, "logps/ref_chosen": -78.94734191894531, "logps/ref_rejected": -106.10554504394531, "logps/rejected": -367.2735595703125, "loss": 1.1395, "margin_dpo/margin_mean": 79.94264221191406, "margin_dpo/margin_std": 127.5152816772461, "step": 412 }, { "KL/chosen_KL_mean": -193.44114685058594, "KL/mean": -237.44496154785156, "KL/rejected_KL_mean": -281.4487609863281, "KL/std": 105.40950775146484, "epoch": 0.6064610866372981, "fcm_dpo/beta": 0.004423927515745163, "fcm_dpo/delta": 0.010950472205877304, "fcm_dpo/margin": 88.00759887695312, "fcm_dpo/q_t": 0.4088040888309479, "grad_norm": 9.673861503601074, "learning_rate": 2.0280175213768205e-07, "logits/chosen": -1.1915881633758545, "logits/rejected": -0.9319154620170593, "logps/chosen": -289.1358642578125, "logps/ref_chosen": -95.69470977783203, "logps/ref_rejected": -107.96085357666016, "logps/rejected": -389.40960693359375, "loss": 1.1071, "margin_dpo/margin_mean": 88.00759887695312, "margin_dpo/margin_std": 125.48699951171875, "step": 413 }, { "KL/chosen_KL_mean": -180.10107421875, "KL/mean": -221.34173583984375, "KL/rejected_KL_mean": -262.5823974609375, "KL/std": 101.55104064941406, "epoch": 0.6079295154185022, "fcm_dpo/beta": 0.004467803984880447, "fcm_dpo/delta": 0.03259623423218727, "fcm_dpo/margin": 82.4813232421875, "fcm_dpo/q_t": 0.4153996706008911, "grad_norm": 12.718391418457031, "learning_rate": 2.0154212744723247e-07, "logits/chosen": -1.438899040222168, "logits/rejected": -1.0733990669250488, "logps/chosen": -268.37774658203125, "logps/ref_chosen": -88.27667236328125, "logps/ref_rejected": -92.87004089355469, "logps/rejected": -355.45245361328125, "loss": 1.1307, "margin_dpo/margin_mean": 82.4813232421875, "margin_dpo/margin_std": 130.08883666992188, "step": 414 }, { "KL/chosen_KL_mean": -194.8232421875, "KL/mean": -229.5859832763672, "KL/rejected_KL_mean": -264.34869384765625, "KL/std": 93.02442932128906, "epoch": 0.6093979441997063, "fcm_dpo/beta": 0.004484872333705425, "fcm_dpo/delta": 0.03682447224855423, "fcm_dpo/margin": 69.52545166015625, "fcm_dpo/q_t": 0.42746877670288086, "grad_norm": 10.712226867675781, "learning_rate": 2.002837796667909e-07, "logits/chosen": -1.6358399391174316, "logits/rejected": -1.2576943635940552, "logps/chosen": -303.7391357421875, "logps/ref_chosen": -108.91590118408203, "logps/ref_rejected": -107.47135925292969, "logps/rejected": -371.820068359375, "loss": 1.1618, "margin_dpo/margin_mean": 69.52545166015625, "margin_dpo/margin_std": 114.09034729003906, "step": 415 }, { "KL/chosen_KL_mean": -194.02548217773438, "KL/mean": -242.18931579589844, "KL/rejected_KL_mean": -290.3531494140625, "KL/std": 107.28054809570312, "epoch": 0.6108663729809104, "fcm_dpo/beta": 0.004463159013539553, "fcm_dpo/delta": -0.0856553316116333, "fcm_dpo/margin": 96.32766723632812, "fcm_dpo/q_t": 0.39990150928497314, "grad_norm": 8.888777732849121, "learning_rate": 1.990267419549914e-07, "logits/chosen": -1.8529837131500244, "logits/rejected": -1.353346824645996, "logps/chosen": -287.42437744140625, "logps/ref_chosen": -93.39888000488281, "logps/ref_rejected": -97.6729736328125, "logps/rejected": -388.026123046875, "loss": 1.0674, "margin_dpo/margin_mean": 96.32767486572266, "margin_dpo/margin_std": 113.21560668945312, "step": 416 }, { "KL/chosen_KL_mean": -179.7425537109375, "KL/mean": -224.8054962158203, "KL/rejected_KL_mean": -269.86846923828125, "KL/std": 99.80001068115234, "epoch": 0.6123348017621145, "fcm_dpo/beta": 0.004430360160768032, "fcm_dpo/delta": 0.0007128175348043442, "fcm_dpo/margin": 90.12593078613281, "fcm_dpo/q_t": 0.4061453938484192, "grad_norm": 8.672005653381348, "learning_rate": 1.9777104743594686e-07, "logits/chosen": -2.450835704803467, "logits/rejected": -1.5716674327850342, "logps/chosen": -263.27789306640625, "logps/ref_chosen": -83.53533172607422, "logps/ref_rejected": -74.44184875488281, "logps/rejected": -344.310302734375, "loss": 1.0753, "margin_dpo/margin_mean": 90.12593078613281, "margin_dpo/margin_std": 102.06039428710938, "step": 417 }, { "KL/chosen_KL_mean": -171.91831970214844, "KL/mean": -228.54393005371094, "KL/rejected_KL_mean": -285.1695556640625, "KL/std": 106.93478393554688, "epoch": 0.6138032305433186, "fcm_dpo/beta": 0.004396216943860054, "fcm_dpo/delta": -0.10364756733179092, "fcm_dpo/margin": 113.251220703125, "fcm_dpo/q_t": 0.38388434052467346, "grad_norm": 10.304213523864746, "learning_rate": 1.965167291983757e-07, "logits/chosen": -2.1975674629211426, "logits/rejected": -1.543036699295044, "logps/chosen": -280.13983154296875, "logps/ref_chosen": -108.22152709960938, "logps/ref_rejected": -111.8646469116211, "logps/rejected": -397.0341796875, "loss": 1.0065, "margin_dpo/margin_mean": 113.251220703125, "margin_dpo/margin_std": 105.12625122070312, "step": 418 }, { "KL/chosen_KL_mean": -179.26296997070312, "KL/mean": -228.87451171875, "KL/rejected_KL_mean": -278.48602294921875, "KL/std": 95.41441345214844, "epoch": 0.6152716593245228, "fcm_dpo/beta": 0.004333334043622017, "fcm_dpo/delta": -0.03156277537345886, "fcm_dpo/margin": 99.2230453491211, "fcm_dpo/q_t": 0.39901256561279297, "grad_norm": 11.327917098999023, "learning_rate": 1.9526382029472988e-07, "logits/chosen": -1.536212682723999, "logits/rejected": -1.0965124368667603, "logps/chosen": -276.4462585449219, "logps/ref_chosen": -97.18328094482422, "logps/ref_rejected": -98.18531799316406, "logps/rejected": -376.67132568359375, "loss": 1.0518, "margin_dpo/margin_mean": 99.22303771972656, "margin_dpo/margin_std": 104.15109252929688, "step": 419 }, { "KL/chosen_KL_mean": -183.4001007080078, "KL/mean": -227.8697509765625, "KL/rejected_KL_mean": -272.3393859863281, "KL/std": 100.98008728027344, "epoch": 0.6167400881057269, "fcm_dpo/beta": 0.004330971743911505, "fcm_dpo/delta": 0.015145653858780861, "fcm_dpo/margin": 88.93927001953125, "fcm_dpo/q_t": 0.41050124168395996, "grad_norm": 14.192485809326172, "learning_rate": 1.9401235374032425e-07, "logits/chosen": -2.2598860263824463, "logits/rejected": -1.1676476001739502, "logps/chosen": -297.7085876464844, "logps/ref_chosen": -114.30848693847656, "logps/ref_rejected": -75.68355560302734, "logps/rejected": -348.02294921875, "loss": 1.0985, "margin_dpo/margin_mean": 88.93927001953125, "margin_dpo/margin_std": 116.28192901611328, "step": 420 }, { "KL/chosen_KL_mean": -174.35287475585938, "KL/mean": -223.04075622558594, "KL/rejected_KL_mean": -271.7286376953125, "KL/std": 103.98757934570312, "epoch": 0.618208516886931, "fcm_dpo/beta": 0.0043023196049034595, "fcm_dpo/delta": -0.01982852630317211, "fcm_dpo/margin": 97.37576293945312, "fcm_dpo/q_t": 0.40179944038391113, "grad_norm": 7.9784626960754395, "learning_rate": 1.9276236251246653e-07, "logits/chosen": -1.8924380540847778, "logits/rejected": -1.5231058597564697, "logps/chosen": -260.23272705078125, "logps/ref_chosen": -85.87985229492188, "logps/ref_rejected": -96.33648681640625, "logps/rejected": -368.06512451171875, "loss": 1.0689, "margin_dpo/margin_mean": 97.37577056884766, "margin_dpo/margin_std": 114.62490844726562, "step": 421 }, { "KL/chosen_KL_mean": -172.6793212890625, "KL/mean": -219.54205322265625, "KL/rejected_KL_mean": -266.40478515625, "KL/std": 96.2136001586914, "epoch": 0.6196769456681351, "fcm_dpo/beta": 0.004305709153413773, "fcm_dpo/delta": -0.0037238094955682755, "fcm_dpo/margin": 93.72547912597656, "fcm_dpo/q_t": 0.4056052267551422, "grad_norm": 11.676365852355957, "learning_rate": 1.9151387954958792e-07, "logits/chosen": -2.4220902919769287, "logits/rejected": -2.059406042098999, "logps/chosen": -273.159912109375, "logps/ref_chosen": -100.48060607910156, "logps/ref_rejected": -94.40821838378906, "logps/rejected": -360.8130187988281, "loss": 1.0737, "margin_dpo/margin_mean": 93.72547912597656, "margin_dpo/margin_std": 107.88322448730469, "step": 422 }, { "KL/chosen_KL_mean": -159.6549072265625, "KL/mean": -207.25613403320312, "KL/rejected_KL_mean": -254.8573760986328, "KL/std": 115.31645202636719, "epoch": 0.6211453744493393, "fcm_dpo/beta": 0.004309993702918291, "fcm_dpo/delta": -0.011030737310647964, "fcm_dpo/margin": 95.20246124267578, "fcm_dpo/q_t": 0.40627768635749817, "grad_norm": 9.54808235168457, "learning_rate": 1.902669377503756e-07, "logits/chosen": -1.8968216180801392, "logits/rejected": -1.7785660028457642, "logps/chosen": -238.10484313964844, "logps/ref_chosen": -78.44993591308594, "logps/ref_rejected": -92.04652404785156, "logps/rejected": -346.9039001464844, "loss": 1.0931, "margin_dpo/margin_mean": 95.20245361328125, "margin_dpo/margin_std": 130.82846069335938, "step": 423 }, { "KL/chosen_KL_mean": -179.8658447265625, "KL/mean": -226.17735290527344, "KL/rejected_KL_mean": -272.4888610839844, "KL/std": 102.47991943359375, "epoch": 0.6226138032305433, "fcm_dpo/beta": 0.004290757235139608, "fcm_dpo/delta": 0.0026815487071871758, "fcm_dpo/margin": 92.62301635742188, "fcm_dpo/q_t": 0.4062444865703583, "grad_norm": 8.440460205078125, "learning_rate": 1.890215699729057e-07, "logits/chosen": -1.7731688022613525, "logits/rejected": -1.1821210384368896, "logps/chosen": -267.5081787109375, "logps/ref_chosen": -87.64236450195312, "logps/ref_rejected": -72.36566162109375, "logps/rejected": -344.8545227050781, "loss": 1.0821, "margin_dpo/margin_mean": 92.62301635742188, "margin_dpo/margin_std": 110.81394958496094, "step": 424 }, { "KL/chosen_KL_mean": -186.88282775878906, "KL/mean": -218.54039001464844, "KL/rejected_KL_mean": -250.1979522705078, "KL/std": 106.28575134277344, "epoch": 0.6240822320117474, "fcm_dpo/beta": 0.004330229014158249, "fcm_dpo/delta": 0.04308256134390831, "fcm_dpo/margin": 63.31511688232422, "fcm_dpo/q_t": 0.4383828341960907, "grad_norm": 10.14073371887207, "learning_rate": 1.8777780903377732e-07, "logits/chosen": -1.6557912826538086, "logits/rejected": -1.5911717414855957, "logps/chosen": -265.4026184082031, "logps/ref_chosen": -78.51979064941406, "logps/ref_rejected": -102.74864196777344, "logps/rejected": -352.94659423828125, "loss": 1.2085, "margin_dpo/margin_mean": 63.31511688232422, "margin_dpo/margin_std": 132.601318359375, "step": 425 }, { "KL/chosen_KL_mean": -159.47183227539062, "KL/mean": -205.91732788085938, "KL/rejected_KL_mean": -252.36279296875, "KL/std": 97.26757049560547, "epoch": 0.6255506607929515, "fcm_dpo/beta": 0.004338660277426243, "fcm_dpo/delta": -0.0032616257667541504, "fcm_dpo/margin": 92.89096069335938, "fcm_dpo/q_t": 0.40564587712287903, "grad_norm": 10.055276870727539, "learning_rate": 1.8653568770724803e-07, "logits/chosen": -1.6178650856018066, "logits/rejected": -0.8660005331039429, "logps/chosen": -267.9776611328125, "logps/ref_chosen": -108.50582885742188, "logps/ref_rejected": -88.300048828125, "logps/rejected": -340.662841796875, "loss": 1.0816, "margin_dpo/margin_mean": 92.89096069335938, "margin_dpo/margin_std": 113.41606140136719, "step": 426 }, { "KL/chosen_KL_mean": -131.5442352294922, "KL/mean": -174.1750946044922, "KL/rejected_KL_mean": -216.8059539794922, "KL/std": 92.49711608886719, "epoch": 0.6270190895741556, "fcm_dpo/beta": 0.0043511101976037025, "fcm_dpo/delta": 0.03008575737476349, "fcm_dpo/margin": 85.26173400878906, "fcm_dpo/q_t": 0.4125143885612488, "grad_norm": 8.564836502075195, "learning_rate": 1.8529523872436977e-07, "logits/chosen": -2.5279462337493896, "logits/rejected": -1.9070940017700195, "logps/chosen": -230.66470336914062, "logps/ref_chosen": -99.12046813964844, "logps/ref_rejected": -85.724609375, "logps/rejected": -302.53057861328125, "loss": 1.0942, "margin_dpo/margin_mean": 85.26173400878906, "margin_dpo/margin_std": 99.58175659179688, "step": 427 }, { "KL/chosen_KL_mean": -188.04348754882812, "KL/mean": -224.72398376464844, "KL/rejected_KL_mean": -261.40447998046875, "KL/std": 104.38056182861328, "epoch": 0.6284875183553598, "fcm_dpo/beta": 0.004351767245680094, "fcm_dpo/delta": -0.002016030717641115, "fcm_dpo/margin": 73.36100006103516, "fcm_dpo/q_t": 0.4255887269973755, "grad_norm": 9.564895629882812, "learning_rate": 1.8405649477212697e-07, "logits/chosen": -1.875936508178711, "logits/rejected": -1.6235854625701904, "logps/chosen": -294.01275634765625, "logps/ref_chosen": -105.96925354003906, "logps/ref_rejected": -109.10218048095703, "logps/rejected": -370.50665283203125, "loss": 1.1544, "margin_dpo/margin_mean": 73.36100769042969, "margin_dpo/margin_std": 116.09918975830078, "step": 428 }, { "KL/chosen_KL_mean": -172.08187866210938, "KL/mean": -218.90805053710938, "KL/rejected_KL_mean": -265.73419189453125, "KL/std": 109.05897521972656, "epoch": 0.6299559471365639, "fcm_dpo/beta": 0.004321882501244545, "fcm_dpo/delta": -0.06914899498224258, "fcm_dpo/margin": 93.65232849121094, "fcm_dpo/q_t": 0.4064915180206299, "grad_norm": 8.121362686157227, "learning_rate": 1.828194884925749e-07, "logits/chosen": -1.9417109489440918, "logits/rejected": -1.2245090007781982, "logps/chosen": -285.62677001953125, "logps/ref_chosen": -113.54487609863281, "logps/ref_rejected": -98.24201965332031, "logps/rejected": -363.9762268066406, "loss": 1.0937, "margin_dpo/margin_mean": 93.65232849121094, "margin_dpo/margin_std": 124.04888916015625, "step": 429 }, { "KL/chosen_KL_mean": -163.50238037109375, "KL/mean": -199.7823486328125, "KL/rejected_KL_mean": -236.06234741210938, "KL/std": 101.12538146972656, "epoch": 0.631424375917768, "fcm_dpo/beta": 0.004299278371036053, "fcm_dpo/delta": 0.01693517528474331, "fcm_dpo/margin": 72.5599365234375, "fcm_dpo/q_t": 0.4275030493736267, "grad_norm": 8.765769958496094, "learning_rate": 1.8158425248197928e-07, "logits/chosen": -1.235914707183838, "logits/rejected": -1.1994497776031494, "logps/chosen": -254.82174682617188, "logps/ref_chosen": -91.31936645507812, "logps/ref_rejected": -110.1096420288086, "logps/rejected": -346.1719970703125, "loss": 1.1596, "margin_dpo/margin_mean": 72.5599365234375, "margin_dpo/margin_std": 117.38682556152344, "step": 430 }, { "KL/chosen_KL_mean": -165.48614501953125, "KL/mean": -208.19583129882812, "KL/rejected_KL_mean": -250.905517578125, "KL/std": 103.6623306274414, "epoch": 0.6328928046989721, "fcm_dpo/beta": 0.004321317188441753, "fcm_dpo/delta": 0.03206493332982063, "fcm_dpo/margin": 85.41938781738281, "fcm_dpo/q_t": 0.41387611627578735, "grad_norm": 9.80251407623291, "learning_rate": 1.8035081928995788e-07, "logits/chosen": -2.067448377609253, "logits/rejected": -1.6924456357955933, "logps/chosen": -258.6673583984375, "logps/ref_chosen": -93.18122100830078, "logps/ref_rejected": -98.13226318359375, "logps/rejected": -349.03778076171875, "loss": 1.113, "margin_dpo/margin_mean": 85.41938781738281, "margin_dpo/margin_std": 117.9261703491211, "step": 431 }, { "KL/chosen_KL_mean": -144.88470458984375, "KL/mean": -196.82659912109375, "KL/rejected_KL_mean": -248.76849365234375, "KL/std": 107.77914428710938, "epoch": 0.6343612334801763, "fcm_dpo/beta": 0.004315475933253765, "fcm_dpo/delta": -0.05062083154916763, "fcm_dpo/margin": 103.88379669189453, "fcm_dpo/q_t": 0.3944205641746521, "grad_norm": 8.59517765045166, "learning_rate": 1.791192214186223e-07, "logits/chosen": -1.3122472763061523, "logits/rejected": -0.7425343990325928, "logps/chosen": -249.31947326660156, "logps/ref_chosen": -104.43478393554688, "logps/ref_rejected": -105.08955383300781, "logps/rejected": -353.8580322265625, "loss": 1.0382, "margin_dpo/margin_mean": 103.88379669189453, "margin_dpo/margin_std": 103.88294219970703, "step": 432 }, { "KL/chosen_KL_mean": -172.14376831054688, "KL/mean": -212.92405700683594, "KL/rejected_KL_mean": -253.70433044433594, "KL/std": 101.35973358154297, "epoch": 0.6358296622613803, "fcm_dpo/beta": 0.004284270573407412, "fcm_dpo/delta": -0.014788562431931496, "fcm_dpo/margin": 81.5605697631836, "fcm_dpo/q_t": 0.4182785153388977, "grad_norm": 8.573363304138184, "learning_rate": 1.7788949132172193e-07, "logits/chosen": -1.595853328704834, "logits/rejected": -1.2731595039367676, "logps/chosen": -261.98699951171875, "logps/ref_chosen": -89.84322357177734, "logps/ref_rejected": -101.73345947265625, "logps/rejected": -355.43780517578125, "loss": 1.1263, "margin_dpo/margin_mean": 81.56056213378906, "margin_dpo/margin_std": 114.823974609375, "step": 433 }, { "KL/chosen_KL_mean": -149.53402709960938, "KL/mean": -195.58148193359375, "KL/rejected_KL_mean": -241.62890625, "KL/std": 112.89448547363281, "epoch": 0.6372980910425844, "fcm_dpo/beta": 0.004284962080419064, "fcm_dpo/delta": 0.005579883232712746, "fcm_dpo/margin": 92.09489440917969, "fcm_dpo/q_t": 0.41094207763671875, "grad_norm": 11.747979164123535, "learning_rate": 1.7666166140378853e-07, "logits/chosen": -2.36212158203125, "logits/rejected": -1.6805628538131714, "logps/chosen": -247.2265625, "logps/ref_chosen": -97.6925277709961, "logps/ref_rejected": -84.09129333496094, "logps/rejected": -325.72021484375, "loss": 1.1017, "margin_dpo/margin_mean": 92.09489440917969, "margin_dpo/margin_std": 131.29226684570312, "step": 434 }, { "KL/chosen_KL_mean": -147.0272979736328, "KL/mean": -194.65455627441406, "KL/rejected_KL_mean": -242.28182983398438, "KL/std": 104.88186645507812, "epoch": 0.6387665198237885, "fcm_dpo/beta": 0.004283071495592594, "fcm_dpo/delta": -0.00834620464593172, "fcm_dpo/margin": 95.2545166015625, "fcm_dpo/q_t": 0.4055781662464142, "grad_norm": 9.863697052001953, "learning_rate": 1.7543576401928218e-07, "logits/chosen": -2.391676187515259, "logits/rejected": -1.94583261013031, "logps/chosen": -233.19921875, "logps/ref_chosen": -86.17192077636719, "logps/ref_rejected": -93.751708984375, "logps/rejected": -336.03350830078125, "loss": 1.0779, "margin_dpo/margin_mean": 95.2545166015625, "margin_dpo/margin_std": 116.85872650146484, "step": 435 }, { "KL/chosen_KL_mean": -122.24266815185547, "KL/mean": -171.98390197753906, "KL/rejected_KL_mean": -221.72512817382812, "KL/std": 88.58323669433594, "epoch": 0.6402349486049926, "fcm_dpo/beta": 0.0042562964372336864, "fcm_dpo/delta": -0.02449946664273739, "fcm_dpo/margin": 99.48246765136719, "fcm_dpo/q_t": 0.3992065191268921, "grad_norm": 9.942447662353516, "learning_rate": 1.742118314717391e-07, "logits/chosen": -1.8173494338989258, "logits/rejected": -0.9608921408653259, "logps/chosen": -228.02976989746094, "logps/ref_chosen": -105.78710174560547, "logps/ref_rejected": -88.62471008300781, "logps/rejected": -310.349853515625, "loss": 1.0434, "margin_dpo/margin_mean": 99.48246765136719, "margin_dpo/margin_std": 91.65057373046875, "step": 436 }, { "KL/chosen_KL_mean": -143.17430114746094, "KL/mean": -183.54501342773438, "KL/rejected_KL_mean": -223.9157257080078, "KL/std": 95.59382629394531, "epoch": 0.6417033773861968, "fcm_dpo/beta": 0.0042947446927428246, "fcm_dpo/delta": 0.05505699664354324, "fcm_dpo/margin": 80.74140930175781, "fcm_dpo/q_t": 0.41714122891426086, "grad_norm": 10.052477836608887, "learning_rate": 1.7298989601292036e-07, "logits/chosen": -1.827838659286499, "logits/rejected": -1.155079960823059, "logps/chosen": -239.23635864257812, "logps/ref_chosen": -96.06204223632812, "logps/ref_rejected": -89.01220703125, "logps/rejected": -312.92791748046875, "loss": 1.1091, "margin_dpo/margin_mean": 80.74140930175781, "margin_dpo/margin_std": 94.30926513671875, "step": 437 }, { "KL/chosen_KL_mean": -148.0892333984375, "KL/mean": -188.77459716796875, "KL/rejected_KL_mean": -229.45993041992188, "KL/std": 96.01412963867188, "epoch": 0.6431718061674009, "fcm_dpo/beta": 0.0042906044982373714, "fcm_dpo/delta": -0.013005274347960949, "fcm_dpo/margin": 81.37069702148438, "fcm_dpo/q_t": 0.4172900915145874, "grad_norm": 9.351896286010742, "learning_rate": 1.7176998984196144e-07, "logits/chosen": -2.1286299228668213, "logits/rejected": -1.3172246217727661, "logps/chosen": -249.94461059570312, "logps/ref_chosen": -101.85537719726562, "logps/ref_rejected": -89.44766235351562, "logps/rejected": -318.9075927734375, "loss": 1.1159, "margin_dpo/margin_mean": 81.37069702148438, "margin_dpo/margin_std": 103.24181365966797, "step": 438 }, { "KL/chosen_KL_mean": -158.1829833984375, "KL/mean": -199.0579376220703, "KL/rejected_KL_mean": -239.93287658691406, "KL/std": 104.05538940429688, "epoch": 0.644640234948605, "fcm_dpo/beta": 0.004260015208274126, "fcm_dpo/delta": -0.035774268209934235, "fcm_dpo/margin": 81.7499008178711, "fcm_dpo/q_t": 0.4178205728530884, "grad_norm": 9.368892669677734, "learning_rate": 1.7055214510452458e-07, "logits/chosen": -1.7336612939834595, "logits/rejected": -1.5019488334655762, "logps/chosen": -239.93861389160156, "logps/ref_chosen": -81.75563049316406, "logps/ref_rejected": -90.58636474609375, "logps/rejected": -330.51922607421875, "loss": 1.1161, "margin_dpo/margin_mean": 81.7499008178711, "margin_dpo/margin_std": 101.83357238769531, "step": 439 }, { "KL/chosen_KL_mean": -141.52011108398438, "KL/mean": -192.32632446289062, "KL/rejected_KL_mean": -243.1325225830078, "KL/std": 105.81680297851562, "epoch": 0.6461086637298091, "fcm_dpo/beta": 0.004220467060804367, "fcm_dpo/delta": -0.030752703547477722, "fcm_dpo/margin": 101.61243438720703, "fcm_dpo/q_t": 0.40036624670028687, "grad_norm": 7.317651748657227, "learning_rate": 1.6933639389195134e-07, "logits/chosen": -2.4706578254699707, "logits/rejected": -1.9770441055297852, "logps/chosen": -247.1611785888672, "logps/ref_chosen": -105.64108276367188, "logps/ref_rejected": -103.4010009765625, "logps/rejected": -346.5335388183594, "loss": 1.0622, "margin_dpo/margin_mean": 101.61243438720703, "margin_dpo/margin_std": 117.44085693359375, "step": 440 }, { "KL/chosen_KL_mean": -182.84115600585938, "KL/mean": -225.94801330566406, "KL/rejected_KL_mean": -269.0549011230469, "KL/std": 100.33851623535156, "epoch": 0.6475770925110133, "fcm_dpo/beta": 0.004250752739608288, "fcm_dpo/delta": 0.03481417894363403, "fcm_dpo/margin": 86.21375274658203, "fcm_dpo/q_t": 0.4153675436973572, "grad_norm": 8.287605285644531, "learning_rate": 1.681227682404166e-07, "logits/chosen": -1.7635142803192139, "logits/rejected": -1.4059213399887085, "logps/chosen": -274.37066650390625, "logps/ref_chosen": -91.529541015625, "logps/ref_rejected": -103.619384765625, "logps/rejected": -372.67425537109375, "loss": 1.1031, "margin_dpo/margin_mean": 86.2137451171875, "margin_dpo/margin_std": 109.54653930664062, "step": 441 }, { "KL/chosen_KL_mean": -178.48822021484375, "KL/mean": -232.74468994140625, "KL/rejected_KL_mean": -287.00115966796875, "KL/std": 105.9219970703125, "epoch": 0.6490455212922174, "fcm_dpo/beta": 0.004226325079798698, "fcm_dpo/delta": -0.061407655477523804, "fcm_dpo/margin": 108.51292419433594, "fcm_dpo/q_t": 0.3918910026550293, "grad_norm": 10.600053787231445, "learning_rate": 1.669113001300851e-07, "logits/chosen": -1.7542593479156494, "logits/rejected": -1.2927069664001465, "logps/chosen": -263.2657775878906, "logps/ref_chosen": -84.77755737304688, "logps/ref_rejected": -83.82415771484375, "logps/rejected": -370.8253173828125, "loss": 1.0273, "margin_dpo/margin_mean": 108.51292419433594, "margin_dpo/margin_std": 106.8646240234375, "step": 442 }, { "KL/chosen_KL_mean": -174.8520965576172, "KL/mean": -208.32086181640625, "KL/rejected_KL_mean": -241.78961181640625, "KL/std": 103.3300552368164, "epoch": 0.6505139500734214, "fcm_dpo/beta": 0.004232651088386774, "fcm_dpo/delta": 0.025125082582235336, "fcm_dpo/margin": 66.9375, "fcm_dpo/q_t": 0.43301093578338623, "grad_norm": 9.790900230407715, "learning_rate": 1.6570202148426815e-07, "logits/chosen": -1.9333646297454834, "logits/rejected": -1.384993076324463, "logps/chosen": -277.5013732910156, "logps/ref_chosen": -102.64927673339844, "logps/ref_rejected": -93.03807067871094, "logps/rejected": -334.82769775390625, "loss": 1.1682, "margin_dpo/margin_mean": 66.93750762939453, "margin_dpo/margin_std": 100.76521301269531, "step": 443 }, { "KL/chosen_KL_mean": -178.84686279296875, "KL/mean": -218.85072326660156, "KL/rejected_KL_mean": -258.85455322265625, "KL/std": 100.89628601074219, "epoch": 0.6519823788546255, "fcm_dpo/beta": 0.004274386912584305, "fcm_dpo/delta": 0.06000368297100067, "fcm_dpo/margin": 80.0076904296875, "fcm_dpo/q_t": 0.4206140637397766, "grad_norm": 10.3822660446167, "learning_rate": 1.6449496416858282e-07, "logits/chosen": -1.6811553239822388, "logits/rejected": -1.516911268234253, "logps/chosen": -266.7666015625, "logps/ref_chosen": -87.91971588134766, "logps/ref_rejected": -103.32345581054688, "logps/rejected": -362.17803955078125, "loss": 1.1325, "margin_dpo/margin_mean": 80.0076904296875, "margin_dpo/margin_std": 116.80731201171875, "step": 444 }, { "KL/chosen_KL_mean": -157.44281005859375, "KL/mean": -203.68724060058594, "KL/rejected_KL_mean": -249.9316864013672, "KL/std": 98.18214416503906, "epoch": 0.6534508076358296, "fcm_dpo/beta": 0.0042739138007164, "fcm_dpo/delta": 0.004733650013804436, "fcm_dpo/margin": 92.4888916015625, "fcm_dpo/q_t": 0.40683087706565857, "grad_norm": 8.112725257873535, "learning_rate": 1.6329015999011182e-07, "logits/chosen": -1.9232900142669678, "logits/rejected": -1.5068702697753906, "logps/chosen": -258.8436584472656, "logps/ref_chosen": -101.40086364746094, "logps/ref_rejected": -99.03790283203125, "logps/rejected": -348.9696044921875, "loss": 1.0771, "margin_dpo/margin_mean": 92.48888397216797, "margin_dpo/margin_std": 103.79328155517578, "step": 445 }, { "KL/chosen_KL_mean": -150.9923095703125, "KL/mean": -192.4937744140625, "KL/rejected_KL_mean": -233.99525451660156, "KL/std": 103.28872680664062, "epoch": 0.6549192364170338, "fcm_dpo/beta": 0.004322713706642389, "fcm_dpo/delta": 0.04259772598743439, "fcm_dpo/margin": 83.00293731689453, "fcm_dpo/q_t": 0.41559872031211853, "grad_norm": 12.37559700012207, "learning_rate": 1.6208764069656578e-07, "logits/chosen": -1.9989771842956543, "logits/rejected": -1.861940860748291, "logps/chosen": -238.41464233398438, "logps/ref_chosen": -87.4223403930664, "logps/ref_rejected": -106.70075988769531, "logps/rejected": -340.6960144042969, "loss": 1.1038, "margin_dpo/margin_mean": 83.00294494628906, "margin_dpo/margin_std": 99.38538360595703, "step": 446 }, { "KL/chosen_KL_mean": -148.73526000976562, "KL/mean": -203.46554565429688, "KL/rejected_KL_mean": -258.19586181640625, "KL/std": 107.6642074584961, "epoch": 0.6563876651982379, "fcm_dpo/beta": 0.004278257489204407, "fcm_dpo/delta": -0.07166311144828796, "fcm_dpo/margin": 109.4605941772461, "fcm_dpo/q_t": 0.39114248752593994, "grad_norm": 8.289060592651367, "learning_rate": 1.608874379754465e-07, "logits/chosen": -2.037332773208618, "logits/rejected": -1.9929592609405518, "logps/chosen": -232.3505096435547, "logps/ref_chosen": -83.6152572631836, "logps/ref_rejected": -104.91239929199219, "logps/rejected": -363.1082763671875, "loss": 1.0381, "margin_dpo/margin_mean": 109.46060180664062, "margin_dpo/margin_std": 123.45143127441406, "step": 447 }, { "KL/chosen_KL_mean": -161.17611694335938, "KL/mean": -213.86087036132812, "KL/rejected_KL_mean": -266.5456237792969, "KL/std": 112.47804260253906, "epoch": 0.657856093979442, "fcm_dpo/beta": 0.004249397665262222, "fcm_dpo/delta": -0.050152819603681564, "fcm_dpo/margin": 105.36949157714844, "fcm_dpo/q_t": 0.3967461884021759, "grad_norm": 9.608505249023438, "learning_rate": 1.5968958345321177e-07, "logits/chosen": -1.7243329286575317, "logits/rejected": -1.4556059837341309, "logps/chosen": -253.7519073486328, "logps/ref_chosen": -92.5757827758789, "logps/ref_rejected": -107.68977355957031, "logps/rejected": -374.2353820800781, "loss": 1.0638, "margin_dpo/margin_mean": 105.36949157714844, "margin_dpo/margin_std": 131.62281799316406, "step": 448 }, { "KL/chosen_KL_mean": -158.67913818359375, "KL/mean": -213.29476928710938, "KL/rejected_KL_mean": -267.9104309082031, "KL/std": 105.8449478149414, "epoch": 0.6593245227606461, "fcm_dpo/beta": 0.004182309843599796, "fcm_dpo/delta": -0.059573158621788025, "fcm_dpo/margin": 109.23129272460938, "fcm_dpo/q_t": 0.39313215017318726, "grad_norm": 8.536856651306152, "learning_rate": 1.584941086944423e-07, "logits/chosen": -1.708990216255188, "logits/rejected": -1.2822885513305664, "logps/chosen": -261.07806396484375, "logps/ref_chosen": -102.39893341064453, "logps/ref_rejected": -95.14886474609375, "logps/rejected": -363.0592956542969, "loss": 1.0413, "margin_dpo/margin_mean": 109.23128509521484, "margin_dpo/margin_std": 121.24258422851562, "step": 449 }, { "KL/chosen_KL_mean": -141.81605529785156, "KL/mean": -192.88380432128906, "KL/rejected_KL_mean": -243.95156860351562, "KL/std": 106.66605377197266, "epoch": 0.6607929515418502, "fcm_dpo/beta": 0.004151148721575737, "fcm_dpo/delta": -0.025080326944589615, "fcm_dpo/margin": 102.135498046875, "fcm_dpo/q_t": 0.400248646736145, "grad_norm": 9.05582332611084, "learning_rate": 1.573010452010098e-07, "logits/chosen": -2.3348100185394287, "logits/rejected": -2.1421189308166504, "logps/chosen": -228.8089141845703, "logps/ref_chosen": -86.99285888671875, "logps/ref_rejected": -108.53203582763672, "logps/rejected": -352.48358154296875, "loss": 1.0617, "margin_dpo/margin_mean": 102.135498046875, "margin_dpo/margin_std": 116.01069641113281, "step": 450 }, { "KL/chosen_KL_mean": -160.21636962890625, "KL/mean": -200.87149047851562, "KL/rejected_KL_mean": -241.526611328125, "KL/std": 105.99018859863281, "epoch": 0.6622613803230544, "fcm_dpo/beta": 0.004193603992462158, "fcm_dpo/delta": 0.06096739321947098, "fcm_dpo/margin": 81.31025695800781, "fcm_dpo/q_t": 0.4217182397842407, "grad_norm": 13.223312377929688, "learning_rate": 1.5611042441124687e-07, "logits/chosen": -2.2851643562316895, "logits/rejected": -1.7675621509552002, "logps/chosen": -247.02764892578125, "logps/ref_chosen": -86.811279296875, "logps/ref_rejected": -79.8555908203125, "logps/rejected": -321.3822021484375, "loss": 1.1477, "margin_dpo/margin_mean": 81.31025695800781, "margin_dpo/margin_std": 132.5671844482422, "step": 451 }, { "KL/chosen_KL_mean": -139.62159729003906, "KL/mean": -185.6328125, "KL/rejected_KL_mean": -231.64402770996094, "KL/std": 100.92298889160156, "epoch": 0.6637298091042585, "fcm_dpo/beta": 0.004152514040470123, "fcm_dpo/delta": -0.057407230138778687, "fcm_dpo/margin": 92.02242279052734, "fcm_dpo/q_t": 0.4089251756668091, "grad_norm": 8.04272747039795, "learning_rate": 1.549222776991186e-07, "logits/chosen": -1.9601504802703857, "logits/rejected": -1.903310775756836, "logps/chosen": -219.00123596191406, "logps/ref_chosen": -79.379638671875, "logps/ref_rejected": -103.71539306640625, "logps/rejected": -335.35943603515625, "loss": 1.081, "margin_dpo/margin_mean": 92.02242279052734, "margin_dpo/margin_std": 93.621337890625, "step": 452 }, { "KL/chosen_KL_mean": -169.2482147216797, "KL/mean": -225.22848510742188, "KL/rejected_KL_mean": -281.2087707519531, "KL/std": 103.44989013671875, "epoch": 0.6651982378854625, "fcm_dpo/beta": 0.004117329604923725, "fcm_dpo/delta": -0.06388219445943832, "fcm_dpo/margin": 111.96056365966797, "fcm_dpo/q_t": 0.39218634366989136, "grad_norm": 12.239067077636719, "learning_rate": 1.5373663637339584e-07, "logits/chosen": -1.9556211233139038, "logits/rejected": -1.4861172437667847, "logps/chosen": -256.94342041015625, "logps/ref_chosen": -87.6951904296875, "logps/ref_rejected": -90.0582275390625, "logps/rejected": -371.2669982910156, "loss": 1.0273, "margin_dpo/margin_mean": 111.96056365966797, "margin_dpo/margin_std": 110.98213958740234, "step": 453 }, { "KL/chosen_KL_mean": -177.6344757080078, "KL/mean": -222.7556610107422, "KL/rejected_KL_mean": -267.8768615722656, "KL/std": 106.05061340332031, "epoch": 0.6666666666666666, "fcm_dpo/beta": 0.004087763838469982, "fcm_dpo/delta": -0.029437892138957977, "fcm_dpo/margin": 90.24238586425781, "fcm_dpo/q_t": 0.4147986173629761, "grad_norm": 9.041144371032715, "learning_rate": 1.5255353167683017e-07, "logits/chosen": -2.0324559211730957, "logits/rejected": -1.5890324115753174, "logps/chosen": -267.2007141113281, "logps/ref_chosen": -89.56623840332031, "logps/ref_rejected": -92.92105102539062, "logps/rejected": -360.79791259765625, "loss": 1.1282, "margin_dpo/margin_mean": 90.24238586425781, "margin_dpo/margin_std": 137.99774169921875, "step": 454 }, { "KL/chosen_KL_mean": -166.79348754882812, "KL/mean": -209.1966552734375, "KL/rejected_KL_mean": -251.59982299804688, "KL/std": 103.95687866210938, "epoch": 0.6681350954478708, "fcm_dpo/beta": 0.00407512579113245, "fcm_dpo/delta": -0.001483845291659236, "fcm_dpo/margin": 84.80633544921875, "fcm_dpo/q_t": 0.4188273847103119, "grad_norm": 9.120612144470215, "learning_rate": 1.5137299478533064e-07, "logits/chosen": -2.023879051208496, "logits/rejected": -1.9858670234680176, "logps/chosen": -244.42347717285156, "logps/ref_chosen": -77.62998962402344, "logps/ref_rejected": -118.97795104980469, "logps/rejected": -370.5777893066406, "loss": 1.1278, "margin_dpo/margin_mean": 84.80635070800781, "margin_dpo/margin_std": 119.13844299316406, "step": 455 }, { "KL/chosen_KL_mean": -168.79946899414062, "KL/mean": -211.44793701171875, "KL/rejected_KL_mean": -254.0963897705078, "KL/std": 105.70134735107422, "epoch": 0.6696035242290749, "fcm_dpo/beta": 0.0041126469150185585, "fcm_dpo/delta": 0.05084359273314476, "fcm_dpo/margin": 85.29690551757812, "fcm_dpo/q_t": 0.41757047176361084, "grad_norm": 11.163372993469238, "learning_rate": 1.5019505680714232e-07, "logits/chosen": -1.9497323036193848, "logits/rejected": -1.8175973892211914, "logps/chosen": -258.4163513183594, "logps/ref_chosen": -89.61686706542969, "logps/ref_rejected": -109.5597152709961, "logps/rejected": -363.6560974121094, "loss": 1.1129, "margin_dpo/margin_mean": 85.29690551757812, "margin_dpo/margin_std": 107.62725830078125, "step": 456 }, { "KL/chosen_KL_mean": -137.1720733642578, "KL/mean": -193.56689453125, "KL/rejected_KL_mean": -249.96173095703125, "KL/std": 105.61326599121094, "epoch": 0.671071953010279, "fcm_dpo/beta": 0.0040517039597034454, "fcm_dpo/delta": -0.061049215495586395, "fcm_dpo/margin": 112.78965759277344, "fcm_dpo/q_t": 0.39104849100112915, "grad_norm": 9.477270126342773, "learning_rate": 1.4901974878202627e-07, "logits/chosen": -2.048971176147461, "logits/rejected": -1.7304811477661133, "logps/chosen": -224.49375915527344, "logps/ref_chosen": -87.32168579101562, "logps/ref_rejected": -90.76660919189453, "logps/rejected": -340.72833251953125, "loss": 1.0192, "margin_dpo/margin_mean": 112.78965759277344, "margin_dpo/margin_std": 94.38288879394531, "step": 457 }, { "KL/chosen_KL_mean": -160.56723022460938, "KL/mean": -201.14279174804688, "KL/rejected_KL_mean": -241.71835327148438, "KL/std": 105.17810821533203, "epoch": 0.6725403817914831, "fcm_dpo/beta": 0.004072901792824268, "fcm_dpo/delta": 0.01642300933599472, "fcm_dpo/margin": 81.15113830566406, "fcm_dpo/q_t": 0.4237171709537506, "grad_norm": 9.095541000366211, "learning_rate": 1.4784710168044212e-07, "logits/chosen": -1.5677192211151123, "logits/rejected": -1.3271763324737549, "logps/chosen": -254.08767700195312, "logps/ref_chosen": -93.52044677734375, "logps/ref_rejected": -103.36898803710938, "logps/rejected": -345.08734130859375, "loss": 1.1415, "margin_dpo/margin_mean": 81.15113067626953, "margin_dpo/margin_std": 122.51338958740234, "step": 458 }, { "KL/chosen_KL_mean": -152.54318237304688, "KL/mean": -204.0995635986328, "KL/rejected_KL_mean": -255.65594482421875, "KL/std": 110.07746887207031, "epoch": 0.6740088105726872, "fcm_dpo/beta": 0.004050368443131447, "fcm_dpo/delta": -0.018888771533966064, "fcm_dpo/margin": 103.11275482177734, "fcm_dpo/q_t": 0.4028448760509491, "grad_norm": 6.887370586395264, "learning_rate": 1.466771464027316e-07, "logits/chosen": -1.6962974071502686, "logits/rejected": -1.4479608535766602, "logps/chosen": -228.2313995361328, "logps/ref_chosen": -75.68820190429688, "logps/ref_rejected": -92.17048645019531, "logps/rejected": -347.826416015625, "loss": 1.0673, "margin_dpo/margin_mean": 103.11275482177734, "margin_dpo/margin_std": 118.2739028930664, "step": 459 }, { "KL/chosen_KL_mean": -156.66201782226562, "KL/mean": -204.0841064453125, "KL/rejected_KL_mean": -251.50619506835938, "KL/std": 106.85321044921875, "epoch": 0.6754772393538914, "fcm_dpo/beta": 0.0040590595453977585, "fcm_dpo/delta": 0.015417791903018951, "fcm_dpo/margin": 94.84416198730469, "fcm_dpo/q_t": 0.4102768301963806, "grad_norm": 10.083240509033203, "learning_rate": 1.4550991377830423e-07, "logits/chosen": -2.106811285018921, "logits/rejected": -2.1325631141662598, "logps/chosen": -237.7799072265625, "logps/ref_chosen": -81.11788940429688, "logps/ref_rejected": -110.31238555908203, "logps/rejected": -361.8185729980469, "loss": 1.0982, "margin_dpo/margin_mean": 94.84416198730469, "margin_dpo/margin_std": 122.52616882324219, "step": 460 }, { "KL/chosen_KL_mean": -192.66212463378906, "KL/mean": -231.67709350585938, "KL/rejected_KL_mean": -270.6921081542969, "KL/std": 93.80046081542969, "epoch": 0.6769456681350955, "fcm_dpo/beta": 0.004091168288141489, "fcm_dpo/delta": 0.01763981208205223, "fcm_dpo/margin": 78.02999114990234, "fcm_dpo/q_t": 0.4252292513847351, "grad_norm": 8.205183982849121, "learning_rate": 1.4434543456482518e-07, "logits/chosen": -1.396677851676941, "logits/rejected": -1.1992204189300537, "logps/chosen": -274.24566650390625, "logps/ref_chosen": -81.58352661132812, "logps/ref_rejected": -93.87710571289062, "logps/rejected": -364.5692138671875, "loss": 1.1446, "margin_dpo/margin_mean": 78.02998352050781, "margin_dpo/margin_std": 113.22684478759766, "step": 461 }, { "KL/chosen_KL_mean": -146.56976318359375, "KL/mean": -195.79876708984375, "KL/rejected_KL_mean": -245.02777099609375, "KL/std": 100.84786987304688, "epoch": 0.6784140969162996, "fcm_dpo/beta": 0.004094731993973255, "fcm_dpo/delta": -0.003325372003018856, "fcm_dpo/margin": 98.45801544189453, "fcm_dpo/q_t": 0.40532612800598145, "grad_norm": 8.431817054748535, "learning_rate": 1.4318373944740484e-07, "logits/chosen": -2.306246757507324, "logits/rejected": -1.8081347942352295, "logps/chosen": -240.768310546875, "logps/ref_chosen": -94.19854736328125, "logps/ref_rejected": -85.63162994384766, "logps/rejected": -330.659423828125, "loss": 1.0702, "margin_dpo/margin_mean": 98.45801544189453, "margin_dpo/margin_std": 108.60719299316406, "step": 462 }, { "KL/chosen_KL_mean": -142.71499633789062, "KL/mean": -201.4064178466797, "KL/rejected_KL_mean": -260.09783935546875, "KL/std": 108.75080871582031, "epoch": 0.6798825256975036, "fcm_dpo/beta": 0.004063591826707125, "fcm_dpo/delta": -0.08109010756015778, "fcm_dpo/margin": 117.3828353881836, "fcm_dpo/q_t": 0.38752663135528564, "grad_norm": 13.197741508483887, "learning_rate": 1.4202485903778976e-07, "logits/chosen": -1.8157784938812256, "logits/rejected": -1.3875007629394531, "logps/chosen": -228.63973999023438, "logps/ref_chosen": -85.92474365234375, "logps/ref_rejected": -96.90184020996094, "logps/rejected": -356.99969482421875, "loss": 1.0233, "margin_dpo/margin_mean": 117.38284301757812, "margin_dpo/margin_std": 115.56355285644531, "step": 463 }, { "KL/chosen_KL_mean": -175.63992309570312, "KL/mean": -223.54788208007812, "KL/rejected_KL_mean": -271.45587158203125, "KL/std": 108.31089782714844, "epoch": 0.6813509544787077, "fcm_dpo/beta": 0.004023114684969187, "fcm_dpo/delta": 0.015042667277157307, "fcm_dpo/margin": 95.8159408569336, "fcm_dpo/q_t": 0.4102011024951935, "grad_norm": 11.25102710723877, "learning_rate": 1.4086882387355658e-07, "logits/chosen": -1.8083809614181519, "logits/rejected": -1.8889503479003906, "logps/chosen": -255.32913208007812, "logps/ref_chosen": -79.68919372558594, "logps/ref_rejected": -107.29232025146484, "logps/rejected": -378.7481689453125, "loss": 1.0962, "margin_dpo/margin_mean": 95.8159408569336, "margin_dpo/margin_std": 123.82083129882812, "step": 464 }, { "KL/chosen_KL_mean": -166.99349975585938, "KL/mean": -223.34901428222656, "KL/rejected_KL_mean": -279.70452880859375, "KL/std": 117.02505493164062, "epoch": 0.6828193832599119, "fcm_dpo/beta": 0.00399835966527462, "fcm_dpo/delta": -0.05313951522111893, "fcm_dpo/margin": 112.71102905273438, "fcm_dpo/q_t": 0.39359086751937866, "grad_norm": 8.355056762695312, "learning_rate": 1.3971566441730714e-07, "logits/chosen": -1.6797990798950195, "logits/rejected": -1.3802953958511353, "logps/chosen": -258.8537292480469, "logps/ref_chosen": -91.8602294921875, "logps/ref_rejected": -118.71001434326172, "logps/rejected": -398.41455078125, "loss": 1.032, "margin_dpo/margin_mean": 112.7110366821289, "margin_dpo/margin_std": 110.83982849121094, "step": 465 }, { "KL/chosen_KL_mean": -148.77413940429688, "KL/mean": -206.149658203125, "KL/rejected_KL_mean": -263.525146484375, "KL/std": 111.46836853027344, "epoch": 0.684287812041116, "fcm_dpo/beta": 0.0039554983377456665, "fcm_dpo/delta": -0.05651950463652611, "fcm_dpo/margin": 114.75102233886719, "fcm_dpo/q_t": 0.3933257758617401, "grad_norm": 6.835327625274658, "learning_rate": 1.3856541105586545e-07, "logits/chosen": -1.3796275854110718, "logits/rejected": -1.0510189533233643, "logps/chosen": -233.4755401611328, "logps/ref_chosen": -84.70140838623047, "logps/ref_rejected": -96.05084228515625, "logps/rejected": -359.57598876953125, "loss": 1.0339, "margin_dpo/margin_mean": 114.75102233886719, "margin_dpo/margin_std": 117.52569580078125, "step": 466 }, { "KL/chosen_KL_mean": -190.7152557373047, "KL/mean": -238.3769073486328, "KL/rejected_KL_mean": -286.0385437011719, "KL/std": 116.53121185302734, "epoch": 0.6857562408223201, "fcm_dpo/beta": 0.003923690877854824, "fcm_dpo/delta": -0.02995622530579567, "fcm_dpo/margin": 95.32328033447266, "fcm_dpo/q_t": 0.41228967905044556, "grad_norm": 12.012237548828125, "learning_rate": 1.3741809409947729e-07, "logits/chosen": -2.1684374809265137, "logits/rejected": -1.5761277675628662, "logps/chosen": -300.0135803222656, "logps/ref_chosen": -109.29832458496094, "logps/ref_rejected": -108.8436508178711, "logps/rejected": -394.8822021484375, "loss": 1.112, "margin_dpo/margin_mean": 95.32328796386719, "margin_dpo/margin_std": 131.83474731445312, "step": 467 }, { "KL/chosen_KL_mean": -147.77432250976562, "KL/mean": -192.05670166015625, "KL/rejected_KL_mean": -236.33908081054688, "KL/std": 114.13752746582031, "epoch": 0.6872246696035242, "fcm_dpo/beta": 0.003950462676584721, "fcm_dpo/delta": 0.051976777613162994, "fcm_dpo/margin": 88.56475067138672, "fcm_dpo/q_t": 0.4197770953178406, "grad_norm": 8.753684043884277, "learning_rate": 1.362737437810114e-07, "logits/chosen": -1.8183279037475586, "logits/rejected": -1.61104416847229, "logps/chosen": -246.09597778320312, "logps/ref_chosen": -98.32164764404297, "logps/ref_rejected": -106.68048095703125, "logps/rejected": -343.01953125, "loss": 1.1304, "margin_dpo/margin_mean": 88.56475067138672, "margin_dpo/margin_std": 131.6105194091797, "step": 468 }, { "KL/chosen_KL_mean": -181.4677276611328, "KL/mean": -235.90357971191406, "KL/rejected_KL_mean": -290.33941650390625, "KL/std": 113.92741394042969, "epoch": 0.6886930983847284, "fcm_dpo/beta": 0.0039282143115997314, "fcm_dpo/delta": -0.02953207865357399, "fcm_dpo/margin": 108.87169647216797, "fcm_dpo/q_t": 0.4000743627548218, "grad_norm": 9.85301685333252, "learning_rate": 1.351323902551631e-07, "logits/chosen": -2.242103099822998, "logits/rejected": -1.823288917541504, "logps/chosen": -278.23193359375, "logps/ref_chosen": -96.76420593261719, "logps/ref_rejected": -109.59500885009766, "logps/rejected": -399.9344482421875, "loss": 1.0614, "margin_dpo/margin_mean": 108.87168884277344, "margin_dpo/margin_std": 123.83409118652344, "step": 469 }, { "KL/chosen_KL_mean": -166.46792602539062, "KL/mean": -216.46983337402344, "KL/rejected_KL_mean": -266.47174072265625, "KL/std": 105.8537826538086, "epoch": 0.6901615271659325, "fcm_dpo/beta": 0.0039438409730792046, "fcm_dpo/delta": 0.005830895621329546, "fcm_dpo/margin": 100.00384521484375, "fcm_dpo/q_t": 0.4067780375480652, "grad_norm": 8.820821762084961, "learning_rate": 1.339940635976592e-07, "logits/chosen": -1.6768912076950073, "logits/rejected": -1.2582461833953857, "logps/chosen": -249.96456909179688, "logps/ref_chosen": -83.49665832519531, "logps/ref_rejected": -88.48578643798828, "logps/rejected": -354.95751953125, "loss": 1.0717, "margin_dpo/margin_mean": 100.00384521484375, "margin_dpo/margin_std": 105.56761169433594, "step": 470 }, { "KL/chosen_KL_mean": -191.78350830078125, "KL/mean": -239.0438232421875, "KL/rejected_KL_mean": -286.30419921875, "KL/std": 105.09850311279297, "epoch": 0.6916299559471366, "fcm_dpo/beta": 0.003960807342082262, "fcm_dpo/delta": 0.026623040437698364, "fcm_dpo/margin": 94.52067565917969, "fcm_dpo/q_t": 0.4116513133049011, "grad_norm": 8.525957107543945, "learning_rate": 1.3285879380446563e-07, "logits/chosen": -1.6159522533416748, "logits/rejected": -1.201261281967163, "logps/chosen": -280.2578125, "logps/ref_chosen": -88.47430419921875, "logps/ref_rejected": -90.48171997070312, "logps/rejected": -376.785888671875, "loss": 1.0886, "margin_dpo/margin_mean": 94.52067565917969, "margin_dpo/margin_std": 106.35237121582031, "step": 471 }, { "KL/chosen_KL_mean": -180.53919982910156, "KL/mean": -236.33090209960938, "KL/rejected_KL_mean": -292.1225891113281, "KL/std": 105.68905639648438, "epoch": 0.6930983847283406, "fcm_dpo/beta": 0.003933281637728214, "fcm_dpo/delta": -0.040896937251091, "fcm_dpo/margin": 111.58338928222656, "fcm_dpo/q_t": 0.397246778011322, "grad_norm": 7.867846965789795, "learning_rate": 1.317266107909975e-07, "logits/chosen": -2.1931662559509277, "logits/rejected": -1.498880386352539, "logps/chosen": -305.77288818359375, "logps/ref_chosen": -125.23370361328125, "logps/ref_rejected": -121.05349731445312, "logps/rejected": -413.17608642578125, "loss": 1.0417, "margin_dpo/margin_mean": 111.58338928222656, "margin_dpo/margin_std": 112.47453308105469, "step": 472 }, { "KL/chosen_KL_mean": -204.4556121826172, "KL/mean": -239.10324096679688, "KL/rejected_KL_mean": -273.7508544921875, "KL/std": 106.93110656738281, "epoch": 0.6945668135095447, "fcm_dpo/beta": 0.003934693988412619, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 69.29525756835938, "fcm_dpo/q_t": 0.4367903470993042, "grad_norm": 9.614045143127441, "learning_rate": 1.3059754439133002e-07, "logits/chosen": -1.9692649841308594, "logits/rejected": -1.4926143884658813, "logps/chosen": -300.0669860839844, "logps/ref_chosen": -95.61137390136719, "logps/ref_rejected": -88.15115356445312, "logps/rejected": -361.9020080566406, "loss": 1.1894, "margin_dpo/margin_mean": 69.29525756835938, "margin_dpo/margin_std": 125.0532455444336, "step": 473 }, { "KL/chosen_KL_mean": -190.392333984375, "KL/mean": -226.40716552734375, "KL/rejected_KL_mean": -262.4219970703125, "KL/std": 101.19680786132812, "epoch": 0.6960352422907489, "fcm_dpo/beta": 0.003933422267436981, "fcm_dpo/delta": -0.003233524737879634, "fcm_dpo/margin": 72.0296630859375, "fcm_dpo/q_t": 0.4336917996406555, "grad_norm": 13.4172945022583, "learning_rate": 1.2947162435741277e-07, "logits/chosen": -1.468416452407837, "logits/rejected": -1.277721881866455, "logps/chosen": -271.8720703125, "logps/ref_chosen": -81.47975158691406, "logps/ref_rejected": -96.46562957763672, "logps/rejected": -358.88763427734375, "loss": 1.185, "margin_dpo/margin_mean": 72.0296630859375, "margin_dpo/margin_std": 123.54573059082031, "step": 474 }, { "KL/chosen_KL_mean": -144.3023681640625, "KL/mean": -199.78546142578125, "KL/rejected_KL_mean": -255.26856994628906, "KL/std": 108.2242431640625, "epoch": 0.697503671071953, "fcm_dpo/beta": 0.0039138151332736015, "fcm_dpo/delta": -0.03584384173154831, "fcm_dpo/margin": 110.9662094116211, "fcm_dpo/q_t": 0.3987465500831604, "grad_norm": 7.769962787628174, "learning_rate": 1.2834888035828596e-07, "logits/chosen": -2.192134380340576, "logits/rejected": -2.140284538269043, "logps/chosen": -218.49835205078125, "logps/ref_chosen": -74.19598388671875, "logps/ref_rejected": -94.69242095947266, "logps/rejected": -349.96099853515625, "loss": 1.0534, "margin_dpo/margin_mean": 110.96621704101562, "margin_dpo/margin_std": 123.66749572753906, "step": 475 }, { "KL/chosen_KL_mean": -168.52392578125, "KL/mean": -218.12692260742188, "KL/rejected_KL_mean": -267.72991943359375, "KL/std": 110.69952392578125, "epoch": 0.6989720998531571, "fcm_dpo/beta": 0.003901264164596796, "fcm_dpo/delta": 0.013379184529185295, "fcm_dpo/margin": 99.20600891113281, "fcm_dpo/q_t": 0.41008588671684265, "grad_norm": 9.65160083770752, "learning_rate": 1.2722934197929802e-07, "logits/chosen": -2.1841890811920166, "logits/rejected": -1.8196310997009277, "logps/chosen": -240.49502563476562, "logps/ref_chosen": -71.97109985351562, "logps/ref_rejected": -80.26224517822266, "logps/rejected": -347.9921875, "loss": 1.0885, "margin_dpo/margin_mean": 99.20600891113281, "margin_dpo/margin_std": 119.98284912109375, "step": 476 }, { "KL/chosen_KL_mean": -169.69662475585938, "KL/mean": -218.6262969970703, "KL/rejected_KL_mean": -267.55596923828125, "KL/std": 97.36880493164062, "epoch": 0.7004405286343612, "fcm_dpo/beta": 0.003927314188331366, "fcm_dpo/delta": 0.016274182125926018, "fcm_dpo/margin": 97.8593521118164, "fcm_dpo/q_t": 0.40946266055107117, "grad_norm": 8.574677467346191, "learning_rate": 1.2611303872132631e-07, "logits/chosen": -2.130237579345703, "logits/rejected": -1.2065376043319702, "logps/chosen": -274.7021789550781, "logps/ref_chosen": -105.00555419921875, "logps/ref_rejected": -81.87843322753906, "logps/rejected": -349.43438720703125, "loss": 1.0925, "margin_dpo/margin_mean": 97.85935974121094, "margin_dpo/margin_std": 121.09370422363281, "step": 477 }, { "KL/chosen_KL_mean": -164.22198486328125, "KL/mean": -215.223388671875, "KL/rejected_KL_mean": -266.22479248046875, "KL/std": 105.84947967529297, "epoch": 0.7019089574155654, "fcm_dpo/beta": 0.003927084617316723, "fcm_dpo/delta": -0.0005975192179903388, "fcm_dpo/margin": 102.00279235839844, "fcm_dpo/q_t": 0.40583863854408264, "grad_norm": 8.494955062866211, "learning_rate": 1.2500000000000005e-07, "logits/chosen": -1.4578558206558228, "logits/rejected": -1.2777230739593506, "logps/chosen": -241.01019287109375, "logps/ref_chosen": -76.7882080078125, "logps/ref_rejected": -90.43994140625, "logps/rejected": -356.66473388671875, "loss": 1.076, "margin_dpo/margin_mean": 102.00279235839844, "margin_dpo/margin_std": 117.43643188476562, "step": 478 }, { "KL/chosen_KL_mean": -166.7762451171875, "KL/mean": -218.04080200195312, "KL/rejected_KL_mean": -269.30535888671875, "KL/std": 107.05646514892578, "epoch": 0.7033773861967695, "fcm_dpo/beta": 0.003938902169466019, "fcm_dpo/delta": -0.004278495907783508, "fcm_dpo/margin": 102.52912902832031, "fcm_dpo/q_t": 0.4045076072216034, "grad_norm": 12.294120788574219, "learning_rate": 1.2389025514492456e-07, "logits/chosen": -1.37725031375885, "logits/rejected": -1.234118938446045, "logps/chosen": -248.1385498046875, "logps/ref_chosen": -81.3623046875, "logps/ref_rejected": -101.09114074707031, "logps/rejected": -370.3965148925781, "loss": 1.0703, "margin_dpo/margin_mean": 102.52912902832031, "margin_dpo/margin_std": 111.4662857055664, "step": 479 }, { "KL/chosen_KL_mean": -194.56231689453125, "KL/mean": -234.51068115234375, "KL/rejected_KL_mean": -274.458984375, "KL/std": 101.8110122680664, "epoch": 0.7048458149779736, "fcm_dpo/beta": 0.003974815830588341, "fcm_dpo/delta": 0.085101418197155, "fcm_dpo/margin": 79.89668273925781, "fcm_dpo/q_t": 0.4255372881889343, "grad_norm": 9.53321361541748, "learning_rate": 1.227838333989088e-07, "logits/chosen": -1.8773475885391235, "logits/rejected": -1.212838053703308, "logps/chosen": -291.33624267578125, "logps/ref_chosen": -96.7739028930664, "logps/ref_rejected": -86.40473937988281, "logps/rejected": -360.8637390136719, "loss": 1.1416, "margin_dpo/margin_mean": 79.89667510986328, "margin_dpo/margin_std": 112.32437133789062, "step": 480 }, { "KL/chosen_KL_mean": -189.54202270507812, "KL/mean": -236.66067504882812, "KL/rejected_KL_mean": -283.7793273925781, "KL/std": 106.41290283203125, "epoch": 0.7063142437591777, "fcm_dpo/beta": 0.0040034460835158825, "fcm_dpo/delta": 0.02362295240163803, "fcm_dpo/margin": 94.23728942871094, "fcm_dpo/q_t": 0.4113905429840088, "grad_norm": 11.179119110107422, "learning_rate": 1.2168076391719489e-07, "logits/chosen": -1.7786369323730469, "logits/rejected": -1.332186222076416, "logps/chosen": -281.2121887207031, "logps/ref_chosen": -91.670166015625, "logps/ref_rejected": -98.69490051269531, "logps/rejected": -382.4742431640625, "loss": 1.0969, "margin_dpo/margin_mean": 94.23728942871094, "margin_dpo/margin_std": 117.689453125, "step": 481 }, { "KL/chosen_KL_mean": -182.7600860595703, "KL/mean": -229.0103759765625, "KL/rejected_KL_mean": -275.2606506347656, "KL/std": 105.2197494506836, "epoch": 0.7077826725403817, "fcm_dpo/beta": 0.004017159342765808, "fcm_dpo/delta": 0.029450150206685066, "fcm_dpo/margin": 92.50057220458984, "fcm_dpo/q_t": 0.4123062491416931, "grad_norm": 12.130163192749023, "learning_rate": 1.2058107576668938e-07, "logits/chosen": -1.5374221801757812, "logits/rejected": -1.1778550148010254, "logps/chosen": -281.28021240234375, "logps/ref_chosen": -98.52011108398438, "logps/ref_rejected": -94.82945251464844, "logps/rejected": -370.090087890625, "loss": 1.1029, "margin_dpo/margin_mean": 92.50057220458984, "margin_dpo/margin_std": 117.57794189453125, "step": 482 }, { "KL/chosen_KL_mean": -189.22280883789062, "KL/mean": -238.18719482421875, "KL/rejected_KL_mean": -287.151611328125, "KL/std": 110.83888244628906, "epoch": 0.7092511013215859, "fcm_dpo/beta": 0.004012971185147762, "fcm_dpo/delta": -0.05088834464550018, "fcm_dpo/margin": 97.92879486083984, "fcm_dpo/q_t": 0.40683647990226746, "grad_norm": 7.473385810852051, "learning_rate": 1.194847979251979e-07, "logits/chosen": -1.9428081512451172, "logits/rejected": -1.2895302772521973, "logps/chosen": -296.3414306640625, "logps/ref_chosen": -107.11860656738281, "logps/ref_rejected": -101.11499786376953, "logps/rejected": -388.2666015625, "loss": 1.0809, "margin_dpo/margin_mean": 97.92880249023438, "margin_dpo/margin_std": 110.62600708007812, "step": 483 }, { "KL/chosen_KL_mean": -172.4029998779297, "KL/mean": -227.1865234375, "KL/rejected_KL_mean": -281.97003173828125, "KL/std": 112.21696472167969, "epoch": 0.71071953010279, "fcm_dpo/beta": 0.003981337416917086, "fcm_dpo/delta": -0.03793150186538696, "fcm_dpo/margin": 109.56703186035156, "fcm_dpo/q_t": 0.397506445646286, "grad_norm": 8.627911567687988, "learning_rate": 1.1839195928066101e-07, "logits/chosen": -2.453545331954956, "logits/rejected": -1.9802464246749878, "logps/chosen": -259.3829345703125, "logps/ref_chosen": -86.97991943359375, "logps/ref_rejected": -90.72367095947266, "logps/rejected": -372.6936950683594, "loss": 1.0542, "margin_dpo/margin_mean": 109.56703186035156, "margin_dpo/margin_std": 122.67124938964844, "step": 484 }, { "KL/chosen_KL_mean": -164.11558532714844, "KL/mean": -223.28280639648438, "KL/rejected_KL_mean": -282.45001220703125, "KL/std": 117.07911682128906, "epoch": 0.7121879588839941, "fcm_dpo/beta": 0.003924118354916573, "fcm_dpo/delta": -0.06747239083051682, "fcm_dpo/margin": 118.33447265625, "fcm_dpo/q_t": 0.3911016583442688, "grad_norm": 8.484262466430664, "learning_rate": 1.1730258863039347e-07, "logits/chosen": -1.7410882711410522, "logits/rejected": -1.3221330642700195, "logps/chosen": -258.17431640625, "logps/ref_chosen": -94.05874633789062, "logps/ref_rejected": -108.56298828125, "logps/rejected": -391.01300048828125, "loss": 1.026, "margin_dpo/margin_mean": 118.33447265625, "margin_dpo/margin_std": 118.95500183105469, "step": 485 }, { "KL/chosen_KL_mean": -188.41806030273438, "KL/mean": -240.04917907714844, "KL/rejected_KL_mean": -291.6802978515625, "KL/std": 112.5208740234375, "epoch": 0.7136563876651982, "fcm_dpo/beta": 0.003916551824659109, "fcm_dpo/delta": -0.004740707576274872, "fcm_dpo/margin": 103.26220703125, "fcm_dpo/q_t": 0.40490737557411194, "grad_norm": 10.068588256835938, "learning_rate": 1.1621671468032493e-07, "logits/chosen": -1.889577865600586, "logits/rejected": -1.1691875457763672, "logps/chosen": -282.1639404296875, "logps/ref_chosen": -93.74588012695312, "logps/ref_rejected": -98.07064819335938, "logps/rejected": -389.7509460449219, "loss": 1.0762, "margin_dpo/margin_mean": 103.26220703125, "margin_dpo/margin_std": 120.4369125366211, "step": 486 }, { "KL/chosen_KL_mean": -182.69142150878906, "KL/mean": -229.45248413085938, "KL/rejected_KL_mean": -276.2135314941406, "KL/std": 107.39639282226562, "epoch": 0.7151248164464024, "fcm_dpo/beta": 0.0039243861101567745, "fcm_dpo/delta": 0.0342482291162014, "fcm_dpo/margin": 93.52210235595703, "fcm_dpo/q_t": 0.4137478470802307, "grad_norm": 9.828581809997559, "learning_rate": 1.1513436604424378e-07, "logits/chosen": -2.1258187294006348, "logits/rejected": -1.8154586553573608, "logps/chosen": -270.7249755859375, "logps/ref_chosen": -88.0335693359375, "logps/ref_rejected": -98.47209930419922, "logps/rejected": -374.68560791015625, "loss": 1.1012, "margin_dpo/margin_mean": 93.52210998535156, "margin_dpo/margin_std": 114.90782165527344, "step": 487 }, { "KL/chosen_KL_mean": -163.84619140625, "KL/mean": -216.0897979736328, "KL/rejected_KL_mean": -268.3333740234375, "KL/std": 113.79476165771484, "epoch": 0.7165932452276065, "fcm_dpo/beta": 0.003930740989744663, "fcm_dpo/delta": -0.011188503354787827, "fcm_dpo/margin": 104.48719024658203, "fcm_dpo/q_t": 0.40240877866744995, "grad_norm": 8.870756149291992, "learning_rate": 1.1405557124304335e-07, "logits/chosen": -1.7000606060028076, "logits/rejected": -1.4414727687835693, "logps/chosen": -248.63583374023438, "logps/ref_chosen": -84.78964233398438, "logps/ref_rejected": -90.2734603881836, "logps/rejected": -358.60687255859375, "loss": 1.0571, "margin_dpo/margin_mean": 104.48719024658203, "margin_dpo/margin_std": 101.5090560913086, "step": 488 }, { "KL/chosen_KL_mean": -185.60153198242188, "KL/mean": -233.37515258789062, "KL/rejected_KL_mean": -281.14874267578125, "KL/std": 104.13800048828125, "epoch": 0.7180616740088106, "fcm_dpo/beta": 0.003939471673220396, "fcm_dpo/delta": 0.024505067616701126, "fcm_dpo/margin": 95.54719543457031, "fcm_dpo/q_t": 0.4118320345878601, "grad_norm": 9.545340538024902, "learning_rate": 1.1298035870396985e-07, "logits/chosen": -1.6859453916549683, "logits/rejected": -1.199491024017334, "logps/chosen": -276.0708312988281, "logps/ref_chosen": -90.46929931640625, "logps/ref_rejected": -86.39761352539062, "logps/rejected": -367.5463562011719, "loss": 1.0943, "margin_dpo/margin_mean": 95.54719543457031, "margin_dpo/margin_std": 115.8781509399414, "step": 489 }, { "KL/chosen_KL_mean": -186.06907653808594, "KL/mean": -230.87518310546875, "KL/rejected_KL_mean": -275.6812744140625, "KL/std": 112.82815551757812, "epoch": 0.7195301027900147, "fcm_dpo/beta": 0.003927274141460657, "fcm_dpo/delta": -0.03976406529545784, "fcm_dpo/margin": 89.61219024658203, "fcm_dpo/q_t": 0.41919487714767456, "grad_norm": 8.340740203857422, "learning_rate": 1.1190875675987355e-07, "logits/chosen": -2.165680408477783, "logits/rejected": -2.245126724243164, "logps/chosen": -271.38922119140625, "logps/ref_chosen": -85.32012939453125, "logps/ref_rejected": -115.99385070800781, "logps/rejected": -391.67510986328125, "loss": 1.1496, "margin_dpo/margin_mean": 89.61219787597656, "margin_dpo/margin_std": 148.9447021484375, "step": 490 }, { "KL/chosen_KL_mean": -189.2417449951172, "KL/mean": -230.4969940185547, "KL/rejected_KL_mean": -271.752197265625, "KL/std": 114.90867614746094, "epoch": 0.7209985315712188, "fcm_dpo/beta": 0.003955123946070671, "fcm_dpo/delta": 0.07616802304983139, "fcm_dpo/margin": 82.51049041748047, "fcm_dpo/q_t": 0.42336469888687134, "grad_norm": 9.988592147827148, "learning_rate": 1.1084079364846241e-07, "logits/chosen": -1.8470979928970337, "logits/rejected": -1.3571316003799438, "logps/chosen": -275.38525390625, "logps/ref_chosen": -86.14352416992188, "logps/ref_rejected": -80.67945861816406, "logps/rejected": -352.43170166015625, "loss": 1.1373, "margin_dpo/margin_mean": 82.51048278808594, "margin_dpo/margin_std": 116.42083740234375, "step": 491 }, { "KL/chosen_KL_mean": -178.52621459960938, "KL/mean": -211.72604370117188, "KL/rejected_KL_mean": -244.92587280273438, "KL/std": 113.08814239501953, "epoch": 0.7224669603524229, "fcm_dpo/beta": 0.0039717028848826885, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 66.39967346191406, "fcm_dpo/q_t": 0.4386589527130127, "grad_norm": 12.74101734161377, "learning_rate": 1.097764975115576e-07, "logits/chosen": -2.1324310302734375, "logits/rejected": -1.7540669441223145, "logps/chosen": -259.6337890625, "logps/ref_chosen": -81.10757446289062, "logps/ref_rejected": -80.75199890136719, "logps/rejected": -325.6778564453125, "loss": 1.2144, "margin_dpo/margin_mean": 66.39967346191406, "margin_dpo/margin_std": 140.76681518554688, "step": 492 }, { "KL/chosen_KL_mean": -175.70169067382812, "KL/mean": -220.04055786132812, "KL/rejected_KL_mean": -264.37945556640625, "KL/std": 101.97767639160156, "epoch": 0.723935389133627, "fcm_dpo/beta": 0.0039587621577084064, "fcm_dpo/delta": -0.016317540779709816, "fcm_dpo/margin": 88.67771911621094, "fcm_dpo/q_t": 0.4165477752685547, "grad_norm": 14.901061058044434, "learning_rate": 1.0871589639435203e-07, "logits/chosen": -2.492276906967163, "logits/rejected": -1.7722182273864746, "logps/chosen": -287.9090270996094, "logps/ref_chosen": -112.20733642578125, "logps/ref_rejected": -93.60719299316406, "logps/rejected": -357.98663330078125, "loss": 1.1124, "margin_dpo/margin_mean": 88.67771911621094, "margin_dpo/margin_std": 109.20489501953125, "step": 493 }, { "KL/chosen_KL_mean": -170.80015563964844, "KL/mean": -228.9921112060547, "KL/rejected_KL_mean": -287.18408203125, "KL/std": 113.13653564453125, "epoch": 0.7254038179148311, "fcm_dpo/beta": 0.003905025077983737, "fcm_dpo/delta": -0.05789130926132202, "fcm_dpo/margin": 116.38390350341797, "fcm_dpo/q_t": 0.39332085847854614, "grad_norm": 9.406898498535156, "learning_rate": 1.0765901824467166e-07, "logits/chosen": -1.3142294883728027, "logits/rejected": -1.2379295825958252, "logps/chosen": -243.91505432128906, "logps/ref_chosen": -73.11489868164062, "logps/ref_rejected": -92.16300201416016, "logps/rejected": -379.3470764160156, "loss": 1.0394, "margin_dpo/margin_mean": 116.38390350341797, "margin_dpo/margin_std": 123.26473999023438, "step": 494 }, { "KL/chosen_KL_mean": -176.23895263671875, "KL/mean": -226.0841827392578, "KL/rejected_KL_mean": -275.9294128417969, "KL/std": 107.29304504394531, "epoch": 0.7268722466960352, "fcm_dpo/beta": 0.003916910383850336, "fcm_dpo/delta": 0.009907922707498074, "fcm_dpo/margin": 99.69044494628906, "fcm_dpo/q_t": 0.40909022092819214, "grad_norm": 10.464844703674316, "learning_rate": 1.0660589091223854e-07, "logits/chosen": -2.159003257751465, "logits/rejected": -1.688874363899231, "logps/chosen": -275.75927734375, "logps/ref_chosen": -99.52031707763672, "logps/ref_rejected": -97.93089294433594, "logps/rejected": -373.86029052734375, "loss": 1.0938, "margin_dpo/margin_mean": 99.69044494628906, "margin_dpo/margin_std": 129.49729919433594, "step": 495 }, { "KL/chosen_KL_mean": -195.28173828125, "KL/mean": -238.1721649169922, "KL/rejected_KL_mean": -281.0625915527344, "KL/std": 107.84174346923828, "epoch": 0.7283406754772394, "fcm_dpo/beta": 0.00392390601336956, "fcm_dpo/delta": 0.007526843808591366, "fcm_dpo/margin": 85.78083801269531, "fcm_dpo/q_t": 0.4199199676513672, "grad_norm": 8.699403762817383, "learning_rate": 1.0555654214793722e-07, "logits/chosen": -2.1817100048065186, "logits/rejected": -1.5394623279571533, "logps/chosen": -303.13848876953125, "logps/ref_chosen": -107.85675048828125, "logps/ref_rejected": -92.77056121826172, "logps/rejected": -373.8331298828125, "loss": 1.1233, "margin_dpo/margin_mean": 85.78083801269531, "margin_dpo/margin_std": 109.09320831298828, "step": 496 }, { "KL/chosen_KL_mean": -208.52532958984375, "KL/mean": -236.30435180664062, "KL/rejected_KL_mean": -264.0833435058594, "KL/std": 109.271728515625, "epoch": 0.7298091042584435, "fcm_dpo/beta": 0.003926859702914953, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 55.558013916015625, "fcm_dpo/q_t": 0.4486031234264374, "grad_norm": 8.046308517456055, "learning_rate": 1.0451099960308374e-07, "logits/chosen": -1.6113924980163574, "logits/rejected": -1.1524041891098022, "logps/chosen": -300.60858154296875, "logps/ref_chosen": -92.08322143554688, "logps/ref_rejected": -81.79503631591797, "logps/rejected": -345.87835693359375, "loss": 1.2321, "margin_dpo/margin_mean": 55.558013916015625, "margin_dpo/margin_std": 116.76002502441406, "step": 497 }, { "KL/chosen_KL_mean": -177.03164672851562, "KL/mean": -223.6384735107422, "KL/rejected_KL_mean": -270.24530029296875, "KL/std": 107.69236755371094, "epoch": 0.7312775330396476, "fcm_dpo/beta": 0.0039444491267204285, "fcm_dpo/delta": 0.03356942534446716, "fcm_dpo/margin": 93.21366882324219, "fcm_dpo/q_t": 0.4133025109767914, "grad_norm": 10.81260871887207, "learning_rate": 1.0346929082869641e-07, "logits/chosen": -1.829862117767334, "logits/rejected": -1.333831787109375, "logps/chosen": -275.22601318359375, "logps/ref_chosen": -98.19437408447266, "logps/ref_rejected": -90.68745422363281, "logps/rejected": -360.9327392578125, "loss": 1.1048, "margin_dpo/margin_mean": 93.21366882324219, "margin_dpo/margin_std": 119.22862243652344, "step": 498 }, { "KL/chosen_KL_mean": -185.80911254882812, "KL/mean": -234.31475830078125, "KL/rejected_KL_mean": -282.82037353515625, "KL/std": 115.52024841308594, "epoch": 0.7327459618208517, "fcm_dpo/beta": 0.003966744989156723, "fcm_dpo/delta": 0.01574990153312683, "fcm_dpo/margin": 97.01126861572266, "fcm_dpo/q_t": 0.4101901650428772, "grad_norm": 9.266758918762207, "learning_rate": 1.0243144327477013e-07, "logits/chosen": -2.2828783988952637, "logits/rejected": -2.2341904640197754, "logps/chosen": -266.8490295410156, "logps/ref_chosen": -81.0399169921875, "logps/ref_rejected": -106.92170715332031, "logps/rejected": -389.7420654296875, "loss": 1.0946, "margin_dpo/margin_mean": 97.01127624511719, "margin_dpo/margin_std": 123.3255615234375, "step": 499 }, { "KL/chosen_KL_mean": -205.74810791015625, "KL/mean": -247.64076232910156, "KL/rejected_KL_mean": -289.53338623046875, "KL/std": 111.3338623046875, "epoch": 0.7342143906020558, "fcm_dpo/beta": 0.0040001594461500645, "fcm_dpo/delta": 0.06714823842048645, "fcm_dpo/margin": 83.78529357910156, "fcm_dpo/q_t": 0.4230141043663025, "grad_norm": 11.359566688537598, "learning_rate": 1.0139748428955333e-07, "logits/chosen": -1.4569916725158691, "logits/rejected": -1.28076171875, "logps/chosen": -294.99615478515625, "logps/ref_chosen": -89.248046875, "logps/ref_rejected": -100.41021728515625, "logps/rejected": -389.943603515625, "loss": 1.1401, "margin_dpo/margin_mean": 83.7853012084961, "margin_dpo/margin_std": 126.78570556640625, "step": 500 }, { "KL/chosen_KL_mean": -179.64697265625, "KL/mean": -226.18873596191406, "KL/rejected_KL_mean": -272.73052978515625, "KL/std": 120.93177795410156, "epoch": 0.73568281938326, "fcm_dpo/beta": 0.003997047431766987, "fcm_dpo/delta": -0.05593404918909073, "fcm_dpo/margin": 93.08353424072266, "fcm_dpo/q_t": 0.4137415289878845, "grad_norm": 8.646526336669922, "learning_rate": 1.0036744111882672e-07, "logits/chosen": -1.61490797996521, "logits/rejected": -1.1807053089141846, "logps/chosen": -270.243896484375, "logps/ref_chosen": -90.596923828125, "logps/ref_rejected": -85.2972640991211, "logps/rejected": -358.02777099609375, "loss": 1.1226, "margin_dpo/margin_mean": 93.08353424072266, "margin_dpo/margin_std": 135.6177978515625, "step": 501 }, { "KL/chosen_KL_mean": -180.65415954589844, "KL/mean": -222.42820739746094, "KL/rejected_KL_mean": -264.2022705078125, "KL/std": 100.51045227050781, "epoch": 0.737151248164464, "fcm_dpo/beta": 0.003974404186010361, "fcm_dpo/delta": -0.0007198267849162221, "fcm_dpo/margin": 83.54811096191406, "fcm_dpo/q_t": 0.42157667875289917, "grad_norm": 9.545127868652344, "learning_rate": 9.934134090518592e-08, "logits/chosen": -1.9886888265609741, "logits/rejected": -1.2634838819503784, "logps/chosen": -284.1248779296875, "logps/ref_chosen": -103.47073364257812, "logps/ref_rejected": -89.04415893554688, "logps/rejected": -353.2464294433594, "loss": 1.1409, "margin_dpo/margin_mean": 83.54811096191406, "margin_dpo/margin_std": 123.66584777832031, "step": 502 }, { "KL/chosen_KL_mean": -179.72842407226562, "KL/mean": -226.67210388183594, "KL/rejected_KL_mean": -273.61578369140625, "KL/std": 107.78350830078125, "epoch": 0.7386196769456681, "fcm_dpo/beta": 0.003964018076658249, "fcm_dpo/delta": -0.02547951601445675, "fcm_dpo/margin": 93.88733673095703, "fcm_dpo/q_t": 0.4123993217945099, "grad_norm": 8.895367622375488, "learning_rate": 9.831921068732571e-08, "logits/chosen": -1.7202941179275513, "logits/rejected": -1.2193692922592163, "logps/chosen": -273.45843505859375, "logps/ref_chosen": -93.72998809814453, "logps/ref_rejected": -87.60896301269531, "logps/rejected": -361.2247314453125, "loss": 1.1028, "margin_dpo/margin_mean": 93.88733673095703, "margin_dpo/margin_std": 118.7103271484375, "step": 503 }, { "KL/chosen_KL_mean": -182.45199584960938, "KL/mean": -242.7974395751953, "KL/rejected_KL_mean": -303.14288330078125, "KL/std": 112.29166412353516, "epoch": 0.7400881057268722, "fcm_dpo/beta": 0.0039139739237725735, "fcm_dpo/delta": -0.07591018080711365, "fcm_dpo/margin": 120.69091796875, "fcm_dpo/q_t": 0.388175904750824, "grad_norm": 9.44067668914795, "learning_rate": 9.730107739932805e-08, "logits/chosen": -2.0505452156066895, "logits/rejected": -1.717842936515808, "logps/chosen": -274.7438659667969, "logps/ref_chosen": -92.2918701171875, "logps/ref_rejected": -109.03765106201172, "logps/rejected": -412.1805419921875, "loss": 1.0156, "margin_dpo/margin_mean": 120.69091796875, "margin_dpo/margin_std": 113.42808532714844, "step": 504 }, { "KL/chosen_KL_mean": -200.13525390625, "KL/mean": -243.42214965820312, "KL/rejected_KL_mean": -286.7090148925781, "KL/std": 111.30079650878906, "epoch": 0.7415565345080763, "fcm_dpo/beta": 0.00392896868288517, "fcm_dpo/delta": 0.061983369290828705, "fcm_dpo/margin": 86.57379150390625, "fcm_dpo/q_t": 0.4198564887046814, "grad_norm": 8.148636817932129, "learning_rate": 9.628696786995188e-08, "logits/chosen": -2.3748321533203125, "logits/rejected": -1.816833734512329, "logps/chosen": -306.41217041015625, "logps/ref_chosen": -106.27693176269531, "logps/ref_rejected": -94.20611572265625, "logps/rejected": -380.9151611328125, "loss": 1.1185, "margin_dpo/margin_mean": 86.57379150390625, "margin_dpo/margin_std": 107.85874938964844, "step": 505 }, { "KL/chosen_KL_mean": -193.41326904296875, "KL/mean": -242.438720703125, "KL/rejected_KL_mean": -291.46417236328125, "KL/std": 105.14840698242188, "epoch": 0.7430249632892805, "fcm_dpo/beta": 0.003937737084925175, "fcm_dpo/delta": 0.014265947043895721, "fcm_dpo/margin": 98.05091857910156, "fcm_dpo/q_t": 0.40805986523628235, "grad_norm": 8.797200202941895, "learning_rate": 9.527690882192635e-08, "logits/chosen": -1.9347736835479736, "logits/rejected": -1.5731432437896729, "logps/chosen": -274.8541259765625, "logps/ref_chosen": -81.44086456298828, "logps/ref_rejected": -85.66439819335938, "logps/rejected": -377.1285705566406, "loss": 1.0831, "margin_dpo/margin_mean": 98.05091857910156, "margin_dpo/margin_std": 109.73526000976562, "step": 506 }, { "KL/chosen_KL_mean": -202.03317260742188, "KL/mean": -251.17965698242188, "KL/rejected_KL_mean": -300.32611083984375, "KL/std": 109.95832824707031, "epoch": 0.7444933920704846, "fcm_dpo/beta": 0.003960990346968174, "fcm_dpo/delta": 0.011093826964497566, "fcm_dpo/margin": 98.29291534423828, "fcm_dpo/q_t": 0.4095962643623352, "grad_norm": 8.922673225402832, "learning_rate": 9.427092687124691e-08, "logits/chosen": -2.125525951385498, "logits/rejected": -1.6822545528411865, "logps/chosen": -301.46014404296875, "logps/ref_chosen": -99.42694854736328, "logps/ref_rejected": -102.25296020507812, "logps/rejected": -402.5790710449219, "loss": 1.0984, "margin_dpo/margin_mean": 98.29291534423828, "margin_dpo/margin_std": 129.6722869873047, "step": 507 }, { "KL/chosen_KL_mean": -194.7322540283203, "KL/mean": -235.767578125, "KL/rejected_KL_mean": -276.8028869628906, "KL/std": 118.11274719238281, "epoch": 0.7459618208516887, "fcm_dpo/beta": 0.004006821662187576, "fcm_dpo/delta": 0.07357528060674667, "fcm_dpo/margin": 82.0706558227539, "fcm_dpo/q_t": 0.4246722459793091, "grad_norm": 15.261529922485352, "learning_rate": 9.326904852647344e-08, "logits/chosen": -2.052635669708252, "logits/rejected": -1.5721744298934937, "logps/chosen": -292.64788818359375, "logps/ref_chosen": -97.9156494140625, "logps/ref_rejected": -103.79782104492188, "logps/rejected": -380.6007080078125, "loss": 1.1593, "margin_dpo/margin_mean": 82.07064819335938, "margin_dpo/margin_std": 137.97544860839844, "step": 508 }, { "KL/chosen_KL_mean": -181.87240600585938, "KL/mean": -220.01315307617188, "KL/rejected_KL_mean": -258.15386962890625, "KL/std": 92.80377960205078, "epoch": 0.7474302496328928, "fcm_dpo/beta": 0.004035960882902145, "fcm_dpo/delta": 0.03529799357056618, "fcm_dpo/margin": 76.28148651123047, "fcm_dpo/q_t": 0.42670390009880066, "grad_norm": 14.2485990524292, "learning_rate": 9.227130018803195e-08, "logits/chosen": -2.026393413543701, "logits/rejected": -1.3643516302108765, "logps/chosen": -282.6022644042969, "logps/ref_chosen": -100.72984313964844, "logps/ref_rejected": -86.943359375, "logps/rejected": -345.09722900390625, "loss": 1.1619, "margin_dpo/margin_mean": 76.281494140625, "margin_dpo/margin_std": 122.8749008178711, "step": 509 }, { "KL/chosen_KL_mean": -215.20278930664062, "KL/mean": -261.70635986328125, "KL/rejected_KL_mean": -308.2099304199219, "KL/std": 111.25944519042969, "epoch": 0.748898678414097, "fcm_dpo/beta": 0.0040697501972317696, "fcm_dpo/delta": 0.02225029282271862, "fcm_dpo/margin": 93.00714874267578, "fcm_dpo/q_t": 0.41105180978775024, "grad_norm": 11.217575073242188, "learning_rate": 9.127770814751932e-08, "logits/chosen": -1.532074213027954, "logits/rejected": -1.3468949794769287, "logps/chosen": -304.3721618652344, "logps/ref_chosen": -89.16938781738281, "logps/ref_rejected": -107.91939544677734, "logps/rejected": -416.12933349609375, "loss": 1.0943, "margin_dpo/margin_mean": 93.00714874267578, "margin_dpo/margin_std": 111.80921173095703, "step": 510 }, { "KL/chosen_KL_mean": -196.1717987060547, "KL/mean": -246.56007385253906, "KL/rejected_KL_mean": -296.9483642578125, "KL/std": 110.18350219726562, "epoch": 0.750367107195301, "fcm_dpo/beta": 0.004070533439517021, "fcm_dpo/delta": -0.010722624137997627, "fcm_dpo/margin": 100.77655792236328, "fcm_dpo/q_t": 0.40484482049942017, "grad_norm": 10.913744926452637, "learning_rate": 9.028829858700973e-08, "logits/chosen": -2.133849859237671, "logits/rejected": -1.566208839416504, "logps/chosen": -285.92156982421875, "logps/ref_chosen": -89.74979400634766, "logps/ref_rejected": -97.73127746582031, "logps/rejected": -394.67962646484375, "loss": 1.0772, "margin_dpo/margin_mean": 100.77656555175781, "margin_dpo/margin_std": 123.26954650878906, "step": 511 }, { "KL/chosen_KL_mean": -185.60037231445312, "KL/mean": -234.26329040527344, "KL/rejected_KL_mean": -282.92620849609375, "KL/std": 107.42489624023438, "epoch": 0.7518355359765051, "fcm_dpo/beta": 0.004080546088516712, "fcm_dpo/delta": 0.0024984925985336304, "fcm_dpo/margin": 97.3258285522461, "fcm_dpo/q_t": 0.40711161494255066, "grad_norm": 13.457165718078613, "learning_rate": 8.930309757836516e-08, "logits/chosen": -1.9062421321868896, "logits/rejected": -1.4717152118682861, "logps/chosen": -271.33551025390625, "logps/ref_chosen": -85.73515319824219, "logps/ref_rejected": -87.6248779296875, "logps/rejected": -370.55108642578125, "loss": 1.0941, "margin_dpo/margin_mean": 97.32583618164062, "margin_dpo/margin_std": 126.97872924804688, "step": 512 }, { "KL/chosen_KL_mean": -197.99903869628906, "KL/mean": -238.67803955078125, "KL/rejected_KL_mean": -279.3570556640625, "KL/std": 102.75556182861328, "epoch": 0.7533039647577092, "fcm_dpo/beta": 0.00405617943033576, "fcm_dpo/delta": -0.006659490987658501, "fcm_dpo/margin": 81.35800170898438, "fcm_dpo/q_t": 0.42128869891166687, "grad_norm": 10.68964672088623, "learning_rate": 8.832213108254863e-08, "logits/chosen": -2.3747973442077637, "logits/rejected": -1.6879041194915771, "logps/chosen": -294.1895446777344, "logps/ref_chosen": -96.19050598144531, "logps/ref_rejected": -82.91773986816406, "logps/rejected": -362.2747802734375, "loss": 1.1396, "margin_dpo/margin_mean": 81.35800170898438, "margin_dpo/margin_std": 116.92046356201172, "step": 513 }, { "KL/chosen_KL_mean": -192.1576690673828, "KL/mean": -236.41180419921875, "KL/rejected_KL_mean": -280.6659851074219, "KL/std": 103.27780151367188, "epoch": 0.7547723935389133, "fcm_dpo/beta": 0.004082049708813429, "fcm_dpo/delta": 0.04013773053884506, "fcm_dpo/margin": 88.50830078125, "fcm_dpo/q_t": 0.41640210151672363, "grad_norm": 10.681291580200195, "learning_rate": 8.734542494893954e-08, "logits/chosen": -1.6041903495788574, "logits/rejected": -1.0222392082214355, "logps/chosen": -289.9813232421875, "logps/ref_chosen": -97.82363891601562, "logps/ref_rejected": -85.36982727050781, "logps/rejected": -366.03582763671875, "loss": 1.116, "margin_dpo/margin_mean": 88.50830078125, "margin_dpo/margin_std": 122.02279663085938, "step": 514 }, { "KL/chosen_KL_mean": -183.2168731689453, "KL/mean": -226.84437561035156, "KL/rejected_KL_mean": -270.4718933105469, "KL/std": 107.51303100585938, "epoch": 0.7562408223201175, "fcm_dpo/beta": 0.004114504437893629, "fcm_dpo/delta": 0.04251883551478386, "fcm_dpo/margin": 87.25503540039062, "fcm_dpo/q_t": 0.4160889983177185, "grad_norm": 11.353057861328125, "learning_rate": 8.637300491465272e-08, "logits/chosen": -1.786031723022461, "logits/rejected": -1.5551931858062744, "logps/chosen": -265.1504211425781, "logps/ref_chosen": -81.93355560302734, "logps/ref_rejected": -93.57656860351562, "logps/rejected": -364.0484619140625, "loss": 1.117, "margin_dpo/margin_mean": 87.25503540039062, "margin_dpo/margin_std": 118.18733215332031, "step": 515 }, { "KL/chosen_KL_mean": -189.4136505126953, "KL/mean": -231.18515014648438, "KL/rejected_KL_mean": -272.95660400390625, "KL/std": 123.27816009521484, "epoch": 0.7577092511013216, "fcm_dpo/beta": 0.004163405857980251, "fcm_dpo/delta": 0.05392815172672272, "fcm_dpo/margin": 83.54295349121094, "fcm_dpo/q_t": 0.42119747400283813, "grad_norm": 12.090835571289062, "learning_rate": 8.540489660386064e-08, "logits/chosen": -2.4288816452026367, "logits/rejected": -2.25951886177063, "logps/chosen": -286.50531005859375, "logps/ref_chosen": -97.09166717529297, "logps/ref_rejected": -118.55667114257812, "logps/rejected": -391.5133056640625, "loss": 1.1555, "margin_dpo/margin_mean": 83.54295349121094, "margin_dpo/margin_std": 146.1177978515625, "step": 516 }, { "KL/chosen_KL_mean": -201.92050170898438, "KL/mean": -253.25572204589844, "KL/rejected_KL_mean": -304.5909423828125, "KL/std": 110.46018981933594, "epoch": 0.7591776798825257, "fcm_dpo/beta": 0.004105024505406618, "fcm_dpo/delta": -0.07673335075378418, "fcm_dpo/margin": 102.67044067382812, "fcm_dpo/q_t": 0.4017278850078583, "grad_norm": 8.936898231506348, "learning_rate": 8.444112552711752e-08, "logits/chosen": -2.0397822856903076, "logits/rejected": -1.625511646270752, "logps/chosen": -295.1070556640625, "logps/ref_chosen": -93.18656921386719, "logps/ref_rejected": -95.5927963256836, "logps/rejected": -400.1837463378906, "loss": 1.0765, "margin_dpo/margin_mean": 102.67042541503906, "margin_dpo/margin_std": 126.3651123046875, "step": 517 }, { "KL/chosen_KL_mean": -210.923583984375, "KL/mean": -248.91769409179688, "KL/rejected_KL_mean": -286.9118347167969, "KL/std": 104.56220245361328, "epoch": 0.7606461086637298, "fcm_dpo/beta": 0.004156820941716433, "fcm_dpo/delta": 0.08690877258777618, "fcm_dpo/margin": 75.9882583618164, "fcm_dpo/q_t": 0.4275084137916565, "grad_norm": 9.773492813110352, "learning_rate": 8.348171708068747e-08, "logits/chosen": -1.7810637950897217, "logits/rejected": -1.446563959121704, "logps/chosen": -297.9449462890625, "logps/ref_chosen": -87.0213623046875, "logps/ref_rejected": -91.24456024169922, "logps/rejected": -378.1564025878906, "loss": 1.1642, "margin_dpo/margin_mean": 75.98825073242188, "margin_dpo/margin_std": 129.84288024902344, "step": 518 }, { "KL/chosen_KL_mean": -194.88018798828125, "KL/mean": -236.708740234375, "KL/rejected_KL_mean": -278.5372619628906, "KL/std": 97.46993255615234, "epoch": 0.762114537444934, "fcm_dpo/beta": 0.0042000580579042435, "fcm_dpo/delta": 0.05042509362101555, "fcm_dpo/margin": 83.65708923339844, "fcm_dpo/q_t": 0.4178110361099243, "grad_norm": 10.447071075439453, "learning_rate": 8.25266965458755e-08, "logits/chosen": -2.0044147968292236, "logits/rejected": -1.6359965801239014, "logps/chosen": -295.5677490234375, "logps/ref_chosen": -100.68755340576172, "logps/ref_rejected": -109.03042602539062, "logps/rejected": -387.56768798828125, "loss": 1.1099, "margin_dpo/margin_mean": 83.65708923339844, "margin_dpo/margin_std": 102.42189025878906, "step": 519 }, { "KL/chosen_KL_mean": -181.1160888671875, "KL/mean": -231.02084350585938, "KL/rejected_KL_mean": -280.925537109375, "KL/std": 109.89045715332031, "epoch": 0.7635829662261381, "fcm_dpo/beta": 0.004199155606329441, "fcm_dpo/delta": -0.020054802298545837, "fcm_dpo/margin": 99.8094711303711, "fcm_dpo/q_t": 0.4011247456073761, "grad_norm": 9.208995819091797, "learning_rate": 8.15760890883607e-08, "logits/chosen": -1.6201674938201904, "logits/rejected": -1.1856615543365479, "logps/chosen": -279.89373779296875, "logps/ref_chosen": -98.77762603759766, "logps/ref_rejected": -106.9457015991211, "logps/rejected": -387.87127685546875, "loss": 1.0594, "margin_dpo/margin_mean": 99.80947875976562, "margin_dpo/margin_std": 107.50337982177734, "step": 520 }, { "KL/chosen_KL_mean": -182.4607391357422, "KL/mean": -233.66900634765625, "KL/rejected_KL_mean": -284.8772888183594, "KL/std": 102.87747192382812, "epoch": 0.7650513950073421, "fcm_dpo/beta": 0.004200609866529703, "fcm_dpo/delta": -0.0318087637424469, "fcm_dpo/margin": 102.41654968261719, "fcm_dpo/q_t": 0.3987743854522705, "grad_norm": 10.099702835083008, "learning_rate": 8.062991975753378e-08, "logits/chosen": -1.9827301502227783, "logits/rejected": -1.6475906372070312, "logps/chosen": -268.81182861328125, "logps/ref_chosen": -86.35108947753906, "logps/ref_rejected": -90.39317321777344, "logps/rejected": -375.27044677734375, "loss": 1.0492, "margin_dpo/margin_mean": 102.41654968261719, "margin_dpo/margin_std": 104.80770874023438, "step": 521 }, { "KL/chosen_KL_mean": -190.17807006835938, "KL/mean": -235.4043426513672, "KL/rejected_KL_mean": -280.630615234375, "KL/std": 112.65061950683594, "epoch": 0.7665198237885462, "fcm_dpo/beta": 0.0041961874812841415, "fcm_dpo/delta": 0.021154247224330902, "fcm_dpo/margin": 90.45255279541016, "fcm_dpo/q_t": 0.41140180826187134, "grad_norm": 9.924357414245605, "learning_rate": 7.968821348583643e-08, "logits/chosen": -1.5827698707580566, "logits/rejected": -1.208116054534912, "logps/chosen": -266.1861267089844, "logps/ref_chosen": -76.008056640625, "logps/ref_rejected": -74.44125366210938, "logps/rejected": -355.0718688964844, "loss": 1.0999, "margin_dpo/margin_mean": 90.45254516601562, "margin_dpo/margin_std": 117.11664581298828, "step": 522 }, { "KL/chosen_KL_mean": -211.0486297607422, "KL/mean": -244.14881896972656, "KL/rejected_KL_mean": -277.2490234375, "KL/std": 110.52507019042969, "epoch": 0.7679882525697503, "fcm_dpo/beta": 0.00423425855115056, "fcm_dpo/delta": 0.04835720360279083, "fcm_dpo/margin": 66.20036315917969, "fcm_dpo/q_t": 0.43549644947052, "grad_norm": 11.738451957702637, "learning_rate": 7.875099508810484e-08, "logits/chosen": -2.0724406242370605, "logits/rejected": -1.381223440170288, "logps/chosen": -307.544921875, "logps/ref_chosen": -96.49627685546875, "logps/ref_rejected": -91.0152816772461, "logps/rejected": -368.2642822265625, "loss": 1.1965, "margin_dpo/margin_mean": 66.20036315917969, "margin_dpo/margin_std": 129.156005859375, "step": 523 }, { "KL/chosen_KL_mean": -192.61854553222656, "KL/mean": -239.88951110839844, "KL/rejected_KL_mean": -287.1604919433594, "KL/std": 110.71673583984375, "epoch": 0.7694566813509545, "fcm_dpo/beta": 0.0042311521247029305, "fcm_dpo/delta": -3.3204443752765656e-05, "fcm_dpo/margin": 94.54196166992188, "fcm_dpo/q_t": 0.4073396325111389, "grad_norm": 12.265958786010742, "learning_rate": 7.781828926091535e-08, "logits/chosen": -2.0376334190368652, "logits/rejected": -1.3248038291931152, "logps/chosen": -296.0240478515625, "logps/ref_chosen": -103.40550994873047, "logps/ref_rejected": -87.91015625, "logps/rejected": -375.0706481933594, "loss": 1.0842, "margin_dpo/margin_mean": 94.54196166992188, "margin_dpo/margin_std": 117.80670928955078, "step": 524 }, { "KL/chosen_KL_mean": -181.10916137695312, "KL/mean": -234.8898162841797, "KL/rejected_KL_mean": -288.67047119140625, "KL/std": 111.06787109375, "epoch": 0.7709251101321586, "fcm_dpo/beta": 0.004136846866458654, "fcm_dpo/delta": -0.1163390725851059, "fcm_dpo/margin": 107.56129455566406, "fcm_dpo/q_t": 0.39781540632247925, "grad_norm": 10.726316452026367, "learning_rate": 7.689012058193384e-08, "logits/chosen": -1.6314878463745117, "logits/rejected": -1.5452654361724854, "logps/chosen": -265.315673828125, "logps/ref_chosen": -84.20648193359375, "logps/ref_rejected": -106.26158905029297, "logps/rejected": -394.93206787109375, "loss": 1.0624, "margin_dpo/margin_mean": 107.5613021850586, "margin_dpo/margin_std": 128.64280700683594, "step": 525 }, { "KL/chosen_KL_mean": -186.1490936279297, "KL/mean": -234.78330993652344, "KL/rejected_KL_mean": -283.41754150390625, "KL/std": 105.11842346191406, "epoch": 0.7723935389133627, "fcm_dpo/beta": 0.004136267118155956, "fcm_dpo/delta": -0.0024262825027108192, "fcm_dpo/margin": 97.2684326171875, "fcm_dpo/q_t": 0.406277060508728, "grad_norm": 8.273813247680664, "learning_rate": 7.596651350926836e-08, "logits/chosen": -1.7758026123046875, "logits/rejected": -1.1259726285934448, "logps/chosen": -289.0032043457031, "logps/ref_chosen": -102.85411071777344, "logps/ref_rejected": -93.65145874023438, "logps/rejected": -377.0689697265625, "loss": 1.0862, "margin_dpo/margin_mean": 97.2684326171875, "margin_dpo/margin_std": 123.83148193359375, "step": 526 }, { "KL/chosen_KL_mean": -188.56680297851562, "KL/mean": -237.99942016601562, "KL/rejected_KL_mean": -287.43206787109375, "KL/std": 110.31617736816406, "epoch": 0.7738619676945668, "fcm_dpo/beta": 0.004128835164010525, "fcm_dpo/delta": -0.008549420163035393, "fcm_dpo/margin": 98.86524963378906, "fcm_dpo/q_t": 0.4029432535171509, "grad_norm": 10.541111946105957, "learning_rate": 7.504749238082414e-08, "logits/chosen": -3.2081172466278076, "logits/rejected": -2.4103713035583496, "logps/chosen": -285.3322448730469, "logps/ref_chosen": -96.76543426513672, "logps/ref_rejected": -85.39518737792969, "logps/rejected": -372.8272399902344, "loss": 1.0571, "margin_dpo/margin_mean": 98.86524963378906, "margin_dpo/margin_std": 96.53399658203125, "step": 527 }, { "KL/chosen_KL_mean": -176.10191345214844, "KL/mean": -224.69422912597656, "KL/rejected_KL_mean": -273.28656005859375, "KL/std": 111.78994750976562, "epoch": 0.775330396475771, "fcm_dpo/beta": 0.004133075010031462, "fcm_dpo/delta": -0.0017875507473945618, "fcm_dpo/margin": 97.18464660644531, "fcm_dpo/q_t": 0.40661191940307617, "grad_norm": 9.107051849365234, "learning_rate": 7.413308141366254e-08, "logits/chosen": -2.2358603477478027, "logits/rejected": -1.7314858436584473, "logps/chosen": -270.266845703125, "logps/ref_chosen": -94.1649169921875, "logps/ref_rejected": -100.372314453125, "logps/rejected": -373.65887451171875, "loss": 1.0895, "margin_dpo/margin_mean": 97.18464660644531, "margin_dpo/margin_std": 126.81500244140625, "step": 528 }, { "KL/chosen_KL_mean": -204.88844299316406, "KL/mean": -249.43350219726562, "KL/rejected_KL_mean": -293.9785461425781, "KL/std": 110.73883056640625, "epoch": 0.7767988252569751, "fcm_dpo/beta": 0.0041540986858308315, "fcm_dpo/delta": 0.03090469352900982, "fcm_dpo/margin": 89.09011840820312, "fcm_dpo/q_t": 0.4150165319442749, "grad_norm": 10.644501686096191, "learning_rate": 7.322330470336313e-08, "logits/chosen": -1.8178691864013672, "logits/rejected": -1.642407774925232, "logps/chosen": -285.66522216796875, "logps/ref_chosen": -80.77680206298828, "logps/ref_rejected": -97.191162109375, "logps/rejected": -391.1697082519531, "loss": 1.1208, "margin_dpo/margin_mean": 89.09011840820312, "margin_dpo/margin_std": 131.6557159423828, "step": 529 }, { "KL/chosen_KL_mean": -188.54885864257812, "KL/mean": -232.77499389648438, "KL/rejected_KL_mean": -277.0010986328125, "KL/std": 106.88856506347656, "epoch": 0.7782672540381792, "fcm_dpo/beta": 0.0041601043194532394, "fcm_dpo/delta": 0.03317118063569069, "fcm_dpo/margin": 88.45223236083984, "fcm_dpo/q_t": 0.4139227569103241, "grad_norm": 9.741031646728516, "learning_rate": 7.231818622338822e-08, "logits/chosen": -1.556344985961914, "logits/rejected": -1.2364283800125122, "logps/chosen": -268.3245849609375, "logps/ref_chosen": -79.7757339477539, "logps/ref_rejected": -92.05390167236328, "logps/rejected": -369.05499267578125, "loss": 1.096, "margin_dpo/margin_mean": 88.45223236083984, "margin_dpo/margin_std": 103.24551391601562, "step": 530 }, { "KL/chosen_KL_mean": -181.57249450683594, "KL/mean": -236.80592346191406, "KL/rejected_KL_mean": -292.03936767578125, "KL/std": 112.72777557373047, "epoch": 0.7797356828193832, "fcm_dpo/beta": 0.004138609394431114, "fcm_dpo/delta": -0.05999011546373367, "fcm_dpo/margin": 110.46685028076172, "fcm_dpo/q_t": 0.39188629388809204, "grad_norm": 11.031170845031738, "learning_rate": 7.141774982445147e-08, "logits/chosen": -1.9628932476043701, "logits/rejected": -1.3742516040802002, "logps/chosen": -260.8916931152344, "logps/ref_chosen": -79.31919860839844, "logps/ref_rejected": -78.10832214355469, "logps/rejected": -370.1476745605469, "loss": 1.0313, "margin_dpo/margin_mean": 110.46684265136719, "margin_dpo/margin_std": 112.10696411132812, "step": 531 }, { "KL/chosen_KL_mean": -194.917236328125, "KL/mean": -255.138427734375, "KL/rejected_KL_mean": -315.359619140625, "KL/std": 107.52310180664062, "epoch": 0.7812041116005873, "fcm_dpo/beta": 0.00408159289509058, "fcm_dpo/delta": -0.09622061997652054, "fcm_dpo/margin": 120.44237518310547, "fcm_dpo/q_t": 0.38491082191467285, "grad_norm": 11.192266464233398, "learning_rate": 7.052201923388953e-08, "logits/chosen": -1.684445858001709, "logits/rejected": -1.059739351272583, "logps/chosen": -296.55413818359375, "logps/ref_chosen": -101.63691711425781, "logps/ref_rejected": -93.34539031982422, "logps/rejected": -408.70501708984375, "loss": 1.0114, "margin_dpo/margin_mean": 120.4423828125, "margin_dpo/margin_std": 118.99960327148438, "step": 532 }, { "KL/chosen_KL_mean": -189.69781494140625, "KL/mean": -224.699951171875, "KL/rejected_KL_mean": -259.7020568847656, "KL/std": 113.68994140625, "epoch": 0.7826725403817915, "fcm_dpo/beta": 0.004050960764288902, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 70.00425720214844, "fcm_dpo/q_t": 0.43451547622680664, "grad_norm": 11.54209041595459, "learning_rate": 6.963101805503646e-08, "logits/chosen": -2.474362850189209, "logits/rejected": -1.880601167678833, "logps/chosen": -280.4444580078125, "logps/ref_chosen": -90.74664306640625, "logps/ref_rejected": -82.88848876953125, "logps/rejected": -342.590576171875, "loss": 1.1807, "margin_dpo/margin_mean": 70.00425720214844, "margin_dpo/margin_std": 121.96046447753906, "step": 533 }, { "KL/chosen_KL_mean": -168.35586547851562, "KL/mean": -220.06646728515625, "KL/rejected_KL_mean": -271.777099609375, "KL/std": 106.7122802734375, "epoch": 0.7841409691629956, "fcm_dpo/beta": 0.004044422414153814, "fcm_dpo/delta": -0.019084319472312927, "fcm_dpo/margin": 103.42121887207031, "fcm_dpo/q_t": 0.40096551179885864, "grad_norm": 8.90018367767334, "learning_rate": 6.874476976660184e-08, "logits/chosen": -1.9054774045944214, "logits/rejected": -1.505403757095337, "logps/chosen": -254.9866943359375, "logps/ref_chosen": -86.6308364868164, "logps/ref_rejected": -84.87196350097656, "logps/rejected": -356.6490478515625, "loss": 1.0545, "margin_dpo/margin_mean": 103.42121887207031, "margin_dpo/margin_std": 105.16766357421875, "step": 534 }, { "KL/chosen_KL_mean": -170.619384765625, "KL/mean": -225.118408203125, "KL/rejected_KL_mean": -279.6174621582031, "KL/std": 107.64976501464844, "epoch": 0.7856093979441997, "fcm_dpo/beta": 0.004028125200420618, "fcm_dpo/delta": -0.04104717820882797, "fcm_dpo/margin": 108.9980697631836, "fcm_dpo/q_t": 0.3977544903755188, "grad_norm": 7.785837650299072, "learning_rate": 6.786329772205246e-08, "logits/chosen": -1.609428882598877, "logits/rejected": -1.42098069190979, "logps/chosen": -260.96478271484375, "logps/ref_chosen": -90.34539031982422, "logps/ref_rejected": -100.7779769897461, "logps/rejected": -380.39544677734375, "loss": 1.0538, "margin_dpo/margin_mean": 108.9980697631836, "margin_dpo/margin_std": 122.8719482421875, "step": 535 }, { "KL/chosen_KL_mean": -168.3565673828125, "KL/mean": -222.52008056640625, "KL/rejected_KL_mean": -276.68359375, "KL/std": 108.9873046875, "epoch": 0.7870778267254038, "fcm_dpo/beta": 0.003975989297032356, "fcm_dpo/delta": -0.03221966326236725, "fcm_dpo/margin": 108.32705688476562, "fcm_dpo/q_t": 0.39833328127861023, "grad_norm": 8.551009178161621, "learning_rate": 6.698662514899638e-08, "logits/chosen": -1.905432939529419, "logits/rejected": -1.6565274000167847, "logps/chosen": -247.31613159179688, "logps/ref_chosen": -78.95956420898438, "logps/ref_rejected": -93.08779907226562, "logps/rejected": -369.77142333984375, "loss": 1.0505, "margin_dpo/margin_mean": 108.32705688476562, "margin_dpo/margin_std": 114.12397766113281, "step": 536 }, { "KL/chosen_KL_mean": -148.21780395507812, "KL/mean": -204.68319702148438, "KL/rejected_KL_mean": -261.14862060546875, "KL/std": 111.81452941894531, "epoch": 0.788546255506608, "fcm_dpo/beta": 0.003969781566411257, "fcm_dpo/delta": -0.050967130810022354, "fcm_dpo/margin": 112.93083190917969, "fcm_dpo/q_t": 0.39411282539367676, "grad_norm": 10.106608390808105, "learning_rate": 6.611477514857114e-08, "logits/chosen": -1.6083147525787354, "logits/rejected": -0.9229831695556641, "logps/chosen": -234.92718505859375, "logps/ref_chosen": -86.70939636230469, "logps/ref_rejected": -78.90766906738281, "logps/rejected": -340.0563049316406, "loss": 1.0358, "margin_dpo/margin_mean": 112.93082427978516, "margin_dpo/margin_std": 109.3245849609375, "step": 537 }, { "KL/chosen_KL_mean": -202.8919677734375, "KL/mean": -248.98818969726562, "KL/rejected_KL_mean": -295.08441162109375, "KL/std": 100.3636474609375, "epoch": 0.7900146842878121, "fcm_dpo/beta": 0.003958378918468952, "fcm_dpo/delta": 0.036391228437423706, "fcm_dpo/margin": 92.19239044189453, "fcm_dpo/q_t": 0.4139641523361206, "grad_norm": 8.873498916625977, "learning_rate": 6.524777069483525e-08, "logits/chosen": -1.7490143775939941, "logits/rejected": -1.2038695812225342, "logps/chosen": -301.3341064453125, "logps/ref_chosen": -98.44214630126953, "logps/ref_rejected": -91.09465026855469, "logps/rejected": -386.1790466308594, "loss": 1.0979, "margin_dpo/margin_mean": 92.19239807128906, "margin_dpo/margin_std": 107.98262023925781, "step": 538 }, { "KL/chosen_KL_mean": -176.15225219726562, "KL/mean": -224.4956817626953, "KL/rejected_KL_mean": -272.839111328125, "KL/std": 109.3220443725586, "epoch": 0.7914831130690162, "fcm_dpo/beta": 0.003976786509156227, "fcm_dpo/delta": 0.016103433445096016, "fcm_dpo/margin": 96.68684387207031, "fcm_dpo/q_t": 0.4100765585899353, "grad_norm": 8.289037704467773, "learning_rate": 6.438563463416221e-08, "logits/chosen": -2.3903229236602783, "logits/rejected": -1.9081597328186035, "logps/chosen": -267.0843505859375, "logps/ref_chosen": -90.93211364746094, "logps/ref_rejected": -97.44546508789062, "logps/rejected": -370.2845458984375, "loss": 1.0969, "margin_dpo/margin_mean": 96.68684387207031, "margin_dpo/margin_std": 125.0517349243164, "step": 539 }, { "KL/chosen_KL_mean": -188.3211669921875, "KL/mean": -247.48739624023438, "KL/rejected_KL_mean": -306.65362548828125, "KL/std": 120.37994384765625, "epoch": 0.7929515418502202, "fcm_dpo/beta": 0.00395231181755662, "fcm_dpo/delta": -0.07105652987957001, "fcm_dpo/margin": 118.33247375488281, "fcm_dpo/q_t": 0.39300185441970825, "grad_norm": 10.326759338378906, "learning_rate": 6.352838968463919e-08, "logits/chosen": -1.9810630083084106, "logits/rejected": -1.7435001134872437, "logps/chosen": -296.1910705566406, "logps/ref_chosen": -107.86990356445312, "logps/ref_rejected": -121.39350128173828, "logps/rejected": -428.047119140625, "loss": 1.0455, "margin_dpo/margin_mean": 118.33245849609375, "margin_dpo/margin_std": 141.44961547851562, "step": 540 }, { "KL/chosen_KL_mean": -208.34991455078125, "KL/mean": -245.27102661132812, "KL/rejected_KL_mean": -282.1921081542969, "KL/std": 99.07437133789062, "epoch": 0.7944199706314243, "fcm_dpo/beta": 0.003920359071344137, "fcm_dpo/delta": -0.001973494188860059, "fcm_dpo/margin": 73.84220886230469, "fcm_dpo/q_t": 0.4318574070930481, "grad_norm": 9.59099292755127, "learning_rate": 6.267605843546767e-08, "logits/chosen": -2.4143664836883545, "logits/rejected": -1.957679271697998, "logps/chosen": -314.91741943359375, "logps/ref_chosen": -106.56748962402344, "logps/ref_rejected": -109.06633758544922, "logps/rejected": -391.2584533691406, "loss": 1.1651, "margin_dpo/margin_mean": 73.84220886230469, "margin_dpo/margin_std": 107.66023254394531, "step": 541 }, { "KL/chosen_KL_mean": -176.13027954101562, "KL/mean": -233.16513061523438, "KL/rejected_KL_mean": -290.199951171875, "KL/std": 112.2148666381836, "epoch": 0.7958883994126285, "fcm_dpo/beta": 0.003893241286277771, "fcm_dpo/delta": -0.046153679490089417, "fcm_dpo/margin": 114.06965637207031, "fcm_dpo/q_t": 0.3952111601829529, "grad_norm": 10.262336730957031, "learning_rate": 6.182866334636888e-08, "logits/chosen": -2.1802868843078613, "logits/rejected": -2.055865526199341, "logps/chosen": -263.49957275390625, "logps/ref_chosen": -87.36929321289062, "logps/ref_rejected": -102.82034301757812, "logps/rejected": -393.02032470703125, "loss": 1.0438, "margin_dpo/margin_mean": 114.06965637207031, "margin_dpo/margin_std": 121.2616958618164, "step": 542 }, { "KL/chosen_KL_mean": -203.35427856445312, "KL/mean": -233.46575927734375, "KL/rejected_KL_mean": -263.5772399902344, "KL/std": 118.56103515625, "epoch": 0.7973568281938326, "fcm_dpo/beta": 0.003922195639461279, "fcm_dpo/delta": 0.04849550127983093, "fcm_dpo/margin": 60.22294235229492, "fcm_dpo/q_t": 0.44569286704063416, "grad_norm": 11.878764152526855, "learning_rate": 6.098622674699147e-08, "logits/chosen": -1.6643638610839844, "logits/rejected": -1.7139687538146973, "logps/chosen": -290.5068664550781, "logps/ref_chosen": -87.152587890625, "logps/ref_rejected": -111.53172302246094, "logps/rejected": -375.10894775390625, "loss": 1.2417, "margin_dpo/margin_mean": 60.22294616699219, "margin_dpo/margin_std": 144.78778076171875, "step": 543 }, { "KL/chosen_KL_mean": -212.8037109375, "KL/mean": -251.02288818359375, "KL/rejected_KL_mean": -289.2420959472656, "KL/std": 111.28211212158203, "epoch": 0.7988252569750367, "fcm_dpo/beta": 0.003956153057515621, "fcm_dpo/delta": 0.043102238327264786, "fcm_dpo/margin": 76.43841552734375, "fcm_dpo/q_t": 0.4294566512107849, "grad_norm": 9.049397468566895, "learning_rate": 6.01487708363232e-08, "logits/chosen": -1.7759820222854614, "logits/rejected": -1.6834551095962524, "logps/chosen": -298.95062255859375, "logps/ref_chosen": -86.14691162109375, "logps/ref_rejected": -106.3939208984375, "logps/rejected": -395.6360168457031, "loss": 1.1571, "margin_dpo/margin_mean": 76.43841552734375, "margin_dpo/margin_std": 117.00006866455078, "step": 544 }, { "KL/chosen_KL_mean": -188.0811767578125, "KL/mean": -235.30410766601562, "KL/rejected_KL_mean": -282.52703857421875, "KL/std": 110.29922485351562, "epoch": 0.8002936857562408, "fcm_dpo/beta": 0.003976769745349884, "fcm_dpo/delta": 0.02528545819222927, "fcm_dpo/margin": 94.44583892822266, "fcm_dpo/q_t": 0.4127165675163269, "grad_norm": 8.903777122497559, "learning_rate": 5.9316317682106294e-08, "logits/chosen": -1.6127533912658691, "logits/rejected": -1.1913080215454102, "logps/chosen": -284.50543212890625, "logps/ref_chosen": -96.42424011230469, "logps/ref_rejected": -100.73405456542969, "logps/rejected": -383.26104736328125, "loss": 1.1161, "margin_dpo/margin_mean": 94.44583129882812, "margin_dpo/margin_std": 136.73757934570312, "step": 545 }, { "KL/chosen_KL_mean": -194.41531372070312, "KL/mean": -236.50909423828125, "KL/rejected_KL_mean": -278.6029052734375, "KL/std": 111.8950424194336, "epoch": 0.801762114537445, "fcm_dpo/beta": 0.004007780458778143, "fcm_dpo/delta": 0.06483136117458344, "fcm_dpo/margin": 84.18757629394531, "fcm_dpo/q_t": 0.4204888939857483, "grad_norm": 8.961593627929688, "learning_rate": 5.848888922025552e-08, "logits/chosen": -1.5761886835098267, "logits/rejected": -1.2249510288238525, "logps/chosen": -280.4364318847656, "logps/ref_chosen": -86.02112579345703, "logps/ref_rejected": -87.37263488769531, "logps/rejected": -365.97552490234375, "loss": 1.1359, "margin_dpo/margin_mean": 84.18758392333984, "margin_dpo/margin_std": 123.34027862548828, "step": 546 }, { "KL/chosen_KL_mean": -186.7025146484375, "KL/mean": -237.83062744140625, "KL/rejected_KL_mean": -288.958740234375, "KL/std": 109.60289764404297, "epoch": 0.8032305433186491, "fcm_dpo/beta": 0.004029825329780579, "fcm_dpo/delta": -0.012673644348978996, "fcm_dpo/margin": 102.25625610351562, "fcm_dpo/q_t": 0.40199214220046997, "grad_norm": 9.157228469848633, "learning_rate": 5.7666507254280265e-08, "logits/chosen": -1.8417205810546875, "logits/rejected": -1.378598690032959, "logps/chosen": -287.353515625, "logps/ref_chosen": -100.6510009765625, "logps/ref_rejected": -97.9117431640625, "logps/rejected": -386.8704833984375, "loss": 1.0538, "margin_dpo/margin_mean": 102.25625610351562, "margin_dpo/margin_std": 97.4776611328125, "step": 547 }, { "KL/chosen_KL_mean": -183.82083129882812, "KL/mean": -225.64892578125, "KL/rejected_KL_mean": -267.47698974609375, "KL/std": 111.69984436035156, "epoch": 0.8046989720998532, "fcm_dpo/beta": 0.0040570092387497425, "fcm_dpo/delta": 0.06271268427371979, "fcm_dpo/margin": 83.6561279296875, "fcm_dpo/q_t": 0.4214785397052765, "grad_norm": 10.480459213256836, "learning_rate": 5.684919345471029e-08, "logits/chosen": -2.3335888385772705, "logits/rejected": -1.980445146560669, "logps/chosen": -279.8509521484375, "logps/ref_chosen": -96.03010559082031, "logps/ref_rejected": -98.41322326660156, "logps/rejected": -365.89019775390625, "loss": 1.1525, "margin_dpo/margin_mean": 83.6561279296875, "margin_dpo/margin_std": 139.59446716308594, "step": 548 }, { "KL/chosen_KL_mean": -173.96072387695312, "KL/mean": -216.42337036132812, "KL/rejected_KL_mean": -258.885986328125, "KL/std": 105.89651489257812, "epoch": 0.8061674008810573, "fcm_dpo/beta": 0.0040628137066960335, "fcm_dpo/delta": -0.014284750446677208, "fcm_dpo/margin": 84.92527770996094, "fcm_dpo/q_t": 0.41891396045684814, "grad_norm": 10.847293853759766, "learning_rate": 5.603696935852426e-08, "logits/chosen": -1.8971278667449951, "logits/rejected": -1.4823839664459229, "logps/chosen": -257.41168212890625, "logps/ref_chosen": -83.45095825195312, "logps/ref_rejected": -81.38023376464844, "logps/rejected": -340.2662353515625, "loss": 1.1409, "margin_dpo/margin_mean": 84.92527770996094, "margin_dpo/margin_std": 130.41908264160156, "step": 549 }, { "KL/chosen_KL_mean": -193.0083770751953, "KL/mean": -234.84326171875, "KL/rejected_KL_mean": -276.67816162109375, "KL/std": 100.83824157714844, "epoch": 0.8076358296622613, "fcm_dpo/beta": 0.004099378362298012, "fcm_dpo/delta": 0.05888357013463974, "fcm_dpo/margin": 83.66975402832031, "fcm_dpo/q_t": 0.4198307693004608, "grad_norm": 9.087468147277832, "learning_rate": 5.5229856368582376e-08, "logits/chosen": -1.5271849632263184, "logits/rejected": -1.3121836185455322, "logps/chosen": -282.60272216796875, "logps/ref_chosen": -89.59434509277344, "logps/ref_rejected": -101.31549835205078, "logps/rejected": -377.99365234375, "loss": 1.1223, "margin_dpo/margin_mean": 83.66976165771484, "margin_dpo/margin_std": 109.50350952148438, "step": 550 }, { "KL/chosen_KL_mean": -171.8982696533203, "KL/mean": -223.13900756835938, "KL/rejected_KL_mean": -274.3797607421875, "KL/std": 115.19548797607422, "epoch": 0.8091042584434655, "fcm_dpo/beta": 0.004074958153069019, "fcm_dpo/delta": -0.018885452300310135, "fcm_dpo/margin": 102.48149108886719, "fcm_dpo/q_t": 0.40341877937316895, "grad_norm": 10.372711181640625, "learning_rate": 5.4427875753062734e-08, "logits/chosen": -1.7168750762939453, "logits/rejected": -1.5784611701965332, "logps/chosen": -266.98040771484375, "logps/ref_chosen": -95.08216094970703, "logps/ref_rejected": -117.7192611694336, "logps/rejected": -392.0990295410156, "loss": 1.0663, "margin_dpo/margin_mean": 102.48149108886719, "margin_dpo/margin_std": 117.40731811523438, "step": 551 }, { "KL/chosen_KL_mean": -162.25375366210938, "KL/mean": -235.99691772460938, "KL/rejected_KL_mean": -309.7400817871094, "KL/std": 124.73002624511719, "epoch": 0.8105726872246696, "fcm_dpo/beta": 0.003944192081689835, "fcm_dpo/delta": -0.19540630280971527, "fcm_dpo/margin": 147.48631286621094, "fcm_dpo/q_t": 0.3634408116340637, "grad_norm": 11.11823558807373, "learning_rate": 5.363104864490034e-08, "logits/chosen": -2.0533602237701416, "logits/rejected": -1.691412091255188, "logps/chosen": -263.6629638671875, "logps/ref_chosen": -101.40918731689453, "logps/ref_rejected": -110.83843994140625, "logps/rejected": -420.5785217285156, "loss": 0.9501, "margin_dpo/margin_mean": 147.48631286621094, "margin_dpo/margin_std": 128.76419067382812, "step": 552 }, { "KL/chosen_KL_mean": -177.9302978515625, "KL/mean": -221.57574462890625, "KL/rejected_KL_mean": -265.22119140625, "KL/std": 115.44457244873047, "epoch": 0.8120411160058737, "fcm_dpo/beta": 0.003971524070948362, "fcm_dpo/delta": 0.05510055273771286, "fcm_dpo/margin": 87.29090881347656, "fcm_dpo/q_t": 0.4185052514076233, "grad_norm": 9.109249114990234, "learning_rate": 5.2839396041230415e-08, "logits/chosen": -1.3791086673736572, "logits/rejected": -1.0422030687332153, "logps/chosen": -275.8941650390625, "logps/ref_chosen": -97.96388244628906, "logps/ref_rejected": -103.57719421386719, "logps/rejected": -368.79840087890625, "loss": 1.1283, "margin_dpo/margin_mean": 87.29090881347656, "margin_dpo/margin_std": 124.11241149902344, "step": 553 }, { "KL/chosen_KL_mean": -186.0308837890625, "KL/mean": -234.49024963378906, "KL/rejected_KL_mean": -282.9495849609375, "KL/std": 116.15403747558594, "epoch": 0.8135095447870778, "fcm_dpo/beta": 0.0039992001838982105, "fcm_dpo/delta": 0.012554768472909927, "fcm_dpo/margin": 96.91871643066406, "fcm_dpo/q_t": 0.41062331199645996, "grad_norm": 8.550159454345703, "learning_rate": 5.205293880283551e-08, "logits/chosen": -1.8397386074066162, "logits/rejected": -1.057518482208252, "logps/chosen": -289.7176818847656, "logps/ref_chosen": -103.68680572509766, "logps/ref_rejected": -92.99210357666016, "logps/rejected": -375.94171142578125, "loss": 1.1214, "margin_dpo/margin_mean": 96.91871643066406, "margin_dpo/margin_std": 149.53387451171875, "step": 554 }, { "KL/chosen_KL_mean": -194.73667907714844, "KL/mean": -239.07911682128906, "KL/rejected_KL_mean": -283.42156982421875, "KL/std": 119.47474670410156, "epoch": 0.8149779735682819, "fcm_dpo/beta": 0.0040138796903193, "fcm_dpo/delta": 0.0456521213054657, "fcm_dpo/margin": 88.68490600585938, "fcm_dpo/q_t": 0.4184246063232422, "grad_norm": 9.095036506652832, "learning_rate": 5.127169765359515e-08, "logits/chosen": -1.6609153747558594, "logits/rejected": -1.5697379112243652, "logps/chosen": -280.9403076171875, "logps/ref_chosen": -86.20362854003906, "logps/ref_rejected": -114.54719543457031, "logps/rejected": -397.96875, "loss": 1.1339, "margin_dpo/margin_mean": 88.68489837646484, "margin_dpo/margin_std": 137.96524047851562, "step": 555 }, { "KL/chosen_KL_mean": -175.3978271484375, "KL/mean": -223.60971069335938, "KL/rejected_KL_mean": -271.82159423828125, "KL/std": 106.0242919921875, "epoch": 0.8164464023494861, "fcm_dpo/beta": 0.004019307438284159, "fcm_dpo/delta": 0.01281764730811119, "fcm_dpo/margin": 96.42373657226562, "fcm_dpo/q_t": 0.4092983603477478, "grad_norm": 10.368645668029785, "learning_rate": 5.049569317994012e-08, "logits/chosen": -1.487354040145874, "logits/rejected": -1.1843937635421753, "logps/chosen": -263.80133056640625, "logps/ref_chosen": -88.40348815917969, "logps/ref_rejected": -106.63911437988281, "logps/rejected": -378.460693359375, "loss": 1.0921, "margin_dpo/margin_mean": 96.42373657226562, "margin_dpo/margin_std": 120.55360412597656, "step": 556 }, { "KL/chosen_KL_mean": -187.70565795898438, "KL/mean": -242.84906005859375, "KL/rejected_KL_mean": -297.9924621582031, "KL/std": 122.87167358398438, "epoch": 0.8179148311306902, "fcm_dpo/beta": 0.003991108387708664, "fcm_dpo/delta": -0.04255310073494911, "fcm_dpo/margin": 110.28684997558594, "fcm_dpo/q_t": 0.398430198431015, "grad_norm": 8.112653732299805, "learning_rate": 4.9724945830310144e-08, "logits/chosen": -1.9215428829193115, "logits/rejected": -1.748028039932251, "logps/chosen": -286.5408935546875, "logps/ref_chosen": -98.83523559570312, "logps/ref_rejected": -115.64888763427734, "logps/rejected": -413.641357421875, "loss": 1.0617, "margin_dpo/margin_mean": 110.28684997558594, "margin_dpo/margin_std": 133.552978515625, "step": 557 }, { "KL/chosen_KL_mean": -183.21401977539062, "KL/mean": -242.38037109375, "KL/rejected_KL_mean": -301.5467224121094, "KL/std": 106.28973388671875, "epoch": 0.8193832599118943, "fcm_dpo/beta": 0.0039487662725150585, "fcm_dpo/delta": -0.07081079483032227, "fcm_dpo/margin": 118.33269500732422, "fcm_dpo/q_t": 0.3897389769554138, "grad_norm": 10.605459213256836, "learning_rate": 4.8959475914614554e-08, "logits/chosen": -2.327383041381836, "logits/rejected": -1.714231252670288, "logps/chosen": -289.2258605957031, "logps/ref_chosen": -106.01183319091797, "logps/ref_rejected": -108.39131927490234, "logps/rejected": -409.93804931640625, "loss": 1.0187, "margin_dpo/margin_mean": 118.33269500732422, "margin_dpo/margin_std": 110.4193115234375, "step": 558 }, { "KL/chosen_KL_mean": -201.95263671875, "KL/mean": -248.3455810546875, "KL/rejected_KL_mean": -294.738525390625, "KL/std": 108.78970336914062, "epoch": 0.8208516886930984, "fcm_dpo/beta": 0.0039668213576078415, "fcm_dpo/delta": 0.03309304639697075, "fcm_dpo/margin": 92.785888671875, "fcm_dpo/q_t": 0.4132770895957947, "grad_norm": 10.393129348754883, "learning_rate": 4.8199303603697614e-08, "logits/chosen": -2.2402353286743164, "logits/rejected": -1.7538548707962036, "logps/chosen": -302.37738037109375, "logps/ref_chosen": -100.42474365234375, "logps/ref_rejected": -100.41270446777344, "logps/rejected": -395.1512451171875, "loss": 1.11, "margin_dpo/margin_mean": 92.785888671875, "margin_dpo/margin_std": 123.98502349853516, "step": 559 }, { "KL/chosen_KL_mean": -189.90618896484375, "KL/mean": -236.27590942382812, "KL/rejected_KL_mean": -282.6456298828125, "KL/std": 106.77798461914062, "epoch": 0.8223201174743024, "fcm_dpo/beta": 0.003984309732913971, "fcm_dpo/delta": 0.03167944774031639, "fcm_dpo/margin": 92.73944091796875, "fcm_dpo/q_t": 0.4136253595352173, "grad_norm": 11.9866304397583, "learning_rate": 4.7444448928806615e-08, "logits/chosen": -1.2863881587982178, "logits/rejected": -0.8358875513076782, "logps/chosen": -277.5417175292969, "logps/ref_chosen": -87.63551330566406, "logps/ref_rejected": -87.37985229492188, "logps/rejected": -370.0255126953125, "loss": 1.1077, "margin_dpo/margin_mean": 92.73944091796875, "margin_dpo/margin_std": 122.90707397460938, "step": 560 }, { "KL/chosen_KL_mean": -199.8811798095703, "KL/mean": -242.90728759765625, "KL/rejected_KL_mean": -285.93341064453125, "KL/std": 105.08248901367188, "epoch": 0.8237885462555066, "fcm_dpo/beta": 0.004034621641039848, "fcm_dpo/delta": 0.05453299731016159, "fcm_dpo/margin": 86.05224609375, "fcm_dpo/q_t": 0.4181227385997772, "grad_norm": 8.237642288208008, "learning_rate": 4.669493178106432e-08, "logits/chosen": -1.7476694583892822, "logits/rejected": -1.721436858177185, "logps/chosen": -274.7625732421875, "logps/ref_chosen": -74.88140106201172, "logps/ref_rejected": -106.04796600341797, "logps/rejected": -391.98138427734375, "loss": 1.1089, "margin_dpo/margin_mean": 86.05224609375, "margin_dpo/margin_std": 101.30625915527344, "step": 561 }, { "KL/chosen_KL_mean": -181.40444946289062, "KL/mean": -224.77960205078125, "KL/rejected_KL_mean": -268.1547546386719, "KL/std": 106.30143737792969, "epoch": 0.8252569750367107, "fcm_dpo/beta": 0.0040319073013961315, "fcm_dpo/delta": -0.007692549843341112, "fcm_dpo/margin": 86.75032043457031, "fcm_dpo/q_t": 0.418799489736557, "grad_norm": 13.258758544921875, "learning_rate": 4.5950771910944596e-08, "logits/chosen": -1.9952281713485718, "logits/rejected": -1.5297197103500366, "logps/chosen": -282.06768798828125, "logps/ref_chosen": -100.66322326660156, "logps/ref_rejected": -102.24087524414062, "logps/rejected": -370.3956298828125, "loss": 1.1345, "margin_dpo/margin_mean": 86.75031280517578, "margin_dpo/margin_std": 132.64466857910156, "step": 562 }, { "KL/chosen_KL_mean": -191.42404174804688, "KL/mean": -227.6953582763672, "KL/rejected_KL_mean": -263.9666748046875, "KL/std": 107.09082794189453, "epoch": 0.8267254038179148, "fcm_dpo/beta": 0.004042648710310459, "fcm_dpo/delta": 0.026570217683911324, "fcm_dpo/margin": 72.5426254272461, "fcm_dpo/q_t": 0.4308703541755676, "grad_norm": 10.151522636413574, "learning_rate": 4.521198892775202e-08, "logits/chosen": -1.7156529426574707, "logits/rejected": -1.3790748119354248, "logps/chosen": -287.6832580566406, "logps/ref_chosen": -96.25921630859375, "logps/ref_rejected": -101.20503997802734, "logps/rejected": -365.17169189453125, "loss": 1.1658, "margin_dpo/margin_mean": 72.5426254272461, "margin_dpo/margin_std": 113.81546020507812, "step": 563 }, { "KL/chosen_KL_mean": -186.30447387695312, "KL/mean": -227.33822631835938, "KL/rejected_KL_mean": -268.3719787597656, "KL/std": 107.06916809082031, "epoch": 0.8281938325991189, "fcm_dpo/beta": 0.00410117581486702, "fcm_dpo/delta": 0.06542927771806717, "fcm_dpo/margin": 82.0675048828125, "fcm_dpo/q_t": 0.42022332549095154, "grad_norm": 11.273946762084961, "learning_rate": 4.447860229910544e-08, "logits/chosen": -2.2707040309906006, "logits/rejected": -1.692732334136963, "logps/chosen": -299.264892578125, "logps/ref_chosen": -112.96040344238281, "logps/ref_rejected": -99.52803802490234, "logps/rejected": -367.9000244140625, "loss": 1.1177, "margin_dpo/margin_mean": 82.0675048828125, "margin_dpo/margin_std": 98.18555450439453, "step": 564 }, { "KL/chosen_KL_mean": -185.06207275390625, "KL/mean": -235.034912109375, "KL/rejected_KL_mean": -285.00775146484375, "KL/std": 116.40463256835938, "epoch": 0.8296622613803231, "fcm_dpo/beta": 0.004095645155757666, "fcm_dpo/delta": -0.009785487316548824, "fcm_dpo/margin": 99.94569396972656, "fcm_dpo/q_t": 0.40508097410202026, "grad_norm": 9.245997428894043, "learning_rate": 4.375063135042445e-08, "logits/chosen": -1.679760456085205, "logits/rejected": -1.174295425415039, "logps/chosen": -280.42767333984375, "logps/ref_chosen": -95.36558532714844, "logps/ref_rejected": -92.21737670898438, "logps/rejected": -377.22515869140625, "loss": 1.0827, "margin_dpo/margin_mean": 99.94569396972656, "margin_dpo/margin_std": 127.49440002441406, "step": 565 }, { "KL/chosen_KL_mean": -192.6268768310547, "KL/mean": -239.96502685546875, "KL/rejected_KL_mean": -287.30316162109375, "KL/std": 118.42991638183594, "epoch": 0.8311306901615272, "fcm_dpo/beta": 0.004114923067390919, "fcm_dpo/delta": 0.010699309408664703, "fcm_dpo/margin": 94.67634582519531, "fcm_dpo/q_t": 0.4082973003387451, "grad_norm": 8.766554832458496, "learning_rate": 4.3028095264420525e-08, "logits/chosen": -1.9097862243652344, "logits/rejected": -1.4794607162475586, "logps/chosen": -291.60009765625, "logps/ref_chosen": -98.97320556640625, "logps/ref_rejected": -110.82090759277344, "logps/rejected": -398.12408447265625, "loss": 1.0905, "margin_dpo/margin_mean": 94.67633819580078, "margin_dpo/margin_std": 117.06986999511719, "step": 566 }, { "KL/chosen_KL_mean": -186.6658935546875, "KL/mean": -228.60211181640625, "KL/rejected_KL_mean": -270.538330078125, "KL/std": 107.04507446289062, "epoch": 0.8325991189427313, "fcm_dpo/beta": 0.00409280089661479, "fcm_dpo/delta": -0.017962271347641945, "fcm_dpo/margin": 83.87245178222656, "fcm_dpo/q_t": 0.41831058263778687, "grad_norm": 9.604767799377441, "learning_rate": 4.231101308059165e-08, "logits/chosen": -2.2468090057373047, "logits/rejected": -1.5719454288482666, "logps/chosen": -279.3695068359375, "logps/ref_chosen": -92.7035903930664, "logps/ref_rejected": -91.22431945800781, "logps/rejected": -361.7626647949219, "loss": 1.1267, "margin_dpo/margin_mean": 83.87245178222656, "margin_dpo/margin_std": 113.43426513671875, "step": 567 }, { "KL/chosen_KL_mean": -176.72360229492188, "KL/mean": -232.8488006591797, "KL/rejected_KL_mean": -288.9739990234375, "KL/std": 112.92816162109375, "epoch": 0.8340675477239354, "fcm_dpo/beta": 0.00404569273814559, "fcm_dpo/delta": -0.05707617476582527, "fcm_dpo/margin": 112.25041198730469, "fcm_dpo/q_t": 0.3934527039527893, "grad_norm": 8.518280982971191, "learning_rate": 4.1599403694720145e-08, "logits/chosen": -1.4681239128112793, "logits/rejected": -1.3411650657653809, "logps/chosen": -249.85586547851562, "logps/ref_chosen": -73.13226318359375, "logps/ref_rejected": -94.95265197753906, "logps/rejected": -383.9266662597656, "loss": 1.0337, "margin_dpo/margin_mean": 112.25041961669922, "margin_dpo/margin_std": 114.32151794433594, "step": 568 }, { "KL/chosen_KL_mean": -187.18194580078125, "KL/mean": -233.02822875976562, "KL/rejected_KL_mean": -278.87451171875, "KL/std": 114.64080810546875, "epoch": 0.8355359765051396, "fcm_dpo/beta": 0.004068131558597088, "fcm_dpo/delta": 0.02796497754752636, "fcm_dpo/margin": 91.69258117675781, "fcm_dpo/q_t": 0.412933886051178, "grad_norm": 12.933995246887207, "learning_rate": 4.089328585837512e-08, "logits/chosen": -2.0733516216278076, "logits/rejected": -1.703305959701538, "logps/chosen": -276.3990478515625, "logps/ref_chosen": -89.21708679199219, "logps/ref_rejected": -86.41105651855469, "logps/rejected": -365.28558349609375, "loss": 1.1232, "margin_dpo/margin_mean": 91.69258117675781, "margin_dpo/margin_std": 137.6352996826172, "step": 569 }, { "KL/chosen_KL_mean": -167.8793487548828, "KL/mean": -226.79827880859375, "KL/rejected_KL_mean": -285.7171936035156, "KL/std": 108.667236328125, "epoch": 0.8370044052863436, "fcm_dpo/beta": 0.004019296728074551, "fcm_dpo/delta": -0.0773601233959198, "fcm_dpo/margin": 117.83782958984375, "fcm_dpo/q_t": 0.3878633379936218, "grad_norm": 8.910812377929688, "learning_rate": 4.019267817841834e-08, "logits/chosen": -2.4476561546325684, "logits/rejected": -1.7825052738189697, "logps/chosen": -262.6421203613281, "logps/ref_chosen": -94.76277160644531, "logps/ref_rejected": -88.846923828125, "logps/rejected": -374.5641174316406, "loss": 1.0175, "margin_dpo/margin_mean": 117.83782958984375, "margin_dpo/margin_std": 113.94918823242188, "step": 570 }, { "KL/chosen_KL_mean": -175.6620330810547, "KL/mean": -227.6282501220703, "KL/rejected_KL_mean": -279.594482421875, "KL/std": 106.80409240722656, "epoch": 0.8384728340675477, "fcm_dpo/beta": 0.004005197435617447, "fcm_dpo/delta": -0.01703745126724243, "fcm_dpo/margin": 103.93246459960938, "fcm_dpo/q_t": 0.40318673849105835, "grad_norm": 12.429949760437012, "learning_rate": 3.9497599116513705e-08, "logits/chosen": -1.5648592710494995, "logits/rejected": -1.4034879207611084, "logps/chosen": -256.4833068847656, "logps/ref_chosen": -80.8212890625, "logps/ref_rejected": -97.01528930664062, "logps/rejected": -376.6097717285156, "loss": 1.0765, "margin_dpo/margin_mean": 103.93246459960938, "margin_dpo/margin_std": 129.15676879882812, "step": 571 }, { "KL/chosen_KL_mean": -179.8299102783203, "KL/mean": -232.33876037597656, "KL/rejected_KL_mean": -284.8476257324219, "KL/std": 113.78169250488281, "epoch": 0.8399412628487518, "fcm_dpo/beta": 0.003977552987635136, "fcm_dpo/delta": -0.01853088289499283, "fcm_dpo/margin": 105.01771545410156, "fcm_dpo/q_t": 0.40315932035446167, "grad_norm": 8.510725021362305, "learning_rate": 3.880806698864086e-08, "logits/chosen": -1.8613808155059814, "logits/rejected": -1.6963881254196167, "logps/chosen": -257.2098388671875, "logps/ref_chosen": -77.37992858886719, "logps/ref_rejected": -90.56597900390625, "logps/rejected": -375.4136047363281, "loss": 1.0667, "margin_dpo/margin_mean": 105.01771545410156, "margin_dpo/margin_std": 121.76112365722656, "step": 572 }, { "KL/chosen_KL_mean": -175.18084716796875, "KL/mean": -223.95648193359375, "KL/rejected_KL_mean": -272.7320861816406, "KL/std": 113.06573486328125, "epoch": 0.8414096916299559, "fcm_dpo/beta": 0.003957781475037336, "fcm_dpo/delta": -0.051512010395526886, "fcm_dpo/margin": 97.55123138427734, "fcm_dpo/q_t": 0.4086564779281616, "grad_norm": 8.708194732666016, "learning_rate": 3.812409996461275e-08, "logits/chosen": -2.1375205516815186, "logits/rejected": -1.9066476821899414, "logps/chosen": -258.8869323730469, "logps/ref_chosen": -83.7060775756836, "logps/ref_rejected": -90.69746398925781, "logps/rejected": -363.4295654296875, "loss": 1.0937, "margin_dpo/margin_mean": 97.55123901367188, "margin_dpo/margin_std": 119.69587707519531, "step": 573 }, { "KL/chosen_KL_mean": -184.34471130371094, "KL/mean": -237.38047790527344, "KL/rejected_KL_mean": -290.41619873046875, "KL/std": 112.2585678100586, "epoch": 0.8428781204111601, "fcm_dpo/beta": 0.003938412293791771, "fcm_dpo/delta": -0.018658364191651344, "fcm_dpo/margin": 106.0715103149414, "fcm_dpo/q_t": 0.4022049307823181, "grad_norm": 9.27223014831543, "learning_rate": 3.74457160675965e-08, "logits/chosen": -2.2239112854003906, "logits/rejected": -1.9022197723388672, "logps/chosen": -269.47418212890625, "logps/ref_chosen": -85.12948608398438, "logps/ref_rejected": -99.05403137207031, "logps/rejected": -389.4702453613281, "loss": 1.0714, "margin_dpo/margin_mean": 106.07151794433594, "margin_dpo/margin_std": 126.59791564941406, "step": 574 }, { "KL/chosen_KL_mean": -190.53359985351562, "KL/mean": -239.09487915039062, "KL/rejected_KL_mean": -287.65618896484375, "KL/std": 114.81494140625, "epoch": 0.8443465491923642, "fcm_dpo/beta": 0.003933538682758808, "fcm_dpo/delta": 0.0186786912381649, "fcm_dpo/margin": 97.12259674072266, "fcm_dpo/q_t": 0.4112199544906616, "grad_norm": 10.575162887573242, "learning_rate": 3.677293317363864e-08, "logits/chosen": -1.2434958219528198, "logits/rejected": -0.7674872279167175, "logps/chosen": -296.304443359375, "logps/ref_chosen": -105.77084350585938, "logps/ref_rejected": -100.94944763183594, "logps/rejected": -388.60565185546875, "loss": 1.1055, "margin_dpo/margin_mean": 97.12259674072266, "margin_dpo/margin_std": 133.65907287597656, "step": 575 }, { "KL/chosen_KL_mean": -188.86343383789062, "KL/mean": -235.95697021484375, "KL/rejected_KL_mean": -283.05047607421875, "KL/std": 106.71722412109375, "epoch": 0.8458149779735683, "fcm_dpo/beta": 0.003959815017879009, "fcm_dpo/delta": 0.02799609862267971, "fcm_dpo/margin": 94.18707275390625, "fcm_dpo/q_t": 0.4126693904399872, "grad_norm": 10.981925010681152, "learning_rate": 3.6105769011194224e-08, "logits/chosen": -2.0085506439208984, "logits/rejected": -1.948533296585083, "logps/chosen": -270.0679626464844, "logps/ref_chosen": -81.20452880859375, "logps/ref_rejected": -105.40940856933594, "logps/rejected": -388.45989990234375, "loss": 1.1099, "margin_dpo/margin_mean": 94.18707275390625, "margin_dpo/margin_std": 128.36178588867188, "step": 576 }, { "KL/chosen_KL_mean": -170.01434326171875, "KL/mean": -228.88552856445312, "KL/rejected_KL_mean": -287.7567138671875, "KL/std": 115.63882446289062, "epoch": 0.8472834067547724, "fcm_dpo/beta": 0.003909149672836065, "fcm_dpo/delta": -0.06362038105726242, "fcm_dpo/margin": 117.74236297607422, "fcm_dpo/q_t": 0.39235496520996094, "grad_norm": 8.003883361816406, "learning_rate": 3.5444241160659304e-08, "logits/chosen": -2.1418039798736572, "logits/rejected": -1.4616312980651855, "logps/chosen": -271.4785461425781, "logps/ref_chosen": -101.46419525146484, "logps/ref_rejected": -89.32637023925781, "logps/rejected": -377.08306884765625, "loss": 1.0412, "margin_dpo/margin_mean": 117.74236297607422, "margin_dpo/margin_std": 131.4029541015625, "step": 577 }, { "KL/chosen_KL_mean": -202.08888244628906, "KL/mean": -245.14999389648438, "KL/rejected_KL_mean": -288.21112060546875, "KL/std": 110.09112548828125, "epoch": 0.8487518355359766, "fcm_dpo/beta": 0.003944209311157465, "fcm_dpo/delta": 0.06246161460876465, "fcm_dpo/margin": 86.12225341796875, "fcm_dpo/q_t": 0.4206528663635254, "grad_norm": 13.475691795349121, "learning_rate": 3.478836705390808e-08, "logits/chosen": -1.2917630672454834, "logits/rejected": -1.0502583980560303, "logps/chosen": -279.38128662109375, "logps/ref_chosen": -77.29241943359375, "logps/ref_rejected": -88.24701690673828, "logps/rejected": -376.4581298828125, "loss": 1.1342, "margin_dpo/margin_mean": 86.12226104736328, "margin_dpo/margin_std": 126.02249145507812, "step": 578 }, { "KL/chosen_KL_mean": -181.1404266357422, "KL/mean": -225.11721801757812, "KL/rejected_KL_mean": -269.09405517578125, "KL/std": 109.78843688964844, "epoch": 0.8502202643171806, "fcm_dpo/beta": 0.00394777674227953, "fcm_dpo/delta": -0.013691302388906479, "fcm_dpo/margin": 87.95361328125, "fcm_dpo/q_t": 0.41843438148498535, "grad_norm": 7.778433799743652, "learning_rate": 3.41381639738331e-08, "logits/chosen": -1.8752148151397705, "logits/rejected": -1.5858491659164429, "logps/chosen": -268.48345947265625, "logps/ref_chosen": -87.34305572509766, "logps/ref_rejected": -102.12100982666016, "logps/rejected": -371.2150573730469, "loss": 1.1196, "margin_dpo/margin_mean": 87.95361328125, "margin_dpo/margin_std": 111.86346435546875, "step": 579 }, { "KL/chosen_KL_mean": -173.6749267578125, "KL/mean": -226.8583526611328, "KL/rejected_KL_mean": -280.04180908203125, "KL/std": 109.77399444580078, "epoch": 0.8516886930983847, "fcm_dpo/beta": 0.0039232028648257256, "fcm_dpo/delta": -0.018368642777204514, "fcm_dpo/margin": 106.36687469482422, "fcm_dpo/q_t": 0.4037545323371887, "grad_norm": 10.025492668151855, "learning_rate": 3.349364905389032e-08, "logits/chosen": -1.6152920722961426, "logits/rejected": -1.3063846826553345, "logps/chosen": -262.60369873046875, "logps/ref_chosen": -88.92879486083984, "logps/ref_rejected": -88.69661712646484, "logps/rejected": -368.7384033203125, "loss": 1.0767, "margin_dpo/margin_mean": 106.36686706542969, "margin_dpo/margin_std": 133.18289184570312, "step": 580 }, { "KL/chosen_KL_mean": -183.19503784179688, "KL/mean": -232.03536987304688, "KL/rejected_KL_mean": -280.875732421875, "KL/std": 110.61803436279297, "epoch": 0.8531571218795888, "fcm_dpo/beta": 0.003946345765143633, "fcm_dpo/delta": 0.01506105437874794, "fcm_dpo/margin": 97.68067932128906, "fcm_dpo/q_t": 0.41012513637542725, "grad_norm": 10.578690528869629, "learning_rate": 3.285483927764726e-08, "logits/chosen": -1.9257314205169678, "logits/rejected": -1.5755860805511475, "logps/chosen": -275.41827392578125, "logps/ref_chosen": -92.22323608398438, "logps/ref_rejected": -99.38943481445312, "logps/rejected": -380.26513671875, "loss": 1.1085, "margin_dpo/margin_mean": 97.68067932128906, "margin_dpo/margin_std": 138.07290649414062, "step": 581 }, { "KL/chosen_KL_mean": -162.97067260742188, "KL/mean": -206.9716796875, "KL/rejected_KL_mean": -250.9727020263672, "KL/std": 115.35728454589844, "epoch": 0.8546255506607929, "fcm_dpo/beta": 0.003923641983419657, "fcm_dpo/delta": -0.054857250303030014, "fcm_dpo/margin": 88.00202941894531, "fcm_dpo/q_t": 0.42033272981643677, "grad_norm": 9.29529094696045, "learning_rate": 3.222175147833556e-08, "logits/chosen": -2.0911173820495605, "logits/rejected": -2.0835142135620117, "logps/chosen": -250.30630493164062, "logps/ref_chosen": -87.33561706542969, "logps/ref_rejected": -115.1544189453125, "logps/rejected": -366.12713623046875, "loss": 1.1347, "margin_dpo/margin_mean": 88.00202941894531, "margin_dpo/margin_std": 127.08179473876953, "step": 582 }, { "KL/chosen_KL_mean": -195.3851318359375, "KL/mean": -244.30120849609375, "KL/rejected_KL_mean": -293.2173156738281, "KL/std": 109.18648529052734, "epoch": 0.856093979441997, "fcm_dpo/beta": 0.0038992201443761587, "fcm_dpo/delta": 0.019070586189627647, "fcm_dpo/margin": 97.83218383789062, "fcm_dpo/q_t": 0.41160455346107483, "grad_norm": 11.060173988342285, "learning_rate": 3.159440233840763e-08, "logits/chosen": -1.4794248342514038, "logits/rejected": -1.2447502613067627, "logps/chosen": -279.1022033691406, "logps/ref_chosen": -83.71707153320312, "logps/ref_rejected": -94.58465576171875, "logps/rejected": -387.8019714355469, "loss": 1.1016, "margin_dpo/margin_mean": 97.83219146728516, "margin_dpo/margin_std": 129.11358642578125, "step": 583 }, { "KL/chosen_KL_mean": -173.88180541992188, "KL/mean": -225.32388305664062, "KL/rejected_KL_mean": -276.7659606933594, "KL/std": 112.76244354248047, "epoch": 0.8575624082232012, "fcm_dpo/beta": 0.00389112439006567, "fcm_dpo/delta": -0.06657686084508896, "fcm_dpo/margin": 102.8841552734375, "fcm_dpo/q_t": 0.40561097860336304, "grad_norm": 7.687736988067627, "learning_rate": 3.0972808389096635e-08, "logits/chosen": -1.858134150505066, "logits/rejected": -1.3956893682479858, "logps/chosen": -268.141357421875, "logps/ref_chosen": -94.25955963134766, "logps/ref_rejected": -104.48269653320312, "logps/rejected": -381.2486572265625, "loss": 1.0889, "margin_dpo/margin_mean": 102.8841552734375, "margin_dpo/margin_std": 127.93247985839844, "step": 584 }, { "KL/chosen_KL_mean": -186.596435546875, "KL/mean": -243.97555541992188, "KL/rejected_KL_mean": -301.35467529296875, "KL/std": 128.8909149169922, "epoch": 0.8590308370044053, "fcm_dpo/beta": 0.0038514507468789816, "fcm_dpo/delta": -0.043976567685604095, "fcm_dpo/margin": 114.75820922851562, "fcm_dpo/q_t": 0.39856576919555664, "grad_norm": 8.535517692565918, "learning_rate": 3.035698600998121e-08, "logits/chosen": -2.046078681945801, "logits/rejected": -1.7791507244110107, "logps/chosen": -272.23321533203125, "logps/ref_chosen": -85.63676452636719, "logps/ref_rejected": -90.28547668457031, "logps/rejected": -391.64013671875, "loss": 1.0731, "margin_dpo/margin_mean": 114.75820922851562, "margin_dpo/margin_std": 152.0629425048828, "step": 585 }, { "KL/chosen_KL_mean": -206.1322784423828, "KL/mean": -244.49789428710938, "KL/rejected_KL_mean": -282.863525390625, "KL/std": 112.7052993774414, "epoch": 0.8604992657856094, "fcm_dpo/beta": 0.0038691554218530655, "fcm_dpo/delta": 0.04906633868813515, "fcm_dpo/margin": 76.73126220703125, "fcm_dpo/q_t": 0.42960894107818604, "grad_norm": 9.505632400512695, "learning_rate": 2.974695142855388e-08, "logits/chosen": -1.964722752571106, "logits/rejected": -1.7065095901489258, "logps/chosen": -289.81341552734375, "logps/ref_chosen": -83.68114471435547, "logps/ref_rejected": -98.35916137695312, "logps/rejected": -381.22271728515625, "loss": 1.1537, "margin_dpo/margin_mean": 76.73126983642578, "margin_dpo/margin_std": 107.5704345703125, "step": 586 }, { "KL/chosen_KL_mean": -166.3143310546875, "KL/mean": -208.69918823242188, "KL/rejected_KL_mean": -251.08404541015625, "KL/std": 109.4501953125, "epoch": 0.8619676945668135, "fcm_dpo/beta": 0.0039144158363342285, "fcm_dpo/delta": 0.07041102647781372, "fcm_dpo/margin": 84.76972198486328, "fcm_dpo/q_t": 0.42214369773864746, "grad_norm": 10.67780590057373, "learning_rate": 2.9142720719793122e-08, "logits/chosen": -2.2781388759613037, "logits/rejected": -1.9919813871383667, "logps/chosen": -243.891845703125, "logps/ref_chosen": -77.57750701904297, "logps/ref_rejected": -89.32032775878906, "logps/rejected": -340.4043884277344, "loss": 1.1289, "margin_dpo/margin_mean": 84.76971435546875, "margin_dpo/margin_std": 113.54153442382812, "step": 587 }, { "KL/chosen_KL_mean": -181.08746337890625, "KL/mean": -220.97515869140625, "KL/rejected_KL_mean": -260.86285400390625, "KL/std": 110.893798828125, "epoch": 0.8634361233480177, "fcm_dpo/beta": 0.0039445785805583, "fcm_dpo/delta": 0.02611854113638401, "fcm_dpo/margin": 79.77537536621094, "fcm_dpo/q_t": 0.4274389147758484, "grad_norm": 13.267123222351074, "learning_rate": 2.8544309805740018e-08, "logits/chosen": -1.8999654054641724, "logits/rejected": -1.863416314125061, "logps/chosen": -264.8917236328125, "logps/ref_chosen": -83.80426025390625, "logps/ref_rejected": -113.11943054199219, "logps/rejected": -373.9822998046875, "loss": 1.1622, "margin_dpo/margin_mean": 79.77537536621094, "margin_dpo/margin_std": 133.38803100585938, "step": 588 }, { "KL/chosen_KL_mean": -193.58914184570312, "KL/mean": -245.51968383789062, "KL/rejected_KL_mean": -297.4501953125, "KL/std": 109.301513671875, "epoch": 0.8649045521292217, "fcm_dpo/beta": 0.003940122202038765, "fcm_dpo/delta": -0.009617887437343597, "fcm_dpo/margin": 103.86106872558594, "fcm_dpo/q_t": 0.40294596552848816, "grad_norm": 8.448515892028809, "learning_rate": 2.7951734455078786e-08, "logits/chosen": -1.565617561340332, "logits/rejected": -1.16941499710083, "logps/chosen": -290.0541687011719, "logps/ref_chosen": -96.46501922607422, "logps/ref_rejected": -116.03719329833984, "logps/rejected": -413.4873962402344, "loss": 1.0591, "margin_dpo/margin_mean": 103.86106872558594, "margin_dpo/margin_std": 104.7183837890625, "step": 589 }, { "KL/chosen_KL_mean": -177.35427856445312, "KL/mean": -229.47930908203125, "KL/rejected_KL_mean": -281.6043395996094, "KL/std": 101.55866241455078, "epoch": 0.8663729809104258, "fcm_dpo/beta": 0.0039449273608624935, "fcm_dpo/delta": -0.01198958232998848, "fcm_dpo/margin": 104.25007629394531, "fcm_dpo/q_t": 0.4031516909599304, "grad_norm": 11.420241355895996, "learning_rate": 2.736501028272095e-08, "logits/chosen": -2.03340482711792, "logits/rejected": -1.8912010192871094, "logps/chosen": -265.7821350097656, "logps/ref_chosen": -88.4278564453125, "logps/ref_rejected": -110.19876861572266, "logps/rejected": -391.8031005859375, "loss": 1.0592, "margin_dpo/margin_mean": 104.25007629394531, "margin_dpo/margin_std": 106.25035095214844, "step": 590 }, { "KL/chosen_KL_mean": -170.3348388671875, "KL/mean": -225.35415649414062, "KL/rejected_KL_mean": -280.37347412109375, "KL/std": 109.25648498535156, "epoch": 0.8678414096916299, "fcm_dpo/beta": 0.003901613410562277, "fcm_dpo/delta": -0.030792508274316788, "fcm_dpo/margin": 110.03866577148438, "fcm_dpo/q_t": 0.39846694469451904, "grad_norm": 7.1532769203186035, "learning_rate": 2.678415274939408e-08, "logits/chosen": -1.966714859008789, "logits/rejected": -1.3548333644866943, "logps/chosen": -258.18450927734375, "logps/ref_chosen": -87.84968566894531, "logps/ref_rejected": -89.57919311523438, "logps/rejected": -369.9526672363281, "loss": 1.0438, "margin_dpo/margin_mean": 110.03866577148438, "margin_dpo/margin_std": 106.27468872070312, "step": 591 }, { "KL/chosen_KL_mean": -200.9137725830078, "KL/mean": -243.66827392578125, "KL/rejected_KL_mean": -286.42279052734375, "KL/std": 106.69731903076172, "epoch": 0.869309838472834, "fcm_dpo/beta": 0.003942975774407387, "fcm_dpo/delta": 0.06500554829835892, "fcm_dpo/margin": 85.50900268554688, "fcm_dpo/q_t": 0.4212019145488739, "grad_norm": 12.713777542114258, "learning_rate": 2.6209177161234442e-08, "logits/chosen": -1.8829023838043213, "logits/rejected": -1.558463215827942, "logps/chosen": -275.8895263671875, "logps/ref_chosen": -74.97573852539062, "logps/ref_rejected": -81.55262756347656, "logps/rejected": -367.97540283203125, "loss": 1.1438, "margin_dpo/margin_mean": 85.50900268554688, "margin_dpo/margin_std": 133.84201049804688, "step": 592 }, { "KL/chosen_KL_mean": -183.4608612060547, "KL/mean": -220.23834228515625, "KL/rejected_KL_mean": -257.01580810546875, "KL/std": 111.59335327148438, "epoch": 0.8707782672540382, "fcm_dpo/beta": 0.003973287530243397, "fcm_dpo/delta": 0.023653369396924973, "fcm_dpo/margin": 73.55494689941406, "fcm_dpo/q_t": 0.43189066648483276, "grad_norm": 13.498023986816406, "learning_rate": 2.564009866938349e-08, "logits/chosen": -0.8610674142837524, "logits/rejected": -0.5588955283164978, "logps/chosen": -255.69989013671875, "logps/ref_chosen": -72.23902893066406, "logps/ref_rejected": -70.09652709960938, "logps/rejected": -327.11236572265625, "loss": 1.1736, "margin_dpo/margin_mean": 73.55494689941406, "margin_dpo/margin_std": 123.48003387451172, "step": 593 }, { "KL/chosen_KL_mean": -164.04953002929688, "KL/mean": -220.51300048828125, "KL/rejected_KL_mean": -276.9764709472656, "KL/std": 108.49273681640625, "epoch": 0.8722466960352423, "fcm_dpo/beta": 0.003957782872021198, "fcm_dpo/delta": -0.049205463379621506, "fcm_dpo/margin": 112.92694091796875, "fcm_dpo/q_t": 0.3944060802459717, "grad_norm": 8.890219688415527, "learning_rate": 2.5076932269588708e-08, "logits/chosen": -1.8701916933059692, "logits/rejected": -1.3391462564468384, "logps/chosen": -254.48724365234375, "logps/ref_chosen": -90.43771362304688, "logps/ref_rejected": -92.267578125, "logps/rejected": -369.2440490722656, "loss": 1.0308, "margin_dpo/margin_mean": 112.92694091796875, "margin_dpo/margin_std": 105.21086120605469, "step": 594 }, { "KL/chosen_KL_mean": -177.05056762695312, "KL/mean": -222.8773193359375, "KL/rejected_KL_mean": -268.7041015625, "KL/std": 108.85829162597656, "epoch": 0.8737151248164464, "fcm_dpo/beta": 0.00396577175706625, "fcm_dpo/delta": 0.03770260140299797, "fcm_dpo/margin": 91.65351867675781, "fcm_dpo/q_t": 0.4157801866531372, "grad_norm": 9.045574188232422, "learning_rate": 2.451969280180849e-08, "logits/chosen": -2.0839295387268066, "logits/rejected": -1.6958184242248535, "logps/chosen": -258.56536865234375, "logps/ref_chosen": -81.51480102539062, "logps/ref_rejected": -86.52047729492188, "logps/rejected": -355.22454833984375, "loss": 1.1135, "margin_dpo/margin_mean": 91.65351867675781, "margin_dpo/margin_std": 124.27123260498047, "step": 595 }, { "KL/chosen_KL_mean": -184.35015869140625, "KL/mean": -225.3868408203125, "KL/rejected_KL_mean": -266.42352294921875, "KL/std": 107.36467742919922, "epoch": 0.8751835535976505, "fcm_dpo/beta": 0.0040108636021614075, "fcm_dpo/delta": 0.07315412908792496, "fcm_dpo/margin": 82.0733642578125, "fcm_dpo/q_t": 0.4241868257522583, "grad_norm": 11.727372169494629, "learning_rate": 2.396839494982103e-08, "logits/chosen": -1.6635912656784058, "logits/rejected": -1.0872061252593994, "logps/chosen": -272.5582275390625, "logps/ref_chosen": -88.20805358886719, "logps/ref_rejected": -86.09375, "logps/rejected": -352.51727294921875, "loss": 1.1558, "margin_dpo/margin_mean": 82.0733642578125, "margin_dpo/margin_std": 136.48329162597656, "step": 596 }, { "KL/chosen_KL_mean": -178.85769653320312, "KL/mean": -234.02279663085938, "KL/rejected_KL_mean": -289.18792724609375, "KL/std": 115.64497375488281, "epoch": 0.8766519823788547, "fcm_dpo/beta": 0.003988821990787983, "fcm_dpo/delta": -0.04211900755763054, "fcm_dpo/margin": 110.33018493652344, "fcm_dpo/q_t": 0.3975440263748169, "grad_norm": 9.088208198547363, "learning_rate": 2.3423053240837514e-08, "logits/chosen": -1.4629243612289429, "logits/rejected": -1.348548173904419, "logps/chosen": -262.4476623535156, "logps/ref_chosen": -83.58998107910156, "logps/ref_rejected": -106.94439697265625, "logps/rejected": -396.13232421875, "loss": 1.0526, "margin_dpo/margin_mean": 110.33019256591797, "margin_dpo/margin_std": 125.17523193359375, "step": 597 }, { "KL/chosen_KL_mean": -170.55517578125, "KL/mean": -217.14370727539062, "KL/rejected_KL_mean": -263.73223876953125, "KL/std": 108.18994140625, "epoch": 0.8781204111600588, "fcm_dpo/beta": 0.003991022706031799, "fcm_dpo/delta": 0.029031306505203247, "fcm_dpo/margin": 93.17707061767578, "fcm_dpo/q_t": 0.4123697876930237, "grad_norm": 9.256612777709961, "learning_rate": 2.2883682045119062e-08, "logits/chosen": -1.9533220529556274, "logits/rejected": -1.641535997390747, "logps/chosen": -260.73779296875, "logps/ref_chosen": -90.18262481689453, "logps/ref_rejected": -94.79856872558594, "logps/rejected": -358.53082275390625, "loss": 1.1081, "margin_dpo/margin_mean": 93.17707824707031, "margin_dpo/margin_std": 122.35414123535156, "step": 598 }, { "KL/chosen_KL_mean": -172.3333740234375, "KL/mean": -213.53013610839844, "KL/rejected_KL_mean": -254.72691345214844, "KL/std": 112.74430847167969, "epoch": 0.8795888399412628, "fcm_dpo/beta": 0.004003605339676142, "fcm_dpo/delta": -0.010556099005043507, "fcm_dpo/margin": 82.39353942871094, "fcm_dpo/q_t": 0.422207236289978, "grad_norm": 8.678912162780762, "learning_rate": 2.2350295575598367e-08, "logits/chosen": -1.4942137002944946, "logits/rejected": -1.3687318563461304, "logps/chosen": -250.1756591796875, "logps/ref_chosen": -77.84226989746094, "logps/ref_rejected": -89.1976318359375, "logps/rejected": -343.924560546875, "loss": 1.1451, "margin_dpo/margin_mean": 82.39353942871094, "margin_dpo/margin_std": 124.41415405273438, "step": 599 }, { "KL/chosen_KL_mean": -193.49203491210938, "KL/mean": -239.5679473876953, "KL/rejected_KL_mean": -285.64385986328125, "KL/std": 106.37391662597656, "epoch": 0.8810572687224669, "fcm_dpo/beta": 0.004024241119623184, "fcm_dpo/delta": 0.030254486948251724, "fcm_dpo/margin": 92.15185546875, "fcm_dpo/q_t": 0.41345566511154175, "grad_norm": 8.820852279663086, "learning_rate": 2.1822907887504932e-08, "logits/chosen": -2.5194473266601562, "logits/rejected": -2.183413505554199, "logps/chosen": -285.6798400878906, "logps/ref_chosen": -92.18781280517578, "logps/ref_rejected": -92.95429992675781, "logps/rejected": -378.5981750488281, "loss": 1.1022, "margin_dpo/margin_mean": 92.15185546875, "margin_dpo/margin_std": 115.8713607788086, "step": 600 }, { "KL/chosen_KL_mean": -195.01119995117188, "KL/mean": -244.45657348632812, "KL/rejected_KL_mean": -293.9019775390625, "KL/std": 108.30038452148438, "epoch": 0.882525697503671, "fcm_dpo/beta": 0.004017959348857403, "fcm_dpo/delta": 0.002610735595226288, "fcm_dpo/margin": 98.89080047607422, "fcm_dpo/q_t": 0.4062908887863159, "grad_norm": 9.566874504089355, "learning_rate": 2.1301532877994742e-08, "logits/chosen": -1.8796908855438232, "logits/rejected": -1.5611186027526855, "logps/chosen": -284.22735595703125, "logps/ref_chosen": -89.21614074707031, "logps/ref_rejected": -100.17054748535156, "logps/rejected": -394.0725402832031, "loss": 1.0699, "margin_dpo/margin_mean": 98.89079284667969, "margin_dpo/margin_std": 103.63615417480469, "step": 601 }, { "KL/chosen_KL_mean": -155.3017578125, "KL/mean": -219.16592407226562, "KL/rejected_KL_mean": -283.03009033203125, "KL/std": 118.75299072265625, "epoch": 0.8839941262848752, "fcm_dpo/beta": 0.003964175470173359, "fcm_dpo/delta": -0.11199070513248444, "fcm_dpo/margin": 127.72828674316406, "fcm_dpo/q_t": 0.3826666474342346, "grad_norm": 9.319535255432129, "learning_rate": 2.0786184285784298e-08, "logits/chosen": -2.0947747230529785, "logits/rejected": -1.8340649604797363, "logps/chosen": -235.359375, "logps/ref_chosen": -80.05760192871094, "logps/ref_rejected": -93.197509765625, "logps/rejected": -376.22760009765625, "loss": 1.0097, "margin_dpo/margin_mean": 127.72828674316406, "margin_dpo/margin_std": 135.908447265625, "step": 602 }, { "KL/chosen_KL_mean": -165.49957275390625, "KL/mean": -218.2401123046875, "KL/rejected_KL_mean": -270.9806213378906, "KL/std": 106.63116455078125, "epoch": 0.8854625550660793, "fcm_dpo/beta": 0.003934741020202637, "fcm_dpo/delta": -0.015692541375756264, "fcm_dpo/margin": 105.48103332519531, "fcm_dpo/q_t": 0.40293973684310913, "grad_norm": 8.120116233825684, "learning_rate": 2.0276875690788204e-08, "logits/chosen": -2.1011710166931152, "logits/rejected": -1.5195586681365967, "logps/chosen": -267.80914306640625, "logps/ref_chosen": -102.30958557128906, "logps/ref_rejected": -108.06884765625, "logps/rejected": -379.0494689941406, "loss": 1.0677, "margin_dpo/margin_mean": 105.48103332519531, "margin_dpo/margin_std": 122.36427307128906, "step": 603 }, { "KL/chosen_KL_mean": -172.99131774902344, "KL/mean": -227.8787841796875, "KL/rejected_KL_mean": -282.7662353515625, "KL/std": 112.15409851074219, "epoch": 0.8869309838472834, "fcm_dpo/beta": 0.0038999803364276886, "fcm_dpo/delta": -0.029628120362758636, "fcm_dpo/margin": 109.77490234375, "fcm_dpo/q_t": 0.39813992381095886, "grad_norm": 8.693439483642578, "learning_rate": 1.977362051376158e-08, "logits/chosen": -1.487715721130371, "logits/rejected": -1.3304961919784546, "logps/chosen": -251.1654052734375, "logps/ref_chosen": -78.17408752441406, "logps/ref_rejected": -99.49610900878906, "logps/rejected": -382.2623291015625, "loss": 1.0522, "margin_dpo/margin_mean": 109.77490234375, "margin_dpo/margin_std": 115.15666198730469, "step": 604 }, { "KL/chosen_KL_mean": -188.95111083984375, "KL/mean": -234.07000732421875, "KL/rejected_KL_mean": -279.1888732910156, "KL/std": 100.14305114746094, "epoch": 0.8883994126284875, "fcm_dpo/beta": 0.0039304220117628574, "fcm_dpo/delta": 0.04701454937458038, "fcm_dpo/margin": 90.23776245117188, "fcm_dpo/q_t": 0.4168788194656372, "grad_norm": 9.627431869506836, "learning_rate": 1.9276432015946446e-08, "logits/chosen": -1.7562696933746338, "logits/rejected": -1.254887342453003, "logps/chosen": -283.72442626953125, "logps/ref_chosen": -94.77332305908203, "logps/ref_rejected": -107.30490112304688, "logps/rejected": -386.4937744140625, "loss": 1.1097, "margin_dpo/margin_mean": 90.23775482177734, "margin_dpo/margin_std": 112.81745910644531, "step": 605 }, { "KL/chosen_KL_mean": -182.39306640625, "KL/mean": -228.991943359375, "KL/rejected_KL_mean": -275.5907897949219, "KL/std": 109.53794860839844, "epoch": 0.8898678414096917, "fcm_dpo/beta": 0.003929854836314917, "fcm_dpo/delta": -0.031113799661397934, "fcm_dpo/margin": 93.19770812988281, "fcm_dpo/q_t": 0.4130397140979767, "grad_norm": 8.124847412109375, "learning_rate": 1.8785323298722093e-08, "logits/chosen": -1.795448899269104, "logits/rejected": -1.406607747077942, "logps/chosen": -270.146484375, "logps/ref_chosen": -87.7533950805664, "logps/ref_rejected": -104.2422103881836, "logps/rejected": -379.8330078125, "loss": 1.1057, "margin_dpo/margin_mean": 93.19770812988281, "margin_dpo/margin_std": 116.86910247802734, "step": 606 }, { "KL/chosen_KL_mean": -176.47531127929688, "KL/mean": -229.19363403320312, "KL/rejected_KL_mean": -281.9119873046875, "KL/std": 117.06456756591797, "epoch": 0.8913362701908958, "fcm_dpo/beta": 0.003912745974957943, "fcm_dpo/delta": -0.013087640516459942, "fcm_dpo/margin": 105.43665313720703, "fcm_dpo/q_t": 0.4045717716217041, "grad_norm": 10.934760093688965, "learning_rate": 1.8300307303259904e-08, "logits/chosen": -1.6783382892608643, "logits/rejected": -1.1429007053375244, "logps/chosen": -264.8043518066406, "logps/ref_chosen": -88.32904052734375, "logps/ref_rejected": -86.76810455322266, "logps/rejected": -368.6800537109375, "loss": 1.091, "margin_dpo/margin_mean": 105.43665313720703, "margin_dpo/margin_std": 144.213134765625, "step": 607 }, { "KL/chosen_KL_mean": -161.14767456054688, "KL/mean": -216.14566040039062, "KL/rejected_KL_mean": -271.1436462402344, "KL/std": 115.4263687133789, "epoch": 0.8928046989720999, "fcm_dpo/beta": 0.0038679465651512146, "fcm_dpo/delta": -0.027507182210683823, "fcm_dpo/margin": 109.99598693847656, "fcm_dpo/q_t": 0.3995211720466614, "grad_norm": 8.010201454162598, "learning_rate": 1.7821396810182437e-08, "logits/chosen": -1.8765852451324463, "logits/rejected": -1.5803370475769043, "logps/chosen": -246.9170379638672, "logps/ref_chosen": -85.76937866210938, "logps/ref_rejected": -100.23281860351562, "logps/rejected": -371.37646484375, "loss": 1.0661, "margin_dpo/margin_mean": 109.99598693847656, "margin_dpo/margin_std": 125.82890319824219, "step": 608 }, { "KL/chosen_KL_mean": -180.2808380126953, "KL/mean": -231.44041442871094, "KL/rejected_KL_mean": -282.5999755859375, "KL/std": 116.58121490478516, "epoch": 0.8942731277533039, "fcm_dpo/beta": 0.0038539059460163116, "fcm_dpo/delta": -0.041400469839572906, "fcm_dpo/margin": 102.319091796875, "fcm_dpo/q_t": 0.40778660774230957, "grad_norm": 10.630335807800293, "learning_rate": 1.7348604439226617e-08, "logits/chosen": -2.297642707824707, "logits/rejected": -1.882304072380066, "logps/chosen": -273.2474365234375, "logps/ref_chosen": -92.9665756225586, "logps/ref_rejected": -95.91818237304688, "logps/rejected": -378.51812744140625, "loss": 1.0897, "margin_dpo/margin_mean": 102.319091796875, "margin_dpo/margin_std": 124.1063232421875, "step": 609 }, { "KL/chosen_KL_mean": -164.69947814941406, "KL/mean": -219.94285583496094, "KL/rejected_KL_mean": -275.186279296875, "KL/std": 110.60639190673828, "epoch": 0.895741556534508, "fcm_dpo/beta": 0.0038321493193507195, "fcm_dpo/delta": -0.024567697197198868, "fcm_dpo/margin": 110.48677825927734, "fcm_dpo/q_t": 0.39954763650894165, "grad_norm": 8.435887336730957, "learning_rate": 1.6881942648911074e-08, "logits/chosen": -1.8060765266418457, "logits/rejected": -1.2700965404510498, "logps/chosen": -259.3997802734375, "logps/ref_chosen": -94.70028686523438, "logps/ref_rejected": -89.68739318847656, "logps/rejected": -364.8736572265625, "loss": 1.0555, "margin_dpo/margin_mean": 110.48677825927734, "margin_dpo/margin_std": 117.20846557617188, "step": 610 }, { "KL/chosen_KL_mean": -174.323974609375, "KL/mean": -228.17726135253906, "KL/rejected_KL_mean": -282.030517578125, "KL/std": 114.42023468017578, "epoch": 0.8972099853157122, "fcm_dpo/beta": 0.003843258135020733, "fcm_dpo/delta": -0.014867845922708511, "fcm_dpo/margin": 107.70655059814453, "fcm_dpo/q_t": 0.4030778408050537, "grad_norm": 9.592172622680664, "learning_rate": 1.6421423736208e-08, "logits/chosen": -2.1820597648620605, "logits/rejected": -1.8455564975738525, "logps/chosen": -261.1072998046875, "logps/ref_chosen": -86.78334045410156, "logps/ref_rejected": -89.84307861328125, "logps/rejected": -371.87359619140625, "loss": 1.0634, "margin_dpo/margin_mean": 107.70654296875, "margin_dpo/margin_std": 116.05413818359375, "step": 611 }, { "KL/chosen_KL_mean": -191.6583251953125, "KL/mean": -247.10220336914062, "KL/rejected_KL_mean": -302.54608154296875, "KL/std": 108.65845489501953, "epoch": 0.8986784140969163, "fcm_dpo/beta": 0.0038062939420342445, "fcm_dpo/delta": -0.023094428703188896, "fcm_dpo/margin": 110.88774108886719, "fcm_dpo/q_t": 0.4003736674785614, "grad_norm": 8.83055591583252, "learning_rate": 1.5967059836219042e-08, "logits/chosen": -2.038139820098877, "logits/rejected": -1.1902828216552734, "logps/chosen": -292.678466796875, "logps/ref_chosen": -101.02015686035156, "logps/ref_rejected": -93.78302764892578, "logps/rejected": -396.3291015625, "loss": 1.052, "margin_dpo/margin_mean": 110.88774108886719, "margin_dpo/margin_std": 113.39448547363281, "step": 612 }, { "KL/chosen_KL_mean": -153.7947998046875, "KL/mean": -216.18380737304688, "KL/rejected_KL_mean": -278.5728454589844, "KL/std": 111.1944351196289, "epoch": 0.9001468428781204, "fcm_dpo/beta": 0.003759459126740694, "fcm_dpo/delta": -0.07265369594097137, "fcm_dpo/margin": 124.77806091308594, "fcm_dpo/q_t": 0.3885994255542755, "grad_norm": 8.025390625, "learning_rate": 1.551886292185553e-08, "logits/chosen": -1.8691550493240356, "logits/rejected": -1.6710984706878662, "logps/chosen": -242.78341674804688, "logps/ref_chosen": -88.98861694335938, "logps/ref_rejected": -109.99551391601562, "logps/rejected": -388.568359375, "loss": 1.0107, "margin_dpo/margin_mean": 124.77806091308594, "margin_dpo/margin_std": 108.96006774902344, "step": 613 }, { "KL/chosen_KL_mean": -192.46804809570312, "KL/mean": -243.35745239257812, "KL/rejected_KL_mean": -294.246826171875, "KL/std": 105.44712829589844, "epoch": 0.9016152716593245, "fcm_dpo/beta": 0.003757667727768421, "fcm_dpo/delta": 0.018245378509163857, "fcm_dpo/margin": 101.77880859375, "fcm_dpo/q_t": 0.4098703861236572, "grad_norm": 9.768272399902344, "learning_rate": 1.507684480352292e-08, "logits/chosen": -1.640364170074463, "logits/rejected": -1.6304938793182373, "logps/chosen": -272.6680908203125, "logps/ref_chosen": -80.20005798339844, "logps/ref_rejected": -109.86239624023438, "logps/rejected": -404.1092529296875, "loss": 1.0972, "margin_dpo/margin_mean": 101.77880859375, "margin_dpo/margin_std": 129.19949340820312, "step": 614 }, { "KL/chosen_KL_mean": -180.46514892578125, "KL/mean": -221.78871154785156, "KL/rejected_KL_mean": -263.1122741699219, "KL/std": 106.55547332763672, "epoch": 0.9030837004405287, "fcm_dpo/beta": 0.003759893123060465, "fcm_dpo/delta": -0.01309838518500328, "fcm_dpo/margin": 82.64714813232422, "fcm_dpo/q_t": 0.4255291819572449, "grad_norm": 8.882323265075684, "learning_rate": 1.4641017128809801e-08, "logits/chosen": -1.9366450309753418, "logits/rejected": -1.547764778137207, "logps/chosen": -280.900390625, "logps/ref_chosen": -100.43525695800781, "logps/ref_rejected": -101.1800537109375, "logps/rejected": -364.2923583984375, "loss": 1.1383, "margin_dpo/margin_mean": 82.64714813232422, "margin_dpo/margin_std": 101.86264038085938, "step": 615 }, { "KL/chosen_KL_mean": -182.88592529296875, "KL/mean": -239.62872314453125, "KL/rejected_KL_mean": -296.3714904785156, "KL/std": 106.27088165283203, "epoch": 0.9045521292217328, "fcm_dpo/beta": 0.003753126133233309, "fcm_dpo/delta": -0.02724025398492813, "fcm_dpo/margin": 113.48554229736328, "fcm_dpo/q_t": 0.39982104301452637, "grad_norm": 8.79699420928955, "learning_rate": 1.4211391382180637e-08, "logits/chosen": -1.8284904956817627, "logits/rejected": -1.3341734409332275, "logps/chosen": -275.37884521484375, "logps/ref_chosen": -92.49292755126953, "logps/ref_rejected": -82.06065368652344, "logps/rejected": -378.43212890625, "loss": 1.0574, "margin_dpo/margin_mean": 113.48554229736328, "margin_dpo/margin_std": 124.32967376708984, "step": 616 }, { "KL/chosen_KL_mean": -189.74118041992188, "KL/mean": -234.5286102294922, "KL/rejected_KL_mean": -279.3160400390625, "KL/std": 103.72169494628906, "epoch": 0.9060205580029369, "fcm_dpo/beta": 0.0037694016937166452, "fcm_dpo/delta": 0.06455539166927338, "fcm_dpo/margin": 89.57487487792969, "fcm_dpo/q_t": 0.420167475938797, "grad_norm": 9.129219055175781, "learning_rate": 1.378797888467345e-08, "logits/chosen": -1.6300756931304932, "logits/rejected": -0.995729923248291, "logps/chosen": -280.83819580078125, "logps/ref_chosen": -91.09699249267578, "logps/ref_rejected": -70.41004943847656, "logps/rejected": -349.7261047363281, "loss": 1.124, "margin_dpo/margin_mean": 89.57487487792969, "margin_dpo/margin_std": 116.30337524414062, "step": 617 }, { "KL/chosen_KL_mean": -191.69598388671875, "KL/mean": -242.4030303955078, "KL/rejected_KL_mean": -293.110107421875, "KL/std": 115.41868591308594, "epoch": 0.9074889867841409, "fcm_dpo/beta": 0.0037880889140069485, "fcm_dpo/delta": 0.016459230333566666, "fcm_dpo/margin": 101.41413116455078, "fcm_dpo/q_t": 0.4089798927307129, "grad_norm": 7.9046854972839355, "learning_rate": 1.3370790793601371e-08, "logits/chosen": -2.1167428493499756, "logits/rejected": -1.760259985923767, "logps/chosen": -293.7165832519531, "logps/ref_chosen": -102.02059936523438, "logps/ref_rejected": -99.80119323730469, "logps/rejected": -392.9112854003906, "loss": 1.0951, "margin_dpo/margin_mean": 101.41413879394531, "margin_dpo/margin_std": 127.3936538696289, "step": 618 }, { "KL/chosen_KL_mean": -185.44357299804688, "KL/mean": -229.78912353515625, "KL/rejected_KL_mean": -274.1346435546875, "KL/std": 113.30085754394531, "epoch": 0.908957415565345, "fcm_dpo/beta": 0.0038301139138638973, "fcm_dpo/delta": 0.06243818253278732, "fcm_dpo/margin": 88.69110870361328, "fcm_dpo/q_t": 0.422113835811615, "grad_norm": 8.41944694519043, "learning_rate": 1.2959838102258535e-08, "logits/chosen": -1.458711862564087, "logits/rejected": -1.092092514038086, "logps/chosen": -275.1849365234375, "logps/ref_chosen": -89.74136352539062, "logps/ref_rejected": -99.90138244628906, "logps/rejected": -374.0360412597656, "loss": 1.1391, "margin_dpo/margin_mean": 88.69111633300781, "margin_dpo/margin_std": 136.2492218017578, "step": 619 }, { "KL/chosen_KL_mean": -176.85470581054688, "KL/mean": -226.03201293945312, "KL/rejected_KL_mean": -275.20928955078125, "KL/std": 118.94052124023438, "epoch": 0.9104258443465492, "fcm_dpo/beta": 0.003787559922784567, "fcm_dpo/delta": -0.07300166040658951, "fcm_dpo/margin": 98.35460662841797, "fcm_dpo/q_t": 0.4120763838291168, "grad_norm": 9.32966136932373, "learning_rate": 1.2555131639630567e-08, "logits/chosen": -2.3457906246185303, "logits/rejected": -1.8342537879943848, "logps/chosen": -261.97900390625, "logps/ref_chosen": -85.12431335449219, "logps/ref_rejected": -85.41253662109375, "logps/rejected": -360.621826171875, "loss": 1.11, "margin_dpo/margin_mean": 98.35459899902344, "margin_dpo/margin_std": 127.18399047851562, "step": 620 }, { "KL/chosen_KL_mean": -181.07896423339844, "KL/mean": -229.0821533203125, "KL/rejected_KL_mean": -277.0853271484375, "KL/std": 115.03120422363281, "epoch": 0.9118942731277533, "fcm_dpo/beta": 0.0037645839620381594, "fcm_dpo/delta": -0.061032526195049286, "fcm_dpo/margin": 96.00635528564453, "fcm_dpo/q_t": 0.4152405858039856, "grad_norm": 8.988188743591309, "learning_rate": 1.2156682070109086e-08, "logits/chosen": -1.9616132974624634, "logits/rejected": -1.6998029947280884, "logps/chosen": -270.327392578125, "logps/ref_chosen": -89.24842071533203, "logps/ref_rejected": -95.46463775634766, "logps/rejected": -372.5499572753906, "loss": 1.1253, "margin_dpo/margin_mean": 96.00636291503906, "margin_dpo/margin_std": 134.9449462890625, "step": 621 }, { "KL/chosen_KL_mean": -178.47454833984375, "KL/mean": -229.943115234375, "KL/rejected_KL_mean": -281.41168212890625, "KL/std": 102.43006896972656, "epoch": 0.9133627019089574, "fcm_dpo/beta": 0.003743324428796768, "fcm_dpo/delta": 0.01522154826670885, "fcm_dpo/margin": 102.93711853027344, "fcm_dpo/q_t": 0.4086582064628601, "grad_norm": 9.605335235595703, "learning_rate": 1.1764499893210878e-08, "logits/chosen": -1.3714749813079834, "logits/rejected": -0.7045179605484009, "logps/chosen": -278.2686767578125, "logps/ref_chosen": -99.79412841796875, "logps/ref_rejected": -90.82821655273438, "logps/rejected": -372.2398986816406, "loss": 1.0798, "margin_dpo/margin_mean": 102.93711853027344, "margin_dpo/margin_std": 111.34817504882812, "step": 622 }, { "KL/chosen_KL_mean": -185.93972778320312, "KL/mean": -222.5531005859375, "KL/rejected_KL_mean": -259.1664733886719, "KL/std": 111.14846801757812, "epoch": 0.9148311306901615, "fcm_dpo/beta": 0.0037793661467731, "fcm_dpo/delta": 0.034983083605766296, "fcm_dpo/margin": 73.22674560546875, "fcm_dpo/q_t": 0.43571192026138306, "grad_norm": 8.993992805480957, "learning_rate": 1.1378595443300998e-08, "logits/chosen": -2.2459628582000732, "logits/rejected": -1.8093518018722534, "logps/chosen": -276.395263671875, "logps/ref_chosen": -90.45555114746094, "logps/ref_rejected": -91.32276153564453, "logps/rejected": -350.4892272949219, "loss": 1.1926, "margin_dpo/margin_mean": 73.22674560546875, "margin_dpo/margin_std": 136.1776123046875, "step": 623 }, { "KL/chosen_KL_mean": -192.54164123535156, "KL/mean": -250.57403564453125, "KL/rejected_KL_mean": -308.6064453125, "KL/std": 110.08511352539062, "epoch": 0.9162995594713657, "fcm_dpo/beta": 0.0037574414163827896, "fcm_dpo/delta": -0.0377613827586174, "fcm_dpo/margin": 116.06480407714844, "fcm_dpo/q_t": 0.3977317810058594, "grad_norm": 9.43755054473877, "learning_rate": 1.0998978889320582e-08, "logits/chosen": -2.629471778869629, "logits/rejected": -1.7353299856185913, "logps/chosen": -302.4168701171875, "logps/ref_chosen": -109.87522888183594, "logps/ref_rejected": -104.77320861816406, "logps/rejected": -413.379638671875, "loss": 1.0505, "margin_dpo/margin_mean": 116.06480407714844, "margin_dpo/margin_std": 126.56492614746094, "step": 624 }, { "KL/chosen_KL_mean": -200.04615783691406, "KL/mean": -241.7044219970703, "KL/rejected_KL_mean": -283.3627014160156, "KL/std": 108.86798095703125, "epoch": 0.9177679882525698, "fcm_dpo/beta": 0.003760346444323659, "fcm_dpo/delta": 0.02503892406821251, "fcm_dpo/margin": 83.31654357910156, "fcm_dpo/q_t": 0.426126629114151, "grad_norm": 8.50216007232666, "learning_rate": 1.0625660234518913e-08, "logits/chosen": -1.551748514175415, "logits/rejected": -1.1699554920196533, "logps/chosen": -287.2143249511719, "logps/ref_chosen": -87.16815948486328, "logps/ref_rejected": -91.86148071289062, "logps/rejected": -375.22418212890625, "loss": 1.1435, "margin_dpo/margin_mean": 83.31653594970703, "margin_dpo/margin_std": 116.60951232910156, "step": 625 }, { "KL/chosen_KL_mean": -187.85755920410156, "KL/mean": -229.61895751953125, "KL/rejected_KL_mean": -271.38037109375, "KL/std": 120.573486328125, "epoch": 0.9192364170337739, "fcm_dpo/beta": 0.0037786171305924654, "fcm_dpo/delta": 0.02343526855111122, "fcm_dpo/margin": 83.5228042602539, "fcm_dpo/q_t": 0.42650681734085083, "grad_norm": 10.359896659851074, "learning_rate": 1.0258649316189721e-08, "logits/chosen": -1.428344488143921, "logits/rejected": -0.9952055215835571, "logps/chosen": -292.081787109375, "logps/ref_chosen": -104.22421264648438, "logps/ref_rejected": -104.1774673461914, "logps/rejected": -375.557861328125, "loss": 1.1545, "margin_dpo/margin_mean": 83.52279663085938, "margin_dpo/margin_std": 131.68812561035156, "step": 626 }, { "KL/chosen_KL_mean": -166.05178833007812, "KL/mean": -219.41494750976562, "KL/rejected_KL_mean": -272.778076171875, "KL/std": 123.51414489746094, "epoch": 0.920704845814978, "fcm_dpo/beta": 0.0037863804027438164, "fcm_dpo/delta": -0.0042802682146430016, "fcm_dpo/margin": 106.72628784179688, "fcm_dpo/q_t": 0.4067443609237671, "grad_norm": 6.791172981262207, "learning_rate": 9.897955805412e-09, "logits/chosen": -1.3849577903747559, "logits/rejected": -1.5382847785949707, "logps/chosen": -240.9864044189453, "logps/ref_chosen": -74.93461608886719, "logps/ref_rejected": -112.57289123535156, "logps/rejected": -385.3509521484375, "loss": 1.0843, "margin_dpo/margin_mean": 106.72628784179688, "margin_dpo/margin_std": 136.26800537109375, "step": 627 }, { "KL/chosen_KL_mean": -177.37405395507812, "KL/mean": -231.8850555419922, "KL/rejected_KL_mean": -286.39605712890625, "KL/std": 111.60227966308594, "epoch": 0.922173274596182, "fcm_dpo/beta": 0.0037779156118631363, "fcm_dpo/delta": -0.012384535744786263, "fcm_dpo/margin": 109.02200317382812, "fcm_dpo/q_t": 0.4034252166748047, "grad_norm": 9.962944030761719, "learning_rate": 9.543589206795238e-09, "logits/chosen": -2.058600425720215, "logits/rejected": -1.7986319065093994, "logps/chosen": -271.06512451171875, "logps/ref_chosen": -93.69107818603516, "logps/ref_rejected": -107.34395599365234, "logps/rejected": -393.739990234375, "loss": 1.0691, "margin_dpo/margin_mean": 109.02200317382812, "margin_dpo/margin_std": 124.29576873779297, "step": 628 }, { "KL/chosen_KL_mean": -190.33973693847656, "KL/mean": -239.97738647460938, "KL/rejected_KL_mean": -289.61505126953125, "KL/std": 112.42868041992188, "epoch": 0.9236417033773862, "fcm_dpo/beta": 0.0037791533395648003, "fcm_dpo/delta": 0.025711238384246826, "fcm_dpo/margin": 99.27531433105469, "fcm_dpo/q_t": 0.4111293852329254, "grad_norm": 9.445576667785645, "learning_rate": 9.19555885822887e-09, "logits/chosen": -2.0779647827148438, "logits/rejected": -1.646055817604065, "logps/chosen": -293.57012939453125, "logps/ref_chosen": -103.23037719726562, "logps/ref_rejected": -97.16841888427734, "logps/rejected": -386.78350830078125, "loss": 1.0958, "margin_dpo/margin_mean": 99.27531433105469, "margin_dpo/margin_std": 119.48643493652344, "step": 629 }, { "KL/chosen_KL_mean": -173.012451171875, "KL/mean": -204.70458984375, "KL/rejected_KL_mean": -236.39675903320312, "KL/std": 104.33016967773438, "epoch": 0.9251101321585903, "fcm_dpo/beta": 0.003814146388322115, "fcm_dpo/delta": 0.05195235460996628, "fcm_dpo/margin": 63.384307861328125, "fcm_dpo/q_t": 0.4427708685398102, "grad_norm": 10.90400505065918, "learning_rate": 8.85387393063622e-09, "logits/chosen": -2.097539186477661, "logits/rejected": -1.5802279710769653, "logps/chosen": -266.90997314453125, "logps/ref_chosen": -93.89755249023438, "logps/ref_rejected": -89.3743896484375, "logps/rejected": -325.7711486816406, "loss": 1.2088, "margin_dpo/margin_mean": 63.384307861328125, "margin_dpo/margin_std": 117.64893341064453, "step": 630 }, { "KL/chosen_KL_mean": -170.91452026367188, "KL/mean": -218.89349365234375, "KL/rejected_KL_mean": -266.8724670410156, "KL/std": 104.21376037597656, "epoch": 0.9265785609397944, "fcm_dpo/beta": 0.0038488118443638086, "fcm_dpo/delta": 0.03186452388763428, "fcm_dpo/margin": 95.95794677734375, "fcm_dpo/q_t": 0.412794828414917, "grad_norm": 8.763140678405762, "learning_rate": 8.518543427732949e-09, "logits/chosen": -2.3164591789245605, "logits/rejected": -1.8770625591278076, "logps/chosen": -258.6853332519531, "logps/ref_chosen": -87.77082061767578, "logps/ref_rejected": -88.68241882324219, "logps/rejected": -355.55487060546875, "loss": 1.1013, "margin_dpo/margin_mean": 95.95794677734375, "margin_dpo/margin_std": 119.26992797851562, "step": 631 }, { "KL/chosen_KL_mean": -166.23529052734375, "KL/mean": -218.92901611328125, "KL/rejected_KL_mean": -271.6227722167969, "KL/std": 109.13834381103516, "epoch": 0.9280469897209985, "fcm_dpo/beta": 0.003838640870526433, "fcm_dpo/delta": -0.00515325739979744, "fcm_dpo/margin": 105.38748931884766, "fcm_dpo/q_t": 0.4058246612548828, "grad_norm": 10.527085304260254, "learning_rate": 8.189576185789637e-09, "logits/chosen": -2.1889047622680664, "logits/rejected": -1.7760957479476929, "logps/chosen": -254.86181640625, "logps/ref_chosen": -88.62652587890625, "logps/ref_rejected": -91.45091247558594, "logps/rejected": -363.07366943359375, "loss": 1.0877, "margin_dpo/margin_mean": 105.38748931884766, "margin_dpo/margin_std": 135.35980224609375, "step": 632 }, { "KL/chosen_KL_mean": -208.9979705810547, "KL/mean": -239.45794677734375, "KL/rejected_KL_mean": -269.9179382324219, "KL/std": 110.80070495605469, "epoch": 0.9295154185022027, "fcm_dpo/beta": 0.0038688089698553085, "fcm_dpo/delta": 0.0369894914329052, "fcm_dpo/margin": 60.91996383666992, "fcm_dpo/q_t": 0.44631657004356384, "grad_norm": 9.852951049804688, "learning_rate": 7.866980873399015e-09, "logits/chosen": -2.2323689460754395, "logits/rejected": -2.044524669647217, "logps/chosen": -290.37237548828125, "logps/ref_chosen": -81.37442016601562, "logps/ref_rejected": -98.62571716308594, "logps/rejected": -368.5436706542969, "loss": 1.245, "margin_dpo/margin_mean": 60.91996383666992, "margin_dpo/margin_std": 149.2567138671875, "step": 633 }, { "KL/chosen_KL_mean": -192.2566680908203, "KL/mean": -232.26593017578125, "KL/rejected_KL_mean": -272.27520751953125, "KL/std": 100.35428619384766, "epoch": 0.9309838472834068, "fcm_dpo/beta": 0.0038733449764549732, "fcm_dpo/delta": -0.025234822183847427, "fcm_dpo/margin": 80.0185317993164, "fcm_dpo/q_t": 0.4262515902519226, "grad_norm": 10.22681713104248, "learning_rate": 7.550765991247654e-09, "logits/chosen": -1.4065275192260742, "logits/rejected": -1.2347018718719482, "logps/chosen": -288.3795166015625, "logps/ref_chosen": -96.12284851074219, "logps/ref_rejected": -112.84780883789062, "logps/rejected": -385.1230163574219, "loss": 1.1424, "margin_dpo/margin_mean": 80.0185317993164, "margin_dpo/margin_std": 102.03579711914062, "step": 634 }, { "KL/chosen_KL_mean": -200.042236328125, "KL/mean": -230.71713256835938, "KL/rejected_KL_mean": -261.39202880859375, "KL/std": 109.91458129882812, "epoch": 0.9324522760646109, "fcm_dpo/beta": 0.003879968076944351, "fcm_dpo/delta": 0.0422615110874176, "fcm_dpo/margin": 61.34981918334961, "fcm_dpo/q_t": 0.44473838806152344, "grad_norm": 11.883227348327637, "learning_rate": 7.240939871891699e-09, "logits/chosen": -1.7093758583068848, "logits/rejected": -1.1638599634170532, "logps/chosen": -298.7263488769531, "logps/ref_chosen": -98.68411254882812, "logps/ref_rejected": -89.89920043945312, "logps/rejected": -351.2912292480469, "loss": 1.2266, "margin_dpo/margin_mean": 61.34981918334961, "margin_dpo/margin_std": 132.36192321777344, "step": 635 }, { "KL/chosen_KL_mean": -178.34039306640625, "KL/mean": -231.24098205566406, "KL/rejected_KL_mean": -284.14154052734375, "KL/std": 112.56794738769531, "epoch": 0.933920704845815, "fcm_dpo/beta": 0.0038938471116125584, "fcm_dpo/delta": -0.012502969242632389, "fcm_dpo/margin": 105.8011703491211, "fcm_dpo/q_t": 0.40304362773895264, "grad_norm": 8.933439254760742, "learning_rate": 6.937510679537628e-09, "logits/chosen": -1.5890717506408691, "logits/rejected": -1.1134685277938843, "logps/chosen": -268.75836181640625, "logps/ref_chosen": -90.41796112060547, "logps/ref_rejected": -87.70687866210938, "logps/rejected": -371.84844970703125, "loss": 1.0618, "margin_dpo/margin_mean": 105.8011703491211, "margin_dpo/margin_std": 112.42809295654297, "step": 636 }, { "KL/chosen_KL_mean": -179.61044311523438, "KL/mean": -236.06961059570312, "KL/rejected_KL_mean": -292.52880859375, "KL/std": 124.97705078125, "epoch": 0.9353891336270191, "fcm_dpo/beta": 0.003869942855089903, "fcm_dpo/delta": -0.03866468742489815, "fcm_dpo/margin": 112.91836547851562, "fcm_dpo/q_t": 0.39703550934791565, "grad_norm": 8.080221176147461, "learning_rate": 6.640486409826785e-09, "logits/chosen": -1.9098118543624878, "logits/rejected": -1.8062106370925903, "logps/chosen": -262.0601501464844, "logps/ref_chosen": -82.44971466064453, "logps/ref_rejected": -104.02860260009766, "logps/rejected": -396.557373046875, "loss": 1.0472, "margin_dpo/margin_mean": 112.91836547851562, "margin_dpo/margin_std": 118.51007080078125, "step": 637 }, { "KL/chosen_KL_mean": -168.32565307617188, "KL/mean": -223.0592041015625, "KL/rejected_KL_mean": -277.79278564453125, "KL/std": 111.24354553222656, "epoch": 0.9368575624082232, "fcm_dpo/beta": 0.0038613637443631887, "fcm_dpo/delta": -0.024044867604970932, "fcm_dpo/margin": 109.46715545654297, "fcm_dpo/q_t": 0.3997325301170349, "grad_norm": 8.261194229125977, "learning_rate": 6.349874889624962e-09, "logits/chosen": -1.5519225597381592, "logits/rejected": -0.8704928159713745, "logps/chosen": -260.2506103515625, "logps/ref_chosen": -91.92498779296875, "logps/ref_rejected": -86.28703308105469, "logps/rejected": -364.079833984375, "loss": 1.0489, "margin_dpo/margin_mean": 109.46715545654297, "margin_dpo/margin_std": 103.84730529785156, "step": 638 }, { "KL/chosen_KL_mean": -223.8411865234375, "KL/mean": -249.5852508544922, "KL/rejected_KL_mean": -275.329345703125, "KL/std": 102.98002624511719, "epoch": 0.9383259911894273, "fcm_dpo/beta": 0.0038381926715373993, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 51.488128662109375, "fcm_dpo/q_t": 0.452871710062027, "grad_norm": 9.00898551940918, "learning_rate": 6.065683776815933e-09, "logits/chosen": -1.3313965797424316, "logits/rejected": -0.5628180503845215, "logps/chosen": -328.3687438964844, "logps/ref_chosen": -104.52755737304688, "logps/ref_rejected": -81.4803466796875, "logps/rejected": -356.8096923828125, "loss": 1.2578, "margin_dpo/margin_mean": 51.48813247680664, "margin_dpo/margin_std": 128.21505737304688, "step": 639 }, { "KL/chosen_KL_mean": -190.1381072998047, "KL/mean": -239.64845275878906, "KL/rejected_KL_mean": -289.1588134765625, "KL/std": 111.77400207519531, "epoch": 0.9397944199706314, "fcm_dpo/beta": 0.0038121978286653757, "fcm_dpo/delta": -0.06818970292806625, "fcm_dpo/margin": 99.02069854736328, "fcm_dpo/q_t": 0.41030749678611755, "grad_norm": 11.37141227722168, "learning_rate": 5.7879205600998296e-09, "logits/chosen": -1.7587177753448486, "logits/rejected": -1.2360551357269287, "logps/chosen": -288.02337646484375, "logps/ref_chosen": -97.88526916503906, "logps/ref_rejected": -112.70502471923828, "logps/rejected": -401.86383056640625, "loss": 1.0948, "margin_dpo/margin_mean": 99.02069854736328, "margin_dpo/margin_std": 112.4079360961914, "step": 640 }, { "KL/chosen_KL_mean": -173.28643798828125, "KL/mean": -224.64605712890625, "KL/rejected_KL_mean": -276.00567626953125, "KL/std": 129.56117248535156, "epoch": 0.9412628487518355, "fcm_dpo/beta": 0.003794408403337002, "fcm_dpo/delta": 0.010643563233315945, "fcm_dpo/margin": 102.71925354003906, "fcm_dpo/q_t": 0.41164323687553406, "grad_norm": 8.372820854187012, "learning_rate": 5.516592558795746e-09, "logits/chosen": -2.2098498344421387, "logits/rejected": -1.6607255935668945, "logps/chosen": -269.73211669921875, "logps/ref_chosen": -96.4456787109375, "logps/ref_rejected": -95.13568878173828, "logps/rejected": -371.141357421875, "loss": 1.1016, "margin_dpo/margin_mean": 102.71925354003906, "margin_dpo/margin_std": 143.9680938720703, "step": 641 }, { "KL/chosen_KL_mean": -192.73406982421875, "KL/mean": -241.2641143798828, "KL/rejected_KL_mean": -289.7941589355469, "KL/std": 111.4757080078125, "epoch": 0.9427312775330396, "fcm_dpo/beta": 0.003820315934717655, "fcm_dpo/delta": 0.030141085386276245, "fcm_dpo/margin": 97.06008911132812, "fcm_dpo/q_t": 0.41512829065322876, "grad_norm": 10.769035339355469, "learning_rate": 5.251706922648868e-09, "logits/chosen": -1.6352812051773071, "logits/rejected": -1.3077454566955566, "logps/chosen": -293.4939270019531, "logps/ref_chosen": -100.75984954833984, "logps/ref_rejected": -114.7076416015625, "logps/rejected": -404.5018005371094, "loss": 1.1146, "margin_dpo/margin_mean": 97.06008911132812, "margin_dpo/margin_std": 138.93240356445312, "step": 642 }, { "KL/chosen_KL_mean": -180.5733642578125, "KL/mean": -226.9500732421875, "KL/rejected_KL_mean": -273.32684326171875, "KL/std": 103.77207946777344, "epoch": 0.9441997063142438, "fcm_dpo/beta": 0.0037874109111726284, "fcm_dpo/delta": -0.03918962553143501, "fcm_dpo/margin": 92.75345611572266, "fcm_dpo/q_t": 0.4163810908794403, "grad_norm": 11.389684677124023, "learning_rate": 4.993270631642038e-09, "logits/chosen": -2.1419332027435303, "logits/rejected": -1.7664439678192139, "logps/chosen": -265.3170166015625, "logps/ref_chosen": -84.74365997314453, "logps/ref_rejected": -94.31842041015625, "logps/rejected": -367.645263671875, "loss": 1.1186, "margin_dpo/margin_mean": 92.75345611572266, "margin_dpo/margin_std": 119.85426330566406, "step": 643 }, { "KL/chosen_KL_mean": -186.76336669921875, "KL/mean": -225.10137939453125, "KL/rejected_KL_mean": -263.43939208984375, "KL/std": 104.70063781738281, "epoch": 0.9456681350954479, "fcm_dpo/beta": 0.0037874109111726284, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 76.67604064941406, "fcm_dpo/q_t": 0.4309181869029999, "grad_norm": 10.11521053314209, "learning_rate": 4.741290495811873e-09, "logits/chosen": -1.7592540979385376, "logits/rejected": -1.4889845848083496, "logps/chosen": -272.08612060546875, "logps/ref_chosen": -85.32274627685547, "logps/ref_rejected": -94.60861206054688, "logps/rejected": -358.04803466796875, "loss": 1.1615, "margin_dpo/margin_mean": 76.67604064941406, "margin_dpo/margin_std": 113.71969604492188, "step": 644 }, { "KL/chosen_KL_mean": -199.71237182617188, "KL/mean": -224.39837646484375, "KL/rejected_KL_mean": -249.08436584472656, "KL/std": 113.5405502319336, "epoch": 0.947136563876652, "fcm_dpo/beta": 0.0037874109111726284, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 49.371978759765625, "fcm_dpo/q_t": 0.4559004604816437, "grad_norm": 9.404606819152832, "learning_rate": 4.495773155069299e-09, "logits/chosen": -1.4555082321166992, "logits/rejected": -1.2892038822174072, "logps/chosen": -282.3026123046875, "logps/ref_chosen": -82.59024047851562, "logps/ref_rejected": -103.00375366210938, "logps/rejected": -352.088134765625, "loss": 1.2669, "margin_dpo/margin_mean": 49.371978759765625, "margin_dpo/margin_std": 129.0969696044922, "step": 645 }, { "KL/chosen_KL_mean": -175.08935546875, "KL/mean": -224.29525756835938, "KL/rejected_KL_mean": -273.50115966796875, "KL/std": 99.33821868896484, "epoch": 0.9486049926578561, "fcm_dpo/beta": 0.0037891233805567026, "fcm_dpo/delta": 0.027975942939519882, "fcm_dpo/margin": 98.41181945800781, "fcm_dpo/q_t": 0.41160231828689575, "grad_norm": 8.314691543579102, "learning_rate": 4.256725079024553e-09, "logits/chosen": -1.9570796489715576, "logits/rejected": -1.3679468631744385, "logps/chosen": -266.2813720703125, "logps/ref_chosen": -91.1920394897461, "logps/ref_rejected": -83.77833557128906, "logps/rejected": -357.27947998046875, "loss": 1.0891, "margin_dpo/margin_mean": 98.41181945800781, "margin_dpo/margin_std": 107.44400024414062, "step": 646 }, { "KL/chosen_KL_mean": -190.22341918945312, "KL/mean": -233.48648071289062, "KL/rejected_KL_mean": -276.74957275390625, "KL/std": 119.07595825195312, "epoch": 0.9500734214390602, "fcm_dpo/beta": 0.0038478460628539324, "fcm_dpo/delta": 0.06937824934720993, "fcm_dpo/margin": 86.52615356445312, "fcm_dpo/q_t": 0.42297041416168213, "grad_norm": 9.62779426574707, "learning_rate": 4.024152566816791e-09, "logits/chosen": -1.6432267427444458, "logits/rejected": -1.5103105306625366, "logps/chosen": -279.06787109375, "logps/ref_chosen": -88.8444595336914, "logps/ref_rejected": -99.49833679199219, "logps/rejected": -376.2478942871094, "loss": 1.1417, "margin_dpo/margin_mean": 86.52615356445312, "margin_dpo/margin_std": 131.43270874023438, "step": 647 }, { "KL/chosen_KL_mean": -176.59988403320312, "KL/mean": -236.47048950195312, "KL/rejected_KL_mean": -296.3410949707031, "KL/std": 122.33714294433594, "epoch": 0.9515418502202643, "fcm_dpo/beta": 0.0038349798414856195, "fcm_dpo/delta": -0.062018170952796936, "fcm_dpo/margin": 119.74122619628906, "fcm_dpo/q_t": 0.3923817574977875, "grad_norm": 9.694408416748047, "learning_rate": 3.798061746947995e-09, "logits/chosen": -1.862655520439148, "logits/rejected": -1.790158748626709, "logps/chosen": -264.447998046875, "logps/ref_chosen": -87.84810638427734, "logps/ref_rejected": -104.67005920410156, "logps/rejected": -401.01116943359375, "loss": 1.0378, "margin_dpo/margin_mean": 119.74121856689453, "margin_dpo/margin_std": 129.68258666992188, "step": 648 }, { "KL/chosen_KL_mean": -162.93673706054688, "KL/mean": -212.46551513671875, "KL/rejected_KL_mean": -261.9942932128906, "KL/std": 106.67803192138672, "epoch": 0.9530102790014684, "fcm_dpo/beta": 0.0037901827599853277, "fcm_dpo/delta": -0.031678035855293274, "fcm_dpo/margin": 99.05757141113281, "fcm_dpo/q_t": 0.410619854927063, "grad_norm": 10.537824630737305, "learning_rate": 3.5784585771215235e-09, "logits/chosen": -2.418762683868408, "logits/rejected": -2.0164315700531006, "logps/chosen": -252.62924194335938, "logps/ref_chosen": -89.6925048828125, "logps/ref_rejected": -88.70658111572266, "logps/rejected": -350.70086669921875, "loss": 1.0856, "margin_dpo/margin_mean": 99.05757141113281, "margin_dpo/margin_std": 104.78544616699219, "step": 649 }, { "KL/chosen_KL_mean": -192.8601531982422, "KL/mean": -239.61412048339844, "KL/rejected_KL_mean": -286.36810302734375, "KL/std": 115.449462890625, "epoch": 0.9544787077826725, "fcm_dpo/beta": 0.0038184935692697763, "fcm_dpo/delta": 0.04448123648762703, "fcm_dpo/margin": 93.5079345703125, "fcm_dpo/q_t": 0.4167550206184387, "grad_norm": 9.346925735473633, "learning_rate": 3.3653488440851253e-09, "logits/chosen": -1.2074940204620361, "logits/rejected": -1.006216287612915, "logps/chosen": -282.790771484375, "logps/ref_chosen": -89.93060302734375, "logps/ref_rejected": -102.61282348632812, "logps/rejected": -388.98089599609375, "loss": 1.1248, "margin_dpo/margin_mean": 93.5079345703125, "margin_dpo/margin_std": 136.7960205078125, "step": 650 }, { "KL/chosen_KL_mean": -159.58489990234375, "KL/mean": -217.26043701171875, "KL/rejected_KL_mean": -274.93597412109375, "KL/std": 111.26205444335938, "epoch": 0.9559471365638766, "fcm_dpo/beta": 0.0037994948215782642, "fcm_dpo/delta": -0.0400543138384819, "fcm_dpo/margin": 115.35110473632812, "fcm_dpo/q_t": 0.3957407474517822, "grad_norm": 8.717374801635742, "learning_rate": 3.158738163478475e-09, "logits/chosen": -2.1445353031158447, "logits/rejected": -1.9700628519058228, "logps/chosen": -238.772216796875, "logps/ref_chosen": -79.18731689453125, "logps/ref_rejected": -105.93333435058594, "logps/rejected": -380.86932373046875, "loss": 1.0371, "margin_dpo/margin_mean": 115.35110473632812, "margin_dpo/margin_std": 109.70521545410156, "step": 651 }, { "KL/chosen_KL_mean": -167.8321533203125, "KL/mean": -215.88595581054688, "KL/rejected_KL_mean": -263.9397277832031, "KL/std": 103.7525634765625, "epoch": 0.9574155653450808, "fcm_dpo/beta": 0.0038213147781789303, "fcm_dpo/delta": 0.03381127491593361, "fcm_dpo/margin": 96.10755157470703, "fcm_dpo/q_t": 0.41279900074005127, "grad_norm": 8.846878051757812, "learning_rate": 2.9586319796851555e-09, "logits/chosen": -2.650205135345459, "logits/rejected": -2.3624677658081055, "logps/chosen": -269.62237548828125, "logps/ref_chosen": -101.79022216796875, "logps/ref_rejected": -116.3245849609375, "logps/rejected": -380.2643127441406, "loss": 1.0867, "margin_dpo/margin_mean": 96.10755157470703, "margin_dpo/margin_std": 98.927978515625, "step": 652 }, { "KL/chosen_KL_mean": -191.9810791015625, "KL/mean": -238.82293701171875, "KL/rejected_KL_mean": -285.664794921875, "KL/std": 113.55856323242188, "epoch": 0.9588839941262849, "fcm_dpo/beta": 0.0038483156822621822, "fcm_dpo/delta": 0.04084649309515953, "fcm_dpo/margin": 93.68368530273438, "fcm_dpo/q_t": 0.413957417011261, "grad_norm": 9.661733627319336, "learning_rate": 2.7650355656892166e-09, "logits/chosen": -2.088801383972168, "logits/rejected": -1.8333606719970703, "logps/chosen": -285.33465576171875, "logps/ref_chosen": -93.35359191894531, "logps/ref_rejected": -109.12324523925781, "logps/rejected": -394.78802490234375, "loss": 1.1029, "margin_dpo/margin_mean": 93.68368530273438, "margin_dpo/margin_std": 111.59762573242188, "step": 653 }, { "KL/chosen_KL_mean": -192.82717895507812, "KL/mean": -233.19229125976562, "KL/rejected_KL_mean": -273.55743408203125, "KL/std": 108.76021575927734, "epoch": 0.960352422907489, "fcm_dpo/beta": 0.0038835632149130106, "fcm_dpo/delta": 0.0426095575094223, "fcm_dpo/margin": 80.73028564453125, "fcm_dpo/q_t": 0.42822974920272827, "grad_norm": 11.199869155883789, "learning_rate": 2.577954022936174e-09, "logits/chosen": -2.3049092292785645, "logits/rejected": -2.1248888969421387, "logps/chosen": -281.94268798828125, "logps/ref_chosen": -89.11553955078125, "logps/ref_rejected": -104.91995239257812, "logps/rejected": -378.4773864746094, "loss": 1.164, "margin_dpo/margin_mean": 80.73028564453125, "margin_dpo/margin_std": 137.359375, "step": 654 }, { "KL/chosen_KL_mean": -175.066162109375, "KL/mean": -223.59652709960938, "KL/rejected_KL_mean": -272.12689208984375, "KL/std": 118.29328155517578, "epoch": 0.9618208516886931, "fcm_dpo/beta": 0.0039000471588224173, "fcm_dpo/delta": 0.022265257313847542, "fcm_dpo/margin": 97.06072998046875, "fcm_dpo/q_t": 0.4109921455383301, "grad_norm": 8.485185623168945, "learning_rate": 2.397392281198729e-09, "logits/chosen": -2.1425085067749023, "logits/rejected": -2.1433873176574707, "logps/chosen": -256.1022644042969, "logps/ref_chosen": -81.03610229492188, "logps/ref_rejected": -102.80233764648438, "logps/rejected": -374.9292297363281, "loss": 1.1021, "margin_dpo/margin_mean": 97.06072998046875, "margin_dpo/margin_std": 126.66535186767578, "step": 655 }, { "KL/chosen_KL_mean": -177.36502075195312, "KL/mean": -239.14007568359375, "KL/rejected_KL_mean": -300.9151306152344, "KL/std": 118.62073516845703, "epoch": 0.9632892804698973, "fcm_dpo/beta": 0.0038512481842190027, "fcm_dpo/delta": -0.07970499992370605, "fcm_dpo/margin": 123.55011749267578, "fcm_dpo/q_t": 0.38876470923423767, "grad_norm": 9.504347801208496, "learning_rate": 2.223355098446622e-09, "logits/chosen": -1.5670292377471924, "logits/rejected": -1.635893702507019, "logps/chosen": -262.69036865234375, "logps/ref_chosen": -85.32534790039062, "logps/ref_rejected": -118.33866119384766, "logps/rejected": -419.2537841796875, "loss": 1.0174, "margin_dpo/margin_mean": 123.55012512207031, "margin_dpo/margin_std": 122.35933685302734, "step": 656 }, { "KL/chosen_KL_mean": -194.00511169433594, "KL/mean": -239.27474975585938, "KL/rejected_KL_mean": -284.54437255859375, "KL/std": 109.36349487304688, "epoch": 0.9647577092511013, "fcm_dpo/beta": 0.003825712949037552, "fcm_dpo/delta": -0.017601385712623596, "fcm_dpo/margin": 90.53927612304688, "fcm_dpo/q_t": 0.4194551706314087, "grad_norm": 10.166887283325195, "learning_rate": 2.055847060721566e-09, "logits/chosen": -1.9024608135223389, "logits/rejected": -1.7356362342834473, "logps/chosen": -274.20281982421875, "logps/ref_chosen": -80.19772338867188, "logps/ref_rejected": -102.581298828125, "logps/rejected": -387.12567138671875, "loss": 1.1321, "margin_dpo/margin_mean": 90.53927612304688, "margin_dpo/margin_std": 131.91270446777344, "step": 657 }, { "KL/chosen_KL_mean": -189.87538146972656, "KL/mean": -232.2349395751953, "KL/rejected_KL_mean": -274.5945129394531, "KL/std": 103.32640075683594, "epoch": 0.9662261380323054, "fcm_dpo/beta": 0.0038302855100482702, "fcm_dpo/delta": 0.011937890201807022, "fcm_dpo/margin": 84.71913146972656, "fcm_dpo/q_t": 0.42307817935943604, "grad_norm": 8.205497741699219, "learning_rate": 1.8948725820160662e-09, "logits/chosen": -2.1717429161071777, "logits/rejected": -1.7159290313720703, "logps/chosen": -284.5098876953125, "logps/ref_chosen": -94.634521484375, "logps/ref_rejected": -101.6316146850586, "logps/rejected": -376.22613525390625, "loss": 1.1324, "margin_dpo/margin_mean": 84.71913146972656, "margin_dpo/margin_std": 110.98983001708984, "step": 658 }, { "KL/chosen_KL_mean": -172.4925537109375, "KL/mean": -223.4534912109375, "KL/rejected_KL_mean": -274.4144287109375, "KL/std": 111.18952941894531, "epoch": 0.9676945668135095, "fcm_dpo/beta": 0.0038269883953034878, "fcm_dpo/delta": 0.010125715285539627, "fcm_dpo/margin": 101.92189025878906, "fcm_dpo/q_t": 0.40981197357177734, "grad_norm": 7.7141618728637695, "learning_rate": 1.7404359041573723e-09, "logits/chosen": -2.16481876373291, "logits/rejected": -1.2066476345062256, "logps/chosen": -285.0484313964844, "logps/ref_chosen": -112.55587005615234, "logps/ref_rejected": -93.9216079711914, "logps/rejected": -368.3360595703125, "loss": 1.1091, "margin_dpo/margin_mean": 101.921875, "margin_dpo/margin_std": 144.13241577148438, "step": 659 }, { "KL/chosen_KL_mean": -180.93466186523438, "KL/mean": -233.36708068847656, "KL/rejected_KL_mean": -285.79949951171875, "KL/std": 108.1478271484375, "epoch": 0.9691629955947136, "fcm_dpo/beta": 0.00384066766127944, "fcm_dpo/delta": -0.0028668222948908806, "fcm_dpo/margin": 104.86483764648438, "fcm_dpo/q_t": 0.4054628312587738, "grad_norm": 11.359251022338867, "learning_rate": 1.592541096695571e-09, "logits/chosen": -1.7729545831680298, "logits/rejected": -1.140892505645752, "logps/chosen": -274.31207275390625, "logps/ref_chosen": -93.37742614746094, "logps/ref_rejected": -81.39482116699219, "logps/rejected": -367.1943054199219, "loss": 1.0727, "margin_dpo/margin_mean": 104.86483764648438, "margin_dpo/margin_std": 119.114501953125, "step": 660 }, { "KL/chosen_KL_mean": -179.06536865234375, "KL/mean": -238.27749633789062, "KL/rejected_KL_mean": -297.4896240234375, "KL/std": 121.45243835449219, "epoch": 0.9706314243759178, "fcm_dpo/beta": 0.003816891461610794, "fcm_dpo/delta": -0.05444643646478653, "fcm_dpo/margin": 118.42427062988281, "fcm_dpo/q_t": 0.3950696587562561, "grad_norm": 9.491766929626465, "learning_rate": 1.4511920567963908e-09, "logits/chosen": -1.7265509366989136, "logits/rejected": -1.2062363624572754, "logps/chosen": -266.9205322265625, "logps/ref_chosen": -87.85516357421875, "logps/ref_rejected": -92.40330505371094, "logps/rejected": -389.8929443359375, "loss": 1.0481, "margin_dpo/margin_mean": 118.42427062988281, "margin_dpo/margin_std": 134.95252990722656, "step": 661 }, { "KL/chosen_KL_mean": -174.8328857421875, "KL/mean": -229.270751953125, "KL/rejected_KL_mean": -283.7086181640625, "KL/std": 105.77983093261719, "epoch": 0.9720998531571219, "fcm_dpo/beta": 0.003797510638833046, "fcm_dpo/delta": -0.014074645936489105, "fcm_dpo/margin": 108.87571716308594, "fcm_dpo/q_t": 0.402312695980072, "grad_norm": 9.805207252502441, "learning_rate": 1.3163925091384532e-09, "logits/chosen": -1.816255807876587, "logits/rejected": -1.2520594596862793, "logps/chosen": -277.6126708984375, "logps/ref_chosen": -102.77980041503906, "logps/ref_rejected": -95.22531127929688, "logps/rejected": -378.9339294433594, "loss": 1.059, "margin_dpo/margin_mean": 108.87571716308594, "margin_dpo/margin_std": 111.89689636230469, "step": 662 }, { "KL/chosen_KL_mean": -176.68136596679688, "KL/mean": -225.45603942871094, "KL/rejected_KL_mean": -274.230712890625, "KL/std": 109.37639617919922, "epoch": 0.973568281938326, "fcm_dpo/beta": 0.00381136080250144, "fcm_dpo/delta": 0.029179099947214127, "fcm_dpo/margin": 97.54936218261719, "fcm_dpo/q_t": 0.41302502155303955, "grad_norm": 11.187478065490723, "learning_rate": 1.1881460058152382e-09, "logits/chosen": -2.202484607696533, "logits/rejected": -1.9862010478973389, "logps/chosen": -273.0279541015625, "logps/ref_chosen": -96.34658813476562, "logps/ref_rejected": -120.52645874023438, "logps/rejected": -394.7571716308594, "loss": 1.1053, "margin_dpo/margin_mean": 97.54936218261719, "margin_dpo/margin_std": 127.24519348144531, "step": 663 }, { "KL/chosen_KL_mean": -196.04647827148438, "KL/mean": -240.283203125, "KL/rejected_KL_mean": -284.5198974609375, "KL/std": 112.3060302734375, "epoch": 0.9750367107195301, "fcm_dpo/beta": 0.003852805122733116, "fcm_dpo/delta": 0.06101434677839279, "fcm_dpo/margin": 88.47343444824219, "fcm_dpo/q_t": 0.4196123480796814, "grad_norm": 8.336725234985352, "learning_rate": 1.066455926241383e-09, "logits/chosen": -1.7888169288635254, "logits/rejected": -1.6194840669631958, "logps/chosen": -287.888916015625, "logps/ref_chosen": -91.84242248535156, "logps/ref_rejected": -111.8366928100586, "logps/rejected": -396.35662841796875, "loss": 1.1192, "margin_dpo/margin_mean": 88.47342681884766, "margin_dpo/margin_std": 110.52880859375, "step": 664 }, { "KL/chosen_KL_mean": -167.45315551757812, "KL/mean": -216.00076293945312, "KL/rejected_KL_mean": -264.54840087890625, "KL/std": 108.16895294189453, "epoch": 0.9765051395007343, "fcm_dpo/beta": 0.0038790619000792503, "fcm_dpo/delta": 0.024137815460562706, "fcm_dpo/margin": 97.09523010253906, "fcm_dpo/q_t": 0.41057807207107544, "grad_norm": 7.814632892608643, "learning_rate": 9.513254770636137e-10, "logits/chosen": -2.469727039337158, "logits/rejected": -2.0466561317443848, "logps/chosen": -255.6393280029297, "logps/ref_chosen": -88.18618774414062, "logps/ref_rejected": -91.9120101928711, "logps/rejected": -356.46038818359375, "loss": 1.0899, "margin_dpo/margin_mean": 97.09523010253906, "margin_dpo/margin_std": 111.64039611816406, "step": 665 }, { "KL/chosen_KL_mean": -183.67776489257812, "KL/mean": -233.32681274414062, "KL/rejected_KL_mean": -282.9758605957031, "KL/std": 106.06681823730469, "epoch": 0.9779735682819384, "fcm_dpo/beta": 0.003875638358294964, "fcm_dpo/delta": 0.015685006976127625, "fcm_dpo/margin": 99.298095703125, "fcm_dpo/q_t": 0.41040295362472534, "grad_norm": 10.720526695251465, "learning_rate": 8.427576920763956e-10, "logits/chosen": -1.6271196603775024, "logits/rejected": -1.2489068508148193, "logps/chosen": -284.65234375, "logps/ref_chosen": -100.97459411621094, "logps/ref_rejected": -101.24992370605469, "logps/rejected": -384.22576904296875, "loss": 1.0965, "margin_dpo/margin_mean": 99.29811096191406, "margin_dpo/margin_std": 126.03630065917969, "step": 666 }, { "KL/chosen_KL_mean": -202.2061767578125, "KL/mean": -252.56402587890625, "KL/rejected_KL_mean": -302.921875, "KL/std": 110.61636352539062, "epoch": 0.9794419970631424, "fcm_dpo/beta": 0.0038884032983332872, "fcm_dpo/delta": 0.008699965663254261, "fcm_dpo/margin": 100.71569061279297, "fcm_dpo/q_t": 0.40710946917533875, "grad_norm": 8.844096183776855, "learning_rate": 7.407554321417764e-10, "logits/chosen": -1.4849872589111328, "logits/rejected": -0.8287320733070374, "logps/chosen": -299.77734375, "logps/ref_chosen": -97.5711669921875, "logps/ref_rejected": -93.58476257324219, "logps/rejected": -396.50665283203125, "loss": 1.0742, "margin_dpo/margin_mean": 100.71569061279297, "margin_dpo/margin_std": 107.05015563964844, "step": 667 }, { "KL/chosen_KL_mean": -177.9842529296875, "KL/mean": -237.2086181640625, "KL/rejected_KL_mean": -296.4330139160156, "KL/std": 118.19255828857422, "epoch": 0.9809104258443465, "fcm_dpo/beta": 0.003863723948597908, "fcm_dpo/delta": -0.06037880480289459, "fcm_dpo/margin": 118.44877624511719, "fcm_dpo/q_t": 0.3940025866031647, "grad_norm": 9.988701820373535, "learning_rate": 6.453213851142225e-10, "logits/chosen": -2.1105642318725586, "logits/rejected": -1.574021577835083, "logps/chosen": -280.559326171875, "logps/ref_chosen": -102.5750503540039, "logps/ref_rejected": -108.81768798828125, "logps/rejected": -405.2507019042969, "loss": 1.0379, "margin_dpo/margin_mean": 118.44877624511719, "margin_dpo/margin_std": 129.19381713867188, "step": 668 }, { "KL/chosen_KL_mean": -162.16372680664062, "KL/mean": -218.8978729248047, "KL/rejected_KL_mean": -275.63201904296875, "KL/std": 114.9798583984375, "epoch": 0.9823788546255506, "fcm_dpo/beta": 0.003828328335657716, "fcm_dpo/delta": -0.03594854846596718, "fcm_dpo/margin": 113.46829223632812, "fcm_dpo/q_t": 0.39817070960998535, "grad_norm": 9.285100936889648, "learning_rate": 5.564580657695939e-10, "logits/chosen": -1.6900508403778076, "logits/rejected": -1.2936570644378662, "logps/chosen": -251.65850830078125, "logps/ref_chosen": -89.49478149414062, "logps/ref_rejected": -82.51950073242188, "logps/rejected": -358.1515197753906, "loss": 1.0509, "margin_dpo/margin_mean": 113.46829223632812, "margin_dpo/margin_std": 123.40389251708984, "step": 669 }, { "KL/chosen_KL_mean": -162.95864868164062, "KL/mean": -225.34881591796875, "KL/rejected_KL_mean": -287.739013671875, "KL/std": 117.26303100585938, "epoch": 0.9838472834067548, "fcm_dpo/beta": 0.003783291671425104, "fcm_dpo/delta": -0.07558497786521912, "fcm_dpo/margin": 124.78034973144531, "fcm_dpo/q_t": 0.38797491788864136, "grad_norm": 7.269352436065674, "learning_rate": 4.741678157389739e-10, "logits/chosen": -1.7142994403839111, "logits/rejected": -1.174939513206482, "logps/chosen": -258.4132385253906, "logps/ref_chosen": -95.45459747314453, "logps/ref_rejected": -101.53292846679688, "logps/rejected": -389.2719421386719, "loss": 1.0206, "margin_dpo/margin_mean": 124.78034973144531, "margin_dpo/margin_std": 122.63932800292969, "step": 670 }, { "KL/chosen_KL_mean": -198.68067932128906, "KL/mean": -241.49562072753906, "KL/rejected_KL_mean": -284.310546875, "KL/std": 108.82093048095703, "epoch": 0.9853157121879589, "fcm_dpo/beta": 0.003759223036468029, "fcm_dpo/delta": -0.010290359146893024, "fcm_dpo/margin": 85.6298828125, "fcm_dpo/q_t": 0.4244227707386017, "grad_norm": 8.976449012756348, "learning_rate": 3.9845280344705245e-10, "logits/chosen": -1.5819687843322754, "logits/rejected": -1.3541139364242554, "logps/chosen": -280.8038024902344, "logps/ref_chosen": -82.12312316894531, "logps/ref_rejected": -90.21969604492188, "logps/rejected": -374.5302734375, "loss": 1.1417, "margin_dpo/margin_mean": 85.6298828125, "margin_dpo/margin_std": 120.84451293945312, "step": 671 }, { "KL/chosen_KL_mean": -198.71026611328125, "KL/mean": -234.968017578125, "KL/rejected_KL_mean": -271.2257995605469, "KL/std": 104.13059997558594, "epoch": 0.986784140969163, "fcm_dpo/beta": 0.003793198149651289, "fcm_dpo/delta": 0.05013387277722359, "fcm_dpo/margin": 72.51554870605469, "fcm_dpo/q_t": 0.43499308824539185, "grad_norm": 12.089276313781738, "learning_rate": 3.293150240547549e-10, "logits/chosen": -2.1074886322021484, "logits/rejected": -1.7845215797424316, "logps/chosen": -288.77215576171875, "logps/ref_chosen": -90.0619125366211, "logps/ref_rejected": -100.45323181152344, "logps/rejected": -371.67901611328125, "loss": 1.1744, "margin_dpo/margin_mean": 72.51554107666016, "margin_dpo/margin_std": 109.86177062988281, "step": 672 }, { "KL/chosen_KL_mean": -177.21229553222656, "KL/mean": -218.13320922851562, "KL/rejected_KL_mean": -259.0541076660156, "KL/std": 107.51701354980469, "epoch": 0.9882525697503671, "fcm_dpo/beta": 0.0038026715628802776, "fcm_dpo/delta": 0.024912934750318527, "fcm_dpo/margin": 81.84182739257812, "fcm_dpo/q_t": 0.4263882637023926, "grad_norm": 10.61828327178955, "learning_rate": 2.6675629940689504e-10, "logits/chosen": -1.853633999824524, "logits/rejected": -1.4833993911743164, "logps/chosen": -256.4754333496094, "logps/ref_chosen": -79.26315307617188, "logps/ref_rejected": -91.34925079345703, "logps/rejected": -350.40338134765625, "loss": 1.1409, "margin_dpo/margin_mean": 81.84182739257812, "margin_dpo/margin_std": 109.56309509277344, "step": 673 }, { "KL/chosen_KL_mean": -166.49276733398438, "KL/mean": -225.67752075195312, "KL/rejected_KL_mean": -284.8623046875, "KL/std": 107.59810638427734, "epoch": 0.9897209985315712, "fcm_dpo/beta": 0.003781170817092061, "fcm_dpo/delta": -0.04985257610678673, "fcm_dpo/margin": 118.36953735351562, "fcm_dpo/q_t": 0.3950941562652588, "grad_norm": 7.943295955657959, "learning_rate": 2.1077827798404725e-10, "logits/chosen": -1.6309033632278442, "logits/rejected": -1.2305643558502197, "logps/chosen": -241.95108032226562, "logps/ref_chosen": -75.45831298828125, "logps/ref_rejected": -76.20362854003906, "logps/rejected": -361.06591796875, "loss": 1.0343, "margin_dpo/margin_mean": 118.36953735351562, "margin_dpo/margin_std": 118.20276641845703, "step": 674 }, { "KL/chosen_KL_mean": -182.51034545898438, "KL/mean": -235.08111572265625, "KL/rejected_KL_mean": -287.65191650390625, "KL/std": 116.70008087158203, "epoch": 0.9911894273127754, "fcm_dpo/beta": 0.003734186990186572, "fcm_dpo/delta": -0.05345737189054489, "fcm_dpo/margin": 105.14154052734375, "fcm_dpo/q_t": 0.40766555070877075, "grad_norm": 10.928277969360352, "learning_rate": 1.6138243485910863e-10, "logits/chosen": -1.285999059677124, "logits/rejected": -0.934632420539856, "logps/chosen": -262.4198913574219, "logps/ref_chosen": -79.90953063964844, "logps/ref_rejected": -81.21824645996094, "logps/rejected": -368.8701477050781, "loss": 1.1019, "margin_dpo/margin_mean": 105.14154815673828, "margin_dpo/margin_std": 143.1363525390625, "step": 675 }, { "KL/chosen_KL_mean": -196.07440185546875, "KL/mean": -247.11036682128906, "KL/rejected_KL_mean": -298.14630126953125, "KL/std": 111.57682800292969, "epoch": 0.9926578560939795, "fcm_dpo/beta": 0.003737333696335554, "fcm_dpo/delta": 0.019213024526834488, "fcm_dpo/margin": 102.07192993164062, "fcm_dpo/q_t": 0.41006457805633545, "grad_norm": 8.219503402709961, "learning_rate": 1.1857007165852472e-10, "logits/chosen": -1.7843862771987915, "logits/rejected": -1.1825398206710815, "logps/chosen": -294.2455139160156, "logps/ref_chosen": -98.17111206054688, "logps/ref_rejected": -95.024658203125, "logps/rejected": -393.1709899902344, "loss": 1.0951, "margin_dpo/margin_mean": 102.07192993164062, "margin_dpo/margin_std": 125.75214385986328, "step": 676 }, { "KL/chosen_KL_mean": -176.21795654296875, "KL/mean": -228.08029174804688, "KL/rejected_KL_mean": -279.942626953125, "KL/std": 110.58978271484375, "epoch": 0.9941262848751835, "fcm_dpo/beta": 0.00374723132699728, "fcm_dpo/delta": 0.011711678467690945, "fcm_dpo/margin": 103.72468566894531, "fcm_dpo/q_t": 0.4079986810684204, "grad_norm": 7.956014633178711, "learning_rate": 8.23423165278725e-11, "logits/chosen": -1.5618531703948975, "logits/rejected": -0.8732852935791016, "logps/chosen": -267.59722900390625, "logps/ref_chosen": -91.37928771972656, "logps/ref_rejected": -82.87776947021484, "logps/rejected": -362.8204040527344, "loss": 1.0759, "margin_dpo/margin_mean": 103.72468566894531, "margin_dpo/margin_std": 108.58404541015625, "step": 677 }, { "KL/chosen_KL_mean": -166.2670440673828, "KL/mean": -227.56236267089844, "KL/rejected_KL_mean": -288.857666015625, "KL/std": 118.49006652832031, "epoch": 0.9955947136563876, "fcm_dpo/beta": 0.0037383511662483215, "fcm_dpo/delta": -0.0611700601875782, "fcm_dpo/margin": 122.59062957763672, "fcm_dpo/q_t": 0.3919582962989807, "grad_norm": 9.8098783493042, "learning_rate": 5.270012410216185e-11, "logits/chosen": -1.7469590902328491, "logits/rejected": -1.4247102737426758, "logps/chosen": -241.91290283203125, "logps/ref_chosen": -75.6458740234375, "logps/ref_rejected": -86.96611022949219, "logps/rejected": -375.82379150390625, "loss": 1.0294, "margin_dpo/margin_mean": 122.59063720703125, "margin_dpo/margin_std": 120.68907165527344, "step": 678 }, { "KL/chosen_KL_mean": -192.85543823242188, "KL/mean": -243.40980529785156, "KL/rejected_KL_mean": -293.96417236328125, "KL/std": 98.76133728027344, "epoch": 0.9970631424375918, "fcm_dpo/beta": 0.0037227459251880646, "fcm_dpo/delta": 0.024527989327907562, "fcm_dpo/margin": 101.1087646484375, "fcm_dpo/q_t": 0.41120100021362305, "grad_norm": 8.339025497436523, "learning_rate": 2.9644275480772416e-11, "logits/chosen": -2.2584304809570312, "logits/rejected": -1.7472279071807861, "logps/chosen": -273.6288757324219, "logps/ref_chosen": -80.77344512939453, "logps/ref_rejected": -82.87850189208984, "logps/rejected": -376.8426818847656, "loss": 1.0813, "margin_dpo/margin_mean": 101.1087646484375, "margin_dpo/margin_std": 105.755126953125, "step": 679 }, { "KL/chosen_KL_mean": -193.2791748046875, "KL/mean": -248.47329711914062, "KL/rejected_KL_mean": -303.66741943359375, "KL/std": 117.02078247070312, "epoch": 0.9985315712187959, "fcm_dpo/beta": 0.0037055439315736294, "fcm_dpo/delta": -0.06578289717435837, "fcm_dpo/margin": 110.38827514648438, "fcm_dpo/q_t": 0.40424782037734985, "grad_norm": 7.081179141998291, "learning_rate": 1.31753782067201e-11, "logits/chosen": -1.9725468158721924, "logits/rejected": -1.643679141998291, "logps/chosen": -300.96209716796875, "logps/ref_chosen": -107.68292999267578, "logps/ref_rejected": -116.0948715209961, "logps/rejected": -419.7623291015625, "loss": 1.0748, "margin_dpo/margin_mean": 110.38825988769531, "margin_dpo/margin_std": 127.57222747802734, "step": 680 }, { "KL/chosen_KL_mean": -176.45477294921875, "KL/mean": -224.28012084960938, "KL/rejected_KL_mean": -272.10546875, "KL/std": 110.61670684814453, "epoch": 1.0, "fcm_dpo/beta": 0.003713160753250122, "fcm_dpo/delta": 0.04634173586964607, "fcm_dpo/margin": 95.65071868896484, "fcm_dpo/q_t": 0.416808545589447, "grad_norm": 9.360736846923828, "learning_rate": 3.2938662507808745e-12, "logits/chosen": -2.351592540740967, "logits/rejected": -2.0445010662078857, "logps/chosen": -269.4658203125, "logps/ref_chosen": -93.01106262207031, "logps/ref_rejected": -94.8221664428711, "logps/rejected": -366.92767333984375, "loss": 1.111, "margin_dpo/margin_mean": 95.65072631835938, "margin_dpo/margin_std": 120.97123718261719, "step": 681 }, { "epoch": 1.0, "step": 681, "total_flos": 0.0, "train_loss": 1.1086520982383807, "train_runtime": 1981.9264, "train_samples_per_second": 21.998, "train_steps_per_second": 0.344 } ], "logging_steps": 1, "max_steps": 681, "num_input_tokens_seen": 0, "num_train_epochs": 1, "save_steps": 50, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": false, "should_training_stop": false }, "attributes": {} } }, "total_flos": 0.0, "train_batch_size": 8, "trial_name": null, "trial_params": null }