diff --git "a/trainer_state.json" "b/trainer_state.json" new file mode 100644--- /dev/null +++ "b/trainer_state.json" @@ -0,0 +1,15706 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.0, + "eval_steps": 100, + "global_step": 681, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "KL/chosen_KL_mean": 0.05419158935546875, + "KL/mean": 0.023573115468025208, + "KL/rejected_KL_mean": -0.007045745849609375, + "KL/std": 0.33097243309020996, + "epoch": 0.0014684287812041115, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.06123492121696472, + "fcm_dpo/q_t": 0.4984707534313202, + "grad_norm": 38.819522857666016, + "learning_rate": 0.0, + "logits/chosen": 0.2677389979362488, + "logits/rejected": 0.6191319227218628, + "logps/chosen": -85.37664031982422, + "logps/ref_chosen": -85.43083190917969, + "logps/ref_rejected": -79.90458679199219, + "logps/rejected": -79.91163635253906, + "loss": 1.3807, + "margin_dpo/margin_mean": 0.061235010623931885, + "margin_dpo/margin_std": 0.4424324035644531, + "step": 1 + }, + { + "KL/chosen_KL_mean": -0.044902801513671875, + "KL/mean": 0.017303764820098877, + "KL/rejected_KL_mean": 0.07951736450195312, + "KL/std": 0.2933448553085327, + "epoch": 0.002936857562408223, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.12442219257354736, + "fcm_dpo/q_t": 0.5031089186668396, + "grad_norm": 29.759681701660156, + "learning_rate": 7.246376811594203e-09, + "logits/chosen": 0.47771453857421875, + "logits/rejected": 0.5908545851707458, + "logps/chosen": -82.11383056640625, + "logps/ref_chosen": -82.06893157958984, + "logps/ref_rejected": -81.65457916259766, + "logps/rejected": -81.57505798339844, + "loss": 1.3992, + "margin_dpo/margin_mean": -0.12442246079444885, + "margin_dpo/margin_std": 0.41358208656311035, + "step": 2 + }, + { + "KL/chosen_KL_mean": 0.01801300048828125, + "KL/mean": 0.009846903383731842, + "KL/rejected_KL_mean": 0.001682281494140625, + "KL/std": 0.29592806100845337, + "epoch": 0.004405286343612335, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.01633407175540924, + "fcm_dpo/q_t": 0.4995919466018677, + "grad_norm": 26.0485897064209, + "learning_rate": 1.4492753623188406e-08, + "logits/chosen": 0.42689090967178345, + "logits/rejected": 0.6502213478088379, + "logps/chosen": -93.79296875, + "logps/ref_chosen": -93.81098937988281, + "logps/ref_rejected": -74.22950744628906, + "logps/rejected": -74.22782135009766, + "loss": 1.3852, + "margin_dpo/margin_mean": 0.016334399580955505, + "margin_dpo/margin_std": 0.43817389011383057, + "step": 3 + }, + { + "KL/chosen_KL_mean": -0.032138824462890625, + "KL/mean": -0.030376777052879333, + "KL/rejected_KL_mean": -0.02861785888671875, + "KL/std": 0.3369988203048706, + "epoch": 0.005873715124816446, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.0035226643085479736, + "fcm_dpo/q_t": 0.5000864267349243, + "grad_norm": 32.7692756652832, + "learning_rate": 2.1739130434782606e-08, + "logits/chosen": 0.6462423801422119, + "logits/rejected": 0.7780641913414001, + "logps/chosen": -87.3246078491211, + "logps/ref_chosen": -87.29246520996094, + "logps/ref_rejected": -93.82425689697266, + "logps/rejected": -93.85287475585938, + "loss": 1.3872, + "margin_dpo/margin_mean": -0.0035224556922912598, + "margin_dpo/margin_std": 0.4537009596824646, + "step": 4 + }, + { + "KL/chosen_KL_mean": -0.055454254150390625, + "KL/mean": -0.045780032873153687, + "KL/rejected_KL_mean": -0.03610992431640625, + "KL/std": 0.3053707480430603, + "epoch": 0.007342143906020558, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.019347012042999268, + "fcm_dpo/q_t": 0.5004826188087463, + "grad_norm": 35.43703842163086, + "learning_rate": 2.898550724637681e-08, + "logits/chosen": 0.38495126366615295, + "logits/rejected": 0.5912345051765442, + "logps/chosen": -89.3922119140625, + "logps/ref_chosen": -89.33676147460938, + "logps/ref_rejected": -88.74783325195312, + "logps/rejected": -88.78394317626953, + "loss": 1.3888, + "margin_dpo/margin_mean": -0.01934686303138733, + "margin_dpo/margin_std": 0.4653624892234802, + "step": 5 + }, + { + "KL/chosen_KL_mean": -0.012523651123046875, + "KL/mean": -0.03238530457019806, + "KL/rejected_KL_mean": -0.05224609375, + "KL/std": 0.31671661138534546, + "epoch": 0.00881057268722467, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.03972157835960388, + "fcm_dpo/q_t": 0.49900656938552856, + "grad_norm": 36.194889068603516, + "learning_rate": 3.6231884057971014e-08, + "logits/chosen": 0.4861461818218231, + "logits/rejected": 0.8017364740371704, + "logps/chosen": -97.33399963378906, + "logps/ref_chosen": -97.32147216796875, + "logps/ref_rejected": -97.88345336914062, + "logps/rejected": -97.93569946289062, + "loss": 1.3829, + "margin_dpo/margin_mean": 0.039721399545669556, + "margin_dpo/margin_std": 0.4631843566894531, + "step": 6 + }, + { + "KL/chosen_KL_mean": -0.01750946044921875, + "KL/mean": -0.010317608714103699, + "KL/rejected_KL_mean": -0.00313568115234375, + "KL/std": 0.3258499503135681, + "epoch": 0.010279001468428781, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.014376074075698853, + "fcm_dpo/q_t": 0.5003604888916016, + "grad_norm": 38.085540771484375, + "learning_rate": 4.347826086956521e-08, + "logits/chosen": 0.4962475597858429, + "logits/rejected": 0.5440775156021118, + "logps/chosen": -86.66604614257812, + "logps/ref_chosen": -86.64852905273438, + "logps/ref_rejected": -109.61618041992188, + "logps/rejected": -109.61931610107422, + "loss": 1.3883, + "margin_dpo/margin_mean": -0.014376431703567505, + "margin_dpo/margin_std": 0.4693182706832886, + "step": 7 + }, + { + "KL/chosen_KL_mean": -0.0348968505859375, + "KL/mean": 0.02407728135585785, + "KL/rejected_KL_mean": 0.0830535888671875, + "KL/std": 0.28958797454833984, + "epoch": 0.011747430249632892, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.11795011162757874, + "fcm_dpo/q_t": 0.5029476284980774, + "grad_norm": 32.91734313964844, + "learning_rate": 5.0724637681159424e-08, + "logits/chosen": 0.4092610478401184, + "logits/rejected": 0.7200788855552673, + "logps/chosen": -89.95858001708984, + "logps/ref_chosen": -89.9236831665039, + "logps/ref_rejected": -86.22804260253906, + "logps/rejected": -86.14498901367188, + "loss": 1.3985, + "margin_dpo/margin_mean": -0.11794936656951904, + "margin_dpo/margin_std": 0.38808369636535645, + "step": 8 + }, + { + "KL/chosen_KL_mean": 0.07097244262695312, + "KL/mean": 0.015693560242652893, + "KL/rejected_KL_mean": -0.039577484130859375, + "KL/std": 0.3280068635940552, + "epoch": 0.013215859030837005, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.11054441332817078, + "fcm_dpo/q_t": 0.4972390830516815, + "grad_norm": 35.82514953613281, + "learning_rate": 5.797101449275362e-08, + "logits/chosen": 0.6291981935501099, + "logits/rejected": 0.8066290616989136, + "logps/chosen": -103.78616333007812, + "logps/ref_chosen": -103.85713195800781, + "logps/ref_rejected": -104.31932067871094, + "logps/rejected": -104.35890197753906, + "loss": 1.3758, + "margin_dpo/margin_mean": 0.11054429411888123, + "margin_dpo/margin_std": 0.46005770564079285, + "step": 9 + }, + { + "KL/chosen_KL_mean": 0.02231597900390625, + "KL/mean": 0.07833842188119888, + "KL/rejected_KL_mean": 0.13436126708984375, + "KL/std": 0.2511771321296692, + "epoch": 0.014684287812041116, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.11205075681209564, + "fcm_dpo/q_t": 0.5027999877929688, + "grad_norm": 33.62577438354492, + "learning_rate": 6.521739130434782e-08, + "logits/chosen": 0.5185714960098267, + "logits/rejected": 0.7311661839485168, + "logps/chosen": -76.18205261230469, + "logps/ref_chosen": -76.20436096191406, + "logps/ref_rejected": -87.152099609375, + "logps/rejected": -87.01774597167969, + "loss": 1.3978, + "margin_dpo/margin_mean": -0.11205106973648071, + "margin_dpo/margin_std": 0.335724413394928, + "step": 10 + }, + { + "KL/chosen_KL_mean": 0.06981277465820312, + "KL/mean": 0.04370792955160141, + "KL/rejected_KL_mean": 0.017604827880859375, + "KL/std": 0.3104771375656128, + "epoch": 0.016152716593245228, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.05220632255077362, + "fcm_dpo/q_t": 0.49869662523269653, + "grad_norm": 36.76396942138672, + "learning_rate": 7.246376811594203e-08, + "logits/chosen": 0.3010936677455902, + "logits/rejected": 0.5179265141487122, + "logps/chosen": -82.29668426513672, + "logps/ref_chosen": -82.36650085449219, + "logps/ref_rejected": -94.26461791992188, + "logps/rejected": -94.24701690673828, + "loss": 1.3817, + "margin_dpo/margin_mean": 0.052206918597221375, + "margin_dpo/margin_std": 0.4737238585948944, + "step": 11 + }, + { + "KL/chosen_KL_mean": -0.011959075927734375, + "KL/mean": -0.013267628848552704, + "KL/rejected_KL_mean": -0.0145721435546875, + "KL/std": 0.34847086668014526, + "epoch": 0.01762114537444934, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.0026122182607650757, + "fcm_dpo/q_t": 0.4999332129955292, + "grad_norm": 40.034034729003906, + "learning_rate": 7.971014492753623e-08, + "logits/chosen": 0.3836405873298645, + "logits/rejected": 0.511702299118042, + "logps/chosen": -99.11746215820312, + "logps/ref_chosen": -99.10549926757812, + "logps/ref_rejected": -110.27140808105469, + "logps/rejected": -110.28598022460938, + "loss": 1.3867, + "margin_dpo/margin_mean": 0.002612382173538208, + "margin_dpo/margin_std": 0.518186628818512, + "step": 12 + }, + { + "KL/chosen_KL_mean": -0.0435791015625, + "KL/mean": -0.024171896278858185, + "KL/rejected_KL_mean": -0.004764556884765625, + "KL/std": 0.2932068705558777, + "epoch": 0.01908957415565345, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.038815781474113464, + "fcm_dpo/q_t": 0.500969648361206, + "grad_norm": 42.037208557128906, + "learning_rate": 8.695652173913042e-08, + "logits/chosen": 0.37952056527137756, + "logits/rejected": 0.6270250082015991, + "logps/chosen": -90.60330200195312, + "logps/ref_chosen": -90.55972290039062, + "logps/ref_rejected": -93.69110107421875, + "logps/rejected": -93.69586181640625, + "loss": 1.3907, + "margin_dpo/margin_mean": -0.03881540894508362, + "margin_dpo/margin_std": 0.4327077269554138, + "step": 13 + }, + { + "KL/chosen_KL_mean": -0.10559463500976562, + "KL/mean": -0.06970568001270294, + "KL/rejected_KL_mean": -0.0338134765625, + "KL/std": 0.3560220003128052, + "epoch": 0.020558002936857563, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.07178828120231628, + "fcm_dpo/q_t": 0.5017932653427124, + "grad_norm": 35.77608871459961, + "learning_rate": 9.420289855072464e-08, + "logits/chosen": 0.9327787160873413, + "logits/rejected": 1.0230083465576172, + "logps/chosen": -99.93276977539062, + "logps/ref_chosen": -99.82717895507812, + "logps/ref_rejected": -108.94200134277344, + "logps/rejected": -108.97581481933594, + "loss": 1.394, + "margin_dpo/margin_mean": -0.07178857922554016, + "margin_dpo/margin_std": 0.43152570724487305, + "step": 14 + }, + { + "KL/chosen_KL_mean": -0.029644012451171875, + "KL/mean": -0.030726686120033264, + "KL/rejected_KL_mean": -0.03180694580078125, + "KL/std": 0.33586567640304565, + "epoch": 0.022026431718061675, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.0021653473377227783, + "fcm_dpo/q_t": 0.4999483525753021, + "grad_norm": 31.695486068725586, + "learning_rate": 1.0144927536231885e-07, + "logits/chosen": 0.4294240176677704, + "logits/rejected": 0.5991781949996948, + "logps/chosen": -78.93961334228516, + "logps/ref_chosen": -78.90997314453125, + "logps/ref_rejected": -90.06234741210938, + "logps/rejected": -90.09414672851562, + "loss": 1.3867, + "margin_dpo/margin_mean": 0.0021652579307556152, + "margin_dpo/margin_std": 0.49874815344810486, + "step": 15 + }, + { + "KL/chosen_KL_mean": 0.01975250244140625, + "KL/mean": -0.0009732842445373535, + "KL/rejected_KL_mean": -0.021701812744140625, + "KL/std": 0.30277085304260254, + "epoch": 0.023494860499265784, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.0414544939994812, + "fcm_dpo/q_t": 0.49896475672721863, + "grad_norm": 34.276187896728516, + "learning_rate": 1.0869565217391303e-07, + "logits/chosen": 0.6241447925567627, + "logits/rejected": 0.8891364336013794, + "logps/chosen": -97.40352630615234, + "logps/ref_chosen": -97.42327880859375, + "logps/ref_rejected": -90.59945678710938, + "logps/rejected": -90.62116241455078, + "loss": 1.3826, + "margin_dpo/margin_mean": 0.04145437479019165, + "margin_dpo/margin_std": 0.41995885968208313, + "step": 16 + }, + { + "KL/chosen_KL_mean": 0.0464630126953125, + "KL/mean": 0.013510197401046753, + "KL/rejected_KL_mean": -0.019443511962890625, + "KL/std": 0.34516268968582153, + "epoch": 0.024963289280469897, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.06591099500656128, + "fcm_dpo/q_t": 0.49835169315338135, + "grad_norm": 35.494384765625, + "learning_rate": 1.1594202898550725e-07, + "logits/chosen": 0.5926298499107361, + "logits/rejected": 0.8359245657920837, + "logps/chosen": -104.31784057617188, + "logps/ref_chosen": -104.36430358886719, + "logps/ref_rejected": -90.46772766113281, + "logps/rejected": -90.48716735839844, + "loss": 1.3804, + "margin_dpo/margin_mean": 0.06591001152992249, + "margin_dpo/margin_std": 0.5128095746040344, + "step": 17 + }, + { + "KL/chosen_KL_mean": 0.056804656982421875, + "KL/mean": 0.0426379069685936, + "KL/rejected_KL_mean": 0.028469085693359375, + "KL/std": 0.32054784893989563, + "epoch": 0.02643171806167401, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.028332993388175964, + "fcm_dpo/q_t": 0.4992940127849579, + "grad_norm": 41.7449951171875, + "learning_rate": 1.2318840579710146e-07, + "logits/chosen": 0.31673505902290344, + "logits/rejected": 0.6540764570236206, + "logps/chosen": -87.03515625, + "logps/ref_chosen": -87.09195709228516, + "logps/ref_rejected": -81.85072326660156, + "logps/rejected": -81.82225036621094, + "loss": 1.384, + "margin_dpo/margin_mean": 0.02833305299282074, + "margin_dpo/margin_std": 0.48277661204338074, + "step": 18 + }, + { + "KL/chosen_KL_mean": 0.02515411376953125, + "KL/mean": -0.008823007345199585, + "KL/rejected_KL_mean": -0.042804718017578125, + "KL/std": 0.317953884601593, + "epoch": 0.027900146842878122, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.06795555353164673, + "fcm_dpo/q_t": 0.4983009099960327, + "grad_norm": 31.350309371948242, + "learning_rate": 1.3043478260869563e-07, + "logits/chosen": 0.4514174461364746, + "logits/rejected": 0.7146160006523132, + "logps/chosen": -105.848388671875, + "logps/ref_chosen": -105.87354278564453, + "logps/ref_rejected": -96.93023681640625, + "logps/rejected": -96.97303771972656, + "loss": 1.38, + "margin_dpo/margin_mean": 0.06795570254325867, + "margin_dpo/margin_std": 0.44763243198394775, + "step": 19 + }, + { + "KL/chosen_KL_mean": 0.013187408447265625, + "KL/mean": 0.021351009607315063, + "KL/rejected_KL_mean": 0.029510498046875, + "KL/std": 0.31396663188934326, + "epoch": 0.02936857562408223, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.01632392406463623, + "fcm_dpo/q_t": 0.5004087090492249, + "grad_norm": 32.772891998291016, + "learning_rate": 1.3768115942028986e-07, + "logits/chosen": 0.21086394786834717, + "logits/rejected": 0.4835556149482727, + "logps/chosen": -90.74493408203125, + "logps/ref_chosen": -90.75811767578125, + "logps/ref_rejected": -85.91232299804688, + "logps/rejected": -85.8828125, + "loss": 1.3885, + "margin_dpo/margin_mean": -0.016323953866958618, + "margin_dpo/margin_std": 0.47240692377090454, + "step": 20 + }, + { + "KL/chosen_KL_mean": 0.0029144287109375, + "KL/mean": -0.015282675623893738, + "KL/rejected_KL_mean": -0.033477783203125, + "KL/std": 0.26727402210235596, + "epoch": 0.030837004405286344, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.03639879822731018, + "fcm_dpo/q_t": 0.499089777469635, + "grad_norm": 31.454870223999023, + "learning_rate": 1.4492753623188405e-07, + "logits/chosen": 0.5349259376525879, + "logits/rejected": 0.7128177881240845, + "logps/chosen": -80.3305435180664, + "logps/ref_chosen": -80.33345794677734, + "logps/ref_rejected": -83.93373107910156, + "logps/rejected": -83.96720886230469, + "loss": 1.3831, + "margin_dpo/margin_mean": 0.03639945387840271, + "margin_dpo/margin_std": 0.3977454900741577, + "step": 21 + }, + { + "KL/chosen_KL_mean": 0.028522491455078125, + "KL/mean": 0.04464656114578247, + "KL/rejected_KL_mean": 0.06076812744140625, + "KL/std": 0.3194689154624939, + "epoch": 0.032305433186490456, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.03225874900817871, + "fcm_dpo/q_t": 0.5008060336112976, + "grad_norm": 42.9687385559082, + "learning_rate": 1.5217391304347825e-07, + "logits/chosen": 0.49087780714035034, + "logits/rejected": 0.6040288805961609, + "logps/chosen": -95.36677551269531, + "logps/ref_chosen": -95.39530181884766, + "logps/ref_rejected": -103.47351837158203, + "logps/rejected": -103.41275024414062, + "loss": 1.3899, + "margin_dpo/margin_mean": -0.03225898742675781, + "margin_dpo/margin_std": 0.3678981065750122, + "step": 22 + }, + { + "KL/chosen_KL_mean": -0.023744583129882812, + "KL/mean": 0.01652967929840088, + "KL/rejected_KL_mean": 0.056812286376953125, + "KL/std": 0.32836759090423584, + "epoch": 0.033773861967694566, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.08055168390274048, + "fcm_dpo/q_t": 0.5020133256912231, + "grad_norm": 31.551267623901367, + "learning_rate": 1.5942028985507245e-07, + "logits/chosen": 0.7624632120132446, + "logits/rejected": 0.943315327167511, + "logps/chosen": -90.6612548828125, + "logps/ref_chosen": -90.63751220703125, + "logps/ref_rejected": -86.59425354003906, + "logps/rejected": -86.53744506835938, + "loss": 1.3949, + "margin_dpo/margin_mean": -0.08055201172828674, + "margin_dpo/margin_std": 0.47492456436157227, + "step": 23 + }, + { + "KL/chosen_KL_mean": 0.03722953796386719, + "KL/mean": -0.0006896853446960449, + "KL/rejected_KL_mean": -0.038608551025390625, + "KL/std": 0.2770400047302246, + "epoch": 0.03524229074889868, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.0758410096168518, + "fcm_dpo/q_t": 0.498104989528656, + "grad_norm": 45.75295639038086, + "learning_rate": 1.6666666666666665e-07, + "logits/chosen": 0.3441356420516968, + "logits/rejected": 0.37922707200050354, + "logps/chosen": -69.88006591796875, + "logps/ref_chosen": -69.91728973388672, + "logps/ref_rejected": -106.60990142822266, + "logps/rejected": -106.64851379394531, + "loss": 1.3791, + "margin_dpo/margin_mean": 0.0758412629365921, + "margin_dpo/margin_std": 0.3807234764099121, + "step": 24 + }, + { + "KL/chosen_KL_mean": 0.03569793701171875, + "KL/mean": -0.004800528287887573, + "KL/rejected_KL_mean": -0.045299530029296875, + "KL/std": 0.39410075545310974, + "epoch": 0.03671071953010279, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.0809900164604187, + "fcm_dpo/q_t": 0.49797898530960083, + "grad_norm": 37.050968170166016, + "learning_rate": 1.7391304347826085e-07, + "logits/chosen": 0.5247936248779297, + "logits/rejected": 0.692723274230957, + "logps/chosen": -80.78978729248047, + "logps/ref_chosen": -80.82548522949219, + "logps/ref_rejected": -95.95710754394531, + "logps/rejected": -96.00241088867188, + "loss": 1.3791, + "margin_dpo/margin_mean": 0.08098980784416199, + "margin_dpo/margin_std": 0.5588604211807251, + "step": 25 + }, + { + "KL/chosen_KL_mean": 0.010463714599609375, + "KL/mean": -0.022343426942825317, + "KL/rejected_KL_mean": -0.055149078369140625, + "KL/std": 0.33425793051719666, + "epoch": 0.0381791483113069, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.06561839580535889, + "fcm_dpo/q_t": 0.49836117029190063, + "grad_norm": 40.846405029296875, + "learning_rate": 1.8115942028985507e-07, + "logits/chosen": 0.31885746121406555, + "logits/rejected": 0.5274312496185303, + "logps/chosen": -88.89070892333984, + "logps/ref_chosen": -88.90116882324219, + "logps/ref_rejected": -109.82819366455078, + "logps/rejected": -109.88334655761719, + "loss": 1.3801, + "margin_dpo/margin_mean": 0.0656188428401947, + "margin_dpo/margin_std": 0.3876963257789612, + "step": 26 + }, + { + "KL/chosen_KL_mean": 0.0208587646484375, + "KL/mean": 0.00705236941576004, + "KL/rejected_KL_mean": -0.006763458251953125, + "KL/std": 0.30177831649780273, + "epoch": 0.039647577092511016, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.02762104570865631, + "fcm_dpo/q_t": 0.4993098974227905, + "grad_norm": 44.29174041748047, + "learning_rate": 1.8840579710144927e-07, + "logits/chosen": 0.055542007088661194, + "logits/rejected": 0.2412649542093277, + "logps/chosen": -77.57514953613281, + "logps/ref_chosen": -77.59600830078125, + "logps/ref_rejected": -103.93850708007812, + "logps/rejected": -103.94526672363281, + "loss": 1.384, + "margin_dpo/margin_mean": 0.02762100100517273, + "margin_dpo/margin_std": 0.43413975834846497, + "step": 27 + }, + { + "KL/chosen_KL_mean": 0.05838775634765625, + "KL/mean": -0.018888160586357117, + "KL/rejected_KL_mean": -0.096160888671875, + "KL/std": 0.3182182013988495, + "epoch": 0.041116005873715125, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.15454670786857605, + "fcm_dpo/q_t": 0.49613893032073975, + "grad_norm": 36.18867874145508, + "learning_rate": 1.9565217391304347e-07, + "logits/chosen": 0.5341115593910217, + "logits/rejected": 0.74845290184021, + "logps/chosen": -102.17018127441406, + "logps/ref_chosen": -102.22856903076172, + "logps/ref_rejected": -96.9594955444336, + "logps/rejected": -97.05565643310547, + "loss": 1.3714, + "margin_dpo/margin_mean": 0.1545475423336029, + "margin_dpo/margin_std": 0.46393883228302, + "step": 28 + }, + { + "KL/chosen_KL_mean": 0.029796600341796875, + "KL/mean": -0.04521440714597702, + "KL/rejected_KL_mean": -0.1202239990234375, + "KL/std": 0.2746419906616211, + "epoch": 0.042584434654919234, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.1500234752893448, + "fcm_dpo/q_t": 0.4962507486343384, + "grad_norm": 42.48250198364258, + "learning_rate": 2.028985507246377e-07, + "logits/chosen": 0.298464834690094, + "logits/rejected": 0.5537115931510925, + "logps/chosen": -88.61725616455078, + "logps/ref_chosen": -88.64704895019531, + "logps/ref_rejected": -102.96011352539062, + "logps/rejected": -103.08033752441406, + "loss": 1.3717, + "margin_dpo/margin_mean": 0.15002469718456268, + "margin_dpo/margin_std": 0.352841854095459, + "step": 29 + }, + { + "KL/chosen_KL_mean": 0.009586334228515625, + "KL/mean": -0.06154375523328781, + "KL/rejected_KL_mean": -0.13267898559570312, + "KL/std": 0.3309122920036316, + "epoch": 0.04405286343612335, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.14225663244724274, + "fcm_dpo/q_t": 0.49644559621810913, + "grad_norm": 38.739402770996094, + "learning_rate": 2.1014492753623187e-07, + "logits/chosen": 0.5708379149436951, + "logits/rejected": 0.6858254671096802, + "logps/chosen": -88.37879943847656, + "logps/ref_chosen": -88.38838195800781, + "logps/ref_rejected": -102.31889343261719, + "logps/rejected": -102.45156860351562, + "loss": 1.3726, + "margin_dpo/margin_mean": 0.14225709438323975, + "margin_dpo/margin_std": 0.4141353368759155, + "step": 30 + }, + { + "KL/chosen_KL_mean": -0.035480499267578125, + "KL/mean": -0.09532057493925095, + "KL/rejected_KL_mean": -0.15515899658203125, + "KL/std": 0.28104445338249207, + "epoch": 0.04552129221732746, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.11968068778514862, + "fcm_dpo/q_t": 0.4970095455646515, + "grad_norm": 31.303373336791992, + "learning_rate": 2.1739130434782607e-07, + "logits/chosen": 0.9836581349372864, + "logits/rejected": 1.2598530054092407, + "logps/chosen": -101.16114044189453, + "logps/ref_chosen": -101.12565612792969, + "logps/ref_rejected": -79.8584213256836, + "logps/rejected": -80.01358032226562, + "loss": 1.3747, + "margin_dpo/margin_mean": 0.11968028545379639, + "margin_dpo/margin_std": 0.3348178267478943, + "step": 31 + }, + { + "KL/chosen_KL_mean": -0.061847686767578125, + "KL/mean": -0.08950134366750717, + "KL/rejected_KL_mean": -0.11715316772460938, + "KL/std": 0.35548287630081177, + "epoch": 0.04698972099853157, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.05530880391597748, + "fcm_dpo/q_t": 0.4986189603805542, + "grad_norm": 35.21878433227539, + "learning_rate": 2.2463768115942027e-07, + "logits/chosen": 0.42202627658843994, + "logits/rejected": 0.5617946982383728, + "logps/chosen": -96.67887878417969, + "logps/ref_chosen": -96.61702728271484, + "logps/ref_rejected": -96.224365234375, + "logps/rejected": -96.34152221679688, + "loss": 1.3815, + "margin_dpo/margin_mean": 0.05530978739261627, + "margin_dpo/margin_std": 0.5153543949127197, + "step": 32 + }, + { + "KL/chosen_KL_mean": 0.001628875732421875, + "KL/mean": -0.04754679650068283, + "KL/rejected_KL_mean": -0.09672164916992188, + "KL/std": 0.31931179761886597, + "epoch": 0.048458149779735685, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.09835340082645416, + "fcm_dpo/q_t": 0.4975426197052002, + "grad_norm": 33.70878982543945, + "learning_rate": 2.318840579710145e-07, + "logits/chosen": 0.6354793906211853, + "logits/rejected": 0.8171831369400024, + "logps/chosen": -81.5193862915039, + "logps/ref_chosen": -81.52101135253906, + "logps/ref_rejected": -93.80595397949219, + "logps/rejected": -93.90267944335938, + "loss": 1.3771, + "margin_dpo/margin_mean": 0.09835365414619446, + "margin_dpo/margin_std": 0.47823020815849304, + "step": 33 + }, + { + "KL/chosen_KL_mean": 0.0430450439453125, + "KL/mean": -0.07708846032619476, + "KL/rejected_KL_mean": -0.19721603393554688, + "KL/std": 0.3642929792404175, + "epoch": 0.049926578560939794, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.24027082324028015, + "fcm_dpo/q_t": 0.4940028190612793, + "grad_norm": 40.969215393066406, + "learning_rate": 2.391304347826087e-07, + "logits/chosen": 0.5381206274032593, + "logits/rejected": 0.7103798389434814, + "logps/chosen": -77.15899658203125, + "logps/ref_chosen": -77.20204162597656, + "logps/ref_rejected": -106.71876525878906, + "logps/rejected": -106.91598510742188, + "loss": 1.3631, + "margin_dpo/margin_mean": 0.24027130007743835, + "margin_dpo/margin_std": 0.4968305826187134, + "step": 34 + }, + { + "KL/chosen_KL_mean": 0.01062774658203125, + "KL/mean": -0.10152420401573181, + "KL/rejected_KL_mean": -0.21367645263671875, + "KL/std": 0.3811684250831604, + "epoch": 0.0513950073421439, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.22430533170700073, + "fcm_dpo/q_t": 0.49439704418182373, + "grad_norm": 42.71918487548828, + "learning_rate": 2.463768115942029e-07, + "logits/chosen": 0.335090696811676, + "logits/rejected": 0.4909539818763733, + "logps/chosen": -77.55972290039062, + "logps/ref_chosen": -77.57035064697266, + "logps/ref_rejected": -112.18855285644531, + "logps/rejected": -112.40222930908203, + "loss": 1.3647, + "margin_dpo/margin_mean": 0.22430622577667236, + "margin_dpo/margin_std": 0.5345590114593506, + "step": 35 + }, + { + "KL/chosen_KL_mean": -0.01268768310546875, + "KL/mean": -0.09679705649614334, + "KL/rejected_KL_mean": -0.180908203125, + "KL/std": 0.38911956548690796, + "epoch": 0.05286343612334802, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.16821442544460297, + "fcm_dpo/q_t": 0.49579930305480957, + "grad_norm": 32.6117057800293, + "learning_rate": 2.536231884057971e-07, + "logits/chosen": 0.4062168002128601, + "logits/rejected": 0.5958766937255859, + "logps/chosen": -83.30093383789062, + "logps/ref_chosen": -83.28823852539062, + "logps/ref_rejected": -91.75741577148438, + "logps/rejected": -91.93832397460938, + "loss": 1.3702, + "margin_dpo/margin_mean": 0.1682145893573761, + "margin_dpo/margin_std": 0.5040839314460754, + "step": 36 + }, + { + "KL/chosen_KL_mean": -0.00870513916015625, + "KL/mean": -0.15830036997795105, + "KL/rejected_KL_mean": -0.30789947509765625, + "KL/std": 0.47221916913986206, + "epoch": 0.05433186490455213, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.2991906404495239, + "fcm_dpo/q_t": 0.4925394058227539, + "grad_norm": 35.66999816894531, + "learning_rate": 2.6086956521739126e-07, + "logits/chosen": 0.9145070314407349, + "logits/rejected": 1.0948659181594849, + "logps/chosen": -94.77980041503906, + "logps/ref_chosen": -94.77108764648438, + "logps/ref_rejected": -85.71726989746094, + "logps/rejected": -86.02516174316406, + "loss": 1.3578, + "margin_dpo/margin_mean": 0.29919081926345825, + "margin_dpo/margin_std": 0.6725693941116333, + "step": 37 + }, + { + "KL/chosen_KL_mean": 0.07264328002929688, + "KL/mean": -0.05678700655698776, + "KL/rejected_KL_mean": -0.18621444702148438, + "KL/std": 0.4086042642593384, + "epoch": 0.055800293685756244, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.25885826349258423, + "fcm_dpo/q_t": 0.4935372769832611, + "grad_norm": 37.20484161376953, + "learning_rate": 2.681159420289855e-07, + "logits/chosen": 0.5482184886932373, + "logits/rejected": 0.6875032186508179, + "logps/chosen": -75.9058609008789, + "logps/ref_chosen": -75.97850036621094, + "logps/ref_rejected": -104.10401916503906, + "logps/rejected": -104.29023742675781, + "loss": 1.3613, + "margin_dpo/margin_mean": 0.2588585615158081, + "margin_dpo/margin_std": 0.5284547805786133, + "step": 38 + }, + { + "KL/chosen_KL_mean": -0.048725128173828125, + "KL/mean": -0.19109302759170532, + "KL/rejected_KL_mean": -0.3334617614746094, + "KL/std": 0.41185247898101807, + "epoch": 0.05726872246696035, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.2847316265106201, + "fcm_dpo/q_t": 0.4928908348083496, + "grad_norm": 36.25815200805664, + "learning_rate": 2.753623188405797e-07, + "logits/chosen": 0.3941457271575928, + "logits/rejected": 0.5761860013008118, + "logps/chosen": -81.23450469970703, + "logps/ref_chosen": -81.18577575683594, + "logps/ref_rejected": -84.1959228515625, + "logps/rejected": -84.52938079833984, + "loss": 1.3589, + "margin_dpo/margin_mean": 0.28473174571990967, + "margin_dpo/margin_std": 0.5670312643051147, + "step": 39 + }, + { + "KL/chosen_KL_mean": 0.04741668701171875, + "KL/mean": -0.0829850435256958, + "KL/rejected_KL_mean": -0.21338653564453125, + "KL/std": 0.35915321111679077, + "epoch": 0.05873715124816446, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.26080721616744995, + "fcm_dpo/q_t": 0.4934840798377991, + "grad_norm": 28.99793243408203, + "learning_rate": 2.8260869565217386e-07, + "logits/chosen": 0.5965007543563843, + "logits/rejected": 0.7917885184288025, + "logps/chosen": -83.28515625, + "logps/ref_chosen": -83.33257293701172, + "logps/ref_rejected": -80.25591278076172, + "logps/rejected": -80.46929931640625, + "loss": 1.3609, + "margin_dpo/margin_mean": 0.2608068287372589, + "margin_dpo/margin_std": 0.4518221616744995, + "step": 40 + }, + { + "KL/chosen_KL_mean": -0.01788330078125, + "KL/mean": -0.14398349821567535, + "KL/rejected_KL_mean": -0.27008819580078125, + "KL/std": 0.43517160415649414, + "epoch": 0.06020558002936858, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.2522120177745819, + "fcm_dpo/q_t": 0.49370720982551575, + "grad_norm": 33.40452575683594, + "learning_rate": 2.898550724637681e-07, + "logits/chosen": 0.6236477494239807, + "logits/rejected": 0.8108532428741455, + "logps/chosen": -93.16654968261719, + "logps/ref_chosen": -93.14866638183594, + "logps/ref_rejected": -102.07920837402344, + "logps/rejected": -102.34929656982422, + "loss": 1.3623, + "margin_dpo/margin_mean": 0.2522123456001282, + "margin_dpo/margin_std": 0.6358495950698853, + "step": 41 + }, + { + "KL/chosen_KL_mean": 0.034381866455078125, + "KL/mean": -0.19486773014068604, + "KL/rejected_KL_mean": -0.4241180419921875, + "KL/std": 0.5133959054946899, + "epoch": 0.06167400881057269, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.4584972858428955, + "fcm_dpo/q_t": 0.48855137825012207, + "grad_norm": 40.651329040527344, + "learning_rate": 2.971014492753623e-07, + "logits/chosen": 0.39264267683029175, + "logits/rejected": 0.5342455506324768, + "logps/chosen": -90.667236328125, + "logps/ref_chosen": -90.70162200927734, + "logps/ref_rejected": -113.8646469116211, + "logps/rejected": -114.28876495361328, + "loss": 1.3418, + "margin_dpo/margin_mean": 0.45849716663360596, + "margin_dpo/margin_std": 0.584077000617981, + "step": 42 + }, + { + "KL/chosen_KL_mean": 0.021514892578125, + "KL/mean": -0.19794650375843048, + "KL/rejected_KL_mean": -0.41741180419921875, + "KL/std": 0.5176049470901489, + "epoch": 0.0631424375917768, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.4389236271381378, + "fcm_dpo/q_t": 0.4890602231025696, + "grad_norm": 37.85871124267578, + "learning_rate": 3.043478260869565e-07, + "logits/chosen": 0.6470640897750854, + "logits/rejected": 0.8401391506195068, + "logps/chosen": -89.62251281738281, + "logps/ref_chosen": -89.64402770996094, + "logps/ref_rejected": -103.95185852050781, + "logps/rejected": -104.3692626953125, + "loss": 1.3443, + "margin_dpo/margin_mean": 0.43892422318458557, + "margin_dpo/margin_std": 0.7477631568908691, + "step": 43 + }, + { + "KL/chosen_KL_mean": -0.026058197021484375, + "KL/mean": -0.2544378638267517, + "KL/rejected_KL_mean": -0.4828224182128906, + "KL/std": 0.5010477304458618, + "epoch": 0.06461086637298091, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.456767737865448, + "fcm_dpo/q_t": 0.48859721422195435, + "grad_norm": 34.155967712402344, + "learning_rate": 3.115942028985507e-07, + "logits/chosen": 0.8367877006530762, + "logits/rejected": 0.9573251605033875, + "logps/chosen": -81.90437316894531, + "logps/ref_chosen": -81.8783187866211, + "logps/ref_rejected": -113.421630859375, + "logps/rejected": -113.90444946289062, + "loss": 1.342, + "margin_dpo/margin_mean": 0.45676785707473755, + "margin_dpo/margin_std": 0.5923409461975098, + "step": 44 + }, + { + "KL/chosen_KL_mean": -0.11049652099609375, + "KL/mean": -0.22296538949012756, + "KL/rejected_KL_mean": -0.3354301452636719, + "KL/std": 0.4330664277076721, + "epoch": 0.06607929515418502, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.22493408620357513, + "fcm_dpo/q_t": 0.49438217282295227, + "grad_norm": 24.40647315979004, + "learning_rate": 3.188405797101449e-07, + "logits/chosen": 0.48652714490890503, + "logits/rejected": 0.5948389172554016, + "logps/chosen": -77.45509338378906, + "logps/ref_chosen": -77.34459686279297, + "logps/ref_rejected": -84.24774169921875, + "logps/rejected": -84.58317565917969, + "loss": 1.3647, + "margin_dpo/margin_mean": 0.224934920668602, + "margin_dpo/margin_std": 0.5600230097770691, + "step": 45 + }, + { + "KL/chosen_KL_mean": 0.03125762939453125, + "KL/mean": -0.21650341153144836, + "KL/rejected_KL_mean": -0.4642601013183594, + "KL/std": 0.5851463079452515, + "epoch": 0.06754772393538913, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.49552083015441895, + "fcm_dpo/q_t": 0.4876311123371124, + "grad_norm": 32.96702575683594, + "learning_rate": 3.260869565217391e-07, + "logits/chosen": 0.4339357316493988, + "logits/rejected": 0.6726645231246948, + "logps/chosen": -90.3095703125, + "logps/ref_chosen": -90.3408203125, + "logps/ref_rejected": -100.676513671875, + "logps/rejected": -101.14077758789062, + "loss": 1.3386, + "margin_dpo/margin_mean": 0.49552121758461, + "margin_dpo/margin_std": 0.7012072205543518, + "step": 46 + }, + { + "KL/chosen_KL_mean": -0.03223419189453125, + "KL/mean": -0.2075231373310089, + "KL/rejected_KL_mean": -0.3828163146972656, + "KL/std": 0.5406893491744995, + "epoch": 0.06901615271659324, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.3505786061286926, + "fcm_dpo/q_t": 0.49125438928604126, + "grad_norm": 34.141780853271484, + "learning_rate": 3.333333333333333e-07, + "logits/chosen": 0.40441077947616577, + "logits/rejected": 0.5829853415489197, + "logps/chosen": -104.44353485107422, + "logps/ref_chosen": -104.41130065917969, + "logps/ref_rejected": -101.25489807128906, + "logps/rejected": -101.63771057128906, + "loss": 1.353, + "margin_dpo/margin_mean": 0.35057854652404785, + "margin_dpo/margin_std": 0.7464984655380249, + "step": 47 + }, + { + "KL/chosen_KL_mean": -0.028476715087890625, + "KL/mean": -0.26390624046325684, + "KL/rejected_KL_mean": -0.49933624267578125, + "KL/std": 0.634148120880127, + "epoch": 0.07048458149779736, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.47085991501808167, + "fcm_dpo/q_t": 0.4882550835609436, + "grad_norm": 38.913291931152344, + "learning_rate": 3.4057971014492755e-07, + "logits/chosen": 0.15821820497512817, + "logits/rejected": 0.35422125458717346, + "logps/chosen": -91.12875366210938, + "logps/ref_chosen": -91.10027313232422, + "logps/ref_rejected": -95.08058166503906, + "logps/rejected": -95.57991027832031, + "loss": 1.3414, + "margin_dpo/margin_mean": 0.47086048126220703, + "margin_dpo/margin_std": 0.8205849528312683, + "step": 48 + }, + { + "KL/chosen_KL_mean": 0.06631088256835938, + "KL/mean": -0.3106277883052826, + "KL/rejected_KL_mean": -0.6875686645507812, + "KL/std": 0.7828932404518127, + "epoch": 0.07195301027900147, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.7538716197013855, + "fcm_dpo/q_t": 0.48122939467430115, + "grad_norm": 39.63159942626953, + "learning_rate": 3.478260869565217e-07, + "logits/chosen": 0.054642464965581894, + "logits/rejected": 0.5203364491462708, + "logps/chosen": -92.93737030029297, + "logps/ref_chosen": -93.00367736816406, + "logps/ref_rejected": -91.74899291992188, + "logps/rejected": -92.43656921386719, + "loss": 1.3149, + "margin_dpo/margin_mean": 0.7538715600967407, + "margin_dpo/margin_std": 0.9957276582717896, + "step": 49 + }, + { + "KL/chosen_KL_mean": -0.1480712890625, + "KL/mean": -0.4775815010070801, + "KL/rejected_KL_mean": -0.8070831298828125, + "KL/std": 0.775669515132904, + "epoch": 0.07342143906020558, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.6590114235877991, + "fcm_dpo/q_t": 0.4835768938064575, + "grad_norm": 33.30721664428711, + "learning_rate": 3.5507246376811595e-07, + "logits/chosen": 0.5360842943191528, + "logits/rejected": 0.7067970037460327, + "logps/chosen": -94.77488708496094, + "logps/ref_chosen": -94.62681579589844, + "logps/ref_rejected": -103.57435607910156, + "logps/rejected": -104.38143920898438, + "loss": 1.3236, + "margin_dpo/margin_mean": 0.6590113639831543, + "margin_dpo/margin_std": 0.9176972508430481, + "step": 50 + }, + { + "KL/chosen_KL_mean": -0.054714202880859375, + "KL/mean": -0.3807927966117859, + "KL/rejected_KL_mean": -0.7068710327148438, + "KL/std": 0.8505513072013855, + "epoch": 0.07488986784140969, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.6521493792533875, + "fcm_dpo/q_t": 0.4837867021560669, + "grad_norm": 29.927831649780273, + "learning_rate": 3.6231884057971015e-07, + "logits/chosen": 0.7329905033111572, + "logits/rejected": 0.8798360824584961, + "logps/chosen": -87.56199645996094, + "logps/ref_chosen": -87.50727844238281, + "logps/ref_rejected": -83.47235107421875, + "logps/rejected": -84.17922973632812, + "loss": 1.3253, + "margin_dpo/margin_mean": 0.6521490812301636, + "margin_dpo/margin_std": 1.0980899333953857, + "step": 51 + }, + { + "KL/chosen_KL_mean": 0.0141448974609375, + "KL/mean": -0.411986768245697, + "KL/rejected_KL_mean": -0.8381233215332031, + "KL/std": 0.778254508972168, + "epoch": 0.0763582966226138, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.8522641658782959, + "fcm_dpo/q_t": 0.47876590490341187, + "grad_norm": 31.903409957885742, + "learning_rate": 3.695652173913043e-07, + "logits/chosen": 0.2494804561138153, + "logits/rejected": 0.449777215719223, + "logps/chosen": -90.61611938476562, + "logps/ref_chosen": -90.63026428222656, + "logps/ref_rejected": -87.0390625, + "logps/rejected": -87.87718200683594, + "loss": 1.305, + "margin_dpo/margin_mean": 0.852264404296875, + "margin_dpo/margin_std": 0.916897714138031, + "step": 52 + }, + { + "KL/chosen_KL_mean": -0.0498809814453125, + "KL/mean": -0.5524184107780457, + "KL/rejected_KL_mean": -1.0549507141113281, + "KL/std": 1.1000334024429321, + "epoch": 0.07782672540381791, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.0050727128982544, + "fcm_dpo/q_t": 0.4751381278038025, + "grad_norm": 39.47797393798828, + "learning_rate": 3.7681159420289855e-07, + "logits/chosen": 0.08327862620353699, + "logits/rejected": 0.19521111249923706, + "logps/chosen": -81.6329574584961, + "logps/ref_chosen": -81.58307647705078, + "logps/ref_rejected": -95.66152954101562, + "logps/rejected": -96.71647644042969, + "loss": 1.2935, + "margin_dpo/margin_mean": 1.0050725936889648, + "margin_dpo/margin_std": 1.3506872653961182, + "step": 53 + }, + { + "KL/chosen_KL_mean": -0.038482666015625, + "KL/mean": -0.5898535251617432, + "KL/rejected_KL_mean": -1.1412277221679688, + "KL/std": 1.2504780292510986, + "epoch": 0.07929515418502203, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.1027414798736572, + "fcm_dpo/q_t": 0.47257959842681885, + "grad_norm": 40.124717712402344, + "learning_rate": 3.8405797101449274e-07, + "logits/chosen": 0.2811519503593445, + "logits/rejected": 0.45574668049812317, + "logps/chosen": -88.94865417480469, + "logps/ref_chosen": -88.91017150878906, + "logps/ref_rejected": -99.1175537109375, + "logps/rejected": -100.25878143310547, + "loss": 1.2826, + "margin_dpo/margin_mean": 1.1027413606643677, + "margin_dpo/margin_std": 1.1984407901763916, + "step": 54 + }, + { + "KL/chosen_KL_mean": 0.14495849609375, + "KL/mean": -0.44793444871902466, + "KL/rejected_KL_mean": -1.0408248901367188, + "KL/std": 1.178361415863037, + "epoch": 0.08076358296622614, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.1857811212539673, + "fcm_dpo/q_t": 0.4706498384475708, + "grad_norm": 35.26246643066406, + "learning_rate": 3.9130434782608694e-07, + "logits/chosen": 0.018801812082529068, + "logits/rejected": 0.3150080442428589, + "logps/chosen": -92.31095886230469, + "logps/ref_chosen": -92.45591735839844, + "logps/ref_rejected": -92.97093200683594, + "logps/rejected": -94.01176452636719, + "loss": 1.2772, + "margin_dpo/margin_mean": 1.1857811212539673, + "margin_dpo/margin_std": 1.4991127252578735, + "step": 55 + }, + { + "KL/chosen_KL_mean": -0.08123016357421875, + "KL/mean": -0.6413096189498901, + "KL/rejected_KL_mean": -1.201385498046875, + "KL/std": 1.1786679029464722, + "epoch": 0.08223201174743025, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.1201488971710205, + "fcm_dpo/q_t": 0.47230932116508484, + "grad_norm": 33.10283279418945, + "learning_rate": 3.9855072463768114e-07, + "logits/chosen": 0.1805080771446228, + "logits/rejected": 0.5188891887664795, + "logps/chosen": -87.31788635253906, + "logps/ref_chosen": -87.23665618896484, + "logps/ref_rejected": -100.64553833007812, + "logps/rejected": -101.846923828125, + "loss": 1.2837, + "margin_dpo/margin_mean": 1.1201484203338623, + "margin_dpo/margin_std": 1.5858896970748901, + "step": 56 + }, + { + "KL/chosen_KL_mean": -0.0514068603515625, + "KL/mean": -0.6366605758666992, + "KL/rejected_KL_mean": -1.2219200134277344, + "KL/std": 1.2971317768096924, + "epoch": 0.08370044052863436, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.1705279350280762, + "fcm_dpo/q_t": 0.4710506796836853, + "grad_norm": 33.408851623535156, + "learning_rate": 4.057971014492754e-07, + "logits/chosen": 0.5278745889663696, + "logits/rejected": 0.7557357549667358, + "logps/chosen": -98.2021484375, + "logps/ref_chosen": -98.15074157714844, + "logps/ref_rejected": -101.05284118652344, + "logps/rejected": -102.2747573852539, + "loss": 1.2793, + "margin_dpo/margin_mean": 1.1705279350280762, + "margin_dpo/margin_std": 1.6327052116394043, + "step": 57 + }, + { + "KL/chosen_KL_mean": 0.21571731567382812, + "KL/mean": -0.5199761986732483, + "KL/rejected_KL_mean": -1.2556610107421875, + "KL/std": 1.4040195941925049, + "epoch": 0.08516886930983847, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.4713754653930664, + "fcm_dpo/q_t": 0.46378639340400696, + "grad_norm": 38.12102508544922, + "learning_rate": 4.1304347826086954e-07, + "logits/chosen": 0.4386252462863922, + "logits/rejected": 0.7860119938850403, + "logps/chosen": -99.36524963378906, + "logps/ref_chosen": -99.58097076416016, + "logps/ref_rejected": -91.22227478027344, + "logps/rejected": -92.47793579101562, + "loss": 1.2535, + "margin_dpo/margin_mean": 1.4713757038116455, + "margin_dpo/margin_std": 1.8455278873443604, + "step": 58 + }, + { + "KL/chosen_KL_mean": 0.03876495361328125, + "KL/mean": -0.6094352006912231, + "KL/rejected_KL_mean": -1.2576370239257812, + "KL/std": 1.2375140190124512, + "epoch": 0.08663729809104258, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.2963955402374268, + "fcm_dpo/q_t": 0.46789902448654175, + "grad_norm": 34.11338424682617, + "learning_rate": 4.2028985507246374e-07, + "logits/chosen": 0.09331036359071732, + "logits/rejected": 0.28541117906570435, + "logps/chosen": -89.76356506347656, + "logps/ref_chosen": -89.80232238769531, + "logps/ref_rejected": -94.27667236328125, + "logps/rejected": -95.53431701660156, + "loss": 1.2667, + "margin_dpo/margin_mean": 1.2963950634002686, + "margin_dpo/margin_std": 1.5107533931732178, + "step": 59 + }, + { + "KL/chosen_KL_mean": 0.030788421630859375, + "KL/mean": -0.5591611862182617, + "KL/rejected_KL_mean": -1.1491165161132812, + "KL/std": 1.2311577796936035, + "epoch": 0.0881057268722467, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.1799085140228271, + "fcm_dpo/q_t": 0.4707183837890625, + "grad_norm": 28.915138244628906, + "learning_rate": 4.2753623188405794e-07, + "logits/chosen": 0.17282041907310486, + "logits/rejected": 0.42514222860336304, + "logps/chosen": -95.12492370605469, + "logps/ref_chosen": -95.15571594238281, + "logps/ref_rejected": -91.3724365234375, + "logps/rejected": -92.52155303955078, + "loss": 1.2764, + "margin_dpo/margin_mean": 1.1799083948135376, + "margin_dpo/margin_std": 1.3647682666778564, + "step": 60 + }, + { + "KL/chosen_KL_mean": -0.2078704833984375, + "KL/mean": -0.7680006623268127, + "KL/rejected_KL_mean": -1.3281364440917969, + "KL/std": 1.3514353036880493, + "epoch": 0.08957415565345081, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.1202651262283325, + "fcm_dpo/q_t": 0.472463458776474, + "grad_norm": 28.457597732543945, + "learning_rate": 4.3478260869565214e-07, + "logits/chosen": 0.1602925956249237, + "logits/rejected": 0.30045902729034424, + "logps/chosen": -85.94017791748047, + "logps/ref_chosen": -85.73230743408203, + "logps/ref_rejected": -97.96575927734375, + "logps/rejected": -99.29389953613281, + "loss": 1.2858, + "margin_dpo/margin_mean": 1.1202644109725952, + "margin_dpo/margin_std": 1.7892072200775146, + "step": 61 + }, + { + "KL/chosen_KL_mean": -0.09750747680664062, + "KL/mean": -0.731864333152771, + "KL/rejected_KL_mean": -1.3662223815917969, + "KL/std": 1.4225494861602783, + "epoch": 0.09104258443465492, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.2687207460403442, + "fcm_dpo/q_t": 0.46867483854293823, + "grad_norm": 28.60158348083496, + "learning_rate": 4.420289855072464e-07, + "logits/chosen": 0.15727725625038147, + "logits/rejected": 0.3492550551891327, + "logps/chosen": -81.73289489746094, + "logps/ref_chosen": -81.63538360595703, + "logps/ref_rejected": -84.03831481933594, + "logps/rejected": -85.40453338623047, + "loss": 1.2729, + "margin_dpo/margin_mean": 1.2687196731567383, + "margin_dpo/margin_std": 1.855647325515747, + "step": 62 + }, + { + "KL/chosen_KL_mean": -0.033748626708984375, + "KL/mean": -0.7986141443252563, + "KL/rejected_KL_mean": -1.5634803771972656, + "KL/std": 1.3721587657928467, + "epoch": 0.09251101321585903, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.5297342538833618, + "fcm_dpo/q_t": 0.46211713552474976, + "grad_norm": 35.054710388183594, + "learning_rate": 4.4927536231884053e-07, + "logits/chosen": 0.5110536217689514, + "logits/rejected": 0.7312678098678589, + "logps/chosen": -103.65780639648438, + "logps/ref_chosen": -103.62405395507812, + "logps/ref_rejected": -103.40303039550781, + "logps/rejected": -104.96650695800781, + "loss": 1.2454, + "margin_dpo/margin_mean": 1.5297338962554932, + "margin_dpo/margin_std": 1.5587282180786133, + "step": 63 + }, + { + "KL/chosen_KL_mean": -0.10355758666992188, + "KL/mean": -0.9556412100791931, + "KL/rejected_KL_mean": -1.8077392578125, + "KL/std": 1.78635835647583, + "epoch": 0.09397944199706314, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.7041810750961304, + "fcm_dpo/q_t": 0.4582006335258484, + "grad_norm": 35.068538665771484, + "learning_rate": 4.5652173913043473e-07, + "logits/chosen": 0.6043516993522644, + "logits/rejected": 0.8259673118591309, + "logps/chosen": -87.10514831542969, + "logps/ref_chosen": -87.0015869140625, + "logps/ref_rejected": -100.5854721069336, + "logps/rejected": -102.3932113647461, + "loss": 1.2341, + "margin_dpo/margin_mean": 1.7041810750961304, + "margin_dpo/margin_std": 2.0286264419555664, + "step": 64 + }, + { + "KL/chosen_KL_mean": -0.0607757568359375, + "KL/mean": -0.9677102565765381, + "KL/rejected_KL_mean": -1.8746452331542969, + "KL/std": 1.6561270952224731, + "epoch": 0.09544787077826726, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.8138678073883057, + "fcm_dpo/q_t": 0.45543304085731506, + "grad_norm": 38.30107116699219, + "learning_rate": 4.63768115942029e-07, + "logits/chosen": -0.06661674380302429, + "logits/rejected": 0.07723595947027206, + "logps/chosen": -91.2826919555664, + "logps/ref_chosen": -91.22191619873047, + "logps/ref_rejected": -115.33553314208984, + "logps/rejected": -117.21017456054688, + "loss": 1.2233, + "margin_dpo/margin_mean": 1.8138679265975952, + "margin_dpo/margin_std": 2.0140886306762695, + "step": 65 + }, + { + "KL/chosen_KL_mean": -0.33434295654296875, + "KL/mean": -0.9711505770683289, + "KL/rejected_KL_mean": -1.6079597473144531, + "KL/std": 1.5632513761520386, + "epoch": 0.09691629955947137, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.2736104726791382, + "fcm_dpo/q_t": 0.46861565113067627, + "grad_norm": 25.61758041381836, + "learning_rate": 4.7101449275362313e-07, + "logits/chosen": 0.30933254957199097, + "logits/rejected": 0.5222980380058289, + "logps/chosen": -84.11856842041016, + "logps/ref_chosen": -83.78422546386719, + "logps/ref_rejected": -82.7520980834961, + "logps/rejected": -84.36006164550781, + "loss": 1.2718, + "margin_dpo/margin_mean": 1.2736105918884277, + "margin_dpo/margin_std": 1.8205499649047852, + "step": 66 + }, + { + "KL/chosen_KL_mean": -0.8533248901367188, + "KL/mean": -1.4257209300994873, + "KL/rejected_KL_mean": -1.9981117248535156, + "KL/std": 2.0455029010772705, + "epoch": 0.09838472834067548, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.1447845697402954, + "fcm_dpo/q_t": 0.4723505675792694, + "grad_norm": 24.28444480895996, + "learning_rate": 4.782608695652174e-07, + "logits/chosen": 0.4081868529319763, + "logits/rejected": 0.6900854110717773, + "logps/chosen": -88.52629089355469, + "logps/ref_chosen": -87.67295837402344, + "logps/ref_rejected": -79.08674621582031, + "logps/rejected": -81.0848617553711, + "loss": 1.2942, + "margin_dpo/margin_mean": 1.144784927368164, + "margin_dpo/margin_std": 2.788841962814331, + "step": 67 + }, + { + "KL/chosen_KL_mean": -0.24430465698242188, + "KL/mean": -1.2685749530792236, + "KL/rejected_KL_mean": -2.2928390502929688, + "KL/std": 2.1893463134765625, + "epoch": 0.09985315712187959, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 2.048532247543335, + "fcm_dpo/q_t": 0.4505225419998169, + "grad_norm": 28.59839630126953, + "learning_rate": 4.855072463768116e-07, + "logits/chosen": 0.24755343794822693, + "logits/rejected": 0.5789214968681335, + "logps/chosen": -97.32315063476562, + "logps/ref_chosen": -97.07884216308594, + "logps/ref_rejected": -84.11872863769531, + "logps/rejected": -86.41156768798828, + "loss": 1.2099, + "margin_dpo/margin_mean": 2.048532009124756, + "margin_dpo/margin_std": 2.707021713256836, + "step": 68 + }, + { + "KL/chosen_KL_mean": -0.33158111572265625, + "KL/mean": -1.3999078273773193, + "KL/rejected_KL_mean": -2.4682350158691406, + "KL/std": 2.3814938068389893, + "epoch": 0.1013215859030837, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 2.1366519927978516, + "fcm_dpo/q_t": 0.4485386610031128, + "grad_norm": 30.916728973388672, + "learning_rate": 4.927536231884058e-07, + "logits/chosen": 0.06354180723428726, + "logits/rejected": 0.17590491473674774, + "logps/chosen": -86.0512924194336, + "logps/ref_chosen": -85.71971130371094, + "logps/ref_rejected": -109.48020935058594, + "logps/rejected": -111.94844055175781, + "loss": 1.2055, + "margin_dpo/margin_mean": 2.1366517543792725, + "margin_dpo/margin_std": 2.973220109939575, + "step": 69 + }, + { + "KL/chosen_KL_mean": -0.6108512878417969, + "KL/mean": -1.9039943218231201, + "KL/rejected_KL_mean": -3.1971397399902344, + "KL/std": 2.7748351097106934, + "epoch": 0.1027900146842878, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 2.586289405822754, + "fcm_dpo/q_t": 0.4384194016456604, + "grad_norm": 29.341352462768555, + "learning_rate": 5e-07, + "logits/chosen": 0.17052993178367615, + "logits/rejected": 0.343351811170578, + "logps/chosen": -95.62080383300781, + "logps/ref_chosen": -95.00994873046875, + "logps/ref_rejected": -96.21272277832031, + "logps/rejected": -99.40986633300781, + "loss": 1.1722, + "margin_dpo/margin_mean": 2.5862884521484375, + "margin_dpo/margin_std": 3.324324131011963, + "step": 70 + }, + { + "KL/chosen_KL_mean": -0.036632537841796875, + "KL/mean": -1.8997746706008911, + "KL/rejected_KL_mean": -3.7629127502441406, + "KL/std": 3.59867525100708, + "epoch": 0.10425844346549193, + "fcm_dpo/beta": 0.10017916560173035, + "fcm_dpo/delta": 0.027692411094903946, + "fcm_dpo/margin": 3.726283073425293, + "fcm_dpo/q_t": 0.4132327437400818, + "grad_norm": 34.35282897949219, + "learning_rate": 4.999967061337492e-07, + "logits/chosen": 0.06169853359460831, + "logits/rejected": 0.4402628540992737, + "logps/chosen": -91.01398468017578, + "logps/ref_chosen": -90.97735595703125, + "logps/ref_rejected": -102.59103393554688, + "logps/rejected": -106.35394287109375, + "loss": 1.0866, + "margin_dpo/margin_mean": 3.726283073425293, + "margin_dpo/margin_std": 4.136852741241455, + "step": 71 + }, + { + "KL/chosen_KL_mean": -0.5575103759765625, + "KL/mean": -2.022162437438965, + "KL/rejected_KL_mean": -3.48681640625, + "KL/std": 2.940340518951416, + "epoch": 0.10572687224669604, + "fcm_dpo/beta": 0.10143917053937912, + "fcm_dpo/delta": 0.04375329613685608, + "fcm_dpo/margin": 2.929307460784912, + "fcm_dpo/q_t": 0.4291730523109436, + "grad_norm": 29.753665924072266, + "learning_rate": 4.999868246217933e-07, + "logits/chosen": -0.42643800377845764, + "logits/rejected": -0.12373970448970795, + "logps/chosen": -98.44903564453125, + "logps/ref_chosen": -97.89152526855469, + "logps/ref_rejected": -100.19171142578125, + "logps/rejected": -103.67852783203125, + "loss": 1.1387, + "margin_dpo/margin_mean": 2.929306983947754, + "margin_dpo/margin_std": 3.3551158905029297, + "step": 72 + }, + { + "KL/chosen_KL_mean": -0.7645721435546875, + "KL/mean": -2.3280248641967773, + "KL/rejected_KL_mean": -3.8914794921875, + "KL/std": 3.8170700073242188, + "epoch": 0.10719530102790015, + "fcm_dpo/beta": 0.10270863771438599, + "fcm_dpo/delta": 0.08143049478530884, + "fcm_dpo/margin": 3.12691330909729, + "fcm_dpo/q_t": 0.42775601148605347, + "grad_norm": 28.466285705566406, + "learning_rate": 4.999703557245192e-07, + "logits/chosen": -0.38065868616104126, + "logits/rejected": -0.07968415319919586, + "logps/chosen": -96.5335922241211, + "logps/ref_chosen": -95.7690200805664, + "logps/ref_rejected": -95.93243408203125, + "logps/rejected": -99.82391357421875, + "loss": 1.1475, + "margin_dpo/margin_mean": 3.126913547515869, + "margin_dpo/margin_std": 4.862720489501953, + "step": 73 + }, + { + "KL/chosen_KL_mean": -0.9901924133300781, + "KL/mean": -2.7506957054138184, + "KL/rejected_KL_mean": -4.511203765869141, + "KL/std": 3.799988269805908, + "epoch": 0.10866372980910426, + "fcm_dpo/beta": 0.10353071987628937, + "fcm_dpo/delta": 0.0368211604654789, + "fcm_dpo/margin": 3.5210113525390625, + "fcm_dpo/q_t": 0.41603541374206543, + "grad_norm": 31.4624080657959, + "learning_rate": 4.999472998758977e-07, + "logits/chosen": -0.16242395341396332, + "logits/rejected": -0.018897918984293938, + "logps/chosen": -79.798583984375, + "logps/ref_chosen": -78.80839538574219, + "logps/ref_rejected": -101.64677429199219, + "logps/rejected": -106.15797424316406, + "loss": 1.106, + "margin_dpo/margin_mean": 3.5210108757019043, + "margin_dpo/margin_std": 4.538544654846191, + "step": 74 + }, + { + "KL/chosen_KL_mean": -1.0562019348144531, + "KL/mean": -3.3269500732421875, + "KL/rejected_KL_mean": -5.597698211669922, + "KL/std": 4.597766876220703, + "epoch": 0.11013215859030837, + "fcm_dpo/beta": 0.10317438840866089, + "fcm_dpo/delta": -0.07197670638561249, + "fcm_dpo/margin": 4.541502952575684, + "fcm_dpo/q_t": 0.3949120342731476, + "grad_norm": 32.846946716308594, + "learning_rate": 4.999176576834721e-07, + "logits/chosen": -0.3875320553779602, + "logits/rejected": -0.18362751603126526, + "logps/chosen": -79.33805847167969, + "logps/ref_chosen": -78.28185272216797, + "logps/ref_rejected": -115.40311431884766, + "logps/rejected": -121.00080871582031, + "loss": 1.0407, + "margin_dpo/margin_mean": 4.541502952575684, + "margin_dpo/margin_std": 5.405215263366699, + "step": 75 + }, + { + "KL/chosen_KL_mean": -1.438232421875, + "KL/mean": -2.984178066253662, + "KL/rejected_KL_mean": -4.530128479003906, + "KL/std": 3.786229133605957, + "epoch": 0.11160058737151249, + "fcm_dpo/beta": 0.10365289449691772, + "fcm_dpo/delta": 0.08214224129915237, + "fcm_dpo/margin": 3.0919015407562256, + "fcm_dpo/q_t": 0.4267314672470093, + "grad_norm": 23.542400360107422, + "learning_rate": 4.998814299283415e-07, + "logits/chosen": -0.6625537276268005, + "logits/rejected": -0.2867332696914673, + "logps/chosen": -89.31536865234375, + "logps/ref_chosen": -87.87713623046875, + "logps/ref_rejected": -85.71968078613281, + "logps/rejected": -90.24980926513672, + "loss": 1.1433, + "margin_dpo/margin_mean": 3.0919017791748047, + "margin_dpo/margin_std": 4.564672470092773, + "step": 76 + }, + { + "KL/chosen_KL_mean": -1.6675910949707031, + "KL/mean": -3.354740619659424, + "KL/rejected_KL_mean": -5.041893005371094, + "KL/std": 4.480142593383789, + "epoch": 0.1130690161527166, + "fcm_dpo/beta": 0.10463721305131912, + "fcm_dpo/delta": 0.048658907413482666, + "fcm_dpo/margin": 3.374298095703125, + "fcm_dpo/q_t": 0.4195900559425354, + "grad_norm": 27.999977111816406, + "learning_rate": 4.998386175651409e-07, + "logits/chosen": -0.21670196950435638, + "logits/rejected": 0.1504397988319397, + "logps/chosen": -101.36793518066406, + "logps/ref_chosen": -99.70034790039062, + "logps/ref_rejected": -98.20576477050781, + "logps/rejected": -103.24765014648438, + "loss": 1.1312, + "margin_dpo/margin_mean": 3.374298334121704, + "margin_dpo/margin_std": 5.108901500701904, + "step": 77 + }, + { + "KL/chosen_KL_mean": -0.7116546630859375, + "KL/mean": -2.511979103088379, + "KL/rejected_KL_mean": -4.312297821044922, + "KL/std": 3.678145170211792, + "epoch": 0.1145374449339207, + "fcm_dpo/beta": 0.10538917779922485, + "fcm_dpo/delta": 0.021346315741539, + "fcm_dpo/margin": 3.600642681121826, + "fcm_dpo/q_t": 0.41206055879592896, + "grad_norm": 26.094635009765625, + "learning_rate": 4.997892217220159e-07, + "logits/chosen": 0.1686553955078125, + "logits/rejected": 0.4075925946235657, + "logps/chosen": -91.00836181640625, + "logps/ref_chosen": -90.29670715332031, + "logps/ref_rejected": -91.13772583007812, + "logps/rejected": -95.45002746582031, + "loss": 1.0902, + "margin_dpo/margin_mean": 3.600642442703247, + "margin_dpo/margin_std": 4.339991092681885, + "step": 78 + }, + { + "KL/chosen_KL_mean": -0.8355979919433594, + "KL/mean": -3.2423648834228516, + "KL/rejected_KL_mean": -5.649135589599609, + "KL/std": 5.327904224395752, + "epoch": 0.11600587371512482, + "fcm_dpo/beta": 0.10404185205698013, + "fcm_dpo/delta": -0.10601618885993958, + "fcm_dpo/margin": 4.813536643981934, + "fcm_dpo/q_t": 0.3878142833709717, + "grad_norm": 27.339481353759766, + "learning_rate": 4.997332437005931e-07, + "logits/chosen": -0.5722988843917847, + "logits/rejected": -0.1865392029285431, + "logps/chosen": -87.21392822265625, + "logps/ref_chosen": -86.37832641601562, + "logps/ref_rejected": -94.10777282714844, + "logps/rejected": -99.75690460205078, + "loss": 1.0278, + "margin_dpo/margin_mean": 4.813536643981934, + "margin_dpo/margin_std": 5.867785453796387, + "step": 79 + }, + { + "KL/chosen_KL_mean": -1.5975379943847656, + "KL/mean": -3.4530553817749023, + "KL/rejected_KL_mean": -5.308574676513672, + "KL/std": 4.671290874481201, + "epoch": 0.11747430249632893, + "fcm_dpo/beta": 0.10355690121650696, + "fcm_dpo/delta": 0.016326699405908585, + "fcm_dpo/margin": 3.7110366821289062, + "fcm_dpo/q_t": 0.41360679268836975, + "grad_norm": 26.259410858154297, + "learning_rate": 4.996706849759452e-07, + "logits/chosen": -0.7483689188957214, + "logits/rejected": -0.39665502309799194, + "logps/chosen": -95.56785583496094, + "logps/ref_chosen": -93.97032165527344, + "logps/ref_rejected": -92.57441711425781, + "logps/rejected": -97.88299560546875, + "loss": 1.1026, + "margin_dpo/margin_mean": 3.7110366821289062, + "margin_dpo/margin_std": 5.134080410003662, + "step": 80 + }, + { + "KL/chosen_KL_mean": -1.1007575988769531, + "KL/mean": -3.7640862464904785, + "KL/rejected_KL_mean": -6.427410125732422, + "KL/std": 5.826144218444824, + "epoch": 0.11894273127753303, + "fcm_dpo/beta": 0.10087832808494568, + "fcm_dpo/delta": -0.1465744972229004, + "fcm_dpo/margin": 5.326655387878418, + "fcm_dpo/q_t": 0.38472476601600647, + "grad_norm": 27.61595344543457, + "learning_rate": 4.996015471965529e-07, + "logits/chosen": -0.569584846496582, + "logits/rejected": -0.14743685722351074, + "logps/chosen": -100.93087768554688, + "logps/ref_chosen": -99.83012390136719, + "logps/ref_rejected": -133.67247009277344, + "logps/rejected": -140.09988403320312, + "loss": 1.0215, + "margin_dpo/margin_mean": 5.326655387878418, + "margin_dpo/margin_std": 6.914580821990967, + "step": 81 + }, + { + "KL/chosen_KL_mean": -2.0198593139648438, + "KL/mean": -3.8523612022399902, + "KL/rejected_KL_mean": -5.6848602294921875, + "KL/std": 4.556976318359375, + "epoch": 0.12041116005873716, + "fcm_dpo/beta": 0.10029402375221252, + "fcm_dpo/delta": -0.037920787930488586, + "fcm_dpo/margin": 3.665005922317505, + "fcm_dpo/q_t": 0.4152594804763794, + "grad_norm": 26.13265609741211, + "learning_rate": 4.995258321842611e-07, + "logits/chosen": -0.048558514565229416, + "logits/rejected": 0.17264960706233978, + "logps/chosen": -85.06584167480469, + "logps/ref_chosen": -83.04598236083984, + "logps/ref_rejected": -94.52595520019531, + "logps/rejected": -100.2108154296875, + "loss": 1.1115, + "margin_dpo/margin_mean": 3.665005922317505, + "margin_dpo/margin_std": 4.93629789352417, + "step": 82 + }, + { + "KL/chosen_KL_mean": -2.360248565673828, + "KL/mean": -4.392603397369385, + "KL/rejected_KL_mean": -6.424961090087891, + "KL/std": 5.152520179748535, + "epoch": 0.12187958883994127, + "fcm_dpo/beta": 0.09982265532016754, + "fcm_dpo/delta": -0.005994098260998726, + "fcm_dpo/margin": 4.064716339111328, + "fcm_dpo/q_t": 0.410980761051178, + "grad_norm": 27.43876075744629, + "learning_rate": 4.994435419342304e-07, + "logits/chosen": -0.32917851209640503, + "logits/rejected": 0.10240879654884338, + "logps/chosen": -94.53646850585938, + "logps/ref_chosen": -92.17621612548828, + "logps/ref_rejected": -107.74464416503906, + "logps/rejected": -114.16960144042969, + "loss": 1.1153, + "margin_dpo/margin_mean": 4.064716339111328, + "margin_dpo/margin_std": 6.495458602905273, + "step": 83 + }, + { + "KL/chosen_KL_mean": -2.9121971130371094, + "KL/mean": -4.227639198303223, + "KL/rejected_KL_mean": -5.543083190917969, + "KL/std": 4.596038818359375, + "epoch": 0.12334801762114538, + "fcm_dpo/beta": 0.09968481957912445, + "fcm_dpo/delta": -0.005473073106259108, + "fcm_dpo/margin": 2.630887031555176, + "fcm_dpo/q_t": 0.4405984878540039, + "grad_norm": 28.258586883544922, + "learning_rate": 4.993546786148857e-07, + "logits/chosen": -0.20731906592845917, + "logits/rejected": 0.20152854919433594, + "logps/chosen": -104.43869018554688, + "logps/ref_chosen": -101.5264892578125, + "logps/ref_rejected": -92.42608642578125, + "logps/rejected": -97.96916961669922, + "loss": 1.2033, + "margin_dpo/margin_mean": 2.630887031555176, + "margin_dpo/margin_std": 4.9484453201293945, + "step": 84 + }, + { + "KL/chosen_KL_mean": -2.679340362548828, + "KL/mean": -4.38173246383667, + "KL/rejected_KL_mean": -6.0841217041015625, + "KL/std": 5.083308696746826, + "epoch": 0.12481644640234948, + "fcm_dpo/beta": 0.09959776699542999, + "fcm_dpo/delta": -0.008740188553929329, + "fcm_dpo/margin": 3.4047844409942627, + "fcm_dpo/q_t": 0.4238542914390564, + "grad_norm": 26.950847625732422, + "learning_rate": 4.992592445678582e-07, + "logits/chosen": -0.49466782808303833, + "logits/rejected": -0.1743880957365036, + "logps/chosen": -98.80671691894531, + "logps/ref_chosen": -96.12738037109375, + "logps/ref_rejected": -85.05519104003906, + "logps/rejected": -91.13931274414062, + "loss": 1.1636, + "margin_dpo/margin_mean": 3.40478515625, + "margin_dpo/margin_std": 6.143662929534912, + "step": 85 + }, + { + "KL/chosen_KL_mean": -2.0633888244628906, + "KL/mean": -4.257645606994629, + "KL/rejected_KL_mean": -6.451904296875, + "KL/std": 5.867281913757324, + "epoch": 0.1262848751835536, + "fcm_dpo/beta": 0.09884131699800491, + "fcm_dpo/delta": -0.035395264625549316, + "fcm_dpo/margin": 4.388519287109375, + "fcm_dpo/q_t": 0.4091922342777252, + "grad_norm": 22.061201095581055, + "learning_rate": 4.991572423079235e-07, + "logits/chosen": -0.56908118724823, + "logits/rejected": -0.3259069323539734, + "logps/chosen": -83.76765441894531, + "logps/ref_chosen": -81.70426177978516, + "logps/ref_rejected": -93.65550231933594, + "logps/rejected": -100.10740661621094, + "loss": 1.1072, + "margin_dpo/margin_mean": 4.388518810272217, + "margin_dpo/margin_std": 7.296498775482178, + "step": 86 + }, + { + "KL/chosen_KL_mean": -2.423919677734375, + "KL/mean": -4.786927223205566, + "KL/rejected_KL_mean": -7.149936676025391, + "KL/std": 6.46231746673584, + "epoch": 0.1277533039647577, + "fcm_dpo/beta": 0.09758473932743073, + "fcm_dpo/delta": -0.06449822336435318, + "fcm_dpo/margin": 4.72601318359375, + "fcm_dpo/q_t": 0.39699968695640564, + "grad_norm": 26.262548446655273, + "learning_rate": 4.990486745229364e-07, + "logits/chosen": -0.9582849740982056, + "logits/rejected": -0.5924580097198486, + "logps/chosen": -95.1098861694336, + "logps/ref_chosen": -92.68596649169922, + "logps/ref_rejected": -102.91818237304688, + "logps/rejected": -110.068115234375, + "loss": 1.1015, + "margin_dpo/margin_mean": 4.726013660430908, + "margin_dpo/margin_std": 7.665117263793945, + "step": 87 + }, + { + "KL/chosen_KL_mean": -2.949321746826172, + "KL/mean": -4.880241394042969, + "KL/rejected_KL_mean": -6.811153411865234, + "KL/std": 6.328596115112305, + "epoch": 0.12922173274596183, + "fcm_dpo/beta": 0.09653878211975098, + "fcm_dpo/delta": -0.051709841936826706, + "fcm_dpo/margin": 3.861827850341797, + "fcm_dpo/q_t": 0.41755035519599915, + "grad_norm": 23.8017578125, + "learning_rate": 4.989335440737586e-07, + "logits/chosen": -0.3058089017868042, + "logits/rejected": -0.1184551939368248, + "logps/chosen": -103.7123031616211, + "logps/ref_chosen": -100.76298522949219, + "logps/ref_rejected": -113.15037536621094, + "logps/rejected": -119.9615249633789, + "loss": 1.1418, + "margin_dpo/margin_mean": 3.861827850341797, + "margin_dpo/margin_std": 6.563701629638672, + "step": 88 + }, + { + "KL/chosen_KL_mean": -2.4566688537597656, + "KL/mean": -4.524205207824707, + "KL/rejected_KL_mean": -6.591743469238281, + "KL/std": 5.3812031745910645, + "epoch": 0.13069016152716592, + "fcm_dpo/beta": 0.09631407260894775, + "fcm_dpo/delta": 0.0016747862100601196, + "fcm_dpo/margin": 4.135067939758301, + "fcm_dpo/q_t": 0.41548407077789307, + "grad_norm": 23.24942970275879, + "learning_rate": 4.988118539941847e-07, + "logits/chosen": -0.7394917607307434, + "logits/rejected": -0.4035339951515198, + "logps/chosen": -92.14775085449219, + "logps/ref_chosen": -89.69108581542969, + "logps/ref_rejected": -88.56832885742188, + "logps/rejected": -95.16007995605469, + "loss": 1.1251, + "margin_dpo/margin_mean": 4.135067939758301, + "margin_dpo/margin_std": 7.14996337890625, + "step": 89 + }, + { + "KL/chosen_KL_mean": -2.1671791076660156, + "KL/mean": -4.8994526863098145, + "KL/rejected_KL_mean": -7.631721496582031, + "KL/std": 6.325523376464844, + "epoch": 0.13215859030837004, + "fcm_dpo/beta": 0.09513455629348755, + "fcm_dpo/delta": -0.1262548714876175, + "fcm_dpo/margin": 5.464543342590332, + "fcm_dpo/q_t": 0.3883175551891327, + "grad_norm": 25.596607208251953, + "learning_rate": 4.986836074908615e-07, + "logits/chosen": -0.47311776876449585, + "logits/rejected": -0.3494108319282532, + "logps/chosen": -83.54973602294922, + "logps/ref_chosen": -81.38255310058594, + "logps/ref_rejected": -117.77714538574219, + "logps/rejected": -125.40885925292969, + "loss": 1.0441, + "margin_dpo/margin_mean": 5.464543342590332, + "margin_dpo/margin_std": 7.588117599487305, + "step": 90 + }, + { + "KL/chosen_KL_mean": -2.4866104125976562, + "KL/mean": -4.783910274505615, + "KL/rejected_KL_mean": -7.081211090087891, + "KL/std": 6.15728759765625, + "epoch": 0.13362701908957417, + "fcm_dpo/beta": 0.09375274181365967, + "fcm_dpo/delta": -0.032131608575582504, + "fcm_dpo/margin": 4.594605445861816, + "fcm_dpo/q_t": 0.4077467620372772, + "grad_norm": 25.023216247558594, + "learning_rate": 4.985488079432037e-07, + "logits/chosen": -0.33563512563705444, + "logits/rejected": 0.08655708283185959, + "logps/chosen": -99.70849609375, + "logps/ref_chosen": -97.22187805175781, + "logps/ref_rejected": -92.97674560546875, + "logps/rejected": -100.05795288085938, + "loss": 1.0999, + "margin_dpo/margin_mean": 4.594604969024658, + "margin_dpo/margin_std": 7.223147869110107, + "step": 91 + }, + { + "KL/chosen_KL_mean": -2.7732505798339844, + "KL/mean": -4.78037166595459, + "KL/rejected_KL_mean": -6.787494659423828, + "KL/std": 6.127893924713135, + "epoch": 0.13509544787077826, + "fcm_dpo/beta": 0.09404444694519043, + "fcm_dpo/delta": 0.023266423493623734, + "fcm_dpo/margin": 4.014244079589844, + "fcm_dpo/q_t": 0.4171380400657654, + "grad_norm": 22.582233428955078, + "learning_rate": 4.984074589033043e-07, + "logits/chosen": -0.7812787294387817, + "logits/rejected": -0.4103330969810486, + "logps/chosen": -87.30352783203125, + "logps/ref_chosen": -84.5302734375, + "logps/ref_rejected": -84.5013198852539, + "logps/rejected": -91.288818359375, + "loss": 1.1295, + "margin_dpo/margin_mean": 4.0142436027526855, + "margin_dpo/margin_std": 6.596063613891602, + "step": 92 + }, + { + "KL/chosen_KL_mean": -2.9890403747558594, + "KL/mean": -4.910215377807617, + "KL/rejected_KL_mean": -6.831398010253906, + "KL/std": 5.791092395782471, + "epoch": 0.13656387665198239, + "fcm_dpo/beta": 0.09467165172100067, + "fcm_dpo/delta": 0.03749629482626915, + "fcm_dpo/margin": 3.842360734939575, + "fcm_dpo/q_t": 0.41992542147636414, + "grad_norm": 25.639062881469727, + "learning_rate": 4.982595640958425e-07, + "logits/chosen": -0.9924514293670654, + "logits/rejected": -0.42984145879745483, + "logps/chosen": -93.23947143554688, + "logps/ref_chosen": -90.25043487548828, + "logps/ref_rejected": -84.0942153930664, + "logps/rejected": -90.92561340332031, + "loss": 1.1562, + "margin_dpo/margin_mean": 3.842360734939575, + "margin_dpo/margin_std": 6.998875617980957, + "step": 93 + }, + { + "KL/chosen_KL_mean": -3.112682342529297, + "KL/mean": -6.050202369689941, + "KL/rejected_KL_mean": -8.987720489501953, + "KL/std": 6.811246871948242, + "epoch": 0.13803230543318648, + "fcm_dpo/beta": 0.09186561405658722, + "fcm_dpo/delta": -0.1502506583929062, + "fcm_dpo/margin": 5.875044345855713, + "fcm_dpo/q_t": 0.38054198026657104, + "grad_norm": 25.209552764892578, + "learning_rate": 4.98105127417984e-07, + "logits/chosen": -0.6663119792938232, + "logits/rejected": -0.36196452379226685, + "logps/chosen": -95.56690216064453, + "logps/ref_chosen": -92.4542236328125, + "logps/ref_rejected": -105.2472915649414, + "logps/rejected": -114.23501586914062, + "loss": 1.0342, + "margin_dpo/margin_mean": 5.875043869018555, + "margin_dpo/margin_std": 7.796442031860352, + "step": 94 + }, + { + "KL/chosen_KL_mean": -2.5027084350585938, + "KL/mean": -4.828878402709961, + "KL/rejected_KL_mean": -7.1550445556640625, + "KL/std": 5.726435661315918, + "epoch": 0.1395007342143906, + "fcm_dpo/beta": 0.0918988585472107, + "fcm_dpo/delta": -0.029026592150330544, + "fcm_dpo/margin": 4.652338981628418, + "fcm_dpo/q_t": 0.4076543152332306, + "grad_norm": 21.351715087890625, + "learning_rate": 4.979441529392784e-07, + "logits/chosen": -0.8635698556900024, + "logits/rejected": -0.5238780975341797, + "logps/chosen": -81.37641906738281, + "logps/ref_chosen": -78.87370300292969, + "logps/ref_rejected": -83.59121704101562, + "logps/rejected": -90.74626159667969, + "loss": 1.0999, + "margin_dpo/margin_mean": 4.652338981628418, + "margin_dpo/margin_std": 7.098349571228027, + "step": 95 + }, + { + "KL/chosen_KL_mean": -2.2016448974609375, + "KL/mean": -5.082788467407227, + "KL/rejected_KL_mean": -7.963932037353516, + "KL/std": 6.861926078796387, + "epoch": 0.14096916299559473, + "fcm_dpo/beta": 0.08951471745967865, + "fcm_dpo/delta": -0.12249662727117538, + "fcm_dpo/margin": 5.7622880935668945, + "fcm_dpo/q_t": 0.3874228298664093, + "grad_norm": 21.74850082397461, + "learning_rate": 4.977766449015534e-07, + "logits/chosen": -0.7713649272918701, + "logits/rejected": -0.3251499533653259, + "logps/chosen": -108.79380798339844, + "logps/ref_chosen": -106.5921630859375, + "logps/ref_rejected": -101.76802825927734, + "logps/rejected": -109.73196411132812, + "loss": 1.0383, + "margin_dpo/margin_mean": 5.7622880935668945, + "margin_dpo/margin_std": 7.617171287536621, + "step": 96 + }, + { + "KL/chosen_KL_mean": -3.3008499145507812, + "KL/mean": -5.859622955322266, + "KL/rejected_KL_mean": -8.418392181396484, + "KL/std": 5.913722038269043, + "epoch": 0.14243759177679882, + "fcm_dpo/beta": 0.08901587128639221, + "fcm_dpo/delta": -0.058812178671360016, + "fcm_dpo/margin": 5.117546081542969, + "fcm_dpo/q_t": 0.3970525860786438, + "grad_norm": 24.633037567138672, + "learning_rate": 4.976026077188012e-07, + "logits/chosen": -0.4443942904472351, + "logits/rejected": -0.035453300923109055, + "logps/chosen": -91.98072814941406, + "logps/ref_chosen": -88.67988586425781, + "logps/ref_rejected": -84.81229400634766, + "logps/rejected": -93.23068237304688, + "loss": 1.0577, + "margin_dpo/margin_mean": 5.117546081542969, + "margin_dpo/margin_std": 6.3041229248046875, + "step": 97 + }, + { + "KL/chosen_KL_mean": -2.5518341064453125, + "KL/mean": -5.429494857788086, + "KL/rejected_KL_mean": -8.30715560913086, + "KL/std": 5.7163214683532715, + "epoch": 0.14390602055800295, + "fcm_dpo/beta": 0.08634430170059204, + "fcm_dpo/delta": -0.1030280590057373, + "fcm_dpo/margin": 5.755323886871338, + "fcm_dpo/q_t": 0.38794711232185364, + "grad_norm": 22.770769119262695, + "learning_rate": 4.974220459770639e-07, + "logits/chosen": -0.6936355829238892, + "logits/rejected": -0.4268031120300293, + "logps/chosen": -94.79432678222656, + "logps/ref_chosen": -92.24249267578125, + "logps/ref_rejected": -101.51948547363281, + "logps/rejected": -109.82664489746094, + "loss": 1.022, + "margin_dpo/margin_mean": 5.755324363708496, + "margin_dpo/margin_std": 6.475984573364258, + "step": 98 + }, + { + "KL/chosen_KL_mean": -1.8986015319824219, + "KL/mean": -5.120244979858398, + "KL/rejected_KL_mean": -8.341896057128906, + "KL/std": 6.887099742889404, + "epoch": 0.14537444933920704, + "fcm_dpo/beta": 0.08430320769548416, + "fcm_dpo/delta": -0.1516081690788269, + "fcm_dpo/margin": 6.443300724029541, + "fcm_dpo/q_t": 0.38580048084259033, + "grad_norm": 22.23921012878418, + "learning_rate": 4.972349644343108e-07, + "logits/chosen": -0.638316810131073, + "logits/rejected": -0.4255262613296509, + "logps/chosen": -74.083251953125, + "logps/ref_chosen": -72.18464660644531, + "logps/ref_rejected": -91.88131713867188, + "logps/rejected": -100.22322082519531, + "loss": 1.0236, + "margin_dpo/margin_mean": 6.443300247192383, + "margin_dpo/margin_std": 8.78098201751709, + "step": 99 + }, + { + "KL/chosen_KL_mean": -2.280132293701172, + "KL/mean": -4.591710090637207, + "KL/rejected_KL_mean": -6.903285980224609, + "KL/std": 5.230842590332031, + "epoch": 0.14684287812041116, + "fcm_dpo/beta": 0.08361945301294327, + "fcm_dpo/delta": 0.013753345236182213, + "fcm_dpo/margin": 4.623149871826172, + "fcm_dpo/q_t": 0.4108346700668335, + "grad_norm": 19.484966278076172, + "learning_rate": 4.970413680203148e-07, + "logits/chosen": -0.6460772752761841, + "logits/rejected": -0.1551763117313385, + "logps/chosen": -91.79396057128906, + "logps/ref_chosen": -89.51382446289062, + "logps/ref_rejected": -81.21713256835938, + "logps/rejected": -88.12041473388672, + "loss": 1.0865, + "margin_dpo/margin_mean": 4.6231489181518555, + "margin_dpo/margin_std": 5.541533946990967, + "step": 100 + }, + { + "KL/chosen_KL_mean": -3.5491943359375, + "KL/mean": -5.64581298828125, + "KL/rejected_KL_mean": -7.742439270019531, + "KL/std": 6.811181545257568, + "epoch": 0.14831130690161526, + "fcm_dpo/beta": 0.08362682163715363, + "fcm_dpo/delta": -0.020485172048211098, + "fcm_dpo/margin": 4.193233013153076, + "fcm_dpo/q_t": 0.4199220538139343, + "grad_norm": 23.749713897705078, + "learning_rate": 4.968412618365215e-07, + "logits/chosen": -0.8543105125427246, + "logits/rejected": -0.5881494283676147, + "logps/chosen": -97.952392578125, + "logps/ref_chosen": -94.4031982421875, + "logps/ref_rejected": -89.91001892089844, + "logps/rejected": -97.65245056152344, + "loss": 1.1539, + "margin_dpo/margin_mean": 4.193233013153076, + "margin_dpo/margin_std": 7.304439544677734, + "step": 101 + }, + { + "KL/chosen_KL_mean": -4.014743804931641, + "KL/mean": -5.838558197021484, + "KL/rejected_KL_mean": -7.662372589111328, + "KL/std": 6.486747741699219, + "epoch": 0.14977973568281938, + "fcm_dpo/beta": 0.08330987393856049, + "fcm_dpo/delta": -0.03804425895214081, + "fcm_dpo/margin": 3.6476352214813232, + "fcm_dpo/q_t": 0.43119144439697266, + "grad_norm": 28.040851593017578, + "learning_rate": 4.966346511559149e-07, + "logits/chosen": -0.9621187448501587, + "logits/rejected": -0.6033914685249329, + "logps/chosen": -95.6241683959961, + "logps/ref_chosen": -91.60942077636719, + "logps/ref_rejected": -75.7689208984375, + "logps/rejected": -83.43128967285156, + "loss": 1.196, + "margin_dpo/margin_mean": 3.6476354598999023, + "margin_dpo/margin_std": 7.342247009277344, + "step": 102 + }, + { + "KL/chosen_KL_mean": -3.1960601806640625, + "KL/mean": -6.231755256652832, + "KL/rejected_KL_mean": -9.267452239990234, + "KL/std": 6.851804733276367, + "epoch": 0.1512481644640235, + "fcm_dpo/beta": 0.08159910887479782, + "fcm_dpo/delta": -0.1006719097495079, + "fcm_dpo/margin": 6.071394920349121, + "fcm_dpo/q_t": 0.3881691098213196, + "grad_norm": 22.51178550720215, + "learning_rate": 4.964215414228785e-07, + "logits/chosen": -0.8078954219818115, + "logits/rejected": -0.41563475131988525, + "logps/chosen": -100.59844970703125, + "logps/ref_chosen": -97.40238952636719, + "logps/ref_rejected": -98.981689453125, + "logps/rejected": -108.24913787841797, + "loss": 1.0431, + "margin_dpo/margin_mean": 6.0713958740234375, + "margin_dpo/margin_std": 7.733498573303223, + "step": 103 + }, + { + "KL/chosen_KL_mean": -2.4227676391601562, + "KL/mean": -5.7053046226501465, + "KL/rejected_KL_mean": -8.987834930419922, + "KL/std": 7.468371391296387, + "epoch": 0.1527165932452276, + "fcm_dpo/beta": 0.07966464757919312, + "fcm_dpo/delta": -0.13002897799015045, + "fcm_dpo/margin": 6.565065860748291, + "fcm_dpo/q_t": 0.3874834179878235, + "grad_norm": 18.731647491455078, + "learning_rate": 4.96201938253052e-07, + "logits/chosen": -0.9786155223846436, + "logits/rejected": -0.6273901462554932, + "logps/chosen": -97.27912902832031, + "logps/ref_chosen": -94.85636138916016, + "logps/ref_rejected": -97.02669525146484, + "logps/rejected": -106.0145263671875, + "loss": 1.0295, + "margin_dpo/margin_mean": 6.565066814422607, + "margin_dpo/margin_std": 8.832313537597656, + "step": 104 + }, + { + "KL/chosen_KL_mean": -3.2622146606445312, + "KL/mean": -6.512721061706543, + "KL/rejected_KL_mean": -9.763225555419922, + "KL/std": 6.803992748260498, + "epoch": 0.15418502202643172, + "fcm_dpo/beta": 0.07827430963516235, + "fcm_dpo/delta": -0.11460494995117188, + "fcm_dpo/margin": 6.501008033752441, + "fcm_dpo/q_t": 0.38681888580322266, + "grad_norm": 21.944747924804688, + "learning_rate": 4.959758474331832e-07, + "logits/chosen": -1.006087303161621, + "logits/rejected": -0.531781017780304, + "logps/chosen": -100.02011108398438, + "logps/ref_chosen": -96.75790405273438, + "logps/ref_rejected": -101.5774917602539, + "logps/rejected": -111.34071350097656, + "loss": 1.0414, + "margin_dpo/margin_mean": 6.501007080078125, + "margin_dpo/margin_std": 8.554824829101562, + "step": 105 + }, + { + "KL/chosen_KL_mean": -3.2737464904785156, + "KL/mean": -6.3534722328186035, + "KL/rejected_KL_mean": -9.433197021484375, + "KL/std": 7.004992961883545, + "epoch": 0.15565345080763582, + "fcm_dpo/beta": 0.07681487500667572, + "fcm_dpo/delta": -0.07672860473394394, + "fcm_dpo/margin": 6.159451007843018, + "fcm_dpo/q_t": 0.39662793278694153, + "grad_norm": 19.68122100830078, + "learning_rate": 4.957432749209755e-07, + "logits/chosen": -0.6537899374961853, + "logits/rejected": -0.21296614408493042, + "logps/chosen": -95.71165466308594, + "logps/ref_chosen": -92.43791198730469, + "logps/ref_rejected": -91.00970458984375, + "logps/rejected": -100.44290161132812, + "loss": 1.0606, + "margin_dpo/margin_mean": 6.159450531005859, + "margin_dpo/margin_std": 8.471151351928711, + "step": 106 + }, + { + "KL/chosen_KL_mean": -2.328693389892578, + "KL/mean": -5.9570417404174805, + "KL/rejected_KL_mean": -9.585391998291016, + "KL/std": 8.865304946899414, + "epoch": 0.15712187958883994, + "fcm_dpo/beta": 0.07456070184707642, + "fcm_dpo/delta": -0.14922761917114258, + "fcm_dpo/margin": 7.256699562072754, + "fcm_dpo/q_t": 0.3894772529602051, + "grad_norm": 19.823720932006836, + "learning_rate": 4.955042268449307e-07, + "logits/chosen": -0.8180956244468689, + "logits/rejected": -0.30201244354248047, + "logps/chosen": -107.45370483398438, + "logps/ref_chosen": -105.12500762939453, + "logps/ref_rejected": -101.1824951171875, + "logps/rejected": -110.76789093017578, + "loss": 1.0426, + "margin_dpo/margin_mean": 7.256699562072754, + "margin_dpo/margin_std": 11.042577743530273, + "step": 107 + }, + { + "KL/chosen_KL_mean": -3.454601287841797, + "KL/mean": -6.853583335876465, + "KL/rejected_KL_mean": -10.252567291259766, + "KL/std": 8.144183158874512, + "epoch": 0.15859030837004406, + "fcm_dpo/beta": 0.07261009514331818, + "fcm_dpo/delta": -0.09935794025659561, + "fcm_dpo/margin": 6.7979631423950195, + "fcm_dpo/q_t": 0.39164412021636963, + "grad_norm": 19.87030601501465, + "learning_rate": 4.952587095041881e-07, + "logits/chosen": -0.9941326975822449, + "logits/rejected": -0.4892861545085907, + "logps/chosen": -92.6639633178711, + "logps/ref_chosen": -89.20936584472656, + "logps/ref_rejected": -101.83784484863281, + "logps/rejected": -112.09041595458984, + "loss": 1.0476, + "margin_dpo/margin_mean": 6.797963619232178, + "margin_dpo/margin_std": 8.928738594055176, + "step": 108 + }, + { + "KL/chosen_KL_mean": -2.7665672302246094, + "KL/mean": -6.983913421630859, + "KL/rejected_KL_mean": -11.20126724243164, + "KL/std": 8.303434371948242, + "epoch": 0.16005873715124816, + "fcm_dpo/beta": 0.07019108533859253, + "fcm_dpo/delta": -0.20492924749851227, + "fcm_dpo/margin": 8.434696197509766, + "fcm_dpo/q_t": 0.37066853046417236, + "grad_norm": 18.741079330444336, + "learning_rate": 4.95006729368358e-07, + "logits/chosen": -0.590620219707489, + "logits/rejected": -0.35671481490135193, + "logps/chosen": -102.95365905761719, + "logps/ref_chosen": -100.18708801269531, + "logps/ref_rejected": -103.53739929199219, + "logps/rejected": -114.73866271972656, + "loss": 0.9878, + "margin_dpo/margin_mean": 8.43469524383545, + "margin_dpo/margin_std": 10.254924774169922, + "step": 109 + }, + { + "KL/chosen_KL_mean": -3.585895538330078, + "KL/mean": -6.090257167816162, + "KL/rejected_KL_mean": -8.594623565673828, + "KL/std": 6.798268795013428, + "epoch": 0.16152716593245228, + "fcm_dpo/beta": 0.06924136728048325, + "fcm_dpo/delta": -0.030235530808568, + "fcm_dpo/margin": 5.008722305297852, + "fcm_dpo/q_t": 0.42062196135520935, + "grad_norm": 20.035903930664062, + "learning_rate": 4.947482930773511e-07, + "logits/chosen": -0.281746506690979, + "logits/rejected": 0.13460886478424072, + "logps/chosen": -110.48161315917969, + "logps/ref_chosen": -106.89572143554688, + "logps/ref_rejected": -85.81120300292969, + "logps/rejected": -94.40582275390625, + "loss": 1.1373, + "margin_dpo/margin_mean": 5.008722305297852, + "margin_dpo/margin_std": 7.700854301452637, + "step": 110 + }, + { + "KL/chosen_KL_mean": -3.2048492431640625, + "KL/mean": -8.13703727722168, + "KL/rejected_KL_mean": -13.069229125976562, + "KL/std": 9.644274711608887, + "epoch": 0.16299559471365638, + "fcm_dpo/beta": 0.06614179909229279, + "fcm_dpo/delta": -0.27217304706573486, + "fcm_dpo/margin": 9.864376068115234, + "fcm_dpo/q_t": 0.3561099171638489, + "grad_norm": 17.213592529296875, + "learning_rate": 4.944834074412042e-07, + "logits/chosen": -1.0692082643508911, + "logits/rejected": -0.7875548601150513, + "logps/chosen": -96.53421783447266, + "logps/ref_chosen": -93.3293685913086, + "logps/ref_rejected": -104.177490234375, + "logps/rejected": -117.24671936035156, + "loss": 0.9458, + "margin_dpo/margin_mean": 9.864374160766602, + "margin_dpo/margin_std": 10.657848358154297, + "step": 111 + }, + { + "KL/chosen_KL_mean": -3.6781959533691406, + "KL/mean": -6.1637067794799805, + "KL/rejected_KL_mean": -8.649219512939453, + "KL/std": 5.5510945320129395, + "epoch": 0.1644640234948605, + "fcm_dpo/beta": 0.06587863713502884, + "fcm_dpo/delta": 0.023251384496688843, + "fcm_dpo/margin": 4.97102165222168, + "fcm_dpo/q_t": 0.4243234395980835, + "grad_norm": 17.179916381835938, + "learning_rate": 4.942120794399002e-07, + "logits/chosen": -1.283064365386963, + "logits/rejected": -0.7845422625541687, + "logps/chosen": -86.42523956298828, + "logps/ref_chosen": -82.74703979492188, + "logps/ref_rejected": -71.5584487915039, + "logps/rejected": -80.20767211914062, + "loss": 1.1436, + "margin_dpo/margin_mean": 4.971022605895996, + "margin_dpo/margin_std": 7.52631950378418, + "step": 112 + }, + { + "KL/chosen_KL_mean": -4.587512969970703, + "KL/mean": -7.3055219650268555, + "KL/rejected_KL_mean": -10.02353286743164, + "KL/std": 7.405384063720703, + "epoch": 0.16593245227606462, + "fcm_dpo/beta": 0.066332146525383, + "fcm_dpo/delta": 0.0408482551574707, + "fcm_dpo/margin": 5.436017036437988, + "fcm_dpo/q_t": 0.4170852601528168, + "grad_norm": 20.547657012939453, + "learning_rate": 4.939343162231841e-07, + "logits/chosen": -0.9216668605804443, + "logits/rejected": -0.3851248025894165, + "logps/chosen": -102.48700714111328, + "logps/ref_chosen": -97.89949035644531, + "logps/ref_rejected": -84.41960144042969, + "logps/rejected": -94.44313049316406, + "loss": 1.1148, + "margin_dpo/margin_mean": 5.436017990112305, + "margin_dpo/margin_std": 7.101157188415527, + "step": 113 + }, + { + "KL/chosen_KL_mean": -4.290248870849609, + "KL/mean": -8.092846870422363, + "KL/rejected_KL_mean": -11.89544677734375, + "KL/std": 8.676340103149414, + "epoch": 0.16740088105726872, + "fcm_dpo/beta": 0.06523959338665009, + "fcm_dpo/delta": -0.10162682086229324, + "fcm_dpo/margin": 7.605199337005615, + "fcm_dpo/q_t": 0.393574059009552, + "grad_norm": 20.153409957885742, + "learning_rate": 4.936501251103751e-07, + "logits/chosen": -1.0858216285705566, + "logits/rejected": -0.5068268775939941, + "logps/chosen": -100.78536987304688, + "logps/ref_chosen": -96.4951171875, + "logps/ref_rejected": -92.01536560058594, + "logps/rejected": -103.91081237792969, + "loss": 1.0683, + "margin_dpo/margin_mean": 7.605199337005615, + "margin_dpo/margin_std": 11.257364273071289, + "step": 114 + }, + { + "KL/chosen_KL_mean": -4.4661102294921875, + "KL/mean": -8.248708724975586, + "KL/rejected_KL_mean": -12.031307220458984, + "KL/std": 9.088540077209473, + "epoch": 0.16886930983847284, + "fcm_dpo/beta": 0.06425650417804718, + "fcm_dpo/delta": -0.09046189486980438, + "fcm_dpo/margin": 7.565197944641113, + "fcm_dpo/q_t": 0.3924819827079773, + "grad_norm": 17.9512996673584, + "learning_rate": 4.933595135901732e-07, + "logits/chosen": -0.9558216333389282, + "logits/rejected": -0.5450321435928345, + "logps/chosen": -107.646240234375, + "logps/ref_chosen": -103.18013000488281, + "logps/ref_rejected": -105.71595764160156, + "logps/rejected": -117.74726867675781, + "loss": 1.0515, + "margin_dpo/margin_mean": 7.565196990966797, + "margin_dpo/margin_std": 9.938637733459473, + "step": 115 + }, + { + "KL/chosen_KL_mean": -3.9439430236816406, + "KL/mean": -7.166526794433594, + "KL/rejected_KL_mean": -10.389106750488281, + "KL/std": 7.188760757446289, + "epoch": 0.17033773861967694, + "fcm_dpo/beta": 0.06402775645256042, + "fcm_dpo/delta": -0.013450777158141136, + "fcm_dpo/margin": 6.445163249969482, + "fcm_dpo/q_t": 0.4059520661830902, + "grad_norm": 17.014379501342773, + "learning_rate": 4.930624893204624e-07, + "logits/chosen": -1.0909994840621948, + "logits/rejected": -0.6614570617675781, + "logps/chosen": -85.1023178100586, + "logps/ref_chosen": -81.15837097167969, + "logps/ref_rejected": -86.30586242675781, + "logps/rejected": -96.69496154785156, + "loss": 1.0807, + "margin_dpo/margin_mean": 6.445164203643799, + "margin_dpo/margin_std": 8.192975997924805, + "step": 116 + }, + { + "KL/chosen_KL_mean": -4.727897644042969, + "KL/mean": -7.359216690063477, + "KL/rejected_KL_mean": -9.99053955078125, + "KL/std": 7.311716556549072, + "epoch": 0.17180616740088106, + "fcm_dpo/beta": 0.06343423575162888, + "fcm_dpo/delta": -0.024360468611121178, + "fcm_dpo/margin": 5.262639045715332, + "fcm_dpo/q_t": 0.4233490824699402, + "grad_norm": 24.08353614807129, + "learning_rate": 4.927590601281083e-07, + "logits/chosen": -0.6956891417503357, + "logits/rejected": -0.27032843232154846, + "logps/chosen": -101.80885314941406, + "logps/ref_chosen": -97.0809555053711, + "logps/ref_rejected": -75.1399154663086, + "logps/rejected": -85.13045501708984, + "loss": 1.1571, + "margin_dpo/margin_mean": 5.262638568878174, + "margin_dpo/margin_std": 8.948646545410156, + "step": 117 + }, + { + "KL/chosen_KL_mean": -3.4797935485839844, + "KL/mean": -7.198921203613281, + "KL/rejected_KL_mean": -10.918048858642578, + "KL/std": 8.009775161743164, + "epoch": 0.17327459618208516, + "fcm_dpo/beta": 0.062702976167202, + "fcm_dpo/delta": -0.06976436078548431, + "fcm_dpo/margin": 7.438255786895752, + "fcm_dpo/q_t": 0.39276593923568726, + "grad_norm": 16.052621841430664, + "learning_rate": 4.924492340087524e-07, + "logits/chosen": -1.125187635421753, + "logits/rejected": -0.7884189486503601, + "logps/chosen": -90.01602935791016, + "logps/ref_chosen": -86.53623962402344, + "logps/ref_rejected": -82.93563842773438, + "logps/rejected": -93.85369110107422, + "loss": 1.0382, + "margin_dpo/margin_mean": 7.438255310058594, + "margin_dpo/margin_std": 8.531219482421875, + "step": 118 + }, + { + "KL/chosen_KL_mean": -5.211345672607422, + "KL/mean": -9.058032989501953, + "KL/rejected_KL_mean": -12.904712677001953, + "KL/std": 9.489660263061523, + "epoch": 0.17474302496328928, + "fcm_dpo/beta": 0.0616828054189682, + "fcm_dpo/delta": -0.07856158912181854, + "fcm_dpo/margin": 7.693366527557373, + "fcm_dpo/q_t": 0.39377036690711975, + "grad_norm": 18.52414894104004, + "learning_rate": 4.92133019126601e-07, + "logits/chosen": -1.3846263885498047, + "logits/rejected": -1.0728895664215088, + "logps/chosen": -109.82042694091797, + "logps/ref_chosen": -104.60908508300781, + "logps/ref_rejected": -109.15780639648438, + "logps/rejected": -122.0625228881836, + "loss": 1.0869, + "margin_dpo/margin_mean": 7.6933674812316895, + "margin_dpo/margin_std": 11.942338943481445, + "step": 119 + }, + { + "KL/chosen_KL_mean": -4.1529083251953125, + "KL/mean": -9.279523849487305, + "KL/rejected_KL_mean": -14.406135559082031, + "KL/std": 11.35304069519043, + "epoch": 0.1762114537444934, + "fcm_dpo/beta": 0.05949506163597107, + "fcm_dpo/delta": -0.22397297620773315, + "fcm_dpo/margin": 10.253229141235352, + "fcm_dpo/q_t": 0.37103092670440674, + "grad_norm": 21.065217971801758, + "learning_rate": 4.918104238142103e-07, + "logits/chosen": -1.2521820068359375, + "logits/rejected": -0.696804404258728, + "logps/chosen": -118.84033203125, + "logps/ref_chosen": -114.68742370605469, + "logps/ref_rejected": -111.18586730957031, + "logps/rejected": -125.59200286865234, + "loss": 1.0193, + "margin_dpo/margin_mean": 10.253229141235352, + "margin_dpo/margin_std": 14.381040573120117, + "step": 120 + }, + { + "KL/chosen_KL_mean": -3.7394447326660156, + "KL/mean": -8.40185832977295, + "KL/rejected_KL_mean": -13.064281463623047, + "KL/std": 8.866586685180664, + "epoch": 0.1776798825256975, + "fcm_dpo/beta": 0.057432860136032104, + "fcm_dpo/delta": -0.14398802816867828, + "fcm_dpo/margin": 9.324836730957031, + "fcm_dpo/q_t": 0.3780592381954193, + "grad_norm": 18.062376022338867, + "learning_rate": 4.91481456572267e-07, + "logits/chosen": -0.9575957655906677, + "logits/rejected": -0.6242144107818604, + "logps/chosen": -95.85610961914062, + "logps/ref_chosen": -92.11666107177734, + "logps/ref_rejected": -114.972900390625, + "logps/rejected": -128.0371856689453, + "loss": 0.9918, + "margin_dpo/margin_mean": 9.324836730957031, + "margin_dpo/margin_std": 9.832696914672852, + "step": 121 + }, + { + "KL/chosen_KL_mean": -4.112579345703125, + "KL/mean": -9.427934646606445, + "KL/rejected_KL_mean": -14.7432861328125, + "KL/std": 9.578864097595215, + "epoch": 0.17914831130690162, + "fcm_dpo/beta": 0.0557304285466671, + "fcm_dpo/delta": -0.20408757030963898, + "fcm_dpo/margin": 10.630702018737793, + "fcm_dpo/q_t": 0.36557281017303467, + "grad_norm": 18.262664794921875, + "learning_rate": 4.911461260693638e-07, + "logits/chosen": -0.9353420734405518, + "logits/rejected": -0.8377484083175659, + "logps/chosen": -88.99015045166016, + "logps/ref_chosen": -84.87757110595703, + "logps/ref_rejected": -111.0829849243164, + "logps/rejected": -125.8262710571289, + "loss": 0.9693, + "margin_dpo/margin_mean": 10.630702018737793, + "margin_dpo/margin_std": 11.238935470581055, + "step": 122 + }, + { + "KL/chosen_KL_mean": -4.574275970458984, + "KL/mean": -8.434359550476074, + "KL/rejected_KL_mean": -12.294441223144531, + "KL/std": 8.846650123596191, + "epoch": 0.18061674008810572, + "fcm_dpo/beta": 0.05413203313946724, + "fcm_dpo/delta": -0.07251356542110443, + "fcm_dpo/margin": 7.720160007476807, + "fcm_dpo/q_t": 0.40424859523773193, + "grad_norm": 15.763967514038086, + "learning_rate": 4.908044411417711e-07, + "logits/chosen": -1.1376043558120728, + "logits/rejected": -0.8168488144874573, + "logps/chosen": -103.05302429199219, + "logps/ref_chosen": -98.47874450683594, + "logps/ref_rejected": -93.85417175292969, + "logps/rejected": -106.14862060546875, + "loss": 1.084, + "margin_dpo/margin_mean": 7.720160007476807, + "margin_dpo/margin_std": 10.183891296386719, + "step": 123 + }, + { + "KL/chosen_KL_mean": -4.424888610839844, + "KL/mean": -9.160396575927734, + "KL/rejected_KL_mean": -13.89590072631836, + "KL/std": 11.099146842956543, + "epoch": 0.18208516886930984, + "fcm_dpo/beta": 0.05327959358692169, + "fcm_dpo/delta": -0.11012198030948639, + "fcm_dpo/margin": 9.471003532409668, + "fcm_dpo/q_t": 0.3945465683937073, + "grad_norm": 16.024564743041992, + "learning_rate": 4.904564107932048e-07, + "logits/chosen": -1.1801691055297852, + "logits/rejected": -0.91695237159729, + "logps/chosen": -107.18778991699219, + "logps/ref_chosen": -102.76290130615234, + "logps/ref_rejected": -123.005615234375, + "logps/rejected": -136.90151977539062, + "loss": 1.066, + "margin_dpo/margin_mean": 9.471002578735352, + "margin_dpo/margin_std": 14.401384353637695, + "step": 124 + }, + { + "KL/chosen_KL_mean": -5.886226654052734, + "KL/mean": -9.330833435058594, + "KL/rejected_KL_mean": -12.775447845458984, + "KL/std": 9.071002960205078, + "epoch": 0.18355359765051396, + "fcm_dpo/beta": 0.05318472534418106, + "fcm_dpo/delta": 0.0348917655646801, + "fcm_dpo/margin": 6.889215469360352, + "fcm_dpo/q_t": 0.41773951053619385, + "grad_norm": 16.47319984436035, + "learning_rate": 4.90102044194588e-07, + "logits/chosen": -0.8794206380844116, + "logits/rejected": -0.6602913737297058, + "logps/chosen": -96.3125, + "logps/ref_chosen": -90.42627716064453, + "logps/ref_rejected": -89.24901580810547, + "logps/rejected": -102.02445983886719, + "loss": 1.1348, + "margin_dpo/margin_mean": 6.889216423034668, + "margin_dpo/margin_std": 11.119590759277344, + "step": 125 + }, + { + "KL/chosen_KL_mean": -4.795345306396484, + "KL/mean": -9.781375885009766, + "KL/rejected_KL_mean": -14.767406463623047, + "KL/std": 9.724905014038086, + "epoch": 0.18502202643171806, + "fcm_dpo/beta": 0.0523233599960804, + "fcm_dpo/delta": -0.1284581869840622, + "fcm_dpo/margin": 9.972065925598145, + "fcm_dpo/q_t": 0.37919655442237854, + "grad_norm": 15.988823890686035, + "learning_rate": 4.897413506838102e-07, + "logits/chosen": -0.8898293972015381, + "logits/rejected": -0.6062628626823425, + "logps/chosen": -99.21823120117188, + "logps/ref_chosen": -94.42288208007812, + "logps/ref_rejected": -103.88156127929688, + "logps/rejected": -118.64896392822266, + "loss": 0.9958, + "margin_dpo/margin_mean": 9.972064971923828, + "margin_dpo/margin_std": 10.190126419067383, + "step": 126 + }, + { + "KL/chosen_KL_mean": -4.822856903076172, + "KL/mean": -8.551148414611816, + "KL/rejected_KL_mean": -12.279441833496094, + "KL/std": 8.983542442321777, + "epoch": 0.18649045521292218, + "fcm_dpo/beta": 0.05218739062547684, + "fcm_dpo/delta": 0.011145832017064095, + "fcm_dpo/margin": 7.4565839767456055, + "fcm_dpo/q_t": 0.41303130984306335, + "grad_norm": 16.119571685791016, + "learning_rate": 4.89374339765481e-07, + "logits/chosen": -0.7476365566253662, + "logits/rejected": -0.39096903800964355, + "logps/chosen": -89.57391357421875, + "logps/ref_chosen": -84.75105285644531, + "logps/ref_rejected": -83.54523468017578, + "logps/rejected": -95.82467651367188, + "loss": 1.1131, + "margin_dpo/margin_mean": 7.4565839767456055, + "margin_dpo/margin_std": 11.260820388793945, + "step": 127 + }, + { + "KL/chosen_KL_mean": -4.778011322021484, + "KL/mean": -8.66218376159668, + "KL/rejected_KL_mean": -12.546350479125977, + "KL/std": 11.10682201385498, + "epoch": 0.18795888399412627, + "fcm_dpo/beta": 0.05228809267282486, + "fcm_dpo/delta": -0.006903614848852158, + "fcm_dpo/margin": 7.768337249755859, + "fcm_dpo/q_t": 0.40844595432281494, + "grad_norm": 18.162736892700195, + "learning_rate": 4.890010211106795e-07, + "logits/chosen": -0.6107750535011292, + "logits/rejected": -0.11944863945245743, + "logps/chosen": -95.39765930175781, + "logps/ref_chosen": -90.61964416503906, + "logps/ref_rejected": -83.10347747802734, + "logps/rejected": -95.64982604980469, + "loss": 1.1287, + "margin_dpo/margin_mean": 7.768337249755859, + "margin_dpo/margin_std": 13.658068656921387, + "step": 128 + }, + { + "KL/chosen_KL_mean": -5.3299407958984375, + "KL/mean": -9.333892822265625, + "KL/rejected_KL_mean": -13.337848663330078, + "KL/std": 8.991896629333496, + "epoch": 0.1894273127753304, + "fcm_dpo/beta": 0.05203121155500412, + "fcm_dpo/delta": -0.017507098615169525, + "fcm_dpo/margin": 8.007903099060059, + "fcm_dpo/q_t": 0.4092479348182678, + "grad_norm": 14.906427383422852, + "learning_rate": 4.88621404556699e-07, + "logits/chosen": -0.797419548034668, + "logits/rejected": -0.48800790309906006, + "logps/chosen": -104.24864959716797, + "logps/ref_chosen": -98.91870880126953, + "logps/ref_rejected": -103.28419494628906, + "logps/rejected": -116.62203979492188, + "loss": 1.1071, + "margin_dpo/margin_mean": 8.007903099060059, + "margin_dpo/margin_std": 12.250895500183105, + "step": 129 + }, + { + "KL/chosen_KL_mean": -4.206462860107422, + "KL/mean": -9.440500259399414, + "KL/rejected_KL_mean": -14.67453384399414, + "KL/std": 10.511327743530273, + "epoch": 0.19089574155653452, + "fcm_dpo/beta": 0.05113845691084862, + "fcm_dpo/delta": -0.14308412373065948, + "fcm_dpo/margin": 10.468074798583984, + "fcm_dpo/q_t": 0.38214167952537537, + "grad_norm": 14.121175765991211, + "learning_rate": 4.882355001067891e-07, + "logits/chosen": -1.132953405380249, + "logits/rejected": -0.8869795203208923, + "logps/chosen": -77.39015197753906, + "logps/ref_chosen": -73.18368530273438, + "logps/ref_rejected": -88.02154541015625, + "logps/rejected": -102.69607543945312, + "loss": 1.0169, + "margin_dpo/margin_mean": 10.468074798583984, + "margin_dpo/margin_std": 12.477448463439941, + "step": 130 + }, + { + "KL/chosen_KL_mean": -3.6156005859375, + "KL/mean": -8.39569091796875, + "KL/rejected_KL_mean": -13.175788879394531, + "KL/std": 10.556526184082031, + "epoch": 0.19236417033773862, + "fcm_dpo/beta": 0.050069913268089294, + "fcm_dpo/delta": -0.08292128890752792, + "fcm_dpo/margin": 9.560185432434082, + "fcm_dpo/q_t": 0.3923855423927307, + "grad_norm": 16.485183715820312, + "learning_rate": 4.878433179298909e-07, + "logits/chosen": -1.0437589883804321, + "logits/rejected": -0.8303720951080322, + "logps/chosen": -77.5720443725586, + "logps/ref_chosen": -73.9564437866211, + "logps/ref_rejected": -94.45582580566406, + "logps/rejected": -107.6316146850586, + "loss": 1.0502, + "margin_dpo/margin_mean": 9.560186386108398, + "margin_dpo/margin_std": 12.052691459655762, + "step": 131 + }, + { + "KL/chosen_KL_mean": -5.328037261962891, + "KL/mean": -10.026786804199219, + "KL/rejected_KL_mean": -14.725532531738281, + "KL/std": 10.929555892944336, + "epoch": 0.19383259911894274, + "fcm_dpo/beta": 0.04888708144426346, + "fcm_dpo/delta": -0.0628490224480629, + "fcm_dpo/margin": 9.39749526977539, + "fcm_dpo/q_t": 0.3952639698982239, + "grad_norm": 16.64833641052246, + "learning_rate": 4.874448683603694e-07, + "logits/chosen": -1.3084355592727661, + "logits/rejected": -0.8912124633789062, + "logps/chosen": -97.76751708984375, + "logps/ref_chosen": -92.43948364257812, + "logps/ref_rejected": -94.13252258300781, + "logps/rejected": -108.85804748535156, + "loss": 1.0639, + "margin_dpo/margin_mean": 9.39749526977539, + "margin_dpo/margin_std": 12.493152618408203, + "step": 132 + }, + { + "KL/chosen_KL_mean": -4.420093536376953, + "KL/mean": -9.39143180847168, + "KL/rejected_KL_mean": -14.362777709960938, + "KL/std": 11.067865371704102, + "epoch": 0.19530102790014683, + "fcm_dpo/beta": 0.04835614562034607, + "fcm_dpo/delta": -0.08480934053659439, + "fcm_dpo/margin": 9.942688941955566, + "fcm_dpo/q_t": 0.3915287256240845, + "grad_norm": 14.990748405456543, + "learning_rate": 4.870401618977415e-07, + "logits/chosen": -1.5841182470321655, + "logits/rejected": -1.1946488618850708, + "logps/chosen": -97.53675842285156, + "logps/ref_chosen": -93.11666107177734, + "logps/ref_rejected": -102.09490966796875, + "logps/rejected": -116.45768737792969, + "loss": 1.0338, + "margin_dpo/margin_mean": 9.94268798828125, + "margin_dpo/margin_std": 11.722248077392578, + "step": 133 + }, + { + "KL/chosen_KL_mean": -4.661293029785156, + "KL/mean": -10.001508712768555, + "KL/rejected_KL_mean": -15.341720581054688, + "KL/std": 10.434558868408203, + "epoch": 0.19676945668135096, + "fcm_dpo/beta": 0.04762697592377663, + "fcm_dpo/delta": -0.11471561342477798, + "fcm_dpo/margin": 10.680419921875, + "fcm_dpo/q_t": 0.3836822509765625, + "grad_norm": 14.525565147399902, + "learning_rate": 4.866292092063986e-07, + "logits/chosen": -1.5968610048294067, + "logits/rejected": -1.2609632015228271, + "logps/chosen": -88.21141815185547, + "logps/ref_chosen": -83.55012512207031, + "logps/ref_rejected": -93.8815689086914, + "logps/rejected": -109.2232894897461, + "loss": 1.0126, + "margin_dpo/margin_mean": 10.680419921875, + "margin_dpo/margin_std": 11.334150314331055, + "step": 134 + }, + { + "KL/chosen_KL_mean": -4.313234329223633, + "KL/mean": -9.659897804260254, + "KL/rejected_KL_mean": -15.006568908691406, + "KL/std": 11.448062896728516, + "epoch": 0.19823788546255505, + "fcm_dpo/beta": 0.04640686511993408, + "fcm_dpo/delta": -0.10114714503288269, + "fcm_dpo/margin": 10.693326950073242, + "fcm_dpo/q_t": 0.3898237943649292, + "grad_norm": 18.776870727539062, + "learning_rate": 4.862120211153265e-07, + "logits/chosen": -1.171066403388977, + "logits/rejected": -1.0541443824768066, + "logps/chosen": -81.89585876464844, + "logps/ref_chosen": -77.58262634277344, + "logps/ref_rejected": -121.05648040771484, + "logps/rejected": -136.06304931640625, + "loss": 1.0323, + "margin_dpo/margin_mean": 10.693326950073242, + "margin_dpo/margin_std": 12.919585227966309, + "step": 135 + }, + { + "KL/chosen_KL_mean": -5.020286560058594, + "KL/mean": -10.215295791625977, + "KL/rejected_KL_mean": -15.410301208496094, + "KL/std": 11.731208801269531, + "epoch": 0.19970631424375918, + "fcm_dpo/beta": 0.04551202431321144, + "fcm_dpo/delta": -0.0765582025051117, + "fcm_dpo/margin": 10.390012741088867, + "fcm_dpo/q_t": 0.39236894249916077, + "grad_norm": 14.227690696716309, + "learning_rate": 4.857886086178193e-07, + "logits/chosen": -1.4899359941482544, + "logits/rejected": -1.057348608970642, + "logps/chosen": -99.19302368164062, + "logps/ref_chosen": -94.1727294921875, + "logps/ref_rejected": -103.26428985595703, + "logps/rejected": -118.67459106445312, + "loss": 1.0329, + "margin_dpo/margin_mean": 10.390012741088867, + "margin_dpo/margin_std": 11.876028060913086, + "step": 136 + }, + { + "KL/chosen_KL_mean": -3.9241867065429688, + "KL/mean": -12.21324348449707, + "KL/rejected_KL_mean": -20.502304077148438, + "KL/std": 15.162540435791016, + "epoch": 0.2011747430249633, + "fcm_dpo/beta": 0.04288076236844063, + "fcm_dpo/delta": -0.33769649267196655, + "fcm_dpo/margin": 16.578121185302734, + "fcm_dpo/q_t": 0.34447985887527466, + "grad_norm": 16.599565505981445, + "learning_rate": 4.853589828711902e-07, + "logits/chosen": -1.4613754749298096, + "logits/rejected": -1.244441270828247, + "logps/chosen": -82.04292297363281, + "logps/ref_chosen": -78.11874389648438, + "logps/ref_rejected": -117.92237091064453, + "logps/rejected": -138.4246826171875, + "loss": 0.8989, + "margin_dpo/margin_mean": 16.5781192779541, + "margin_dpo/margin_std": 16.622516632080078, + "step": 137 + }, + { + "KL/chosen_KL_mean": -6.602939605712891, + "KL/mean": -11.714762687683105, + "KL/rejected_KL_mean": -16.826583862304688, + "KL/std": 11.099853515625, + "epoch": 0.2026431718061674, + "fcm_dpo/beta": 0.04212722182273865, + "fcm_dpo/delta": -0.03219844400882721, + "fcm_dpo/margin": 10.223644256591797, + "fcm_dpo/q_t": 0.40117987990379333, + "grad_norm": 13.062910079956055, + "learning_rate": 4.849231551964771e-07, + "logits/chosen": -1.0434367656707764, + "logits/rejected": -0.7851938605308533, + "logps/chosen": -99.4328842163086, + "logps/ref_chosen": -92.82994079589844, + "logps/ref_rejected": -98.64840698242188, + "logps/rejected": -115.47499084472656, + "loss": 1.0624, + "margin_dpo/margin_mean": 10.223644256591797, + "margin_dpo/margin_std": 12.262227058410645, + "step": 138 + }, + { + "KL/chosen_KL_mean": -4.611412048339844, + "KL/mean": -10.478314399719238, + "KL/rejected_KL_mean": -16.345218658447266, + "KL/std": 12.116909980773926, + "epoch": 0.20411160058737152, + "fcm_dpo/beta": 0.04147449880838394, + "fcm_dpo/delta": -0.09138133376836777, + "fcm_dpo/margin": 11.733804702758789, + "fcm_dpo/q_t": 0.3868618905544281, + "grad_norm": 13.359282493591309, + "learning_rate": 4.844811370781446e-07, + "logits/chosen": -1.4438704252243042, + "logits/rejected": -1.124290108680725, + "logps/chosen": -83.61442565917969, + "logps/ref_chosen": -79.00301361083984, + "logps/ref_rejected": -85.82733154296875, + "logps/rejected": -102.17254638671875, + "loss": 1.0373, + "margin_dpo/margin_mean": 11.733804702758789, + "margin_dpo/margin_std": 14.385666847229004, + "step": 139 + }, + { + "KL/chosen_KL_mean": -5.039573669433594, + "KL/mean": -10.38720703125, + "KL/rejected_KL_mean": -15.734840393066406, + "KL/std": 11.379945755004883, + "epoch": 0.2055800293685756, + "fcm_dpo/beta": 0.041134219616651535, + "fcm_dpo/delta": -0.04176412522792816, + "fcm_dpo/margin": 10.695271492004395, + "fcm_dpo/q_t": 0.39817529916763306, + "grad_norm": 12.633871078491211, + "learning_rate": 4.840329401637809e-07, + "logits/chosen": -1.629274845123291, + "logits/rejected": -1.2231906652450562, + "logps/chosen": -93.39202117919922, + "logps/ref_chosen": -88.35244750976562, + "logps/ref_rejected": -90.57476806640625, + "logps/rejected": -106.30961608886719, + "loss": 1.0519, + "margin_dpo/margin_mean": 10.695271492004395, + "margin_dpo/margin_std": 12.148259162902832, + "step": 140 + }, + { + "KL/chosen_KL_mean": -6.828273773193359, + "KL/mean": -12.909975051879883, + "KL/rejected_KL_mean": -18.991676330566406, + "KL/std": 14.93606948852539, + "epoch": 0.20704845814977973, + "fcm_dpo/beta": 0.04045747220516205, + "fcm_dpo/delta": -0.09682976454496384, + "fcm_dpo/margin": 12.163406372070312, + "fcm_dpo/q_t": 0.38734182715415955, + "grad_norm": 15.835349082946777, + "learning_rate": 4.83578576263792e-07, + "logits/chosen": -1.2295082807540894, + "logits/rejected": -0.8325952887535095, + "logps/chosen": -114.04843139648438, + "logps/ref_chosen": -107.22016143798828, + "logps/ref_rejected": -104.71223449707031, + "logps/rejected": -123.70390319824219, + "loss": 1.0373, + "margin_dpo/margin_mean": 12.163406372070312, + "margin_dpo/margin_std": 15.071775436401367, + "step": 141 + }, + { + "KL/chosen_KL_mean": -6.979434967041016, + "KL/mean": -12.973379135131836, + "KL/rejected_KL_mean": -18.967323303222656, + "KL/std": 13.900266647338867, + "epoch": 0.20851688693098386, + "fcm_dpo/beta": 0.039698414504528046, + "fcm_dpo/delta": -0.07988056540489197, + "fcm_dpo/margin": 11.98788833618164, + "fcm_dpo/q_t": 0.39354777336120605, + "grad_norm": 12.661876678466797, + "learning_rate": 4.83118057351089e-07, + "logits/chosen": -1.1199415922164917, + "logits/rejected": -0.8576666116714478, + "logps/chosen": -95.70838165283203, + "logps/ref_chosen": -88.72894287109375, + "logps/ref_rejected": -99.73038482666016, + "logps/rejected": -118.69770812988281, + "loss": 1.0381, + "margin_dpo/margin_mean": 11.98788833618164, + "margin_dpo/margin_std": 14.585189819335938, + "step": 142 + }, + { + "KL/chosen_KL_mean": -5.2153167724609375, + "KL/mean": -10.112159729003906, + "KL/rejected_KL_mean": -15.00899887084961, + "KL/std": 11.589612007141113, + "epoch": 0.20998531571218795, + "fcm_dpo/beta": 0.039522238075733185, + "fcm_dpo/delta": 0.013181895017623901, + "fcm_dpo/margin": 9.793678283691406, + "fcm_dpo/q_t": 0.4106484651565552, + "grad_norm": 12.86412239074707, + "learning_rate": 4.826513955607734e-07, + "logits/chosen": -1.5898847579956055, + "logits/rejected": -1.1357622146606445, + "logps/chosen": -90.71989440917969, + "logps/ref_chosen": -85.50457763671875, + "logps/ref_rejected": -85.8056640625, + "logps/rejected": -100.81465911865234, + "loss": 1.0973, + "margin_dpo/margin_mean": 9.793678283691406, + "margin_dpo/margin_std": 12.777687072753906, + "step": 143 + }, + { + "KL/chosen_KL_mean": -6.343677520751953, + "KL/mean": -11.225152969360352, + "KL/rejected_KL_mean": -16.10663604736328, + "KL/std": 12.24761962890625, + "epoch": 0.21145374449339208, + "fcm_dpo/beta": 0.03950557857751846, + "fcm_dpo/delta": -0.050757840275764465, + "fcm_dpo/margin": 9.762959480285645, + "fcm_dpo/q_t": 0.4123876690864563, + "grad_norm": 16.281400680541992, + "learning_rate": 4.821786031898176e-07, + "logits/chosen": -1.4441888332366943, + "logits/rejected": -0.898901104927063, + "logps/chosen": -111.03076934814453, + "logps/ref_chosen": -104.68708801269531, + "logps/ref_rejected": -87.71908569335938, + "logps/rejected": -103.82571411132812, + "loss": 1.1221, + "margin_dpo/margin_mean": 9.762960433959961, + "margin_dpo/margin_std": 15.188072204589844, + "step": 144 + }, + { + "KL/chosen_KL_mean": -5.382896423339844, + "KL/mean": -11.173028945922852, + "KL/rejected_KL_mean": -16.96316146850586, + "KL/std": 11.726531982421875, + "epoch": 0.21292217327459617, + "fcm_dpo/beta": 0.0389803871512413, + "fcm_dpo/delta": -0.05385002866387367, + "fcm_dpo/margin": 11.5802640914917, + "fcm_dpo/q_t": 0.3956826329231262, + "grad_norm": 13.102893829345703, + "learning_rate": 4.816996926967401e-07, + "logits/chosen": -1.0844286680221558, + "logits/rejected": -0.49777746200561523, + "logps/chosen": -93.90278625488281, + "logps/ref_chosen": -88.51988983154297, + "logps/ref_rejected": -83.76513671875, + "logps/rejected": -100.72830200195312, + "loss": 1.0462, + "margin_dpo/margin_mean": 11.580265045166016, + "margin_dpo/margin_std": 13.240138053894043, + "step": 145 + }, + { + "KL/chosen_KL_mean": -5.799015045166016, + "KL/mean": -12.270423889160156, + "KL/rejected_KL_mean": -18.741836547851562, + "KL/std": 12.897621154785156, + "epoch": 0.2143906020558003, + "fcm_dpo/beta": 0.038127027451992035, + "fcm_dpo/delta": -0.09929933398962021, + "fcm_dpo/margin": 12.942817687988281, + "fcm_dpo/q_t": 0.3859749734401703, + "grad_norm": 12.930309295654297, + "learning_rate": 4.812146767012779e-07, + "logits/chosen": -1.420979380607605, + "logits/rejected": -0.751975953578949, + "logps/chosen": -108.70487976074219, + "logps/ref_chosen": -102.90586853027344, + "logps/ref_rejected": -88.9493637084961, + "logps/rejected": -107.69120025634766, + "loss": 1.0203, + "margin_dpo/margin_mean": 12.942819595336914, + "margin_dpo/margin_std": 14.13513469696045, + "step": 146 + }, + { + "KL/chosen_KL_mean": -5.618473052978516, + "KL/mean": -12.281608581542969, + "KL/rejected_KL_mean": -18.944747924804688, + "KL/std": 12.288924217224121, + "epoch": 0.21585903083700442, + "fcm_dpo/beta": 0.0375375859439373, + "fcm_dpo/delta": -0.10548540949821472, + "fcm_dpo/margin": 13.326274871826172, + "fcm_dpo/q_t": 0.3839772939682007, + "grad_norm": 12.124649047851562, + "learning_rate": 4.807235679840536e-07, + "logits/chosen": -1.4902106523513794, + "logits/rejected": -0.9062970876693726, + "logps/chosen": -94.88066101074219, + "logps/ref_chosen": -89.26219177246094, + "logps/ref_rejected": -76.50011444091797, + "logps/rejected": -95.44486236572266, + "loss": 1.003, + "margin_dpo/margin_mean": 13.326274871826172, + "margin_dpo/margin_std": 13.225658416748047, + "step": 147 + }, + { + "KL/chosen_KL_mean": -4.025173187255859, + "KL/mean": -10.219099044799805, + "KL/rejected_KL_mean": -16.41303253173828, + "KL/std": 12.871241569519043, + "epoch": 0.2173274596182085, + "fcm_dpo/beta": 0.037258125841617584, + "fcm_dpo/delta": -0.06528796255588531, + "fcm_dpo/margin": 12.387852668762207, + "fcm_dpo/q_t": 0.39699316024780273, + "grad_norm": 12.853314399719238, + "learning_rate": 4.802263794862384e-07, + "logits/chosen": -1.6103897094726562, + "logits/rejected": -1.215408444404602, + "logps/chosen": -95.78805541992188, + "logps/ref_chosen": -91.76287841796875, + "logps/ref_rejected": -110.38552856445312, + "logps/rejected": -126.7985610961914, + "loss": 1.0535, + "margin_dpo/margin_mean": 12.387852668762207, + "margin_dpo/margin_std": 14.648200035095215, + "step": 148 + }, + { + "KL/chosen_KL_mean": -7.165077209472656, + "KL/mean": -13.094527244567871, + "KL/rejected_KL_mean": -19.023963928222656, + "KL/std": 13.681352615356445, + "epoch": 0.21879588839941264, + "fcm_dpo/beta": 0.03651132062077522, + "fcm_dpo/delta": -0.03489532321691513, + "fcm_dpo/margin": 11.85888671875, + "fcm_dpo/q_t": 0.4054684638977051, + "grad_norm": 12.661170959472656, + "learning_rate": 4.797231243092118e-07, + "logits/chosen": -1.0527235269546509, + "logits/rejected": -0.7100017666816711, + "logps/chosen": -100.64909362792969, + "logps/ref_chosen": -93.48401641845703, + "logps/ref_rejected": -105.6436996459961, + "logps/rejected": -124.66766357421875, + "loss": 1.0861, + "margin_dpo/margin_mean": 11.858888626098633, + "margin_dpo/margin_std": 17.029895782470703, + "step": 149 + }, + { + "KL/chosen_KL_mean": -5.740791320800781, + "KL/mean": -11.940508842468262, + "KL/rejected_KL_mean": -18.140228271484375, + "KL/std": 14.490161895751953, + "epoch": 0.22026431718061673, + "fcm_dpo/beta": 0.03621049225330353, + "fcm_dpo/delta": -0.05155959725379944, + "fcm_dpo/margin": 12.399433135986328, + "fcm_dpo/q_t": 0.3988327085971832, + "grad_norm": 11.37544059753418, + "learning_rate": 4.792138157142157e-07, + "logits/chosen": -1.078065276145935, + "logits/rejected": -0.8071381449699402, + "logps/chosen": -81.51947784423828, + "logps/ref_chosen": -75.7786865234375, + "logps/ref_rejected": -90.02676391601562, + "logps/rejected": -108.1669921875, + "loss": 1.0623, + "margin_dpo/margin_mean": 12.399433135986328, + "margin_dpo/margin_std": 16.125131607055664, + "step": 150 + }, + { + "KL/chosen_KL_mean": -7.623424530029297, + "KL/mean": -13.627882957458496, + "KL/rejected_KL_mean": -19.632335662841797, + "KL/std": 14.161249160766602, + "epoch": 0.22173274596182085, + "fcm_dpo/beta": 0.0360143817961216, + "fcm_dpo/delta": -0.033974550664424896, + "fcm_dpo/margin": 12.008913040161133, + "fcm_dpo/q_t": 0.40218406915664673, + "grad_norm": 13.809005737304688, + "learning_rate": 4.786984671220053e-07, + "logits/chosen": -2.040083646774292, + "logits/rejected": -1.446272373199463, + "logps/chosen": -110.35765075683594, + "logps/ref_chosen": -102.73422241210938, + "logps/ref_rejected": -106.03781127929688, + "logps/rejected": -125.67015075683594, + "loss": 1.0737, + "margin_dpo/margin_mean": 12.008913040161133, + "margin_dpo/margin_std": 16.029272079467773, + "step": 151 + }, + { + "KL/chosen_KL_mean": -4.768917083740234, + "KL/mean": -12.093599319458008, + "KL/rejected_KL_mean": -19.41828155517578, + "KL/std": 14.003814697265625, + "epoch": 0.22320117474302498, + "fcm_dpo/beta": 0.035251080989837646, + "fcm_dpo/delta": -0.12301421165466309, + "fcm_dpo/margin": 14.649358749389648, + "fcm_dpo/q_t": 0.3814336359500885, + "grad_norm": 12.385575294494629, + "learning_rate": 4.78177092112495e-07, + "logits/chosen": -1.3468904495239258, + "logits/rejected": -0.9493122100830078, + "logps/chosen": -102.35966491699219, + "logps/ref_chosen": -97.59074401855469, + "logps/ref_rejected": -110.49913024902344, + "logps/rejected": -129.91741943359375, + "loss": 1.0088, + "margin_dpo/margin_mean": 14.649358749389648, + "margin_dpo/margin_std": 16.299846649169922, + "step": 152 + }, + { + "KL/chosen_KL_mean": -7.291164398193359, + "KL/mean": -12.519810676574707, + "KL/rejected_KL_mean": -17.748458862304688, + "KL/std": 13.610466003417969, + "epoch": 0.22466960352422907, + "fcm_dpo/beta": 0.03525884449481964, + "fcm_dpo/delta": 0.0324854850769043, + "fcm_dpo/margin": 10.457305908203125, + "fcm_dpo/q_t": 0.41564375162124634, + "grad_norm": 12.339768409729004, + "learning_rate": 4.776497044244016e-07, + "logits/chosen": -1.7729458808898926, + "logits/rejected": -1.4528738260269165, + "logps/chosen": -94.31676483154297, + "logps/ref_chosen": -87.02560424804688, + "logps/ref_rejected": -91.06854248046875, + "logps/rejected": -108.81700134277344, + "loss": 1.1235, + "margin_dpo/margin_mean": 10.457305908203125, + "margin_dpo/margin_std": 15.588835716247559, + "step": 153 + }, + { + "KL/chosen_KL_mean": -5.521923065185547, + "KL/mean": -14.028284072875977, + "KL/rejected_KL_mean": -22.534645080566406, + "KL/std": 14.900197982788086, + "epoch": 0.2261380323054332, + "fcm_dpo/beta": 0.034528039395809174, + "fcm_dpo/delta": -0.19871015846729279, + "fcm_dpo/margin": 17.012727737426758, + "fcm_dpo/q_t": 0.36560943722724915, + "grad_norm": 12.075468063354492, + "learning_rate": 4.771163179548808e-07, + "logits/chosen": -1.2888035774230957, + "logits/rejected": -0.959321141242981, + "logps/chosen": -96.62759399414062, + "logps/ref_chosen": -91.10566711425781, + "logps/ref_rejected": -111.37238311767578, + "logps/rejected": -133.9070281982422, + "loss": 0.9544, + "margin_dpo/margin_mean": 17.01272964477539, + "margin_dpo/margin_std": 15.812162399291992, + "step": 154 + }, + { + "KL/chosen_KL_mean": -7.637653350830078, + "KL/mean": -13.939414978027344, + "KL/rejected_KL_mean": -20.241172790527344, + "KL/std": 15.43907356262207, + "epoch": 0.2276064610866373, + "fcm_dpo/beta": 0.03364599868655205, + "fcm_dpo/delta": -0.025743480771780014, + "fcm_dpo/margin": 12.603522300720215, + "fcm_dpo/q_t": 0.40080979466438293, + "grad_norm": 14.663453102111816, + "learning_rate": 4.7657694675916247e-07, + "logits/chosen": -1.446264386177063, + "logits/rejected": -1.0660717487335205, + "logps/chosen": -102.19862365722656, + "logps/ref_chosen": -94.56097412109375, + "logps/ref_rejected": -84.77359008789062, + "logps/rejected": -105.01476287841797, + "loss": 1.1017, + "margin_dpo/margin_mean": 12.603523254394531, + "margin_dpo/margin_std": 19.157184600830078, + "step": 155 + }, + { + "KL/chosen_KL_mean": -9.37765121459961, + "KL/mean": -15.051112174987793, + "KL/rejected_KL_mean": -20.724571228027344, + "KL/std": 13.716203689575195, + "epoch": 0.2290748898678414, + "fcm_dpo/beta": 0.03350888937711716, + "fcm_dpo/delta": -0.038376789540052414, + "fcm_dpo/margin": 11.34692096710205, + "fcm_dpo/q_t": 0.4143773913383484, + "grad_norm": 13.021503448486328, + "learning_rate": 4.7603160505017893e-07, + "logits/chosen": -1.6705093383789062, + "logits/rejected": -1.1727981567382812, + "logps/chosen": -104.03303527832031, + "logps/ref_chosen": -94.65538787841797, + "logps/ref_rejected": -84.41934204101562, + "logps/rejected": -105.14391326904297, + "loss": 1.1178, + "margin_dpo/margin_mean": 11.34692096710205, + "margin_dpo/margin_std": 16.84894371032715, + "step": 156 + }, + { + "KL/chosen_KL_mean": -8.810501098632812, + "KL/mean": -16.74010467529297, + "KL/rejected_KL_mean": -24.66971206665039, + "KL/std": 14.787250518798828, + "epoch": 0.2305433186490455, + "fcm_dpo/beta": 0.03291456401348114, + "fcm_dpo/delta": -0.1285938024520874, + "fcm_dpo/margin": 15.85920524597168, + "fcm_dpo/q_t": 0.3789578080177307, + "grad_norm": 13.065104484558105, + "learning_rate": 4.7548030719819154e-07, + "logits/chosen": -1.9881575107574463, + "logits/rejected": -1.5299711227416992, + "logps/chosen": -105.82289123535156, + "logps/ref_chosen": -97.01239013671875, + "logps/ref_rejected": -111.70133972167969, + "logps/rejected": -136.37106323242188, + "loss": 0.9921, + "margin_dpo/margin_mean": 15.859204292297363, + "margin_dpo/margin_std": 15.631433486938477, + "step": 157 + }, + { + "KL/chosen_KL_mean": -7.910472869873047, + "KL/mean": -16.244686126708984, + "KL/rejected_KL_mean": -24.578895568847656, + "KL/std": 15.939573287963867, + "epoch": 0.23201174743024963, + "fcm_dpo/beta": 0.03203149139881134, + "fcm_dpo/delta": -0.14129498600959778, + "fcm_dpo/margin": 16.668426513671875, + "fcm_dpo/q_t": 0.3779933750629425, + "grad_norm": 12.011423110961914, + "learning_rate": 4.7492306773041136e-07, + "logits/chosen": -1.4900768995285034, + "logits/rejected": -1.2238035202026367, + "logps/chosen": -102.6234359741211, + "logps/ref_chosen": -94.71296691894531, + "logps/ref_rejected": -120.93733978271484, + "logps/rejected": -145.5162353515625, + "loss": 0.9892, + "margin_dpo/margin_mean": 16.668426513671875, + "margin_dpo/margin_std": 17.173080444335938, + "step": 158 + }, + { + "KL/chosen_KL_mean": -7.4810791015625, + "KL/mean": -14.905092239379883, + "KL/rejected_KL_mean": -22.329105377197266, + "KL/std": 15.53576946258545, + "epoch": 0.23348017621145375, + "fcm_dpo/beta": 0.03140883520245552, + "fcm_dpo/delta": -0.06963557749986649, + "fcm_dpo/margin": 14.848034858703613, + "fcm_dpo/q_t": 0.39328068494796753, + "grad_norm": 11.008720397949219, + "learning_rate": 4.743599013306165e-07, + "logits/chosen": -1.860887050628662, + "logits/rejected": -1.351900577545166, + "logps/chosen": -115.6993179321289, + "logps/ref_chosen": -108.2182388305664, + "logps/ref_rejected": -94.83901977539062, + "logps/rejected": -117.16812896728516, + "loss": 1.0437, + "margin_dpo/margin_mean": 14.84803581237793, + "margin_dpo/margin_std": 17.740581512451172, + "step": 159 + }, + { + "KL/chosen_KL_mean": -6.347347259521484, + "KL/mean": -13.680153846740723, + "KL/rejected_KL_mean": -21.012958526611328, + "KL/std": 14.236058235168457, + "epoch": 0.23494860499265785, + "fcm_dpo/beta": 0.030983250588178635, + "fcm_dpo/delta": -0.05716680362820625, + "fcm_dpo/margin": 14.665613174438477, + "fcm_dpo/q_t": 0.39353251457214355, + "grad_norm": 11.312978744506836, + "learning_rate": 4.737908228387656e-07, + "logits/chosen": -1.736264705657959, + "logits/rejected": -1.1935105323791504, + "logps/chosen": -104.51075744628906, + "logps/ref_chosen": -98.16340637207031, + "logps/ref_rejected": -102.28097534179688, + "logps/rejected": -123.29393768310547, + "loss": 1.0346, + "margin_dpo/margin_mean": 14.665614128112793, + "margin_dpo/margin_std": 15.253125190734863, + "step": 160 + }, + { + "KL/chosen_KL_mean": -8.444732666015625, + "KL/mean": -15.035680770874023, + "KL/rejected_KL_mean": -21.626625061035156, + "KL/std": 13.177711486816406, + "epoch": 0.23641703377386197, + "fcm_dpo/beta": 0.030808940529823303, + "fcm_dpo/delta": -0.006680246442556381, + "fcm_dpo/margin": 13.18189811706543, + "fcm_dpo/q_t": 0.40630465745925903, + "grad_norm": 11.289678573608398, + "learning_rate": 4.7321584725060594e-07, + "logits/chosen": -1.7891913652420044, + "logits/rejected": -1.293304443359375, + "logps/chosen": -95.03660583496094, + "logps/ref_chosen": -86.59187316894531, + "logps/ref_rejected": -89.01643371582031, + "logps/rejected": -110.64305877685547, + "loss": 1.0872, + "margin_dpo/margin_mean": 13.18189811706543, + "margin_dpo/margin_std": 17.073577880859375, + "step": 161 + }, + { + "KL/chosen_KL_mean": -7.480491638183594, + "KL/mean": -14.86019515991211, + "KL/rejected_KL_mean": -22.23989486694336, + "KL/std": 15.275270462036133, + "epoch": 0.23788546255506607, + "fcm_dpo/beta": 0.030585046857595444, + "fcm_dpo/delta": -0.05412214249372482, + "fcm_dpo/margin": 14.759407997131348, + "fcm_dpo/q_t": 0.3945804834365845, + "grad_norm": 10.866828918457031, + "learning_rate": 4.7263498971727905e-07, + "logits/chosen": -1.2355608940124512, + "logits/rejected": -0.8285146951675415, + "logps/chosen": -104.88337707519531, + "logps/ref_chosen": -97.40288543701172, + "logps/ref_rejected": -95.86923217773438, + "logps/rejected": -118.109130859375, + "loss": 1.0426, + "margin_dpo/margin_mean": 14.759408950805664, + "margin_dpo/margin_std": 16.299522399902344, + "step": 162 + }, + { + "KL/chosen_KL_mean": -7.325176239013672, + "KL/mean": -16.010160446166992, + "KL/rejected_KL_mean": -24.695148468017578, + "KL/std": 17.343318939208984, + "epoch": 0.2393538913362702, + "fcm_dpo/beta": 0.02993404120206833, + "fcm_dpo/delta": -0.12696653604507446, + "fcm_dpo/margin": 17.36997413635254, + "fcm_dpo/q_t": 0.3817980885505676, + "grad_norm": 9.801036834716797, + "learning_rate": 4.720482655449212e-07, + "logits/chosen": -1.6288502216339111, + "logits/rejected": -1.2274677753448486, + "logps/chosen": -89.50056457519531, + "logps/ref_chosen": -82.1753921508789, + "logps/ref_rejected": -82.86473846435547, + "logps/rejected": -107.55989074707031, + "loss": 1.0006, + "margin_dpo/margin_mean": 17.36997413635254, + "margin_dpo/margin_std": 18.650039672851562, + "step": 163 + }, + { + "KL/chosen_KL_mean": -11.081611633300781, + "KL/mean": -17.008907318115234, + "KL/rejected_KL_mean": -22.936203002929688, + "KL/std": 13.699360847473145, + "epoch": 0.24082232011747431, + "fcm_dpo/beta": 0.02955673262476921, + "fcm_dpo/delta": -0.0430353507399559, + "fcm_dpo/margin": 11.854592323303223, + "fcm_dpo/q_t": 0.41718682646751404, + "grad_norm": 11.487072944641113, + "learning_rate": 4.714556901942599e-07, + "logits/chosen": -1.407117486000061, + "logits/rejected": -0.9362297058105469, + "logps/chosen": -101.55587768554688, + "logps/ref_chosen": -90.47427368164062, + "logps/ref_rejected": -87.02809143066406, + "logps/rejected": -109.96429443359375, + "loss": 1.1175, + "margin_dpo/margin_mean": 11.854593276977539, + "margin_dpo/margin_std": 15.119787216186523, + "step": 164 + }, + { + "KL/chosen_KL_mean": -7.5335235595703125, + "KL/mean": -16.14388656616211, + "KL/rejected_KL_mean": -24.754249572753906, + "KL/std": 16.31938362121582, + "epoch": 0.2422907488986784, + "fcm_dpo/beta": 0.029155373573303223, + "fcm_dpo/delta": -0.10733498632907867, + "fcm_dpo/margin": 17.220731735229492, + "fcm_dpo/q_t": 0.386141836643219, + "grad_norm": 9.381265640258789, + "learning_rate": 4.708572792802069e-07, + "logits/chosen": -1.7909035682678223, + "logits/rejected": -1.3680275678634644, + "logps/chosen": -94.84675598144531, + "logps/ref_chosen": -87.313232421875, + "logps/ref_rejected": -81.902587890625, + "logps/rejected": -106.65682983398438, + "loss": 1.0092, + "margin_dpo/margin_mean": 17.22072982788086, + "margin_dpo/margin_std": 18.515459060668945, + "step": 165 + }, + { + "KL/chosen_KL_mean": -12.25210952758789, + "KL/mean": -20.740781784057617, + "KL/rejected_KL_mean": -29.229454040527344, + "KL/std": 17.86658477783203, + "epoch": 0.24375917767988253, + "fcm_dpo/beta": 0.028545517474412918, + "fcm_dpo/delta": -0.08894811570644379, + "fcm_dpo/margin": 16.97733497619629, + "fcm_dpo/q_t": 0.3932517170906067, + "grad_norm": 10.706637382507324, + "learning_rate": 4.702530485714461e-07, + "logits/chosen": -1.5442438125610352, + "logits/rejected": -1.3437598943710327, + "logps/chosen": -93.83818054199219, + "logps/ref_chosen": -81.58606719970703, + "logps/ref_rejected": -103.16517639160156, + "logps/rejected": -132.39462280273438, + "loss": 1.0388, + "margin_dpo/margin_mean": 16.97733497619629, + "margin_dpo/margin_std": 21.5998592376709, + "step": 166 + }, + { + "KL/chosen_KL_mean": -10.866371154785156, + "KL/mean": -20.000240325927734, + "KL/rejected_KL_mean": -29.134113311767578, + "KL/std": 16.47540855407715, + "epoch": 0.24522760646108663, + "fcm_dpo/beta": 0.027795948088169098, + "fcm_dpo/delta": -0.11465759575366974, + "fcm_dpo/margin": 18.26773452758789, + "fcm_dpo/q_t": 0.3816662132740021, + "grad_norm": 9.923656463623047, + "learning_rate": 4.6964301399001877e-07, + "logits/chosen": -1.185351848602295, + "logits/rejected": -1.0142467021942139, + "logps/chosen": -94.42803955078125, + "logps/ref_chosen": -83.56167602539062, + "logps/ref_rejected": -99.05305480957031, + "logps/rejected": -128.18716430664062, + "loss": 0.9928, + "margin_dpo/margin_mean": 18.26773452758789, + "margin_dpo/margin_std": 16.833255767822266, + "step": 167 + }, + { + "KL/chosen_KL_mean": -10.428199768066406, + "KL/mean": -18.22240447998047, + "KL/rejected_KL_mean": -26.01660919189453, + "KL/std": 16.315227508544922, + "epoch": 0.24669603524229075, + "fcm_dpo/beta": 0.027679800987243652, + "fcm_dpo/delta": -0.03289950639009476, + "fcm_dpo/margin": 15.588400840759277, + "fcm_dpo/q_t": 0.39915919303894043, + "grad_norm": 10.774555206298828, + "learning_rate": 4.690271916109034e-07, + "logits/chosen": -1.5796266794204712, + "logits/rejected": -1.2353956699371338, + "logps/chosen": -90.35943603515625, + "logps/ref_chosen": -79.93123626708984, + "logps/ref_rejected": -80.10216522216797, + "logps/rejected": -106.1187744140625, + "loss": 1.0489, + "margin_dpo/margin_mean": 15.588400840759277, + "margin_dpo/margin_std": 16.35304832458496, + "step": 168 + }, + { + "KL/chosen_KL_mean": -9.043933868408203, + "KL/mean": -18.00758171081543, + "KL/rejected_KL_mean": -26.97122573852539, + "KL/std": 17.62850570678711, + "epoch": 0.24816446402349487, + "fcm_dpo/beta": 0.0271998792886734, + "fcm_dpo/delta": -0.09217746555805206, + "fcm_dpo/margin": 17.927284240722656, + "fcm_dpo/q_t": 0.3879433572292328, + "grad_norm": 10.464804649353027, + "learning_rate": 4.6840559766159235e-07, + "logits/chosen": -1.6549845933914185, + "logits/rejected": -1.4354002475738525, + "logps/chosen": -91.94763946533203, + "logps/ref_chosen": -82.90370178222656, + "logps/ref_rejected": -90.12191009521484, + "logps/rejected": -117.0931396484375, + "loss": 1.0164, + "margin_dpo/margin_mean": 17.92728614807129, + "margin_dpo/margin_std": 18.866226196289062, + "step": 169 + }, + { + "KL/chosen_KL_mean": -10.63247299194336, + "KL/mean": -18.46246337890625, + "KL/rejected_KL_mean": -26.292449951171875, + "KL/std": 14.761564254760742, + "epoch": 0.24963289280469897, + "fcm_dpo/beta": 0.026878267526626587, + "fcm_dpo/delta": -0.022340044379234314, + "fcm_dpo/margin": 15.659981727600098, + "fcm_dpo/q_t": 0.40010008215904236, + "grad_norm": 10.76467227935791, + "learning_rate": 4.6777824852166437e-07, + "logits/chosen": -1.5600578784942627, + "logits/rejected": -1.3475031852722168, + "logps/chosen": -85.61444091796875, + "logps/ref_chosen": -74.98197174072266, + "logps/ref_rejected": -82.16560363769531, + "logps/rejected": -108.45805358886719, + "loss": 1.0478, + "margin_dpo/margin_mean": 15.659982681274414, + "margin_dpo/margin_std": 14.7039794921875, + "step": 170 + }, + { + "KL/chosen_KL_mean": -13.5615234375, + "KL/mean": -22.261917114257812, + "KL/rejected_KL_mean": -30.96231460571289, + "KL/std": 21.234651565551758, + "epoch": 0.2511013215859031, + "fcm_dpo/beta": 0.02671811357140541, + "fcm_dpo/delta": -0.06803969293832779, + "fcm_dpo/margin": 17.400794982910156, + "fcm_dpo/q_t": 0.39949217438697815, + "grad_norm": 12.291194915771484, + "learning_rate": 4.6714516072235273e-07, + "logits/chosen": -1.721679925918579, + "logits/rejected": -1.1742870807647705, + "logps/chosen": -132.54005432128906, + "logps/ref_chosen": -118.97853088378906, + "logps/ref_rejected": -115.76930236816406, + "logps/rejected": -146.73162841796875, + "loss": 1.0711, + "margin_dpo/margin_mean": 17.400794982910156, + "margin_dpo/margin_std": 25.43389320373535, + "step": 171 + }, + { + "KL/chosen_KL_mean": -14.527019500732422, + "KL/mean": -22.23581314086914, + "KL/rejected_KL_mean": -29.944602966308594, + "KL/std": 15.254207611083984, + "epoch": 0.2525697503671072, + "fcm_dpo/beta": 0.026501990854740143, + "fcm_dpo/delta": -0.009108584374189377, + "fcm_dpo/margin": 15.417583465576172, + "fcm_dpo/q_t": 0.4019724130630493, + "grad_norm": 10.160428047180176, + "learning_rate": 4.6650635094610966e-07, + "logits/chosen": -2.3624086380004883, + "logits/rejected": -1.6074013710021973, + "logps/chosen": -115.6808090209961, + "logps/ref_chosen": -101.15379333496094, + "logps/ref_rejected": -91.00568389892578, + "logps/rejected": -120.95028686523438, + "loss": 1.046, + "margin_dpo/margin_mean": 15.417583465576172, + "margin_dpo/margin_std": 12.492011070251465, + "step": 172 + }, + { + "KL/chosen_KL_mean": -13.005489349365234, + "KL/mean": -19.82510757446289, + "KL/rejected_KL_mean": -26.64472198486328, + "KL/std": 14.4640474319458, + "epoch": 0.2540381791483113, + "fcm_dpo/beta": 0.026738755404949188, + "fcm_dpo/delta": 0.036543648689985275, + "fcm_dpo/margin": 13.63923454284668, + "fcm_dpo/q_t": 0.4138786494731903, + "grad_norm": 9.789052963256836, + "learning_rate": 4.6586183602616687e-07, + "logits/chosen": -1.6370983123779297, + "logits/rejected": -1.1626412868499756, + "logps/chosen": -112.06217956542969, + "logps/ref_chosen": -99.05668640136719, + "logps/ref_rejected": -83.8507308959961, + "logps/rejected": -110.49545288085938, + "loss": 1.0944, + "margin_dpo/margin_mean": 13.63923454284668, + "margin_dpo/margin_std": 15.29142951965332, + "step": 173 + }, + { + "KL/chosen_KL_mean": -12.65925407409668, + "KL/mean": -22.08783721923828, + "KL/rejected_KL_mean": -31.51641845703125, + "KL/std": 19.73651885986328, + "epoch": 0.2555066079295154, + "fcm_dpo/beta": 0.026379603892564774, + "fcm_dpo/delta": -0.10244762897491455, + "fcm_dpo/margin": 18.857162475585938, + "fcm_dpo/q_t": 0.3849840462207794, + "grad_norm": 11.376117706298828, + "learning_rate": 4.652116329460919e-07, + "logits/chosen": -1.8383495807647705, + "logits/rejected": -1.7962310314178467, + "logps/chosen": -93.06816101074219, + "logps/ref_chosen": -80.40890502929688, + "logps/ref_rejected": -108.20171356201172, + "logps/rejected": -139.7181396484375, + "loss": 1.0113, + "margin_dpo/margin_mean": 18.857162475585938, + "margin_dpo/margin_std": 19.820348739624023, + "step": 174 + }, + { + "KL/chosen_KL_mean": -16.196422576904297, + "KL/mean": -25.205842971801758, + "KL/rejected_KL_mean": -34.21526336669922, + "KL/std": 17.551250457763672, + "epoch": 0.25697503671071953, + "fcm_dpo/beta": 0.02599494531750679, + "fcm_dpo/delta": -0.07170958817005157, + "fcm_dpo/margin": 18.018842697143555, + "fcm_dpo/q_t": 0.38958460092544556, + "grad_norm": 11.104044914245605, + "learning_rate": 4.645557588393406e-07, + "logits/chosen": -1.010321855545044, + "logits/rejected": -0.7336099743843079, + "logps/chosen": -98.91291809082031, + "logps/ref_chosen": -82.71649169921875, + "logps/ref_rejected": -96.66712951660156, + "logps/rejected": -130.88238525390625, + "loss": 1.0182, + "margin_dpo/margin_mean": 18.018844604492188, + "margin_dpo/margin_std": 16.789112091064453, + "step": 175 + }, + { + "KL/chosen_KL_mean": -13.433433532714844, + "KL/mean": -22.715320587158203, + "KL/rejected_KL_mean": -31.997203826904297, + "KL/std": 19.08074951171875, + "epoch": 0.25844346549192365, + "fcm_dpo/beta": 0.025528497993946075, + "fcm_dpo/delta": -0.07761147618293762, + "fcm_dpo/margin": 18.563766479492188, + "fcm_dpo/q_t": 0.39184433221817017, + "grad_norm": 10.83478832244873, + "learning_rate": 4.638942309888058e-07, + "logits/chosen": -1.50587797164917, + "logits/rejected": -1.3449994325637817, + "logps/chosen": -93.06273651123047, + "logps/ref_chosen": -79.62930297851562, + "logps/ref_rejected": -102.6885986328125, + "logps/rejected": -134.685791015625, + "loss": 1.0394, + "margin_dpo/margin_mean": 18.563766479492188, + "margin_dpo/margin_std": 21.974594116210938, + "step": 176 + }, + { + "KL/chosen_KL_mean": -15.007614135742188, + "KL/mean": -25.763689041137695, + "KL/rejected_KL_mean": -36.5197639465332, + "KL/std": 21.865760803222656, + "epoch": 0.2599118942731278, + "fcm_dpo/beta": 0.024997854605317116, + "fcm_dpo/delta": -0.14533759653568268, + "fcm_dpo/margin": 21.512149810791016, + "fcm_dpo/q_t": 0.3792595863342285, + "grad_norm": 9.867096900939941, + "learning_rate": 4.6322706682636137e-07, + "logits/chosen": -1.5718889236450195, + "logits/rejected": -1.228896975517273, + "logps/chosen": -107.56871032714844, + "logps/ref_chosen": -92.56109619140625, + "logps/ref_rejected": -103.33839416503906, + "logps/rejected": -139.858154296875, + "loss": 0.9895, + "margin_dpo/margin_mean": 21.512149810791016, + "margin_dpo/margin_std": 22.594806671142578, + "step": 177 + }, + { + "KL/chosen_KL_mean": -13.884201049804688, + "KL/mean": -26.73963737487793, + "KL/rejected_KL_mean": -39.595062255859375, + "KL/std": 21.40091323852539, + "epoch": 0.26138032305433184, + "fcm_dpo/beta": 0.02386431209743023, + "fcm_dpo/delta": -0.22777138650417328, + "fcm_dpo/margin": 25.71085548400879, + "fcm_dpo/q_t": 0.3595629036426544, + "grad_norm": 10.852680206298828, + "learning_rate": 4.6255428393240354e-07, + "logits/chosen": -1.8012843132019043, + "logits/rejected": -1.5873453617095947, + "logps/chosen": -98.14999389648438, + "logps/ref_chosen": -84.26579284667969, + "logps/ref_rejected": -110.79730987548828, + "logps/rejected": -150.39236450195312, + "loss": 0.9296, + "margin_dpo/margin_mean": 25.710853576660156, + "margin_dpo/margin_std": 23.14470672607422, + "step": 178 + }, + { + "KL/chosen_KL_mean": -17.639938354492188, + "KL/mean": -25.848388671875, + "KL/rejected_KL_mean": -34.05683517456055, + "KL/std": 20.54976463317871, + "epoch": 0.26284875183553597, + "fcm_dpo/beta": 0.023576252162456512, + "fcm_dpo/delta": 0.01329508051276207, + "fcm_dpo/margin": 16.416900634765625, + "fcm_dpo/q_t": 0.41209518909454346, + "grad_norm": 10.426217079162598, + "learning_rate": 4.6187590003538724e-07, + "logits/chosen": -1.7180531024932861, + "logits/rejected": -1.2977184057235718, + "logps/chosen": -111.0879135131836, + "logps/ref_chosen": -93.4479751586914, + "logps/ref_rejected": -94.25541687011719, + "logps/rejected": -128.312255859375, + "loss": 1.1112, + "margin_dpo/margin_mean": 16.416898727416992, + "margin_dpo/margin_std": 23.931289672851562, + "step": 179 + }, + { + "KL/chosen_KL_mean": -14.733882904052734, + "KL/mean": -25.410295486450195, + "KL/rejected_KL_mean": -36.086708068847656, + "KL/std": 19.89807891845703, + "epoch": 0.2643171806167401, + "fcm_dpo/beta": 0.02324799634516239, + "fcm_dpo/delta": -0.10189928114414215, + "fcm_dpo/margin": 21.352828979492188, + "fcm_dpo/q_t": 0.38459378480911255, + "grad_norm": 12.427611351013184, + "learning_rate": 4.611919330113591e-07, + "logits/chosen": -1.5371224880218506, + "logits/rejected": -1.3066015243530273, + "logps/chosen": -103.56942749023438, + "logps/ref_chosen": -88.83554077148438, + "logps/ref_rejected": -103.90696716308594, + "logps/rejected": -139.99368286132812, + "loss": 1.0001, + "margin_dpo/margin_mean": 21.352828979492188, + "margin_dpo/margin_std": 20.412736892700195, + "step": 180 + }, + { + "KL/chosen_KL_mean": -14.737850189208984, + "KL/mean": -22.852394104003906, + "KL/rejected_KL_mean": -30.966934204101562, + "KL/std": 18.975448608398438, + "epoch": 0.2657856093979442, + "fcm_dpo/beta": 0.023292385041713715, + "fcm_dpo/delta": 0.022812366485595703, + "fcm_dpo/margin": 16.229084014892578, + "fcm_dpo/q_t": 0.4129781126976013, + "grad_norm": 9.517890930175781, + "learning_rate": 4.605024008834863e-07, + "logits/chosen": -1.6671667098999023, + "logits/rejected": -1.2102280855178833, + "logps/chosen": -93.97181701660156, + "logps/ref_chosen": -79.23396301269531, + "logps/ref_rejected": -69.17442321777344, + "logps/rejected": -100.141357421875, + "loss": 1.0944, + "margin_dpo/margin_mean": 16.229084014892578, + "margin_dpo/margin_std": 20.26224136352539, + "step": 181 + }, + { + "KL/chosen_KL_mean": -18.547344207763672, + "KL/mean": -29.14194107055664, + "KL/rejected_KL_mean": -39.736534118652344, + "KL/std": 21.912866592407227, + "epoch": 0.26725403817914833, + "fcm_dpo/beta": 0.02291329950094223, + "fcm_dpo/delta": -0.09038232266902924, + "fcm_dpo/margin": 21.189189910888672, + "fcm_dpo/q_t": 0.3889124393463135, + "grad_norm": 10.108099937438965, + "learning_rate": 4.598073218215817e-07, + "logits/chosen": -1.3922648429870605, + "logits/rejected": -1.1766209602355957, + "logps/chosen": -94.62953186035156, + "logps/ref_chosen": -76.08218383789062, + "logps/ref_rejected": -94.9686050415039, + "logps/rejected": -134.70513916015625, + "loss": 1.0278, + "margin_dpo/margin_mean": 21.189189910888672, + "margin_dpo/margin_std": 24.110326766967773, + "step": 182 + }, + { + "KL/chosen_KL_mean": -19.209930419921875, + "KL/mean": -29.572113037109375, + "KL/rejected_KL_mean": -39.934295654296875, + "KL/std": 20.031349182128906, + "epoch": 0.2687224669603524, + "fcm_dpo/beta": 0.02255667932331562, + "fcm_dpo/delta": -0.07131802290678024, + "fcm_dpo/margin": 20.724355697631836, + "fcm_dpo/q_t": 0.3906700611114502, + "grad_norm": 10.490107536315918, + "learning_rate": 4.5910671414162484e-07, + "logits/chosen": -1.5747482776641846, + "logits/rejected": -0.9683206081390381, + "logps/chosen": -113.11783599853516, + "logps/ref_chosen": -93.90790557861328, + "logps/ref_rejected": -83.56061553955078, + "logps/rejected": -123.49491119384766, + "loss": 1.0237, + "margin_dpo/margin_mean": 20.724355697631836, + "margin_dpo/margin_std": 20.56180191040039, + "step": 183 + }, + { + "KL/chosen_KL_mean": -17.145397186279297, + "KL/mean": -24.17635726928711, + "KL/rejected_KL_mean": -31.207317352294922, + "KL/std": 20.696651458740234, + "epoch": 0.2701908957415565, + "fcm_dpo/beta": 0.022848699241876602, + "fcm_dpo/delta": 0.08124689012765884, + "fcm_dpo/margin": 14.061912536621094, + "fcm_dpo/q_t": 0.42483752965927124, + "grad_norm": 9.723278045654297, + "learning_rate": 4.5840059630527985e-07, + "logits/chosen": -1.792701005935669, + "logits/rejected": -1.5211716890335083, + "logps/chosen": -104.0133056640625, + "logps/ref_chosen": -86.86790466308594, + "logps/ref_rejected": -84.28456115722656, + "logps/rejected": -115.49187469482422, + "loss": 1.1526, + "margin_dpo/margin_mean": 14.06191349029541, + "margin_dpo/margin_std": 22.382007598876953, + "step": 184 + }, + { + "KL/chosen_KL_mean": -18.211639404296875, + "KL/mean": -26.628517150878906, + "KL/rejected_KL_mean": -35.04539489746094, + "KL/std": 20.271533966064453, + "epoch": 0.27165932452276065, + "fcm_dpo/beta": 0.023033609613776207, + "fcm_dpo/delta": 0.012558795511722565, + "fcm_dpo/margin": 16.833759307861328, + "fcm_dpo/q_t": 0.41145211458206177, + "grad_norm": 10.948394775390625, + "learning_rate": 4.5768898691940836e-07, + "logits/chosen": -2.1860392093658447, + "logits/rejected": -1.571969747543335, + "logps/chosen": -112.23306274414062, + "logps/ref_chosen": -94.02142333984375, + "logps/ref_rejected": -81.45668029785156, + "logps/rejected": -116.5020751953125, + "loss": 1.1032, + "margin_dpo/margin_mean": 16.833759307861328, + "margin_dpo/margin_std": 23.41136932373047, + "step": 185 + }, + { + "KL/chosen_KL_mean": -22.441036224365234, + "KL/mean": -31.718746185302734, + "KL/rejected_KL_mean": -40.996456146240234, + "KL/std": 19.75735092163086, + "epoch": 0.27312775330396477, + "fcm_dpo/beta": 0.022826526314020157, + "fcm_dpo/delta": -0.024883225560188293, + "fcm_dpo/margin": 18.555416107177734, + "fcm_dpo/q_t": 0.4012763798236847, + "grad_norm": 11.483257293701172, + "learning_rate": 4.5697190473557947e-07, + "logits/chosen": -2.2207369804382324, + "logits/rejected": -1.296262502670288, + "logps/chosen": -136.63272094726562, + "logps/ref_chosen": -114.19168090820312, + "logps/ref_rejected": -93.33912658691406, + "logps/rejected": -134.33558654785156, + "loss": 1.0633, + "margin_dpo/margin_mean": 18.555418014526367, + "margin_dpo/margin_std": 21.36487579345703, + "step": 186 + }, + { + "KL/chosen_KL_mean": -19.209117889404297, + "KL/mean": -28.090518951416016, + "KL/rejected_KL_mean": -36.971920013427734, + "KL/std": 22.092451095581055, + "epoch": 0.2745961820851689, + "fcm_dpo/beta": 0.02291002869606018, + "fcm_dpo/delta": -0.007384965196251869, + "fcm_dpo/margin": 17.76280975341797, + "fcm_dpo/q_t": 0.4070413112640381, + "grad_norm": 10.197686195373535, + "learning_rate": 4.5624936864957555e-07, + "logits/chosen": -1.8901567459106445, + "logits/rejected": -1.6576085090637207, + "logps/chosen": -96.07866668701172, + "logps/ref_chosen": -76.86955261230469, + "logps/ref_rejected": -88.16494750976562, + "logps/rejected": -125.1368637084961, + "loss": 1.0915, + "margin_dpo/margin_mean": 17.762807846069336, + "margin_dpo/margin_std": 24.168514251708984, + "step": 187 + }, + { + "KL/chosen_KL_mean": -14.895027160644531, + "KL/mean": -24.94672966003418, + "KL/rejected_KL_mean": -34.998435974121094, + "KL/std": 21.68739128112793, + "epoch": 0.27606461086637296, + "fcm_dpo/beta": 0.02256125584244728, + "fcm_dpo/delta": -0.05659133940935135, + "fcm_dpo/margin": 20.103404998779297, + "fcm_dpo/q_t": 0.3929678797721863, + "grad_norm": 10.544901847839355, + "learning_rate": 4.5552139770089454e-07, + "logits/chosen": -2.15828800201416, + "logits/rejected": -1.919528603553772, + "logps/chosen": -94.08982849121094, + "logps/ref_chosen": -79.1948013305664, + "logps/ref_rejected": -96.82889556884766, + "logps/rejected": -131.82733154296875, + "loss": 1.0365, + "margin_dpo/margin_mean": 20.103404998779297, + "margin_dpo/margin_std": 20.449508666992188, + "step": 188 + }, + { + "KL/chosen_KL_mean": -19.69805145263672, + "KL/mean": -30.020755767822266, + "KL/rejected_KL_mean": -40.34346389770508, + "KL/std": 21.775554656982422, + "epoch": 0.2775330396475771, + "fcm_dpo/beta": 0.022475745528936386, + "fcm_dpo/delta": -0.06736817210912704, + "fcm_dpo/margin": 20.645416259765625, + "fcm_dpo/q_t": 0.39483535289764404, + "grad_norm": 9.479969024658203, + "learning_rate": 4.5478801107224794e-07, + "logits/chosen": -1.806870698928833, + "logits/rejected": -1.166749119758606, + "logps/chosen": -103.59085083007812, + "logps/ref_chosen": -83.8927993774414, + "logps/ref_rejected": -79.46589660644531, + "logps/rejected": -119.80935668945312, + "loss": 1.0431, + "margin_dpo/margin_mean": 20.645416259765625, + "margin_dpo/margin_std": 24.183279037475586, + "step": 189 + }, + { + "KL/chosen_KL_mean": -15.783485412597656, + "KL/mean": -28.141357421875, + "KL/rejected_KL_mean": -40.499237060546875, + "KL/std": 24.050844192504883, + "epoch": 0.2790014684287812, + "fcm_dpo/beta": 0.021788340061903, + "fcm_dpo/delta": -0.14646783471107483, + "fcm_dpo/margin": 24.71575355529785, + "fcm_dpo/q_t": 0.3746356666088104, + "grad_norm": 10.152409553527832, + "learning_rate": 4.5404922808905543e-07, + "logits/chosen": -1.8220771551132202, + "logits/rejected": -1.2215911149978638, + "logps/chosen": -119.2249526977539, + "logps/ref_chosen": -103.44146728515625, + "logps/ref_rejected": -109.21502685546875, + "logps/rejected": -149.71426391601562, + "loss": 0.9751, + "margin_dpo/margin_mean": 24.71575355529785, + "margin_dpo/margin_std": 22.809528350830078, + "step": 190 + }, + { + "KL/chosen_KL_mean": -15.049739837646484, + "KL/mean": -29.086788177490234, + "KL/rejected_KL_mean": -43.12383270263672, + "KL/std": 26.152400970458984, + "epoch": 0.28046989720998533, + "fcm_dpo/beta": 0.021027402952313423, + "fcm_dpo/delta": -0.20201078057289124, + "fcm_dpo/margin": 28.074092864990234, + "fcm_dpo/q_t": 0.3653144836425781, + "grad_norm": 11.1878080368042, + "learning_rate": 4.5330506821893565e-07, + "logits/chosen": -1.4565821886062622, + "logits/rejected": -1.1412721872329712, + "logps/chosen": -114.83525085449219, + "logps/ref_chosen": -99.78551483154297, + "logps/ref_rejected": -116.00028228759766, + "logps/rejected": -159.12411499023438, + "loss": 0.9504, + "margin_dpo/margin_mean": 28.074092864990234, + "margin_dpo/margin_std": 26.740522384643555, + "step": 191 + }, + { + "KL/chosen_KL_mean": -24.489482879638672, + "KL/mean": -34.255035400390625, + "KL/rejected_KL_mean": -44.020572662353516, + "KL/std": 20.901046752929688, + "epoch": 0.28193832599118945, + "fcm_dpo/beta": 0.02073926106095314, + "fcm_dpo/delta": -0.005302004981786013, + "fcm_dpo/margin": 19.53109359741211, + "fcm_dpo/q_t": 0.40668633580207825, + "grad_norm": 11.313491821289062, + "learning_rate": 4.5255555107119336e-07, + "logits/chosen": -1.6908667087554932, + "logits/rejected": -1.4428727626800537, + "logps/chosen": -125.34980773925781, + "logps/ref_chosen": -100.86032104492188, + "logps/ref_rejected": -109.44500732421875, + "logps/rejected": -153.465576171875, + "loss": 1.081, + "margin_dpo/margin_mean": 19.53109359741211, + "margin_dpo/margin_std": 24.421367645263672, + "step": 192 + }, + { + "KL/chosen_KL_mean": -20.182708740234375, + "KL/mean": -28.216341018676758, + "KL/rejected_KL_mean": -36.249977111816406, + "KL/std": 22.246002197265625, + "epoch": 0.2834067547723935, + "fcm_dpo/beta": 0.020767875015735626, + "fcm_dpo/delta": 0.006026466842740774, + "fcm_dpo/margin": 16.067264556884766, + "fcm_dpo/q_t": 0.42281368374824524, + "grad_norm": 10.591568946838379, + "learning_rate": 4.5180069639630236e-07, + "logits/chosen": -2.098865270614624, + "logits/rejected": -1.5169203281402588, + "logps/chosen": -113.12102508544922, + "logps/ref_chosen": -92.93831634521484, + "logps/ref_rejected": -83.10917663574219, + "logps/rejected": -119.35916137695312, + "loss": 1.1554, + "margin_dpo/margin_mean": 16.067264556884766, + "margin_dpo/margin_std": 26.86675262451172, + "step": 193 + }, + { + "KL/chosen_KL_mean": -20.637863159179688, + "KL/mean": -29.951847076416016, + "KL/rejected_KL_mean": -39.26583480834961, + "KL/std": 22.802433013916016, + "epoch": 0.28487518355359764, + "fcm_dpo/beta": 0.020880235359072685, + "fcm_dpo/delta": 0.011257613077759743, + "fcm_dpo/margin": 18.627979278564453, + "fcm_dpo/q_t": 0.4104408025741577, + "grad_norm": 9.059990882873535, + "learning_rate": 4.510405240853854e-07, + "logits/chosen": -2.1468214988708496, + "logits/rejected": -1.750035285949707, + "logps/chosen": -84.42943572998047, + "logps/ref_chosen": -63.79157257080078, + "logps/ref_rejected": -68.72665405273438, + "logps/rejected": -107.99249267578125, + "loss": 1.0945, + "margin_dpo/margin_mean": 18.627979278564453, + "margin_dpo/margin_std": 24.174999237060547, + "step": 194 + }, + { + "KL/chosen_KL_mean": -19.76596450805664, + "KL/mean": -32.27701187133789, + "KL/rejected_KL_mean": -44.78805160522461, + "KL/std": 23.19590950012207, + "epoch": 0.28634361233480177, + "fcm_dpo/beta": 0.020515337586402893, + "fcm_dpo/delta": -0.11929795145988464, + "fcm_dpo/margin": 25.0220947265625, + "fcm_dpo/q_t": 0.38101837038993835, + "grad_norm": 9.40182113647461, + "learning_rate": 4.5027505416968985e-07, + "logits/chosen": -1.3792896270751953, + "logits/rejected": -1.2623875141143799, + "logps/chosen": -103.65142822265625, + "logps/ref_chosen": -83.88545989990234, + "logps/ref_rejected": -104.00125122070312, + "logps/rejected": -148.789306640625, + "loss": 1.0065, + "margin_dpo/margin_mean": 25.022092819213867, + "margin_dpo/margin_std": 26.55062484741211, + "step": 195 + }, + { + "KL/chosen_KL_mean": -23.175148010253906, + "KL/mean": -34.83287811279297, + "KL/rejected_KL_mean": -46.4906120300293, + "KL/std": 24.843109130859375, + "epoch": 0.2878120411160059, + "fcm_dpo/beta": 0.020115695893764496, + "fcm_dpo/delta": -0.07241245359182358, + "fcm_dpo/margin": 23.31546401977539, + "fcm_dpo/q_t": 0.38920408487319946, + "grad_norm": 9.05499267578125, + "learning_rate": 4.495043068200599e-07, + "logits/chosen": -1.4280736446380615, + "logits/rejected": -0.9839767217636108, + "logps/chosen": -112.57650756835938, + "logps/ref_chosen": -89.4013671875, + "logps/ref_rejected": -83.34735107421875, + "logps/rejected": -129.83795166015625, + "loss": 1.0194, + "margin_dpo/margin_mean": 23.31546401977539, + "margin_dpo/margin_std": 22.447128295898438, + "step": 196 + }, + { + "KL/chosen_KL_mean": -23.49209976196289, + "KL/mean": -30.68783187866211, + "KL/rejected_KL_mean": -37.8835563659668, + "KL/std": 20.021530151367188, + "epoch": 0.28928046989721, + "fcm_dpo/beta": 0.020043786615133286, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 14.391450881958008, + "fcm_dpo/q_t": 0.4311813712120056, + "grad_norm": 9.273676872253418, + "learning_rate": 4.4872830234640493e-07, + "logits/chosen": -1.269303798675537, + "logits/rejected": -0.9314581751823425, + "logps/chosen": -110.33280944824219, + "logps/ref_chosen": -86.84070587158203, + "logps/ref_rejected": -82.79289245605469, + "logps/rejected": -120.67645263671875, + "loss": 1.1617, + "margin_dpo/margin_mean": 14.391451835632324, + "margin_dpo/margin_std": 21.266876220703125, + "step": 197 + }, + { + "KL/chosen_KL_mean": -27.17208480834961, + "KL/mean": -38.82708740234375, + "KL/rejected_KL_mean": -50.48210144042969, + "KL/std": 23.719057083129883, + "epoch": 0.2907488986784141, + "fcm_dpo/beta": 0.019858654588460922, + "fcm_dpo/delta": -0.06592979282140732, + "fcm_dpo/margin": 23.310012817382812, + "fcm_dpo/q_t": 0.39266514778137207, + "grad_norm": 10.997335433959961, + "learning_rate": 4.479470611971645e-07, + "logits/chosen": -1.7973957061767578, + "logits/rejected": -1.6265045404434204, + "logps/chosen": -112.22032928466797, + "logps/ref_chosen": -85.04824829101562, + "logps/ref_rejected": -104.10365295410156, + "logps/rejected": -154.58575439453125, + "loss": 1.03, + "margin_dpo/margin_mean": 23.31001091003418, + "margin_dpo/margin_std": 24.526268005371094, + "step": 198 + }, + { + "KL/chosen_KL_mean": -26.518173217773438, + "KL/mean": -36.141231536865234, + "KL/rejected_KL_mean": -45.764286041259766, + "KL/std": 23.040512084960938, + "epoch": 0.2922173274596182, + "fcm_dpo/beta": 0.019856173545122147, + "fcm_dpo/delta": 0.018514636904001236, + "fcm_dpo/margin": 19.246116638183594, + "fcm_dpo/q_t": 0.41099148988723755, + "grad_norm": 9.613306045532227, + "learning_rate": 4.471606039587695e-07, + "logits/chosen": -1.7797781229019165, + "logits/rejected": -1.525411605834961, + "logps/chosen": -108.30487060546875, + "logps/ref_chosen": -81.78669738769531, + "logps/ref_rejected": -90.790771484375, + "logps/rejected": -136.5550537109375, + "loss": 1.0949, + "margin_dpo/margin_mean": 19.246116638183594, + "margin_dpo/margin_std": 24.207881927490234, + "step": 199 + }, + { + "KL/chosen_KL_mean": -25.89590835571289, + "KL/mean": -37.32862854003906, + "KL/rejected_KL_mean": -48.76133346557617, + "KL/std": 24.73147964477539, + "epoch": 0.2936857562408223, + "fcm_dpo/beta": 0.019621796905994415, + "fcm_dpo/delta": -0.051508162170648575, + "fcm_dpo/margin": 22.86542510986328, + "fcm_dpo/q_t": 0.3969246447086334, + "grad_norm": 9.600115776062012, + "learning_rate": 4.4636895135509966e-07, + "logits/chosen": -1.6544878482818604, + "logits/rejected": -1.111642599105835, + "logps/chosen": -110.16400146484375, + "logps/ref_chosen": -84.26809692382812, + "logps/ref_rejected": -86.96786499023438, + "logps/rejected": -135.7292022705078, + "loss": 1.0611, + "margin_dpo/margin_mean": 22.86542510986328, + "margin_dpo/margin_std": 28.457225799560547, + "step": 200 + }, + { + "KL/chosen_KL_mean": -27.968212127685547, + "KL/mean": -39.04167175292969, + "KL/rejected_KL_mean": -50.11512756347656, + "KL/std": 26.870845794677734, + "epoch": 0.29515418502202645, + "fcm_dpo/beta": 0.01959862932562828, + "fcm_dpo/delta": -0.03564136475324631, + "fcm_dpo/margin": 22.14691925048828, + "fcm_dpo/q_t": 0.4006343185901642, + "grad_norm": 9.832130432128906, + "learning_rate": 4.455721242469372e-07, + "logits/chosen": -1.5167807340621948, + "logits/rejected": -1.1362334489822388, + "logps/chosen": -135.74070739746094, + "logps/ref_chosen": -107.77249145507812, + "logps/ref_rejected": -119.79248046875, + "logps/rejected": -169.90760803222656, + "loss": 1.075, + "margin_dpo/margin_mean": 22.14691925048828, + "margin_dpo/margin_std": 29.343774795532227, + "step": 201 + }, + { + "KL/chosen_KL_mean": -26.233989715576172, + "KL/mean": -36.52886962890625, + "KL/rejected_KL_mean": -46.82374572753906, + "KL/std": 22.212574005126953, + "epoch": 0.2966226138032305, + "fcm_dpo/beta": 0.019527671858668327, + "fcm_dpo/delta": -0.0021812673658132553, + "fcm_dpo/margin": 20.589757919311523, + "fcm_dpo/q_t": 0.40716424584388733, + "grad_norm": 10.090387344360352, + "learning_rate": 4.4477014363141755e-07, + "logits/chosen": -1.749312162399292, + "logits/rejected": -1.6240307092666626, + "logps/chosen": -102.2064437866211, + "logps/ref_chosen": -75.97245788574219, + "logps/ref_rejected": -94.4599838256836, + "logps/rejected": -141.28372192382812, + "loss": 1.0825, + "margin_dpo/margin_mean": 20.58975601196289, + "margin_dpo/margin_std": 25.11485481262207, + "step": 202 + }, + { + "KL/chosen_KL_mean": -23.40564727783203, + "KL/mean": -35.04082489013672, + "KL/rejected_KL_mean": -46.67601013183594, + "KL/std": 23.13131332397461, + "epoch": 0.29809104258443464, + "fcm_dpo/beta": 0.019376806914806366, + "fcm_dpo/delta": -0.05327478423714638, + "fcm_dpo/margin": 23.270366668701172, + "fcm_dpo/q_t": 0.39427393674850464, + "grad_norm": 9.077618598937988, + "learning_rate": 4.439630306414758e-07, + "logits/chosen": -1.965078353881836, + "logits/rejected": -1.5162969827651978, + "logps/chosen": -118.372802734375, + "logps/ref_chosen": -94.96715545654297, + "logps/ref_rejected": -92.8876724243164, + "logps/rejected": -139.56369018554688, + "loss": 1.0361, + "margin_dpo/margin_mean": 23.270366668701172, + "margin_dpo/margin_std": 23.98265838623047, + "step": 203 + }, + { + "KL/chosen_KL_mean": -29.340232849121094, + "KL/mean": -39.35368347167969, + "KL/rejected_KL_mean": -49.36712646484375, + "KL/std": 23.16506576538086, + "epoch": 0.29955947136563876, + "fcm_dpo/beta": 0.019382819533348083, + "fcm_dpo/delta": 0.0121227465569973, + "fcm_dpo/margin": 20.026893615722656, + "fcm_dpo/q_t": 0.4106913208961487, + "grad_norm": 9.94970417022705, + "learning_rate": 4.431508065452897e-07, + "logits/chosen": -1.9286906719207764, + "logits/rejected": -1.274007797241211, + "logps/chosen": -145.69741821289062, + "logps/ref_chosen": -116.35719299316406, + "logps/ref_rejected": -93.39759063720703, + "logps/rejected": -142.76470947265625, + "loss": 1.1013, + "margin_dpo/margin_mean": 20.026893615722656, + "margin_dpo/margin_std": 26.898170471191406, + "step": 204 + }, + { + "KL/chosen_KL_mean": -27.196041107177734, + "KL/mean": -40.15739440917969, + "KL/rejected_KL_mean": -53.118751525878906, + "KL/std": 25.728809356689453, + "epoch": 0.3010279001468429, + "fcm_dpo/beta": 0.019023960456252098, + "fcm_dpo/delta": -0.09826859086751938, + "fcm_dpo/margin": 25.922714233398438, + "fcm_dpo/q_t": 0.3828265070915222, + "grad_norm": 8.933135032653809, + "learning_rate": 4.4233349274571974e-07, + "logits/chosen": -2.452038288116455, + "logits/rejected": -2.041184425354004, + "logps/chosen": -116.05538940429688, + "logps/ref_chosen": -88.85934448242188, + "logps/ref_rejected": -91.8544921875, + "logps/rejected": -144.97323608398438, + "loss": 0.9938, + "margin_dpo/margin_mean": 25.922714233398438, + "margin_dpo/margin_std": 21.94232940673828, + "step": 205 + }, + { + "KL/chosen_KL_mean": -27.82909393310547, + "KL/mean": -39.71849060058594, + "KL/rejected_KL_mean": -51.60789108276367, + "KL/std": 24.90050506591797, + "epoch": 0.302496328928047, + "fcm_dpo/beta": 0.01876676455140114, + "fcm_dpo/delta": -0.048801764845848083, + "fcm_dpo/margin": 23.77880096435547, + "fcm_dpo/q_t": 0.3938303589820862, + "grad_norm": 10.346532821655273, + "learning_rate": 4.415111107797445e-07, + "logits/chosen": -1.6061272621154785, + "logits/rejected": -1.459729552268982, + "logps/chosen": -104.37544250488281, + "logps/ref_chosen": -76.54634857177734, + "logps/ref_rejected": -102.95314025878906, + "logps/rejected": -154.56103515625, + "loss": 1.0278, + "margin_dpo/margin_mean": 23.77880096435547, + "margin_dpo/margin_std": 21.266902923583984, + "step": 206 + }, + { + "KL/chosen_KL_mean": -29.162586212158203, + "KL/mean": -42.509910583496094, + "KL/rejected_KL_mean": -55.85722732543945, + "KL/std": 30.608707427978516, + "epoch": 0.3039647577092511, + "fcm_dpo/beta": 0.018516261130571365, + "fcm_dpo/delta": -0.09914683550596237, + "fcm_dpo/margin": 26.694644927978516, + "fcm_dpo/q_t": 0.3901873230934143, + "grad_norm": 10.37959098815918, + "learning_rate": 4.4068368231789365e-07, + "logits/chosen": -1.9148609638214111, + "logits/rejected": -1.4796404838562012, + "logps/chosen": -115.39423370361328, + "logps/ref_chosen": -86.23164367675781, + "logps/ref_rejected": -90.65512084960938, + "logps/rejected": -146.51234436035156, + "loss": 1.0456, + "margin_dpo/margin_mean": 26.69464683532715, + "margin_dpo/margin_std": 35.2462272644043, + "step": 207 + }, + { + "KL/chosen_KL_mean": -30.22491455078125, + "KL/mean": -43.85370635986328, + "KL/rejected_KL_mean": -57.48249816894531, + "KL/std": 27.858169555664062, + "epoch": 0.3054331864904552, + "fcm_dpo/beta": 0.018213478848338127, + "fcm_dpo/delta": -0.1014234870672226, + "fcm_dpo/margin": 27.25757598876953, + "fcm_dpo/q_t": 0.38294517993927, + "grad_norm": 10.183534622192383, + "learning_rate": 4.398512291636768e-07, + "logits/chosen": -1.8578312397003174, + "logits/rejected": -1.5064001083374023, + "logps/chosen": -124.38446044921875, + "logps/ref_chosen": -94.1595458984375, + "logps/ref_rejected": -100.96233367919922, + "logps/rejected": -158.44482421875, + "loss": 1.0027, + "margin_dpo/margin_mean": 27.257577896118164, + "margin_dpo/margin_std": 25.58877944946289, + "step": 208 + }, + { + "KL/chosen_KL_mean": -25.645763397216797, + "KL/mean": -37.72901916503906, + "KL/rejected_KL_mean": -49.8122673034668, + "KL/std": 24.893207550048828, + "epoch": 0.3069016152716593, + "fcm_dpo/beta": 0.01788686215877533, + "fcm_dpo/delta": -0.034120529890060425, + "fcm_dpo/margin": 24.16650390625, + "fcm_dpo/q_t": 0.3998543918132782, + "grad_norm": 9.04115104675293, + "learning_rate": 4.3901377325300857e-07, + "logits/chosen": -2.3702800273895264, + "logits/rejected": -1.9165239334106445, + "logps/chosen": -109.81632995605469, + "logps/ref_chosen": -84.17056274414062, + "logps/ref_rejected": -87.61955261230469, + "logps/rejected": -137.43182373046875, + "loss": 1.0607, + "margin_dpo/margin_mean": 24.16650390625, + "margin_dpo/margin_std": 28.215757369995117, + "step": 209 + }, + { + "KL/chosen_KL_mean": -29.096385955810547, + "KL/mean": -40.607025146484375, + "KL/rejected_KL_mean": -52.1176643371582, + "KL/std": 26.404951095581055, + "epoch": 0.30837004405286345, + "fcm_dpo/beta": 0.01786329597234726, + "fcm_dpo/delta": -0.011780261993408203, + "fcm_dpo/margin": 23.02127456665039, + "fcm_dpo/q_t": 0.4044468402862549, + "grad_norm": 8.486321449279785, + "learning_rate": 4.381713366536311e-07, + "logits/chosen": -1.9202561378479004, + "logits/rejected": -1.53126060962677, + "logps/chosen": -110.26756286621094, + "logps/ref_chosen": -81.17117309570312, + "logps/ref_rejected": -84.17478942871094, + "logps/rejected": -136.29244995117188, + "loss": 1.0696, + "margin_dpo/margin_mean": 23.02127456665039, + "margin_dpo/margin_std": 26.58807373046875, + "step": 210 + }, + { + "KL/chosen_KL_mean": -32.55935287475586, + "KL/mean": -43.85276794433594, + "KL/rejected_KL_mean": -55.146183013916016, + "KL/std": 27.905582427978516, + "epoch": 0.30983847283406757, + "fcm_dpo/beta": 0.01790366694331169, + "fcm_dpo/delta": -0.004649447277188301, + "fcm_dpo/margin": 22.586833953857422, + "fcm_dpo/q_t": 0.4066464304924011, + "grad_norm": 10.353544235229492, + "learning_rate": 4.373239415645323e-07, + "logits/chosen": -2.105949640274048, + "logits/rejected": -1.484708547592163, + "logps/chosen": -141.27206420898438, + "logps/ref_chosen": -108.71270751953125, + "logps/ref_rejected": -93.55564880371094, + "logps/rejected": -148.70184326171875, + "loss": 1.081, + "margin_dpo/margin_mean": 22.58683204650879, + "margin_dpo/margin_std": 28.160480499267578, + "step": 211 + }, + { + "KL/chosen_KL_mean": -22.877002716064453, + "KL/mean": -39.357173919677734, + "KL/rejected_KL_mean": -55.83734130859375, + "KL/std": 31.732728958129883, + "epoch": 0.31130690161527164, + "fcm_dpo/beta": 0.01734536699950695, + "fcm_dpo/delta": -0.1826426386833191, + "fcm_dpo/margin": 32.9603385925293, + "fcm_dpo/q_t": 0.3686683177947998, + "grad_norm": 9.021686553955078, + "learning_rate": 4.3647161031536086e-07, + "logits/chosen": -1.8028687238693237, + "logits/rejected": -1.3849356174468994, + "logps/chosen": -121.23895263671875, + "logps/ref_chosen": -98.36195373535156, + "logps/ref_rejected": -109.88999938964844, + "logps/rejected": -165.7273406982422, + "loss": 0.9615, + "margin_dpo/margin_mean": 32.96033477783203, + "margin_dpo/margin_std": 31.690784454345703, + "step": 212 + }, + { + "KL/chosen_KL_mean": -26.760753631591797, + "KL/mean": -39.19908905029297, + "KL/rejected_KL_mean": -51.637420654296875, + "KL/std": 28.971054077148438, + "epoch": 0.31277533039647576, + "fcm_dpo/beta": 0.01711571030318737, + "fcm_dpo/delta": -0.027080245316028595, + "fcm_dpo/margin": 24.876659393310547, + "fcm_dpo/q_t": 0.4012739062309265, + "grad_norm": 11.205018043518066, + "learning_rate": 4.3561436536583774e-07, + "logits/chosen": -1.945738434791565, + "logits/rejected": -1.1565263271331787, + "logps/chosen": -134.81607055664062, + "logps/ref_chosen": -108.05531311035156, + "logps/ref_rejected": -100.14414978027344, + "logps/rejected": -151.7815704345703, + "loss": 1.0737, + "margin_dpo/margin_mean": 24.87665557861328, + "margin_dpo/margin_std": 31.63521957397461, + "step": 213 + }, + { + "KL/chosen_KL_mean": -28.489566802978516, + "KL/mean": -40.16156005859375, + "KL/rejected_KL_mean": -51.83355712890625, + "KL/std": 30.93045425415039, + "epoch": 0.3142437591776799, + "fcm_dpo/beta": 0.01714509166777134, + "fcm_dpo/delta": -0.0002644965425133705, + "fcm_dpo/margin": 23.34398651123047, + "fcm_dpo/q_t": 0.40931522846221924, + "grad_norm": 8.833428382873535, + "learning_rate": 4.3475222930516473e-07, + "logits/chosen": -2.026113748550415, + "logits/rejected": -1.9088587760925293, + "logps/chosen": -106.29430389404297, + "logps/ref_chosen": -77.80473327636719, + "logps/ref_rejected": -84.27578735351562, + "logps/rejected": -136.10934448242188, + "loss": 1.1037, + "margin_dpo/margin_mean": 23.3439884185791, + "margin_dpo/margin_std": 33.94078063964844, + "step": 214 + }, + { + "KL/chosen_KL_mean": -24.645503997802734, + "KL/mean": -40.430931091308594, + "KL/rejected_KL_mean": -56.21635055541992, + "KL/std": 30.444000244140625, + "epoch": 0.315712187958884, + "fcm_dpo/beta": 0.016785655170679092, + "fcm_dpo/delta": -0.13718369603157043, + "fcm_dpo/margin": 31.570846557617188, + "fcm_dpo/q_t": 0.37727463245391846, + "grad_norm": 8.714178085327148, + "learning_rate": 4.3388522485142885e-07, + "logits/chosen": -1.6730263233184814, + "logits/rejected": -1.193305492401123, + "logps/chosen": -109.7593994140625, + "logps/ref_chosen": -85.1138916015625, + "logps/ref_rejected": -96.86151123046875, + "logps/rejected": -153.07786560058594, + "loss": 0.9818, + "margin_dpo/margin_mean": 31.570846557617188, + "margin_dpo/margin_std": 29.977745056152344, + "step": 215 + }, + { + "KL/chosen_KL_mean": -26.72441864013672, + "KL/mean": -40.45320129394531, + "KL/rejected_KL_mean": -54.18199157714844, + "KL/std": 27.406570434570312, + "epoch": 0.31718061674008813, + "fcm_dpo/beta": 0.016590487211942673, + "fcm_dpo/delta": -0.05830288678407669, + "fcm_dpo/margin": 27.45757293701172, + "fcm_dpo/q_t": 0.39406317472457886, + "grad_norm": 8.250274658203125, + "learning_rate": 4.330133748510036e-07, + "logits/chosen": -1.9427597522735596, + "logits/rejected": -1.5463614463806152, + "logps/chosen": -107.31672668457031, + "logps/ref_chosen": -80.59230041503906, + "logps/ref_rejected": -81.41983795166016, + "logps/rejected": -135.60183715820312, + "loss": 1.0346, + "margin_dpo/margin_mean": 27.45757293701172, + "margin_dpo/margin_std": 28.32803726196289, + "step": 216 + }, + { + "KL/chosen_KL_mean": -32.52296829223633, + "KL/mean": -43.812591552734375, + "KL/rejected_KL_mean": -55.10221481323242, + "KL/std": 28.377426147460938, + "epoch": 0.3186490455212922, + "fcm_dpo/beta": 0.016561094671487808, + "fcm_dpo/delta": 0.026995552703738213, + "fcm_dpo/margin": 22.579242706298828, + "fcm_dpo/q_t": 0.4147772490978241, + "grad_norm": 10.1908597946167, + "learning_rate": 4.3213670227794757e-07, + "logits/chosen": -2.124011516571045, + "logits/rejected": -1.8209376335144043, + "logps/chosen": -125.99555206298828, + "logps/ref_chosen": -93.47257995605469, + "logps/ref_rejected": -103.488525390625, + "logps/rejected": -158.59072875976562, + "loss": 1.1129, + "margin_dpo/margin_mean": 22.579242706298828, + "margin_dpo/margin_std": 32.31659698486328, + "step": 217 + }, + { + "KL/chosen_KL_mean": -32.00503921508789, + "KL/mean": -45.61492919921875, + "KL/rejected_KL_mean": -59.224830627441406, + "KL/std": 28.565357208251953, + "epoch": 0.3201174743024963, + "fcm_dpo/beta": 0.01639879308640957, + "fcm_dpo/delta": -0.04879341274499893, + "fcm_dpo/margin": 27.21978759765625, + "fcm_dpo/q_t": 0.39750951528549194, + "grad_norm": 8.804474830627441, + "learning_rate": 4.3125523023339815e-07, + "logits/chosen": -1.317617416381836, + "logits/rejected": -0.9291328191757202, + "logps/chosen": -121.06387329101562, + "logps/ref_chosen": -89.05883026123047, + "logps/ref_rejected": -94.30680847167969, + "logps/rejected": -153.53164672851562, + "loss": 1.0623, + "margin_dpo/margin_mean": 27.21978759765625, + "margin_dpo/margin_std": 34.204444885253906, + "step": 218 + }, + { + "KL/chosen_KL_mean": -28.777660369873047, + "KL/mean": -40.968997955322266, + "KL/rejected_KL_mean": -53.16034698486328, + "KL/std": 33.21924591064453, + "epoch": 0.32158590308370044, + "fcm_dpo/beta": 0.01631225273013115, + "fcm_dpo/delta": -0.053448427468538284, + "fcm_dpo/margin": 24.382694244384766, + "fcm_dpo/q_t": 0.4085707664489746, + "grad_norm": 9.802894592285156, + "learning_rate": 4.303689819449636e-07, + "logits/chosen": -1.6217575073242188, + "logits/rejected": -1.154639720916748, + "logps/chosen": -129.78500366210938, + "logps/ref_chosen": -101.00733947753906, + "logps/ref_rejected": -92.46794128417969, + "logps/rejected": -145.6282958984375, + "loss": 1.1063, + "margin_dpo/margin_mean": 24.382692337036133, + "margin_dpo/margin_std": 35.364593505859375, + "step": 219 + }, + { + "KL/chosen_KL_mean": -29.587989807128906, + "KL/mean": -41.061065673828125, + "KL/rejected_KL_mean": -52.534149169921875, + "KL/std": 27.626510620117188, + "epoch": 0.32305433186490456, + "fcm_dpo/beta": 0.016299601644277573, + "fcm_dpo/delta": 0.02697490155696869, + "fcm_dpo/margin": 22.946155548095703, + "fcm_dpo/q_t": 0.41226431727409363, + "grad_norm": 10.13264274597168, + "learning_rate": 4.2947798076611047e-07, + "logits/chosen": -1.6190385818481445, + "logits/rejected": -1.276505708694458, + "logps/chosen": -125.12520599365234, + "logps/ref_chosen": -95.53721618652344, + "logps/ref_rejected": -94.30703735351562, + "logps/rejected": -146.8411865234375, + "loss": 1.0947, + "margin_dpo/margin_mean": 22.946155548095703, + "margin_dpo/margin_std": 27.265180587768555, + "step": 220 + }, + { + "KL/chosen_KL_mean": -27.93502426147461, + "KL/mean": -45.54465866088867, + "KL/rejected_KL_mean": -63.154293060302734, + "KL/std": 34.48823165893555, + "epoch": 0.3245227606461087, + "fcm_dpo/beta": 0.016002081334590912, + "fcm_dpo/delta": -0.17308306694030762, + "fcm_dpo/margin": 35.219268798828125, + "fcm_dpo/q_t": 0.3696337938308716, + "grad_norm": 9.928122520446777, + "learning_rate": 4.285822501755485e-07, + "logits/chosen": -1.3340513706207275, + "logits/rejected": -1.1993815898895264, + "logps/chosen": -110.77989196777344, + "logps/ref_chosen": -82.84486389160156, + "logps/ref_rejected": -110.81179809570312, + "logps/rejected": -173.96609497070312, + "loss": 0.9595, + "margin_dpo/margin_mean": 35.219268798828125, + "margin_dpo/margin_std": 30.34347915649414, + "step": 221 + }, + { + "KL/chosen_KL_mean": -35.79014587402344, + "KL/mean": -50.069541931152344, + "KL/rejected_KL_mean": -64.34893798828125, + "KL/std": 33.09998321533203, + "epoch": 0.32599118942731276, + "fcm_dpo/beta": 0.015705151483416557, + "fcm_dpo/delta": -0.050947126001119614, + "fcm_dpo/margin": 28.558786392211914, + "fcm_dpo/q_t": 0.3975737690925598, + "grad_norm": 9.994668006896973, + "learning_rate": 4.276818137766118e-07, + "logits/chosen": -2.027966022491455, + "logits/rejected": -1.6006577014923096, + "logps/chosen": -130.93212890625, + "logps/ref_chosen": -95.14198303222656, + "logps/ref_rejected": -106.80441284179688, + "logps/rejected": -171.15335083007812, + "loss": 1.075, + "margin_dpo/margin_mean": 28.558786392211914, + "margin_dpo/margin_std": 38.62836456298828, + "step": 222 + }, + { + "KL/chosen_KL_mean": -35.09429168701172, + "KL/mean": -45.19826889038086, + "KL/rejected_KL_mean": -55.302249908447266, + "KL/std": 29.226844787597656, + "epoch": 0.3274596182085169, + "fcm_dpo/beta": 0.015683729201555252, + "fcm_dpo/delta": 0.027387013658881187, + "fcm_dpo/margin": 20.20795440673828, + "fcm_dpo/q_t": 0.42626410722732544, + "grad_norm": 9.692822456359863, + "learning_rate": 4.2677669529663686e-07, + "logits/chosen": -2.1440961360931396, + "logits/rejected": -1.7515130043029785, + "logps/chosen": -120.66940307617188, + "logps/ref_chosen": -85.57511138916016, + "logps/ref_rejected": -86.45237731933594, + "logps/rejected": -141.754638671875, + "loss": 1.1548, + "margin_dpo/margin_mean": 20.20795440673828, + "margin_dpo/margin_std": 32.085304260253906, + "step": 223 + }, + { + "KL/chosen_KL_mean": -32.0039176940918, + "KL/mean": -46.522911071777344, + "KL/rejected_KL_mean": -61.041908264160156, + "KL/std": 32.32599639892578, + "epoch": 0.328928046989721, + "fcm_dpo/beta": 0.015548836439847946, + "fcm_dpo/delta": -0.05398253723978996, + "fcm_dpo/margin": 29.037982940673828, + "fcm_dpo/q_t": 0.39580273628234863, + "grad_norm": 9.209607124328613, + "learning_rate": 4.2586691858633747e-07, + "logits/chosen": -1.727626085281372, + "logits/rejected": -1.264815330505371, + "logps/chosen": -114.72772216796875, + "logps/ref_chosen": -82.72380065917969, + "logps/ref_rejected": -82.59538269042969, + "logps/rejected": -143.63729858398438, + "loss": 1.049, + "margin_dpo/margin_mean": 29.037982940673828, + "margin_dpo/margin_std": 33.81342315673828, + "step": 224 + }, + { + "KL/chosen_KL_mean": -35.956302642822266, + "KL/mean": -50.97644805908203, + "KL/rejected_KL_mean": -65.99658966064453, + "KL/std": 33.929115295410156, + "epoch": 0.3303964757709251, + "fcm_dpo/beta": 0.015419520437717438, + "fcm_dpo/delta": -0.06630764901638031, + "fcm_dpo/margin": 30.0402889251709, + "fcm_dpo/q_t": 0.39472413063049316, + "grad_norm": 10.009520530700684, + "learning_rate": 4.249525076191759e-07, + "logits/chosen": -2.102365016937256, + "logits/rejected": -1.6344006061553955, + "logps/chosen": -131.6339874267578, + "logps/ref_chosen": -95.67768859863281, + "logps/ref_rejected": -105.09687805175781, + "logps/rejected": -171.09347534179688, + "loss": 1.0543, + "margin_dpo/margin_mean": 30.0402889251709, + "margin_dpo/margin_std": 38.083045959472656, + "step": 225 + }, + { + "KL/chosen_KL_mean": -29.665660858154297, + "KL/mean": -44.15570831298828, + "KL/rejected_KL_mean": -58.64575958251953, + "KL/std": 33.27996826171875, + "epoch": 0.33186490455212925, + "fcm_dpo/beta": 0.015142346732318401, + "fcm_dpo/delta": -0.04130810871720314, + "fcm_dpo/margin": 28.98008918762207, + "fcm_dpo/q_t": 0.3973340094089508, + "grad_norm": 8.312088012695312, + "learning_rate": 4.2403348649073167e-07, + "logits/chosen": -1.6455552577972412, + "logits/rejected": -1.0894885063171387, + "logps/chosen": -123.1265869140625, + "logps/ref_chosen": -93.46092224121094, + "logps/ref_rejected": -86.7017593383789, + "logps/rejected": -145.34751892089844, + "loss": 1.0442, + "margin_dpo/margin_mean": 28.98008918762207, + "margin_dpo/margin_std": 29.74630355834961, + "step": 226 + }, + { + "KL/chosen_KL_mean": -40.30400466918945, + "KL/mean": -55.16033935546875, + "KL/rejected_KL_mean": -70.01666259765625, + "KL/std": 32.16132354736328, + "epoch": 0.3333333333333333, + "fcm_dpo/beta": 0.015080028213560581, + "fcm_dpo/delta": -0.05030110478401184, + "fcm_dpo/margin": 29.712661743164062, + "fcm_dpo/q_t": 0.3975616693496704, + "grad_norm": 9.174317359924316, + "learning_rate": 4.2310987941806615e-07, + "logits/chosen": -1.498534917831421, + "logits/rejected": -1.1503180265426636, + "logps/chosen": -133.11827087402344, + "logps/ref_chosen": -92.81427001953125, + "logps/ref_rejected": -104.73693084716797, + "logps/rejected": -174.75360107421875, + "loss": 1.065, + "margin_dpo/margin_mean": 29.712661743164062, + "margin_dpo/margin_std": 38.15606689453125, + "step": 227 + }, + { + "KL/chosen_KL_mean": -37.99431610107422, + "KL/mean": -49.339935302734375, + "KL/rejected_KL_mean": -60.6855583190918, + "KL/std": 30.187368392944336, + "epoch": 0.33480176211453744, + "fcm_dpo/beta": 0.01515631377696991, + "fcm_dpo/delta": 0.058105990290641785, + "fcm_dpo/margin": 22.69123077392578, + "fcm_dpo/q_t": 0.4199024438858032, + "grad_norm": 9.403865814208984, + "learning_rate": 4.2218171073908463e-07, + "logits/chosen": -1.9922826290130615, + "logits/rejected": -1.5676965713500977, + "logps/chosen": -132.03143310546875, + "logps/ref_chosen": -94.03712463378906, + "logps/ref_rejected": -96.02151489257812, + "logps/rejected": -156.7070770263672, + "loss": 1.1353, + "margin_dpo/margin_mean": 22.691232681274414, + "margin_dpo/margin_std": 34.02064514160156, + "step": 228 + }, + { + "KL/chosen_KL_mean": -33.82004928588867, + "KL/mean": -52.36869812011719, + "KL/rejected_KL_mean": -70.9173583984375, + "KL/std": 36.59883499145508, + "epoch": 0.33627019089574156, + "fcm_dpo/beta": 0.014859197661280632, + "fcm_dpo/delta": -0.1599859595298767, + "fcm_dpo/margin": 37.09730529785156, + "fcm_dpo/q_t": 0.37526413798332214, + "grad_norm": 9.268818855285645, + "learning_rate": 4.212490049118951e-07, + "logits/chosen": -2.0356123447418213, + "logits/rejected": -1.42457115650177, + "logps/chosen": -129.39772033691406, + "logps/ref_chosen": -95.57766723632812, + "logps/ref_rejected": -89.17379760742188, + "logps/rejected": -160.09115600585938, + "loss": 0.9875, + "margin_dpo/margin_mean": 37.09730529785156, + "margin_dpo/margin_std": 40.07815170288086, + "step": 229 + }, + { + "KL/chosen_KL_mean": -35.235069274902344, + "KL/mean": -54.34660339355469, + "KL/rejected_KL_mean": -73.4581298828125, + "KL/std": 36.1552734375, + "epoch": 0.3377386196769457, + "fcm_dpo/beta": 0.014383679255843163, + "fcm_dpo/delta": -0.1585429459810257, + "fcm_dpo/margin": 38.22306442260742, + "fcm_dpo/q_t": 0.3721032738685608, + "grad_norm": 10.260030746459961, + "learning_rate": 4.203117865141635e-07, + "logits/chosen": -1.8191139698028564, + "logits/rejected": -1.758512258529663, + "logps/chosen": -98.94869995117188, + "logps/ref_chosen": -63.713626861572266, + "logps/ref_rejected": -91.90870666503906, + "logps/rejected": -165.36683654785156, + "loss": 0.9715, + "margin_dpo/margin_mean": 38.223060607910156, + "margin_dpo/margin_std": 36.017547607421875, + "step": 230 + }, + { + "KL/chosen_KL_mean": -38.84626770019531, + "KL/mean": -55.4265022277832, + "KL/rejected_KL_mean": -72.00672912597656, + "KL/std": 34.953025817871094, + "epoch": 0.3392070484581498, + "fcm_dpo/beta": 0.014057666063308716, + "fcm_dpo/delta": -0.07013809680938721, + "fcm_dpo/margin": 33.16046905517578, + "fcm_dpo/q_t": 0.3900415301322937, + "grad_norm": 8.214221000671387, + "learning_rate": 4.1937008024246625e-07, + "logits/chosen": -1.631027102470398, + "logits/rejected": -1.0523672103881836, + "logps/chosen": -134.30194091796875, + "logps/ref_chosen": -95.45567321777344, + "logps/ref_rejected": -80.95568084716797, + "logps/rejected": -152.96240234375, + "loss": 1.0173, + "margin_dpo/margin_mean": 33.16046905517578, + "margin_dpo/margin_std": 30.208650588989258, + "step": 231 + }, + { + "KL/chosen_KL_mean": -41.343650817871094, + "KL/mean": -55.100006103515625, + "KL/rejected_KL_mean": -68.85635375976562, + "KL/std": 32.287376403808594, + "epoch": 0.3406754772393539, + "fcm_dpo/beta": 0.014084059745073318, + "fcm_dpo/delta": 0.013005422428250313, + "fcm_dpo/margin": 27.512710571289062, + "fcm_dpo/q_t": 0.4092198610305786, + "grad_norm": 9.009343147277832, + "learning_rate": 4.1842391091163933e-07, + "logits/chosen": -1.3948099613189697, + "logits/rejected": -0.9005382061004639, + "logps/chosen": -138.24090576171875, + "logps/ref_chosen": -96.89726257324219, + "logps/ref_rejected": -89.76461791992188, + "logps/rejected": -158.6209716796875, + "loss": 1.086, + "margin_dpo/margin_mean": 27.512710571289062, + "margin_dpo/margin_std": 32.424991607666016, + "step": 232 + }, + { + "KL/chosen_KL_mean": -42.68800354003906, + "KL/mean": -62.00927734375, + "KL/rejected_KL_mean": -81.33053588867188, + "KL/std": 40.835723876953125, + "epoch": 0.342143906020558, + "fcm_dpo/beta": 0.013795394450426102, + "fcm_dpo/delta": -0.1408371776342392, + "fcm_dpo/margin": 38.64254379272461, + "fcm_dpo/q_t": 0.3769005537033081, + "grad_norm": 9.590836524963379, + "learning_rate": 4.174733034541245e-07, + "logits/chosen": -1.6353737115859985, + "logits/rejected": -1.4302332401275635, + "logps/chosen": -131.7383270263672, + "logps/ref_chosen": -89.05032348632812, + "logps/ref_rejected": -112.75917053222656, + "logps/rejected": -194.08970642089844, + "loss": 0.991, + "margin_dpo/margin_mean": 38.64254379272461, + "margin_dpo/margin_std": 39.588863372802734, + "step": 233 + }, + { + "KL/chosen_KL_mean": -41.83635711669922, + "KL/mean": -62.46691131591797, + "KL/rejected_KL_mean": -83.09745788574219, + "KL/std": 39.073936462402344, + "epoch": 0.3436123348017621, + "fcm_dpo/beta": 0.013297352939844131, + "fcm_dpo/delta": -0.15955108404159546, + "fcm_dpo/margin": 41.261104583740234, + "fcm_dpo/q_t": 0.3744843602180481, + "grad_norm": 8.52673625946045, + "learning_rate": 4.165182829193126e-07, + "logits/chosen": -1.558506965637207, + "logits/rejected": -1.4900007247924805, + "logps/chosen": -116.15472412109375, + "logps/ref_chosen": -74.318359375, + "logps/ref_rejected": -106.38758850097656, + "logps/rejected": -189.48504638671875, + "loss": 0.9866, + "margin_dpo/margin_mean": 41.261104583740234, + "margin_dpo/margin_std": 42.67225646972656, + "step": 234 + }, + { + "KL/chosen_KL_mean": -50.23050308227539, + "KL/mean": -65.73869323730469, + "KL/rejected_KL_mean": -81.24687194824219, + "KL/std": 35.055206298828125, + "epoch": 0.34508076358296624, + "fcm_dpo/beta": 0.013276444748044014, + "fcm_dpo/delta": -0.01230689324438572, + "fcm_dpo/margin": 31.016374588012695, + "fcm_dpo/q_t": 0.40316057205200195, + "grad_norm": 8.82662296295166, + "learning_rate": 4.1555887447288255e-07, + "logits/chosen": -1.70002019405365, + "logits/rejected": -1.1506282091140747, + "logps/chosen": -148.44754028320312, + "logps/ref_chosen": -98.217041015625, + "logps/ref_rejected": -97.24677276611328, + "logps/rejected": -178.49365234375, + "loss": 1.0618, + "margin_dpo/margin_mean": 31.016376495361328, + "margin_dpo/margin_std": 33.02741241455078, + "step": 235 + }, + { + "KL/chosen_KL_mean": -42.19235610961914, + "KL/mean": -62.37560272216797, + "KL/rejected_KL_mean": -82.55885314941406, + "KL/std": 44.101383209228516, + "epoch": 0.3465491923641703, + "fcm_dpo/beta": 0.013020765967667103, + "fcm_dpo/delta": -0.1323883831501007, + "fcm_dpo/margin": 40.36650085449219, + "fcm_dpo/q_t": 0.3842836618423462, + "grad_norm": 9.429834365844727, + "learning_rate": 4.1459510339613946e-07, + "logits/chosen": -1.4334239959716797, + "logits/rejected": -1.4379512071609497, + "logps/chosen": -121.03009033203125, + "logps/ref_chosen": -78.83773040771484, + "logps/ref_rejected": -109.06343078613281, + "logps/rejected": -191.62228393554688, + "loss": 1.01, + "margin_dpo/margin_mean": 40.36650085449219, + "margin_dpo/margin_std": 47.81488037109375, + "step": 236 + }, + { + "KL/chosen_KL_mean": -50.26822280883789, + "KL/mean": -64.86390686035156, + "KL/rejected_KL_mean": -79.45960235595703, + "KL/std": 38.93791198730469, + "epoch": 0.34801762114537443, + "fcm_dpo/beta": 0.012950649484992027, + "fcm_dpo/delta": 0.022814584895968437, + "fcm_dpo/margin": 29.191373825073242, + "fcm_dpo/q_t": 0.416756808757782, + "grad_norm": 8.735091209411621, + "learning_rate": 4.136269950853473e-07, + "logits/chosen": -1.8456820249557495, + "logits/rejected": -1.507851243019104, + "logps/chosen": -135.47952270507812, + "logps/ref_chosen": -85.21128845214844, + "logps/ref_rejected": -99.90999603271484, + "logps/rejected": -179.36959838867188, + "loss": 1.1409, + "margin_dpo/margin_mean": 29.191375732421875, + "margin_dpo/margin_std": 50.543724060058594, + "step": 237 + }, + { + "KL/chosen_KL_mean": -54.99638748168945, + "KL/mean": -72.25209045410156, + "KL/rejected_KL_mean": -89.50779724121094, + "KL/std": 40.485992431640625, + "epoch": 0.34948604992657856, + "fcm_dpo/beta": 0.012862354516983032, + "fcm_dpo/delta": -0.04601479694247246, + "fcm_dpo/margin": 34.51141357421875, + "fcm_dpo/q_t": 0.3984161913394928, + "grad_norm": 8.139162063598633, + "learning_rate": 4.126545750510605e-07, + "logits/chosen": -1.2899951934814453, + "logits/rejected": -1.141869068145752, + "logps/chosen": -133.72763061523438, + "logps/ref_chosen": -78.73123168945312, + "logps/ref_rejected": -95.41840362548828, + "logps/rejected": -184.92620849609375, + "loss": 1.0573, + "margin_dpo/margin_mean": 34.511409759521484, + "margin_dpo/margin_std": 41.94647979736328, + "step": 238 + }, + { + "KL/chosen_KL_mean": -49.67461013793945, + "KL/mean": -65.66973876953125, + "KL/rejected_KL_mean": -81.66487884521484, + "KL/std": 36.1599235534668, + "epoch": 0.3509544787077827, + "fcm_dpo/beta": 0.012679128907620907, + "fcm_dpo/delta": -0.065593421459198, + "fcm_dpo/margin": 31.990262985229492, + "fcm_dpo/q_t": 0.4057391881942749, + "grad_norm": 8.987502098083496, + "learning_rate": 4.116778689174514e-07, + "logits/chosen": -2.1470353603363037, + "logits/rejected": -1.692811369895935, + "logps/chosen": -142.27554321289062, + "logps/ref_chosen": -92.60093688964844, + "logps/ref_rejected": -100.51769256591797, + "logps/rejected": -182.1825714111328, + "loss": 1.0857, + "margin_dpo/margin_mean": 31.990262985229492, + "margin_dpo/margin_std": 40.13188171386719, + "step": 239 + }, + { + "KL/chosen_KL_mean": -51.856868743896484, + "KL/mean": -66.01220703125, + "KL/rejected_KL_mean": -80.16754150390625, + "KL/std": 36.49290466308594, + "epoch": 0.3524229074889868, + "fcm_dpo/beta": 0.012731213122606277, + "fcm_dpo/delta": 0.04104982316493988, + "fcm_dpo/margin": 28.3106689453125, + "fcm_dpo/q_t": 0.4163438677787781, + "grad_norm": 10.008689880371094, + "learning_rate": 4.106969024216348e-07, + "logits/chosen": -1.9556775093078613, + "logits/rejected": -1.5861685276031494, + "logps/chosen": -138.01663208007812, + "logps/ref_chosen": -86.15977478027344, + "logps/ref_rejected": -80.45567321777344, + "logps/rejected": -160.62319946289062, + "loss": 1.118, + "margin_dpo/margin_mean": 28.3106689453125, + "margin_dpo/margin_std": 39.285972595214844, + "step": 240 + }, + { + "KL/chosen_KL_mean": -47.5243034362793, + "KL/mean": -72.66444396972656, + "KL/rejected_KL_mean": -97.80459594726562, + "KL/std": 41.57490539550781, + "epoch": 0.35389133627019087, + "fcm_dpo/beta": 0.012402208521962166, + "fcm_dpo/delta": -0.23778831958770752, + "fcm_dpo/margin": 50.280296325683594, + "fcm_dpo/q_t": 0.35778650641441345, + "grad_norm": 8.693832397460938, + "learning_rate": 4.097117014129903e-07, + "logits/chosen": -1.7208023071289062, + "logits/rejected": -1.036415934562683, + "logps/chosen": -148.57025146484375, + "logps/ref_chosen": -101.04594421386719, + "logps/ref_rejected": -94.04934692382812, + "logps/rejected": -191.85394287109375, + "loss": 0.9329, + "margin_dpo/margin_mean": 50.280296325683594, + "margin_dpo/margin_std": 46.47297668457031, + "step": 241 + }, + { + "KL/chosen_KL_mean": -56.00395584106445, + "KL/mean": -74.20791625976562, + "KL/rejected_KL_mean": -92.41185760498047, + "KL/std": 37.318153381347656, + "epoch": 0.355359765051395, + "fcm_dpo/beta": 0.012145690619945526, + "fcm_dpo/delta": -0.04419766739010811, + "fcm_dpo/margin": 36.407894134521484, + "fcm_dpo/q_t": 0.3966217041015625, + "grad_norm": 8.951445579528809, + "learning_rate": 4.087222918524807e-07, + "logits/chosen": -1.9276995658874512, + "logits/rejected": -1.2692008018493652, + "logps/chosen": -151.67662048339844, + "logps/ref_chosen": -95.67266845703125, + "logps/ref_rejected": -90.65454864501953, + "logps/rejected": -183.06640625, + "loss": 1.0525, + "margin_dpo/margin_mean": 36.40789031982422, + "margin_dpo/margin_std": 41.32374954223633, + "step": 242 + }, + { + "KL/chosen_KL_mean": -55.74171447753906, + "KL/mean": -73.79441833496094, + "KL/rejected_KL_mean": -91.84712219238281, + "KL/std": 40.06411361694336, + "epoch": 0.3568281938325991, + "fcm_dpo/beta": 0.011970823630690575, + "fcm_dpo/delta": -0.09341253340244293, + "fcm_dpo/margin": 36.10541534423828, + "fcm_dpo/q_t": 0.40096449851989746, + "grad_norm": 8.581452369689941, + "learning_rate": 4.07728699811968e-07, + "logits/chosen": -1.9148585796356201, + "logits/rejected": -1.0913387537002563, + "logps/chosen": -153.7731170654297, + "logps/ref_chosen": -98.03140258789062, + "logps/ref_rejected": -83.18806457519531, + "logps/rejected": -175.03518676757812, + "loss": 1.081, + "margin_dpo/margin_mean": 36.10541534423828, + "margin_dpo/margin_std": 47.55536651611328, + "step": 243 + }, + { + "KL/chosen_KL_mean": -58.18250274658203, + "KL/mean": -77.11045837402344, + "KL/rejected_KL_mean": -96.03840637207031, + "KL/std": 43.899818420410156, + "epoch": 0.35829662261380324, + "fcm_dpo/beta": 0.011755963787436485, + "fcm_dpo/delta": -0.04726836830377579, + "fcm_dpo/margin": 37.85589599609375, + "fcm_dpo/q_t": 0.39697739481925964, + "grad_norm": 8.456755638122559, + "learning_rate": 4.067309514735267e-07, + "logits/chosen": -2.235485315322876, + "logits/rejected": -1.9078938961029053, + "logps/chosen": -147.076416015625, + "logps/ref_chosen": -88.8939208984375, + "logps/ref_rejected": -102.57278442382812, + "logps/rejected": -198.61117553710938, + "loss": 1.0505, + "margin_dpo/margin_mean": 37.85589599609375, + "margin_dpo/margin_std": 43.53936767578125, + "step": 244 + }, + { + "KL/chosen_KL_mean": -49.47932815551758, + "KL/mean": -69.46237182617188, + "KL/rejected_KL_mean": -89.44541931152344, + "KL/std": 40.532493591308594, + "epoch": 0.35976505139500736, + "fcm_dpo/beta": 0.0116469357162714, + "fcm_dpo/delta": -0.06862208247184753, + "fcm_dpo/margin": 39.96609878540039, + "fcm_dpo/q_t": 0.39160656929016113, + "grad_norm": 7.924368858337402, + "learning_rate": 4.057290731287531e-07, + "logits/chosen": -2.3287272453308105, + "logits/rejected": -1.4995757341384888, + "logps/chosen": -153.67332458496094, + "logps/ref_chosen": -104.19400024414062, + "logps/ref_rejected": -92.65645599365234, + "logps/rejected": -182.10186767578125, + "loss": 1.029, + "margin_dpo/margin_mean": 39.96609878540039, + "margin_dpo/margin_std": 41.65141296386719, + "step": 245 + }, + { + "KL/chosen_KL_mean": -64.39878845214844, + "KL/mean": -80.34806823730469, + "KL/rejected_KL_mean": -96.29736328125, + "KL/std": 39.891204833984375, + "epoch": 0.36123348017621143, + "fcm_dpo/beta": 0.011543956585228443, + "fcm_dpo/delta": -0.018736502155661583, + "fcm_dpo/margin": 31.898571014404297, + "fcm_dpo/q_t": 0.41454970836639404, + "grad_norm": 8.051033020019531, + "learning_rate": 4.047230911780736e-07, + "logits/chosen": -1.9757585525512695, + "logits/rejected": -1.4433469772338867, + "logps/chosen": -167.61782836914062, + "logps/ref_chosen": -103.21904754638672, + "logps/ref_rejected": -90.99221801757812, + "logps/rejected": -187.28958129882812, + "loss": 1.1101, + "margin_dpo/margin_mean": 31.898571014404297, + "margin_dpo/margin_std": 43.04576873779297, + "step": 246 + }, + { + "KL/chosen_KL_mean": -58.211830139160156, + "KL/mean": -84.24368286132812, + "KL/rejected_KL_mean": -110.27554321289062, + "KL/std": 44.67066192626953, + "epoch": 0.36270190895741555, + "fcm_dpo/beta": 0.01116771437227726, + "fcm_dpo/delta": -0.19386975467205048, + "fcm_dpo/margin": 52.06371307373047, + "fcm_dpo/q_t": 0.3671821355819702, + "grad_norm": 8.201760292053223, + "learning_rate": 4.0371303213004814e-07, + "logits/chosen": -2.1689987182617188, + "logits/rejected": -1.9922220706939697, + "logps/chosen": -145.20620727539062, + "logps/ref_chosen": -86.99437713623047, + "logps/ref_rejected": -111.33802795410156, + "logps/rejected": -221.61355590820312, + "loss": 0.9595, + "margin_dpo/margin_mean": 52.06371307373047, + "margin_dpo/margin_std": 50.39811706542969, + "step": 247 + }, + { + "KL/chosen_KL_mean": -67.36201477050781, + "KL/mean": -84.65104675292969, + "KL/rejected_KL_mean": -101.9400863647461, + "KL/std": 39.40555953979492, + "epoch": 0.3641703377386197, + "fcm_dpo/beta": 0.011161180213093758, + "fcm_dpo/delta": 0.014418689534068108, + "fcm_dpo/margin": 34.578086853027344, + "fcm_dpo/q_t": 0.40965962409973145, + "grad_norm": 8.75762939453125, + "learning_rate": 4.0269892260067197e-07, + "logits/chosen": -1.566325068473816, + "logits/rejected": -1.5425187349319458, + "logps/chosen": -142.14752197265625, + "logps/ref_chosen": -74.7855224609375, + "logps/ref_rejected": -98.27689361572266, + "logps/rejected": -200.21697998046875, + "loss": 1.086, + "margin_dpo/margin_mean": 34.578086853027344, + "margin_dpo/margin_std": 40.4761848449707, + "step": 248 + }, + { + "KL/chosen_KL_mean": -74.44125366210938, + "KL/mean": -90.29890441894531, + "KL/rejected_KL_mean": -106.15655517578125, + "KL/std": 42.263832092285156, + "epoch": 0.3656387665198238, + "fcm_dpo/beta": 0.011107325553894043, + "fcm_dpo/delta": -0.013334457762539387, + "fcm_dpo/margin": 31.715316772460938, + "fcm_dpo/q_t": 0.4185119569301605, + "grad_norm": 9.714157104492188, + "learning_rate": 4.0168078931267426e-07, + "logits/chosen": -2.2583506107330322, + "logits/rejected": -1.7227230072021484, + "logps/chosen": -170.14505004882812, + "logps/ref_chosen": -95.70379638671875, + "logps/ref_rejected": -87.14646911621094, + "logps/rejected": -193.30303955078125, + "loss": 1.1341, + "margin_dpo/margin_mean": 31.715316772460938, + "margin_dpo/margin_std": 48.34738540649414, + "step": 249 + }, + { + "KL/chosen_KL_mean": -66.18019104003906, + "KL/mean": -84.00900268554688, + "KL/rejected_KL_mean": -101.83782958984375, + "KL/std": 42.873809814453125, + "epoch": 0.3671071953010279, + "fcm_dpo/beta": 0.011063864454627037, + "fcm_dpo/delta": 0.005284234881401062, + "fcm_dpo/margin": 35.65764617919922, + "fcm_dpo/q_t": 0.40786194801330566, + "grad_norm": 8.91695499420166, + "learning_rate": 4.006586590948141e-07, + "logits/chosen": -1.9831795692443848, + "logits/rejected": -1.0479307174682617, + "logps/chosen": -180.23239135742188, + "logps/ref_chosen": -114.05221557617188, + "logps/ref_rejected": -81.08767700195312, + "logps/rejected": -182.92550659179688, + "loss": 1.0959, + "margin_dpo/margin_mean": 35.65764617919922, + "margin_dpo/margin_std": 45.877777099609375, + "step": 250 + }, + { + "KL/chosen_KL_mean": -66.26416015625, + "KL/mean": -82.0506591796875, + "KL/rejected_KL_mean": -97.83717346191406, + "KL/std": 40.05830383300781, + "epoch": 0.368575624082232, + "fcm_dpo/beta": 0.01105173397809267, + "fcm_dpo/delta": -0.03037196956574917, + "fcm_dpo/margin": 31.572999954223633, + "fcm_dpo/q_t": 0.41971632838249207, + "grad_norm": 9.536917686462402, + "learning_rate": 3.9963255888117325e-07, + "logits/chosen": -1.7843058109283447, + "logits/rejected": -1.071089267730713, + "logps/chosen": -163.97544860839844, + "logps/ref_chosen": -97.71128845214844, + "logps/ref_rejected": -83.52742004394531, + "logps/rejected": -181.36459350585938, + "loss": 1.1369, + "margin_dpo/margin_mean": 31.573001861572266, + "margin_dpo/margin_std": 48.03728103637695, + "step": 251 + }, + { + "KL/chosen_KL_mean": -66.95657348632812, + "KL/mean": -82.80113220214844, + "KL/rejected_KL_mean": -98.64569091796875, + "KL/std": 42.470863342285156, + "epoch": 0.3700440528634361, + "fcm_dpo/beta": 0.011149127036333084, + "fcm_dpo/delta": 0.04827743023633957, + "fcm_dpo/margin": 31.68912696838379, + "fcm_dpo/q_t": 0.4175231456756592, + "grad_norm": 9.4552640914917, + "learning_rate": 3.9860251571044666e-07, + "logits/chosen": -1.9976928234100342, + "logits/rejected": -1.4521489143371582, + "logps/chosen": -175.94268798828125, + "logps/ref_chosen": -108.9861068725586, + "logps/ref_rejected": -91.56423950195312, + "logps/rejected": -190.20993041992188, + "loss": 1.1253, + "margin_dpo/margin_mean": 31.68912696838379, + "margin_dpo/margin_std": 45.009246826171875, + "step": 252 + }, + { + "KL/chosen_KL_mean": -73.47262573242188, + "KL/mean": -88.30828857421875, + "KL/rejected_KL_mean": -103.1439437866211, + "KL/std": 46.262168884277344, + "epoch": 0.37151248164464024, + "fcm_dpo/beta": 0.011135118082165718, + "fcm_dpo/delta": -0.021412089467048645, + "fcm_dpo/margin": 29.67132568359375, + "fcm_dpo/q_t": 0.42164188623428345, + "grad_norm": 9.413656234741211, + "learning_rate": 3.9756855672522986e-07, + "logits/chosen": -2.139962911605835, + "logits/rejected": -1.935544490814209, + "logps/chosen": -173.68893432617188, + "logps/ref_chosen": -100.21630096435547, + "logps/ref_rejected": -105.67670440673828, + "logps/rejected": -208.82064819335938, + "loss": 1.1329, + "margin_dpo/margin_mean": 29.671321868896484, + "margin_dpo/margin_std": 39.909759521484375, + "step": 253 + }, + { + "KL/chosen_KL_mean": -70.78187561035156, + "KL/mean": -86.1494140625, + "KL/rejected_KL_mean": -101.5169448852539, + "KL/std": 43.68431854248047, + "epoch": 0.37298091042584436, + "fcm_dpo/beta": 0.01108785904943943, + "fcm_dpo/delta": -0.01054837554693222, + "fcm_dpo/margin": 30.735069274902344, + "fcm_dpo/q_t": 0.4197158217430115, + "grad_norm": 9.133874893188477, + "learning_rate": 3.965307091713037e-07, + "logits/chosen": -2.117870330810547, + "logits/rejected": -1.5947855710983276, + "logps/chosen": -169.51705932617188, + "logps/ref_chosen": -98.73518371582031, + "logps/ref_rejected": -93.73825073242188, + "logps/rejected": -195.25518798828125, + "loss": 1.1394, + "margin_dpo/margin_mean": 30.735069274902344, + "margin_dpo/margin_std": 46.54502487182617, + "step": 254 + }, + { + "KL/chosen_KL_mean": -71.57162475585938, + "KL/mean": -95.09077453613281, + "KL/rejected_KL_mean": -118.60993957519531, + "KL/std": 49.213096618652344, + "epoch": 0.3744493392070485, + "fcm_dpo/beta": 0.010904960334300995, + "fcm_dpo/delta": -0.1189572736620903, + "fcm_dpo/margin": 47.03831100463867, + "fcm_dpo/q_t": 0.38124898076057434, + "grad_norm": 9.016454696655273, + "learning_rate": 3.954890003969163e-07, + "logits/chosen": -1.624290943145752, + "logits/rejected": -1.2691798210144043, + "logps/chosen": -161.95419311523438, + "logps/ref_chosen": -90.382568359375, + "logps/ref_rejected": -97.07624816894531, + "logps/rejected": -215.68618774414062, + "loss": 1.0063, + "margin_dpo/margin_mean": 47.03831481933594, + "margin_dpo/margin_std": 49.886207580566406, + "step": 255 + }, + { + "KL/chosen_KL_mean": -70.11643981933594, + "KL/mean": -91.02188873291016, + "KL/rejected_KL_mean": -111.92735290527344, + "KL/std": 51.18582534790039, + "epoch": 0.37591776798825255, + "fcm_dpo/beta": 0.01076935138553381, + "fcm_dpo/delta": -0.052641574293375015, + "fcm_dpo/margin": 41.8109016418457, + "fcm_dpo/q_t": 0.39821550250053406, + "grad_norm": 9.411343574523926, + "learning_rate": 3.944434578520628e-07, + "logits/chosen": -1.8807837963104248, + "logits/rejected": -1.5417015552520752, + "logps/chosen": -158.86924743652344, + "logps/ref_chosen": -88.7528076171875, + "logps/ref_rejected": -98.49383544921875, + "logps/rejected": -210.42117309570312, + "loss": 1.0817, + "margin_dpo/margin_mean": 41.81090545654297, + "margin_dpo/margin_std": 59.800994873046875, + "step": 256 + }, + { + "KL/chosen_KL_mean": -71.79951477050781, + "KL/mean": -93.05375671386719, + "KL/rejected_KL_mean": -114.30799865722656, + "KL/std": 52.42218780517578, + "epoch": 0.37738619676945667, + "fcm_dpo/beta": 0.010478893294930458, + "fcm_dpo/delta": -0.11083962023258209, + "fcm_dpo/margin": 42.50849151611328, + "fcm_dpo/q_t": 0.3968268036842346, + "grad_norm": 8.39576244354248, + "learning_rate": 3.933941090877615e-07, + "logits/chosen": -1.7643895149230957, + "logits/rejected": -1.3534268140792847, + "logps/chosen": -154.60302734375, + "logps/ref_chosen": -82.80352020263672, + "logps/ref_rejected": -85.8677978515625, + "logps/rejected": -200.17578125, + "loss": 1.0616, + "margin_dpo/margin_mean": 42.50849151611328, + "margin_dpo/margin_std": 50.7281494140625, + "step": 257 + }, + { + "KL/chosen_KL_mean": -75.10973358154297, + "KL/mean": -94.32515716552734, + "KL/rejected_KL_mean": -113.54057312011719, + "KL/std": 49.886898040771484, + "epoch": 0.3788546255506608, + "fcm_dpo/beta": 0.010417868383228779, + "fcm_dpo/delta": -0.05857791751623154, + "fcm_dpo/margin": 38.430850982666016, + "fcm_dpo/q_t": 0.4060943126678467, + "grad_norm": 9.19739818572998, + "learning_rate": 3.923409817553284e-07, + "logits/chosen": -2.39119553565979, + "logits/rejected": -2.0479345321655273, + "logps/chosen": -165.2974853515625, + "logps/ref_chosen": -90.187744140625, + "logps/ref_rejected": -103.47068786621094, + "logps/rejected": -217.01126098632812, + "loss": 1.0873, + "margin_dpo/margin_mean": 38.43085479736328, + "margin_dpo/margin_std": 47.67232131958008, + "step": 258 + }, + { + "KL/chosen_KL_mean": -81.25570678710938, + "KL/mean": -97.72335815429688, + "KL/rejected_KL_mean": -114.19100952148438, + "KL/std": 50.4412727355957, + "epoch": 0.3803230543318649, + "fcm_dpo/beta": 0.010299797169864178, + "fcm_dpo/delta": -0.027616608887910843, + "fcm_dpo/margin": 32.935298919677734, + "fcm_dpo/q_t": 0.420768141746521, + "grad_norm": 8.87748908996582, + "learning_rate": 3.9128410360564793e-07, + "logits/chosen": -1.2344012260437012, + "logits/rejected": -0.9772180318832397, + "logps/chosen": -172.02825927734375, + "logps/ref_chosen": -90.77254486083984, + "logps/ref_rejected": -94.58816528320312, + "logps/rejected": -208.7791748046875, + "loss": 1.1585, + "margin_dpo/margin_mean": 32.935298919677734, + "margin_dpo/margin_std": 56.87708282470703, + "step": 259 + }, + { + "KL/chosen_KL_mean": -83.33119201660156, + "KL/mean": -97.52334594726562, + "KL/rejected_KL_mean": -111.71551513671875, + "KL/std": 50.935203552246094, + "epoch": 0.38179148311306904, + "fcm_dpo/beta": 0.010396572761237621, + "fcm_dpo/delta": 0.04675988852977753, + "fcm_dpo/margin": 28.384326934814453, + "fcm_dpo/q_t": 0.4314086437225342, + "grad_norm": 10.557072639465332, + "learning_rate": 3.9022350248844246e-07, + "logits/chosen": -1.667799472808838, + "logits/rejected": -1.6990429162979126, + "logps/chosen": -158.92388916015625, + "logps/ref_chosen": -75.59269714355469, + "logps/ref_rejected": -100.84554290771484, + "logps/rejected": -212.56106567382812, + "loss": 1.1916, + "margin_dpo/margin_mean": 28.384326934814453, + "margin_dpo/margin_std": 55.32361602783203, + "step": 260 + }, + { + "KL/chosen_KL_mean": -64.40402221679688, + "KL/mean": -88.78857421875, + "KL/rejected_KL_mean": -113.17312622070312, + "KL/std": 51.38774871826172, + "epoch": 0.3832599118942731, + "fcm_dpo/beta": 0.010184694081544876, + "fcm_dpo/delta": -0.1028708964586258, + "fcm_dpo/margin": 48.76910400390625, + "fcm_dpo/q_t": 0.3848683834075928, + "grad_norm": 8.344627380371094, + "learning_rate": 3.891592063515376e-07, + "logits/chosen": -2.0689573287963867, + "logits/rejected": -1.705404281616211, + "logps/chosen": -158.658935546875, + "logps/ref_chosen": -94.25491333007812, + "logps/ref_rejected": -93.65699005126953, + "logps/rejected": -206.83010864257812, + "loss": 1.0113, + "margin_dpo/margin_mean": 48.769107818603516, + "margin_dpo/margin_std": 50.05211639404297, + "step": 261 + }, + { + "KL/chosen_KL_mean": -69.95246887207031, + "KL/mean": -93.08432006835938, + "KL/rejected_KL_mean": -116.21617126464844, + "KL/std": 54.20262908935547, + "epoch": 0.38472834067547723, + "fcm_dpo/beta": 0.010046085342764854, + "fcm_dpo/delta": -0.06842421740293503, + "fcm_dpo/margin": 46.263694763183594, + "fcm_dpo/q_t": 0.3925879895687103, + "grad_norm": 8.307077407836914, + "learning_rate": 3.880912432401264e-07, + "logits/chosen": -1.6251637935638428, + "logits/rejected": -1.0782082080841064, + "logps/chosen": -155.2197723388672, + "logps/ref_chosen": -85.26730346679688, + "logps/ref_rejected": -90.82609558105469, + "logps/rejected": -207.04226684570312, + "loss": 1.0439, + "margin_dpo/margin_mean": 46.263694763183594, + "margin_dpo/margin_std": 54.11823654174805, + "step": 262 + }, + { + "KL/chosen_KL_mean": -56.42060852050781, + "KL/mean": -87.4281997680664, + "KL/rejected_KL_mean": -118.435791015625, + "KL/std": 58.96156311035156, + "epoch": 0.38619676945668135, + "fcm_dpo/beta": 0.009745280258357525, + "fcm_dpo/delta": -0.21718664467334747, + "fcm_dpo/margin": 62.01518249511719, + "fcm_dpo/q_t": 0.36159005761146545, + "grad_norm": 8.110187530517578, + "learning_rate": 3.870196412960302e-07, + "logits/chosen": -2.079312801361084, + "logits/rejected": -1.5224335193634033, + "logps/chosen": -152.1785125732422, + "logps/ref_chosen": -95.75790405273438, + "logps/ref_rejected": -101.83377075195312, + "logps/rejected": -220.26956176757812, + "loss": 0.9562, + "margin_dpo/margin_mean": 62.01518249511719, + "margin_dpo/margin_std": 62.88063049316406, + "step": 263 + }, + { + "KL/chosen_KL_mean": -65.93309020996094, + "KL/mean": -92.15192413330078, + "KL/rejected_KL_mean": -118.37074279785156, + "KL/std": 55.61305236816406, + "epoch": 0.3876651982378855, + "fcm_dpo/beta": 0.009461034089326859, + "fcm_dpo/delta": -0.10133536159992218, + "fcm_dpo/margin": 52.43765640258789, + "fcm_dpo/q_t": 0.384945809841156, + "grad_norm": 8.082246780395508, + "learning_rate": 3.8594442875695665e-07, + "logits/chosen": -2.0536587238311768, + "logits/rejected": -1.7670159339904785, + "logps/chosen": -156.55575561523438, + "logps/ref_chosen": -90.6226577758789, + "logps/ref_rejected": -100.32554626464844, + "logps/rejected": -218.6962890625, + "loss": 1.0171, + "margin_dpo/margin_mean": 52.437660217285156, + "margin_dpo/margin_std": 56.434661865234375, + "step": 264 + }, + { + "KL/chosen_KL_mean": -69.6064224243164, + "KL/mean": -92.77266693115234, + "KL/rejected_KL_mean": -115.93890380859375, + "KL/std": 56.830081939697266, + "epoch": 0.3891336270190896, + "fcm_dpo/beta": 0.009339086711406708, + "fcm_dpo/delta": -0.03467705845832825, + "fcm_dpo/margin": 46.33248519897461, + "fcm_dpo/q_t": 0.4013739824295044, + "grad_norm": 8.375724792480469, + "learning_rate": 3.848656339557562e-07, + "logits/chosen": -2.082062244415283, + "logits/rejected": -1.6759120225906372, + "logps/chosen": -161.978759765625, + "logps/ref_chosen": -92.37232971191406, + "logps/ref_rejected": -94.62757873535156, + "logps/rejected": -210.5664825439453, + "loss": 1.0774, + "margin_dpo/margin_mean": 46.33248519897461, + "margin_dpo/margin_std": 61.289188385009766, + "step": 265 + }, + { + "KL/chosen_KL_mean": -81.2215576171875, + "KL/mean": -102.11510467529297, + "KL/rejected_KL_mean": -123.00863647460938, + "KL/std": 55.078590393066406, + "epoch": 0.39060205580029367, + "fcm_dpo/beta": 0.009357543662190437, + "fcm_dpo/delta": 0.009292619302868843, + "fcm_dpo/margin": 41.787078857421875, + "fcm_dpo/q_t": 0.41033831238746643, + "grad_norm": 10.080767631530762, + "learning_rate": 3.8378328531967507e-07, + "logits/chosen": -2.2015106678009033, + "logits/rejected": -1.3638226985931396, + "logps/chosen": -183.4215850830078, + "logps/ref_chosen": -102.20002746582031, + "logps/ref_rejected": -74.36642456054688, + "logps/rejected": -197.37506103515625, + "loss": 1.1048, + "margin_dpo/margin_mean": 41.787078857421875, + "margin_dpo/margin_std": 58.27301025390625, + "step": 266 + }, + { + "KL/chosen_KL_mean": -72.06327819824219, + "KL/mean": -96.59632110595703, + "KL/rejected_KL_mean": -121.12936401367188, + "KL/std": 55.48495864868164, + "epoch": 0.3920704845814978, + "fcm_dpo/beta": 0.009264141321182251, + "fcm_dpo/delta": -0.05764763057231903, + "fcm_dpo/margin": 49.066078186035156, + "fcm_dpo/q_t": 0.39444106817245483, + "grad_norm": 8.82445240020752, + "learning_rate": 3.8269741136960646e-07, + "logits/chosen": -1.9128168821334839, + "logits/rejected": -1.3005517721176147, + "logps/chosen": -176.34927368164062, + "logps/ref_chosen": -104.28599548339844, + "logps/ref_rejected": -95.98719024658203, + "logps/rejected": -217.11654663085938, + "loss": 1.0404, + "margin_dpo/margin_mean": 49.06608200073242, + "margin_dpo/margin_std": 53.13525390625, + "step": 267 + }, + { + "KL/chosen_KL_mean": -71.71980285644531, + "KL/mean": -100.52813720703125, + "KL/rejected_KL_mean": -129.33645629882812, + "KL/std": 55.80779266357422, + "epoch": 0.3935389133627019, + "fcm_dpo/beta": 0.009052696637809277, + "fcm_dpo/delta": -0.12931329011917114, + "fcm_dpo/margin": 57.616661071777344, + "fcm_dpo/q_t": 0.3780568242073059, + "grad_norm": 9.015339851379395, + "learning_rate": 3.8160804071933894e-07, + "logits/chosen": -1.6295409202575684, + "logits/rejected": -1.4765753746032715, + "logps/chosen": -158.416015625, + "logps/ref_chosen": -86.69622039794922, + "logps/ref_rejected": -109.19184112548828, + "logps/rejected": -238.52828979492188, + "loss": 0.9905, + "margin_dpo/margin_mean": 57.616661071777344, + "margin_dpo/margin_std": 55.16349411010742, + "step": 268 + }, + { + "KL/chosen_KL_mean": -76.3770751953125, + "KL/mean": -104.35591125488281, + "KL/rejected_KL_mean": -132.33473205566406, + "KL/std": 59.76306915283203, + "epoch": 0.39500734214390604, + "fcm_dpo/beta": 0.008916812017560005, + "fcm_dpo/delta": -0.10403021425008774, + "fcm_dpo/margin": 55.957672119140625, + "fcm_dpo/q_t": 0.3841118812561035, + "grad_norm": 8.239960670471191, + "learning_rate": 3.8051520207480204e-07, + "logits/chosen": -1.8281971216201782, + "logits/rejected": -1.3735320568084717, + "logps/chosen": -181.348876953125, + "logps/ref_chosen": -104.97181701660156, + "logps/ref_rejected": -112.47647094726562, + "logps/rejected": -244.8112030029297, + "loss": 1.0093, + "margin_dpo/margin_mean": 55.957672119140625, + "margin_dpo/margin_std": 57.233211517333984, + "step": 269 + }, + { + "KL/chosen_KL_mean": -84.89106750488281, + "KL/mean": -108.30947875976562, + "KL/rejected_KL_mean": -131.72787475585938, + "KL/std": 58.81729507446289, + "epoch": 0.3964757709251101, + "fcm_dpo/beta": 0.008856823667883873, + "fcm_dpo/delta": -0.015634391456842422, + "fcm_dpo/margin": 46.836814880371094, + "fcm_dpo/q_t": 0.40438681840896606, + "grad_norm": 8.963788986206055, + "learning_rate": 3.794189242333106e-07, + "logits/chosen": -2.467940330505371, + "logits/rejected": -2.057849884033203, + "logps/chosen": -185.96490478515625, + "logps/ref_chosen": -101.07383728027344, + "logps/ref_rejected": -117.75289154052734, + "logps/rejected": -249.48077392578125, + "loss": 1.0975, + "margin_dpo/margin_mean": 46.83681106567383, + "margin_dpo/margin_std": 66.76217651367188, + "step": 270 + }, + { + "KL/chosen_KL_mean": -70.38786315917969, + "KL/mean": -99.63778686523438, + "KL/rejected_KL_mean": -128.88772583007812, + "KL/std": 55.40803527832031, + "epoch": 0.39794419970631423, + "fcm_dpo/beta": 0.008662492968142033, + "fcm_dpo/delta": -0.11264573782682419, + "fcm_dpo/margin": 58.49987030029297, + "fcm_dpo/q_t": 0.38125231862068176, + "grad_norm": 8.793933868408203, + "learning_rate": 3.7831923608280514e-07, + "logits/chosen": -1.5933396816253662, + "logits/rejected": -1.145467758178711, + "logps/chosen": -167.11245727539062, + "logps/ref_chosen": -96.72459411621094, + "logps/ref_rejected": -98.5244140625, + "logps/rejected": -227.41213989257812, + "loss": 1.0142, + "margin_dpo/margin_mean": 58.49987030029297, + "margin_dpo/margin_std": 63.44873809814453, + "step": 271 + }, + { + "KL/chosen_KL_mean": -60.55150604248047, + "KL/mean": -88.66481018066406, + "KL/rejected_KL_mean": -116.77812194824219, + "KL/std": 61.343833923339844, + "epoch": 0.39941262848751835, + "fcm_dpo/beta": 0.008516614325344563, + "fcm_dpo/delta": -0.08284822106361389, + "fcm_dpo/margin": 56.226619720458984, + "fcm_dpo/q_t": 0.3901873826980591, + "grad_norm": 8.260068893432617, + "learning_rate": 3.772161666010912e-07, + "logits/chosen": -1.4849441051483154, + "logits/rejected": -1.3740112781524658, + "logps/chosen": -141.52871704101562, + "logps/ref_chosen": -80.97721862792969, + "logps/ref_rejected": -108.55535888671875, + "logps/rejected": -225.33348083496094, + "loss": 1.0387, + "margin_dpo/margin_mean": 56.226619720458984, + "margin_dpo/margin_std": 66.62297821044922, + "step": 272 + }, + { + "KL/chosen_KL_mean": -83.14810943603516, + "KL/mean": -111.72622680664062, + "KL/rejected_KL_mean": -140.30435180664062, + "KL/std": 63.00804138183594, + "epoch": 0.4008810572687225, + "fcm_dpo/beta": 0.008320118300616741, + "fcm_dpo/delta": -0.08099813759326935, + "fcm_dpo/margin": 57.15624237060547, + "fcm_dpo/q_t": 0.3903518319129944, + "grad_norm": 8.189800262451172, + "learning_rate": 3.761097448550755e-07, + "logits/chosen": -1.490915298461914, + "logits/rejected": -1.0723785161972046, + "logps/chosen": -175.37271118164062, + "logps/ref_chosen": -92.22460174560547, + "logps/ref_rejected": -97.3630599975586, + "logps/rejected": -237.66741943359375, + "loss": 1.0532, + "margin_dpo/margin_mean": 57.15624237060547, + "margin_dpo/margin_std": 71.69188690185547, + "step": 273 + }, + { + "KL/chosen_KL_mean": -84.57870483398438, + "KL/mean": -107.02603912353516, + "KL/rejected_KL_mean": -129.473388671875, + "KL/std": 61.506961822509766, + "epoch": 0.4023494860499266, + "fcm_dpo/beta": 0.00835760310292244, + "fcm_dpo/delta": 0.025692790746688843, + "fcm_dpo/margin": 44.89469528198242, + "fcm_dpo/q_t": 0.4126904010772705, + "grad_norm": 12.220967292785645, + "learning_rate": 3.75e-07, + "logits/chosen": -1.8258918523788452, + "logits/rejected": -1.4234145879745483, + "logps/chosen": -166.839599609375, + "logps/ref_chosen": -82.26089477539062, + "logps/ref_rejected": -83.87699890136719, + "logps/rejected": -213.3503875732422, + "loss": 1.0993, + "margin_dpo/margin_mean": 44.894691467285156, + "margin_dpo/margin_std": 56.543540954589844, + "step": 274 + }, + { + "KL/chosen_KL_mean": -71.63504028320312, + "KL/mean": -96.9306640625, + "KL/rejected_KL_mean": -122.22628784179688, + "KL/std": 60.558570861816406, + "epoch": 0.40381791483113066, + "fcm_dpo/beta": 0.008400652557611465, + "fcm_dpo/delta": -0.0265403613448143, + "fcm_dpo/margin": 50.59123611450195, + "fcm_dpo/q_t": 0.4022720158100128, + "grad_norm": 9.108122825622559, + "learning_rate": 3.738869612786737e-07, + "logits/chosen": -2.0346293449401855, + "logits/rejected": -1.709783911705017, + "logps/chosen": -151.32200622558594, + "logps/ref_chosen": -79.68696594238281, + "logps/ref_rejected": -98.7509765625, + "logps/rejected": -220.97726440429688, + "loss": 1.0955, + "margin_dpo/margin_mean": 50.59123611450195, + "margin_dpo/margin_std": 72.06424713134766, + "step": 275 + }, + { + "KL/chosen_KL_mean": -85.46694946289062, + "KL/mean": -112.68284606933594, + "KL/rejected_KL_mean": -139.8987579345703, + "KL/std": 61.15618133544922, + "epoch": 0.4052863436123348, + "fcm_dpo/beta": 0.008227425627410412, + "fcm_dpo/delta": -0.05128801241517067, + "fcm_dpo/margin": 54.43180465698242, + "fcm_dpo/q_t": 0.3975059986114502, + "grad_norm": 9.379951477050781, + "learning_rate": 3.7277065802070204e-07, + "logits/chosen": -2.101893663406372, + "logits/rejected": -1.5263922214508057, + "logps/chosen": -172.00665283203125, + "logps/ref_chosen": -86.53970336914062, + "logps/ref_rejected": -77.85394287109375, + "logps/rejected": -217.75270080566406, + "loss": 1.0668, + "margin_dpo/margin_mean": 54.43180847167969, + "margin_dpo/margin_std": 69.58363342285156, + "step": 276 + }, + { + "KL/chosen_KL_mean": -92.11480712890625, + "KL/mean": -111.81443786621094, + "KL/rejected_KL_mean": -131.51406860351562, + "KL/std": 59.30174255371094, + "epoch": 0.4067547723935389, + "fcm_dpo/beta": 0.00826319307088852, + "fcm_dpo/delta": 0.002128008520230651, + "fcm_dpo/margin": 39.399261474609375, + "fcm_dpo/q_t": 0.4241357445716858, + "grad_norm": 9.081292152404785, + "learning_rate": 3.71651119641714e-07, + "logits/chosen": -2.0738203525543213, + "logits/rejected": -1.658502459526062, + "logps/chosen": -176.35891723632812, + "logps/ref_chosen": -84.24411010742188, + "logps/ref_rejected": -98.83421325683594, + "logps/rejected": -230.34829711914062, + "loss": 1.1479, + "margin_dpo/margin_mean": 39.399261474609375, + "margin_dpo/margin_std": 60.69454574584961, + "step": 277 + }, + { + "KL/chosen_KL_mean": -88.09430694580078, + "KL/mean": -113.26759338378906, + "KL/rejected_KL_mean": -138.44088745117188, + "KL/std": 63.547183990478516, + "epoch": 0.40822320117474303, + "fcm_dpo/beta": 0.008229157887399197, + "fcm_dpo/delta": -0.015029809437692165, + "fcm_dpo/margin": 50.346580505371094, + "fcm_dpo/q_t": 0.40309515595436096, + "grad_norm": 10.935393333435059, + "learning_rate": 3.705283756425872e-07, + "logits/chosen": -2.0817856788635254, + "logits/rejected": -1.8657386302947998, + "logps/chosen": -170.52618408203125, + "logps/ref_chosen": -82.431884765625, + "logps/ref_rejected": -97.85691833496094, + "logps/rejected": -236.2978057861328, + "loss": 1.0819, + "margin_dpo/margin_mean": 50.346580505371094, + "margin_dpo/margin_std": 64.09684753417969, + "step": 278 + }, + { + "KL/chosen_KL_mean": -90.63104248046875, + "KL/mean": -121.99627685546875, + "KL/rejected_KL_mean": -153.3614959716797, + "KL/std": 67.62579345703125, + "epoch": 0.40969162995594716, + "fcm_dpo/beta": 0.008074283599853516, + "fcm_dpo/delta": -0.11279429495334625, + "fcm_dpo/margin": 62.73046112060547, + "fcm_dpo/q_t": 0.3830791115760803, + "grad_norm": 10.03566837310791, + "learning_rate": 3.6940245560867e-07, + "logits/chosen": -1.8028552532196045, + "logits/rejected": -1.5892622470855713, + "logps/chosen": -175.79904174804688, + "logps/ref_chosen": -85.16799926757812, + "logps/ref_rejected": -94.12664794921875, + "logps/rejected": -247.48814392089844, + "loss": 1.0114, + "margin_dpo/margin_mean": 62.73046112060547, + "margin_dpo/margin_std": 67.60432434082031, + "step": 279 + }, + { + "KL/chosen_KL_mean": -93.82257080078125, + "KL/mean": -124.91815948486328, + "KL/rejected_KL_mean": -156.01376342773438, + "KL/std": 61.90663146972656, + "epoch": 0.4111600587371512, + "fcm_dpo/beta": 0.007951401174068451, + "fcm_dpo/delta": -0.09931014478206635, + "fcm_dpo/margin": 62.19118881225586, + "fcm_dpo/q_t": 0.3849431872367859, + "grad_norm": 8.708633422851562, + "learning_rate": 3.6827338920900253e-07, + "logits/chosen": -1.8133509159088135, + "logits/rejected": -1.6158638000488281, + "logps/chosen": -179.67898559570312, + "logps/ref_chosen": -85.85641479492188, + "logps/ref_rejected": -104.11859130859375, + "logps/rejected": -260.1323547363281, + "loss": 1.0142, + "margin_dpo/margin_mean": 62.191192626953125, + "margin_dpo/margin_std": 64.64448547363281, + "step": 280 + }, + { + "KL/chosen_KL_mean": -100.50975036621094, + "KL/mean": -131.24822998046875, + "KL/rejected_KL_mean": -161.9866943359375, + "KL/std": 71.71414184570312, + "epoch": 0.41262848751835535, + "fcm_dpo/beta": 0.0077834296971559525, + "fcm_dpo/delta": -0.08270800113677979, + "fcm_dpo/margin": 61.4769401550293, + "fcm_dpo/q_t": 0.39019155502319336, + "grad_norm": 7.265856742858887, + "learning_rate": 3.6714120619553435e-07, + "logits/chosen": -2.1806912422180176, + "logits/rejected": -1.4496757984161377, + "logps/chosen": -187.06057739257812, + "logps/ref_chosen": -86.55082702636719, + "logps/ref_rejected": -88.62866973876953, + "logps/rejected": -250.6153564453125, + "loss": 1.0364, + "margin_dpo/margin_mean": 61.47693634033203, + "margin_dpo/margin_std": 72.03846740722656, + "step": 281 + }, + { + "KL/chosen_KL_mean": -97.80078887939453, + "KL/mean": -136.38632202148438, + "KL/rejected_KL_mean": -174.97186279296875, + "KL/std": 69.38382720947266, + "epoch": 0.41409691629955947, + "fcm_dpo/beta": 0.007540118880569935, + "fcm_dpo/delta": -0.19331520795822144, + "fcm_dpo/margin": 77.17108154296875, + "fcm_dpo/q_t": 0.36470791697502136, + "grad_norm": 8.835838317871094, + "learning_rate": 3.660059364023408e-07, + "logits/chosen": -1.7645771503448486, + "logits/rejected": -1.3814326524734497, + "logps/chosen": -202.90589904785156, + "logps/ref_chosen": -105.10511016845703, + "logps/ref_rejected": -102.85336303710938, + "logps/rejected": -277.82525634765625, + "loss": 0.9485, + "margin_dpo/margin_mean": 77.17108154296875, + "margin_dpo/margin_std": 69.40178680419922, + "step": 282 + }, + { + "KL/chosen_KL_mean": -112.97630310058594, + "KL/mean": -140.03945922851562, + "KL/rejected_KL_mean": -167.10260009765625, + "KL/std": 69.02311706542969, + "epoch": 0.4155653450807636, + "fcm_dpo/beta": 0.007465697359293699, + "fcm_dpo/delta": -0.004264989867806435, + "fcm_dpo/margin": 54.126319885253906, + "fcm_dpo/q_t": 0.4070722162723541, + "grad_norm": 9.946036338806152, + "learning_rate": 3.6486760974483685e-07, + "logits/chosen": -2.327394962310791, + "logits/rejected": -1.9982503652572632, + "logps/chosen": -208.02890014648438, + "logps/ref_chosen": -95.05259704589844, + "logps/ref_rejected": -103.54454803466797, + "logps/rejected": -270.64715576171875, + "loss": 1.099, + "margin_dpo/margin_mean": 54.126319885253906, + "margin_dpo/margin_std": 76.44354248046875, + "step": 283 + }, + { + "KL/chosen_KL_mean": -102.30014038085938, + "KL/mean": -136.99880981445312, + "KL/rejected_KL_mean": -171.697509765625, + "KL/std": 68.04427337646484, + "epoch": 0.4170337738619677, + "fcm_dpo/beta": 0.007310959044843912, + "fcm_dpo/delta": -0.11388857662677765, + "fcm_dpo/margin": 69.39735412597656, + "fcm_dpo/q_t": 0.38020601868629456, + "grad_norm": 9.636674880981445, + "learning_rate": 3.6372625621898863e-07, + "logits/chosen": -1.767768144607544, + "logits/rejected": -1.4314764738082886, + "logps/chosen": -189.96661376953125, + "logps/ref_chosen": -87.66647338867188, + "logps/ref_rejected": -98.75103759765625, + "logps/rejected": -270.44854736328125, + "loss": 0.9887, + "margin_dpo/margin_mean": 69.39735412597656, + "margin_dpo/margin_std": 61.032047271728516, + "step": 284 + }, + { + "KL/chosen_KL_mean": -114.56338500976562, + "KL/mean": -138.91604614257812, + "KL/rejected_KL_mean": -163.26869201660156, + "KL/std": 62.78680419921875, + "epoch": 0.4185022026431718, + "fcm_dpo/beta": 0.007360531948506832, + "fcm_dpo/delta": 0.042854174971580505, + "fcm_dpo/margin": 48.705291748046875, + "fcm_dpo/q_t": 0.4151710867881775, + "grad_norm": 9.950057983398438, + "learning_rate": 3.625819059005228e-07, + "logits/chosen": -2.6420364379882812, + "logits/rejected": -2.376955032348633, + "logps/chosen": -208.99642944335938, + "logps/ref_chosen": -94.43303680419922, + "logps/ref_rejected": -104.07194519042969, + "logps/rejected": -267.34063720703125, + "loss": 1.1066, + "margin_dpo/margin_mean": 48.705291748046875, + "margin_dpo/margin_std": 59.84783935546875, + "step": 285 + }, + { + "KL/chosen_KL_mean": -118.46646118164062, + "KL/mean": -151.51300048828125, + "KL/rejected_KL_mean": -184.55950927734375, + "KL/std": 66.89373779296875, + "epoch": 0.4199706314243759, + "fcm_dpo/beta": 0.00725883524864912, + "fcm_dpo/delta": -0.08393802493810654, + "fcm_dpo/margin": 66.09306335449219, + "fcm_dpo/q_t": 0.38787949085235596, + "grad_norm": 8.2915620803833, + "learning_rate": 3.614345889441346e-07, + "logits/chosen": -2.117314100265503, + "logits/rejected": -1.6650042533874512, + "logps/chosen": -222.18685913085938, + "logps/ref_chosen": -103.72039794921875, + "logps/ref_rejected": -96.25775909423828, + "logps/rejected": -280.8172607421875, + "loss": 1.024, + "margin_dpo/margin_mean": 66.09307098388672, + "margin_dpo/margin_std": 69.53059387207031, + "step": 286 + }, + { + "KL/chosen_KL_mean": -136.50839233398438, + "KL/mean": -155.32199096679688, + "KL/rejected_KL_mean": -174.13558959960938, + "KL/std": 60.80683517456055, + "epoch": 0.42143906020558003, + "fcm_dpo/beta": 0.007252223789691925, + "fcm_dpo/delta": 0.01974192075431347, + "fcm_dpo/margin": 37.62720489501953, + "fcm_dpo/q_t": 0.43733546137809753, + "grad_norm": 8.115185737609863, + "learning_rate": 3.6028433558269275e-07, + "logits/chosen": -2.132850170135498, + "logits/rejected": -1.5235352516174316, + "logps/chosen": -230.39828491210938, + "logps/ref_chosen": -93.88988494873047, + "logps/ref_rejected": -83.33365631103516, + "logps/rejected": -257.46923828125, + "loss": 1.1979, + "margin_dpo/margin_mean": 37.62720489501953, + "margin_dpo/margin_std": 70.94268035888672, + "step": 287 + }, + { + "KL/chosen_KL_mean": -128.83718872070312, + "KL/mean": -163.05276489257812, + "KL/rejected_KL_mean": -197.26834106445312, + "KL/std": 69.10311126708984, + "epoch": 0.42290748898678415, + "fcm_dpo/beta": 0.007152985781431198, + "fcm_dpo/delta": -0.09431672096252441, + "fcm_dpo/margin": 68.43115997314453, + "fcm_dpo/q_t": 0.38811296224594116, + "grad_norm": 7.692909240722656, + "learning_rate": 3.5913117612644327e-07, + "logits/chosen": -1.9543590545654297, + "logits/rejected": -1.5476887226104736, + "logps/chosen": -216.9932098388672, + "logps/ref_chosen": -88.15602111816406, + "logps/ref_rejected": -93.28195190429688, + "logps/rejected": -290.55029296875, + "loss": 1.0251, + "margin_dpo/margin_mean": 68.43116760253906, + "margin_dpo/margin_std": 77.25398254394531, + "step": 288 + }, + { + "KL/chosen_KL_mean": -125.1337890625, + "KL/mean": -159.77838134765625, + "KL/rejected_KL_mean": -194.4229736328125, + "KL/std": 72.15738677978516, + "epoch": 0.4243759177679883, + "fcm_dpo/beta": 0.007046650163829327, + "fcm_dpo/delta": -0.09268496930599213, + "fcm_dpo/margin": 69.2891845703125, + "fcm_dpo/q_t": 0.3900146484375, + "grad_norm": 7.289619445800781, + "learning_rate": 3.5797514096221024e-07, + "logits/chosen": -2.028315544128418, + "logits/rejected": -1.8274290561676025, + "logps/chosen": -200.52671813964844, + "logps/ref_chosen": -75.39292907714844, + "logps/ref_rejected": -93.15428161621094, + "logps/rejected": -287.5772705078125, + "loss": 1.0345, + "margin_dpo/margin_mean": 69.28919219970703, + "margin_dpo/margin_std": 83.13552856445312, + "step": 289 + }, + { + "KL/chosen_KL_mean": -134.67037963867188, + "KL/mean": -170.7037353515625, + "KL/rejected_KL_mean": -206.73707580566406, + "KL/std": 76.6993408203125, + "epoch": 0.42584434654919234, + "fcm_dpo/beta": 0.00693401787430048, + "fcm_dpo/delta": -0.10508199781179428, + "fcm_dpo/margin": 72.06668090820312, + "fcm_dpo/q_t": 0.3863818049430847, + "grad_norm": 8.005448341369629, + "learning_rate": 3.568162605525952e-07, + "logits/chosen": -1.3975692987442017, + "logits/rejected": -1.442676067352295, + "logps/chosen": -222.71237182617188, + "logps/ref_chosen": -88.04197692871094, + "logps/ref_rejected": -123.212158203125, + "logps/rejected": -329.94921875, + "loss": 1.0212, + "margin_dpo/margin_mean": 72.06668090820312, + "margin_dpo/margin_std": 80.52359008789062, + "step": 290 + }, + { + "KL/chosen_KL_mean": -126.65396881103516, + "KL/mean": -157.83328247070312, + "KL/rejected_KL_mean": -189.0126190185547, + "KL/std": 72.67715454101562, + "epoch": 0.42731277533039647, + "fcm_dpo/beta": 0.006835309322923422, + "fcm_dpo/delta": -0.027408163994550705, + "fcm_dpo/margin": 62.35863494873047, + "fcm_dpo/q_t": 0.40002545714378357, + "grad_norm": 8.919699668884277, + "learning_rate": 3.5565456543517485e-07, + "logits/chosen": -2.2441418170928955, + "logits/rejected": -1.7710058689117432, + "logps/chosen": -220.74920654296875, + "logps/ref_chosen": -94.09524536132812, + "logps/ref_rejected": -96.05006408691406, + "logps/rejected": -285.06268310546875, + "loss": 1.0625, + "margin_dpo/margin_mean": 62.35863494873047, + "margin_dpo/margin_std": 72.18458557128906, + "step": 291 + }, + { + "KL/chosen_KL_mean": -131.94564819335938, + "KL/mean": -163.62689208984375, + "KL/rejected_KL_mean": -195.30810546875, + "KL/std": 70.27583312988281, + "epoch": 0.4287812041116006, + "fcm_dpo/beta": 0.006783951073884964, + "fcm_dpo/delta": -0.03119458258152008, + "fcm_dpo/margin": 63.362449645996094, + "fcm_dpo/q_t": 0.39915913343429565, + "grad_norm": 8.372907638549805, + "learning_rate": 3.5449008622169583e-07, + "logits/chosen": -2.0467820167541504, + "logits/rejected": -1.6140203475952148, + "logps/chosen": -220.19607543945312, + "logps/ref_chosen": -88.25041198730469, + "logps/ref_rejected": -96.41764068603516, + "logps/rejected": -291.72576904296875, + "loss": 1.0483, + "margin_dpo/margin_mean": 63.362449645996094, + "margin_dpo/margin_std": 65.77003479003906, + "step": 292 + }, + { + "KL/chosen_KL_mean": -131.47671508789062, + "KL/mean": -156.93624877929688, + "KL/rejected_KL_mean": -182.39581298828125, + "KL/std": 74.41465759277344, + "epoch": 0.4302496328928047, + "fcm_dpo/beta": 0.006768643390387297, + "fcm_dpo/delta": -0.008427482098340988, + "fcm_dpo/margin": 50.919090270996094, + "fcm_dpo/q_t": 0.42108622193336487, + "grad_norm": 7.748796463012695, + "learning_rate": 3.5332285359726846e-07, + "logits/chosen": -2.3342254161834717, + "logits/rejected": -1.762291431427002, + "logps/chosen": -218.85325622558594, + "logps/ref_chosen": -87.37654113769531, + "logps/ref_rejected": -85.75579833984375, + "logps/rejected": -268.151611328125, + "loss": 1.1374, + "margin_dpo/margin_mean": 50.919090270996094, + "margin_dpo/margin_std": 78.06587982177734, + "step": 293 + }, + { + "KL/chosen_KL_mean": -134.5666046142578, + "KL/mean": -155.5863037109375, + "KL/rejected_KL_mean": -176.6060333251953, + "KL/std": 69.97149658203125, + "epoch": 0.43171806167400884, + "fcm_dpo/beta": 0.006784706376492977, + "fcm_dpo/delta": 0.03208325430750847, + "fcm_dpo/margin": 42.03943634033203, + "fcm_dpo/q_t": 0.4344545304775238, + "grad_norm": 8.094246864318848, + "learning_rate": 3.5215289831955786e-07, + "logits/chosen": -1.7406806945800781, + "logits/rejected": -1.5746405124664307, + "logps/chosen": -208.07455444335938, + "logps/ref_chosen": -73.50794982910156, + "logps/ref_rejected": -88.08877563476562, + "logps/rejected": -264.69482421875, + "loss": 1.1972, + "margin_dpo/margin_mean": 42.03943634033203, + "margin_dpo/margin_std": 81.99540710449219, + "step": 294 + }, + { + "KL/chosen_KL_mean": -127.7253646850586, + "KL/mean": -160.01303100585938, + "KL/rejected_KL_mean": -192.30072021484375, + "KL/std": 71.88095092773438, + "epoch": 0.4331864904552129, + "fcm_dpo/beta": 0.006742920260876417, + "fcm_dpo/delta": -0.037477537989616394, + "fcm_dpo/margin": 64.57534790039062, + "fcm_dpo/q_t": 0.39943987131118774, + "grad_norm": 8.162626266479492, + "learning_rate": 3.509802512179737e-07, + "logits/chosen": -1.652780532836914, + "logits/rejected": -1.49755859375, + "logps/chosen": -205.4908447265625, + "logps/ref_chosen": -77.76548767089844, + "logps/ref_rejected": -94.24726867675781, + "logps/rejected": -286.5479736328125, + "loss": 1.0618, + "margin_dpo/margin_mean": 64.57534790039062, + "margin_dpo/margin_std": 77.56766510009766, + "step": 295 + }, + { + "KL/chosen_KL_mean": -137.2755126953125, + "KL/mean": -171.8076171875, + "KL/rejected_KL_mean": -206.33970642089844, + "KL/std": 72.12962341308594, + "epoch": 0.434654919236417, + "fcm_dpo/beta": 0.006669612601399422, + "fcm_dpo/delta": -0.06399206817150116, + "fcm_dpo/margin": 69.06419372558594, + "fcm_dpo/q_t": 0.3919883668422699, + "grad_norm": 11.0608491897583, + "learning_rate": 3.498049431928577e-07, + "logits/chosen": -2.4091262817382812, + "logits/rejected": -1.9138436317443848, + "logps/chosen": -235.13192749023438, + "logps/ref_chosen": -97.85641479492188, + "logps/ref_rejected": -100.81631469726562, + "logps/rejected": -307.156005859375, + "loss": 1.0342, + "margin_dpo/margin_mean": 69.06419372558594, + "margin_dpo/margin_std": 72.87110900878906, + "step": 296 + }, + { + "KL/chosen_KL_mean": -139.03265380859375, + "KL/mean": -168.15167236328125, + "KL/rejected_KL_mean": -197.27072143554688, + "KL/std": 72.32681274414062, + "epoch": 0.43612334801762115, + "fcm_dpo/beta": 0.006693677511066198, + "fcm_dpo/delta": 0.010481396690011024, + "fcm_dpo/margin": 58.23806381225586, + "fcm_dpo/q_t": 0.4106457233428955, + "grad_norm": 11.073972702026367, + "learning_rate": 3.486270052146694e-07, + "logits/chosen": -1.2457356452941895, + "logits/rejected": -0.9378162622451782, + "logps/chosen": -227.59848022460938, + "logps/ref_chosen": -88.56583404541016, + "logps/ref_rejected": -101.55656433105469, + "logps/rejected": -298.8272705078125, + "loss": 1.0969, + "margin_dpo/margin_mean": 58.23806381225586, + "margin_dpo/margin_std": 77.65412902832031, + "step": 297 + }, + { + "KL/chosen_KL_mean": -117.33425903320312, + "KL/mean": -154.497314453125, + "KL/rejected_KL_mean": -191.6603546142578, + "KL/std": 78.61767578125, + "epoch": 0.43759177679882527, + "fcm_dpo/beta": 0.006604637950658798, + "fcm_dpo/delta": -0.09547838568687439, + "fcm_dpo/margin": 74.32611083984375, + "fcm_dpo/q_t": 0.38755857944488525, + "grad_norm": 11.828282356262207, + "learning_rate": 3.474464683231698e-07, + "logits/chosen": -2.0623486042022705, + "logits/rejected": -2.0400123596191406, + "logps/chosen": -212.21469116210938, + "logps/ref_chosen": -94.88043975830078, + "logps/ref_rejected": -122.31101989746094, + "logps/rejected": -313.97137451171875, + "loss": 1.0178, + "margin_dpo/margin_mean": 74.32611083984375, + "margin_dpo/margin_std": 79.70893096923828, + "step": 298 + }, + { + "KL/chosen_KL_mean": -122.30178833007812, + "KL/mean": -159.39028930664062, + "KL/rejected_KL_mean": -196.478759765625, + "KL/std": 75.82727813720703, + "epoch": 0.4390602055800294, + "fcm_dpo/beta": 0.006472492590546608, + "fcm_dpo/delta": -0.08419737219810486, + "fcm_dpo/margin": 74.1769790649414, + "fcm_dpo/q_t": 0.38858693838119507, + "grad_norm": 9.64562702178955, + "learning_rate": 3.462633636266041e-07, + "logits/chosen": -1.1129204034805298, + "logits/rejected": -0.8912756443023682, + "logps/chosen": -202.71014404296875, + "logps/ref_chosen": -80.40835571289062, + "logps/ref_rejected": -89.53717041015625, + "logps/rejected": -286.01593017578125, + "loss": 1.0237, + "margin_dpo/margin_mean": 74.1769790649414, + "margin_dpo/margin_std": 79.17498779296875, + "step": 299 + }, + { + "KL/chosen_KL_mean": -129.1915740966797, + "KL/mean": -159.88795471191406, + "KL/rejected_KL_mean": -190.58432006835938, + "KL/std": 79.55728912353516, + "epoch": 0.44052863436123346, + "fcm_dpo/beta": 0.006427213549613953, + "fcm_dpo/delta": 0.005332227796316147, + "fcm_dpo/margin": 61.39275360107422, + "fcm_dpo/q_t": 0.4087502062320709, + "grad_norm": 8.07475471496582, + "learning_rate": 3.4507772230088147e-07, + "logits/chosen": -1.635288119316101, + "logits/rejected": -1.4246439933776855, + "logps/chosen": -217.35047912597656, + "logps/ref_chosen": -88.15890502929688, + "logps/ref_rejected": -100.93919372558594, + "logps/rejected": -291.52349853515625, + "loss": 1.1024, + "margin_dpo/margin_mean": 61.39275360107422, + "margin_dpo/margin_std": 85.14608764648438, + "step": 300 + }, + { + "KL/chosen_KL_mean": -118.31253814697266, + "KL/mean": -156.63668823242188, + "KL/rejected_KL_mean": -194.9608154296875, + "KL/std": 77.95761108398438, + "epoch": 0.4419970631424376, + "fcm_dpo/beta": 0.006336267106235027, + "fcm_dpo/delta": -0.09099871665239334, + "fcm_dpo/margin": 76.64825439453125, + "fcm_dpo/q_t": 0.3882323205471039, + "grad_norm": 9.345137596130371, + "learning_rate": 3.4388957558875316e-07, + "logits/chosen": -1.7852119207382202, + "logits/rejected": -1.355362892150879, + "logps/chosen": -203.81744384765625, + "logps/ref_chosen": -85.50491333007812, + "logps/ref_rejected": -98.91683959960938, + "logps/rejected": -293.8776550292969, + "loss": 1.0314, + "margin_dpo/margin_mean": 76.64825439453125, + "margin_dpo/margin_std": 87.937744140625, + "step": 301 + }, + { + "KL/chosen_KL_mean": -114.86396026611328, + "KL/mean": -147.6676788330078, + "KL/rejected_KL_mean": -180.47140502929688, + "KL/std": 80.09053802490234, + "epoch": 0.4434654919236417, + "fcm_dpo/beta": 0.006313342601060867, + "fcm_dpo/delta": -0.014902262017130852, + "fcm_dpo/margin": 65.60742950439453, + "fcm_dpo/q_t": 0.4030223488807678, + "grad_norm": 8.341588020324707, + "learning_rate": 3.426989547989902e-07, + "logits/chosen": -1.2355989217758179, + "logits/rejected": -1.059578776359558, + "logps/chosen": -194.31436157226562, + "logps/ref_chosen": -79.45040893554688, + "logps/ref_rejected": -103.0591049194336, + "logps/rejected": -283.530517578125, + "loss": 1.0722, + "margin_dpo/margin_mean": 65.60742950439453, + "margin_dpo/margin_std": 78.26306915283203, + "step": 302 + }, + { + "KL/chosen_KL_mean": -107.10307312011719, + "KL/mean": -142.83676147460938, + "KL/rejected_KL_mean": -178.57046508789062, + "KL/std": 74.27822875976562, + "epoch": 0.44493392070484583, + "fcm_dpo/beta": 0.006287604570388794, + "fcm_dpo/delta": -0.051686663180589676, + "fcm_dpo/margin": 71.46737670898438, + "fcm_dpo/q_t": 0.3948175311088562, + "grad_norm": 8.15832805633545, + "learning_rate": 3.4150589130555773e-07, + "logits/chosen": -1.8444539308547974, + "logits/rejected": -1.384230375289917, + "logps/chosen": -202.7013702392578, + "logps/ref_chosen": -95.59829711914062, + "logps/ref_rejected": -92.0645751953125, + "logps/rejected": -270.6350402832031, + "loss": 1.0464, + "margin_dpo/margin_mean": 71.4673843383789, + "margin_dpo/margin_std": 79.21310424804688, + "step": 303 + }, + { + "KL/chosen_KL_mean": -105.83399963378906, + "KL/mean": -143.13702392578125, + "KL/rejected_KL_mean": -180.44003295898438, + "KL/std": 77.14070129394531, + "epoch": 0.44640234948604995, + "fcm_dpo/beta": 0.006196558475494385, + "fcm_dpo/delta": -0.06529698520898819, + "fcm_dpo/margin": 74.60604858398438, + "fcm_dpo/q_t": 0.3918082118034363, + "grad_norm": 8.952920913696289, + "learning_rate": 3.403104165467883e-07, + "logits/chosen": -2.0361664295196533, + "logits/rejected": -1.6098146438598633, + "logps/chosen": -205.28713989257812, + "logps/ref_chosen": -99.4531478881836, + "logps/ref_rejected": -98.08226013183594, + "logps/rejected": -278.5223083496094, + "loss": 1.0336, + "margin_dpo/margin_mean": 74.60604858398438, + "margin_dpo/margin_std": 79.76287841796875, + "step": 304 + }, + { + "KL/chosen_KL_mean": -115.79327392578125, + "KL/mean": -151.1674346923828, + "KL/rejected_KL_mean": -186.54159545898438, + "KL/std": 80.5308837890625, + "epoch": 0.447870778267254, + "fcm_dpo/beta": 0.006108911242336035, + "fcm_dpo/delta": -0.03442249447107315, + "fcm_dpo/margin": 70.74832153320312, + "fcm_dpo/q_t": 0.39917391538619995, + "grad_norm": 7.5331597328186035, + "learning_rate": 3.391125620245535e-07, + "logits/chosen": -1.8691487312316895, + "logits/rejected": -1.3629894256591797, + "logps/chosen": -219.05667114257812, + "logps/ref_chosen": -103.26339721679688, + "logps/ref_rejected": -97.82897186279297, + "logps/rejected": -284.37054443359375, + "loss": 1.0702, + "margin_dpo/margin_mean": 70.74832153320312, + "margin_dpo/margin_std": 86.82716369628906, + "step": 305 + }, + { + "KL/chosen_KL_mean": -104.38932800292969, + "KL/mean": -145.96383666992188, + "KL/rejected_KL_mean": -187.538330078125, + "KL/std": 76.4020004272461, + "epoch": 0.44933920704845814, + "fcm_dpo/beta": 0.006018854212015867, + "fcm_dpo/delta": -0.1061825156211853, + "fcm_dpo/margin": 83.14901733398438, + "fcm_dpo/q_t": 0.3811030685901642, + "grad_norm": 9.700840950012207, + "learning_rate": 3.3791235930343417e-07, + "logits/chosen": -2.1679553985595703, + "logits/rejected": -1.6040107011795044, + "logps/chosen": -204.6931610107422, + "logps/ref_chosen": -100.3038330078125, + "logps/ref_rejected": -92.23086547851562, + "logps/rejected": -279.76922607421875, + "loss": 0.9912, + "margin_dpo/margin_mean": 83.14901733398438, + "margin_dpo/margin_std": 70.9165267944336, + "step": 306 + }, + { + "KL/chosen_KL_mean": -109.17141723632812, + "KL/mean": -140.42274475097656, + "KL/rejected_KL_mean": -171.67405700683594, + "KL/std": 77.14170837402344, + "epoch": 0.45080763582966227, + "fcm_dpo/beta": 0.006004738621413708, + "fcm_dpo/delta": 0.02550465799868107, + "fcm_dpo/margin": 62.50264358520508, + "fcm_dpo/q_t": 0.4135735034942627, + "grad_norm": 8.71324634552002, + "learning_rate": 3.367098400098881e-07, + "logits/chosen": -1.8640072345733643, + "logits/rejected": -1.5111415386199951, + "logps/chosen": -205.82118225097656, + "logps/ref_chosen": -96.64976501464844, + "logps/ref_rejected": -93.34860229492188, + "logps/rejected": -265.02264404296875, + "loss": 1.1178, + "margin_dpo/margin_mean": 62.50264358520508, + "margin_dpo/margin_std": 91.26786041259766, + "step": 307 + }, + { + "KL/chosen_KL_mean": -114.31552124023438, + "KL/mean": -145.63380432128906, + "KL/rejected_KL_mean": -176.95208740234375, + "KL/std": 78.7996597290039, + "epoch": 0.4522760646108664, + "fcm_dpo/beta": 0.005973818711936474, + "fcm_dpo/delta": -0.04912327229976654, + "fcm_dpo/margin": 62.63658142089844, + "fcm_dpo/q_t": 0.41376030445098877, + "grad_norm": 10.472161293029785, + "learning_rate": 3.355050358314172e-07, + "logits/chosen": -1.5214544534683228, + "logits/rejected": -1.2289180755615234, + "logps/chosen": -200.04019165039062, + "logps/ref_chosen": -85.72467041015625, + "logps/ref_rejected": -86.42317962646484, + "logps/rejected": -263.3752746582031, + "loss": 1.1221, + "margin_dpo/margin_mean": 62.63658142089844, + "margin_dpo/margin_std": 91.94395446777344, + "step": 308 + }, + { + "KL/chosen_KL_mean": -128.42152404785156, + "KL/mean": -156.42520141601562, + "KL/rejected_KL_mean": -184.42886352539062, + "KL/std": 83.79991149902344, + "epoch": 0.45374449339207046, + "fcm_dpo/beta": 0.006035213358700275, + "fcm_dpo/delta": 0.06410080194473267, + "fcm_dpo/margin": 56.00735855102539, + "fcm_dpo/q_t": 0.4215248227119446, + "grad_norm": 11.60377025604248, + "learning_rate": 3.3429797851573183e-07, + "logits/chosen": -1.9197332859039307, + "logits/rejected": -1.5002105236053467, + "logps/chosen": -211.57298278808594, + "logps/ref_chosen": -83.15145874023438, + "logps/ref_rejected": -86.58602142333984, + "logps/rejected": -271.014892578125, + "loss": 1.1401, + "margin_dpo/margin_mean": 56.00735855102539, + "margin_dpo/margin_std": 83.6112060546875, + "step": 309 + }, + { + "KL/chosen_KL_mean": -116.37451934814453, + "KL/mean": -147.36968994140625, + "KL/rejected_KL_mean": -178.3648681640625, + "KL/std": 76.15359497070312, + "epoch": 0.4552129221732746, + "fcm_dpo/beta": 0.006068498827517033, + "fcm_dpo/delta": 0.024749252945184708, + "fcm_dpo/margin": 61.990352630615234, + "fcm_dpo/q_t": 0.41219815611839294, + "grad_norm": 8.85814380645752, + "learning_rate": 3.3308869986991487e-07, + "logits/chosen": -2.462876319885254, + "logits/rejected": -2.066950559616089, + "logps/chosen": -206.11251831054688, + "logps/ref_chosen": -89.73799133300781, + "logps/ref_rejected": -87.07083129882812, + "logps/rejected": -265.4356994628906, + "loss": 1.1014, + "margin_dpo/margin_mean": 61.99034881591797, + "margin_dpo/margin_std": 80.15214538574219, + "step": 310 + }, + { + "KL/chosen_KL_mean": -134.4075927734375, + "KL/mean": -174.58982849121094, + "KL/rejected_KL_mean": -214.77206420898438, + "KL/std": 77.66455841064453, + "epoch": 0.4566813509544787, + "fcm_dpo/beta": 0.006006065756082535, + "fcm_dpo/delta": -0.08682767301797867, + "fcm_dpo/margin": 80.36447143554688, + "fcm_dpo/q_t": 0.38752785325050354, + "grad_norm": 9.148625373840332, + "learning_rate": 3.3187723175958346e-07, + "logits/chosen": -1.7456867694854736, + "logits/rejected": -1.0247340202331543, + "logps/chosen": -222.93402099609375, + "logps/ref_chosen": -88.52644348144531, + "logps/ref_rejected": -82.26608276367188, + "logps/rejected": -297.03814697265625, + "loss": 1.0124, + "margin_dpo/margin_mean": 80.3644790649414, + "margin_dpo/margin_std": 79.02332305908203, + "step": 311 + }, + { + "KL/chosen_KL_mean": -130.44659423828125, + "KL/mean": -163.57403564453125, + "KL/rejected_KL_mean": -196.7014923095703, + "KL/std": 85.45756530761719, + "epoch": 0.4581497797356828, + "fcm_dpo/beta": 0.005996899679303169, + "fcm_dpo/delta": 0.0025873929262161255, + "fcm_dpo/margin": 66.2549057006836, + "fcm_dpo/q_t": 0.40717554092407227, + "grad_norm": 9.376420974731445, + "learning_rate": 3.306636061080487e-07, + "logits/chosen": -1.6214861869812012, + "logits/rejected": -1.098444938659668, + "logps/chosen": -210.8472900390625, + "logps/ref_chosen": -80.40069580078125, + "logps/ref_rejected": -82.4471435546875, + "logps/rejected": -279.14862060546875, + "loss": 1.0856, + "margin_dpo/margin_mean": 66.2549057006836, + "margin_dpo/margin_std": 81.75543975830078, + "step": 312 + }, + { + "KL/chosen_KL_mean": -124.4384765625, + "KL/mean": -157.04403686523438, + "KL/rejected_KL_mean": -189.6496124267578, + "KL/std": 83.68032836914062, + "epoch": 0.45961820851688695, + "fcm_dpo/beta": 0.006004225462675095, + "fcm_dpo/delta": 0.008609505370259285, + "fcm_dpo/margin": 65.21114349365234, + "fcm_dpo/q_t": 0.40929821133613586, + "grad_norm": 9.61097526550293, + "learning_rate": 3.2944785489547537e-07, + "logits/chosen": -2.390892505645752, + "logits/rejected": -1.9862465858459473, + "logps/chosen": -208.58038330078125, + "logps/ref_chosen": -84.14190673828125, + "logps/ref_rejected": -93.42684173583984, + "logps/rejected": -283.0764465332031, + "loss": 1.1017, + "margin_dpo/margin_mean": 65.21115112304688, + "margin_dpo/margin_std": 89.83334350585938, + "step": 313 + }, + { + "KL/chosen_KL_mean": -126.9460678100586, + "KL/mean": -158.96875, + "KL/rejected_KL_mean": -190.99139404296875, + "KL/std": 83.79634094238281, + "epoch": 0.461086637298091, + "fcm_dpo/beta": 0.005988355726003647, + "fcm_dpo/delta": 0.017022665590047836, + "fcm_dpo/margin": 64.04533386230469, + "fcm_dpo/q_t": 0.4116547703742981, + "grad_norm": 10.217790603637695, + "learning_rate": 3.2823001015803857e-07, + "logits/chosen": -2.0049421787261963, + "logits/rejected": -1.7885639667510986, + "logps/chosen": -215.75579833984375, + "logps/ref_chosen": -88.80972290039062, + "logps/ref_rejected": -104.15892028808594, + "logps/rejected": -295.15032958984375, + "loss": 1.1001, + "margin_dpo/margin_mean": 64.04534149169922, + "margin_dpo/margin_std": 84.52468872070312, + "step": 314 + }, + { + "KL/chosen_KL_mean": -122.24842071533203, + "KL/mean": -155.42909240722656, + "KL/rejected_KL_mean": -188.60977172851562, + "KL/std": 88.0732192993164, + "epoch": 0.46255506607929514, + "fcm_dpo/beta": 0.006007509771734476, + "fcm_dpo/delta": 0.001381636131554842, + "fcm_dpo/margin": 66.36135864257812, + "fcm_dpo/q_t": 0.4093374013900757, + "grad_norm": 8.968510627746582, + "learning_rate": 3.270101039870797e-07, + "logits/chosen": -1.5781664848327637, + "logits/rejected": -1.4049468040466309, + "logps/chosen": -203.0421142578125, + "logps/ref_chosen": -80.793701171875, + "logps/ref_rejected": -92.13711547851562, + "logps/rejected": -280.74688720703125, + "loss": 1.1129, + "margin_dpo/margin_mean": 66.36135864257812, + "margin_dpo/margin_std": 100.99237060546875, + "step": 315 + }, + { + "KL/chosen_KL_mean": -124.93754577636719, + "KL/mean": -160.77418518066406, + "KL/rejected_KL_mean": -196.61085510253906, + "KL/std": 86.6968002319336, + "epoch": 0.46402349486049926, + "fcm_dpo/beta": 0.005990965291857719, + "fcm_dpo/delta": -0.030701998621225357, + "fcm_dpo/margin": 71.67329406738281, + "fcm_dpo/q_t": 0.40148815512657166, + "grad_norm": 9.289080619812012, + "learning_rate": 3.2578816852826086e-07, + "logits/chosen": -1.7152100801467896, + "logits/rejected": -1.716172695159912, + "logps/chosen": -209.12179565429688, + "logps/ref_chosen": -84.18424987792969, + "logps/ref_rejected": -105.6908187866211, + "logps/rejected": -302.3016662597656, + "loss": 1.0704, + "margin_dpo/margin_mean": 71.67329406738281, + "margin_dpo/margin_std": 90.77346801757812, + "step": 316 + }, + { + "KL/chosen_KL_mean": -130.26666259765625, + "KL/mean": -174.43026733398438, + "KL/rejected_KL_mean": -218.5938720703125, + "KL/std": 95.59025573730469, + "epoch": 0.4654919236417034, + "fcm_dpo/beta": 0.005883518606424332, + "fcm_dpo/delta": -0.1260424256324768, + "fcm_dpo/margin": 88.32722473144531, + "fcm_dpo/q_t": 0.38176554441452026, + "grad_norm": 10.334203720092773, + "learning_rate": 3.2456423598071783e-07, + "logits/chosen": -1.7716938257217407, + "logits/rejected": -1.2840113639831543, + "logps/chosen": -218.14015197753906, + "logps/ref_chosen": -87.87348937988281, + "logps/ref_rejected": -108.26519775390625, + "logps/rejected": -326.85906982421875, + "loss": 1.0172, + "margin_dpo/margin_mean": 88.32722473144531, + "margin_dpo/margin_std": 102.13755798339844, + "step": 317 + }, + { + "KL/chosen_KL_mean": -127.30432891845703, + "KL/mean": -166.45045471191406, + "KL/rejected_KL_mean": -205.59658813476562, + "KL/std": 80.25211334228516, + "epoch": 0.4669603524229075, + "fcm_dpo/beta": 0.005772769451141357, + "fcm_dpo/delta": -0.05450741946697235, + "fcm_dpo/margin": 78.29225158691406, + "fcm_dpo/q_t": 0.39487743377685547, + "grad_norm": 10.423051834106445, + "learning_rate": 3.233383385962115e-07, + "logits/chosen": -2.327599287033081, + "logits/rejected": -1.6279561519622803, + "logps/chosen": -225.50985717773438, + "logps/ref_chosen": -98.20553588867188, + "logps/ref_rejected": -88.13629150390625, + "logps/rejected": -293.7328796386719, + "loss": 1.0458, + "margin_dpo/margin_mean": 78.29225158691406, + "margin_dpo/margin_std": 88.71526336669922, + "step": 318 + }, + { + "KL/chosen_KL_mean": -109.34317016601562, + "KL/mean": -152.42117309570312, + "KL/rejected_KL_mean": -195.49920654296875, + "KL/std": 83.83075714111328, + "epoch": 0.4684287812041116, + "fcm_dpo/beta": 0.00567377544939518, + "fcm_dpo/delta": -0.09361159056425095, + "fcm_dpo/margin": 86.1560287475586, + "fcm_dpo/q_t": 0.38663536310195923, + "grad_norm": 10.118432998657227, + "learning_rate": 3.2211050867827805e-07, + "logits/chosen": -1.8486950397491455, + "logits/rejected": -1.7804803848266602, + "logps/chosen": -191.84654235839844, + "logps/ref_chosen": -82.50337219238281, + "logps/ref_rejected": -118.65068054199219, + "logps/rejected": -314.14990234375, + "loss": 1.0223, + "margin_dpo/margin_mean": 86.15603637695312, + "margin_dpo/margin_std": 94.27151489257812, + "step": 319 + }, + { + "KL/chosen_KL_mean": -111.82501220703125, + "KL/mean": -146.86178588867188, + "KL/rejected_KL_mean": -181.89852905273438, + "KL/std": 84.91935729980469, + "epoch": 0.4698972099853157, + "fcm_dpo/beta": 0.005651119165122509, + "fcm_dpo/delta": 0.00413152202963829, + "fcm_dpo/margin": 70.07352447509766, + "fcm_dpo/q_t": 0.40768054127693176, + "grad_norm": 8.290468215942383, + "learning_rate": 3.208807785813777e-07, + "logits/chosen": -1.9670660495758057, + "logits/rejected": -1.7768141031265259, + "logps/chosen": -198.96224975585938, + "logps/ref_chosen": -87.13722229003906, + "logps/ref_rejected": -103.22412109375, + "logps/rejected": -285.1226501464844, + "loss": 1.0807, + "margin_dpo/margin_mean": 70.07352447509766, + "margin_dpo/margin_std": 82.98087310791016, + "step": 320 + }, + { + "KL/chosen_KL_mean": -125.65251922607422, + "KL/mean": -166.46595764160156, + "KL/rejected_KL_mean": -207.27938842773438, + "KL/std": 87.3461685180664, + "epoch": 0.4713656387665198, + "fcm_dpo/beta": 0.005591265857219696, + "fcm_dpo/delta": -0.05955302342772484, + "fcm_dpo/margin": 81.62687683105469, + "fcm_dpo/q_t": 0.3923654556274414, + "grad_norm": 8.765765190124512, + "learning_rate": 3.1964918071004217e-07, + "logits/chosen": -1.909494400024414, + "logits/rejected": -1.5170204639434814, + "logps/chosen": -218.75718688964844, + "logps/ref_chosen": -93.10466766357422, + "logps/ref_rejected": -97.4681396484375, + "logps/rejected": -304.7475280761719, + "loss": 1.0382, + "margin_dpo/margin_mean": 81.62687683105469, + "margin_dpo/margin_std": 87.35755920410156, + "step": 321 + }, + { + "KL/chosen_KL_mean": -116.93401336669922, + "KL/mean": -160.39947509765625, + "KL/rejected_KL_mean": -203.8649139404297, + "KL/std": 86.72179412841797, + "epoch": 0.47283406754772395, + "fcm_dpo/beta": 0.005490246694535017, + "fcm_dpo/delta": -0.08251510560512543, + "fcm_dpo/margin": 86.9309310913086, + "fcm_dpo/q_t": 0.3887970447540283, + "grad_norm": 13.106282234191895, + "learning_rate": 3.184157475180207e-07, + "logits/chosen": -1.781909704208374, + "logits/rejected": -1.427588701248169, + "logps/chosen": -209.12692260742188, + "logps/ref_chosen": -92.19291687011719, + "logps/ref_rejected": -100.89321899414062, + "logps/rejected": -304.75811767578125, + "loss": 1.0304, + "margin_dpo/margin_mean": 86.9309310913086, + "margin_dpo/margin_std": 93.81636810302734, + "step": 322 + }, + { + "KL/chosen_KL_mean": -134.53424072265625, + "KL/mean": -171.5132293701172, + "KL/rejected_KL_mean": -208.49220275878906, + "KL/std": 87.30924987792969, + "epoch": 0.47430249632892807, + "fcm_dpo/beta": 0.005486944690346718, + "fcm_dpo/delta": -0.006131318397819996, + "fcm_dpo/margin": 73.95794677734375, + "fcm_dpo/q_t": 0.4063902199268341, + "grad_norm": 11.118532180786133, + "learning_rate": 3.171805115074251e-07, + "logits/chosen": -2.0952534675598145, + "logits/rejected": -1.7223830223083496, + "logps/chosen": -214.8063507080078, + "logps/ref_chosen": -80.27210235595703, + "logps/ref_rejected": -80.3033447265625, + "logps/rejected": -288.7955322265625, + "loss": 1.0806, + "margin_dpo/margin_mean": 73.95794677734375, + "margin_dpo/margin_std": 91.36995697021484, + "step": 323 + }, + { + "KL/chosen_KL_mean": -134.28904724121094, + "KL/mean": -172.3663787841797, + "KL/rejected_KL_mean": -210.44369506835938, + "KL/std": 87.37994384765625, + "epoch": 0.47577092511013214, + "fcm_dpo/beta": 0.005488350056111813, + "fcm_dpo/delta": -0.018764495849609375, + "fcm_dpo/margin": 76.1546401977539, + "fcm_dpo/q_t": 0.4014281630516052, + "grad_norm": 8.76894474029541, + "learning_rate": 3.1594350522787295e-07, + "logits/chosen": -1.72676420211792, + "logits/rejected": -1.0805652141571045, + "logps/chosen": -230.01840209960938, + "logps/ref_chosen": -95.72935485839844, + "logps/ref_rejected": -94.16924285888672, + "logps/rejected": -304.6129150390625, + "loss": 1.0603, + "margin_dpo/margin_mean": 76.1546401977539, + "margin_dpo/margin_std": 82.4627685546875, + "step": 324 + }, + { + "KL/chosen_KL_mean": -123.01898193359375, + "KL/mean": -164.42840576171875, + "KL/rejected_KL_mean": -205.83782958984375, + "KL/std": 83.34144592285156, + "epoch": 0.47723935389133626, + "fcm_dpo/beta": 0.00543711893260479, + "fcm_dpo/delta": -0.05263667553663254, + "fcm_dpo/margin": 82.81886291503906, + "fcm_dpo/q_t": 0.394151896238327, + "grad_norm": 7.848616123199463, + "learning_rate": 3.147047612756302e-07, + "logits/chosen": -1.790490746498108, + "logits/rejected": -1.0584759712219238, + "logps/chosen": -221.72584533691406, + "logps/ref_chosen": -98.70686340332031, + "logps/ref_rejected": -76.40809631347656, + "logps/rejected": -282.2459411621094, + "loss": 1.0391, + "margin_dpo/margin_mean": 82.81885528564453, + "margin_dpo/margin_std": 86.77519226074219, + "step": 325 + }, + { + "KL/chosen_KL_mean": -137.7166748046875, + "KL/mean": -174.91812133789062, + "KL/rejected_KL_mean": -212.11956787109375, + "KL/std": 83.25706481933594, + "epoch": 0.4787077826725404, + "fcm_dpo/beta": 0.005412403494119644, + "fcm_dpo/delta": -0.0028189420700073242, + "fcm_dpo/margin": 74.40288543701172, + "fcm_dpo/q_t": 0.4054105281829834, + "grad_norm": 7.6784586906433105, + "learning_rate": 3.134643122927519e-07, + "logits/chosen": -2.5124287605285645, + "logits/rejected": -1.8499565124511719, + "logps/chosen": -242.37799072265625, + "logps/ref_chosen": -104.66130828857422, + "logps/ref_rejected": -86.82537841796875, + "logps/rejected": -298.9449462890625, + "loss": 1.08, + "margin_dpo/margin_mean": 74.40289306640625, + "margin_dpo/margin_std": 89.81488037109375, + "step": 326 + }, + { + "KL/chosen_KL_mean": -132.90896606445312, + "KL/mean": -168.94619750976562, + "KL/rejected_KL_mean": -204.98342895507812, + "KL/std": 76.07303619384766, + "epoch": 0.4801762114537445, + "fcm_dpo/beta": 0.005408043973147869, + "fcm_dpo/delta": 0.01050557941198349, + "fcm_dpo/margin": 72.07447814941406, + "fcm_dpo/q_t": 0.40942442417144775, + "grad_norm": 8.35208797454834, + "learning_rate": 3.1222219096622264e-07, + "logits/chosen": -1.8414480686187744, + "logits/rejected": -1.2603416442871094, + "logps/chosen": -240.3491973876953, + "logps/ref_chosen": -107.44023895263672, + "logps/ref_rejected": -117.05323028564453, + "logps/rejected": -322.03668212890625, + "loss": 1.085, + "margin_dpo/margin_mean": 72.0744857788086, + "margin_dpo/margin_std": 86.0423583984375, + "step": 327 + }, + { + "KL/chosen_KL_mean": -129.3306884765625, + "KL/mean": -176.6916046142578, + "KL/rejected_KL_mean": -224.05252075195312, + "KL/std": 90.3766098022461, + "epoch": 0.48164464023494863, + "fcm_dpo/beta": 0.0053399354219436646, + "fcm_dpo/delta": -0.11138816177845001, + "fcm_dpo/margin": 94.72184753417969, + "fcm_dpo/q_t": 0.3810993432998657, + "grad_norm": 9.81959342956543, + "learning_rate": 3.1097843002709427e-07, + "logits/chosen": -1.6090867519378662, + "logits/rejected": -1.307708740234375, + "logps/chosen": -215.94137573242188, + "logps/ref_chosen": -86.6106948852539, + "logps/ref_rejected": -97.40188598632812, + "logps/rejected": -321.45440673828125, + "loss": 0.9933, + "margin_dpo/margin_mean": 94.72184753417969, + "margin_dpo/margin_std": 86.73387145996094, + "step": 328 + }, + { + "KL/chosen_KL_mean": -135.27142333984375, + "KL/mean": -172.5872802734375, + "KL/rejected_KL_mean": -209.9031524658203, + "KL/std": 87.32764434814453, + "epoch": 0.4831130690161527, + "fcm_dpo/beta": 0.005290657747536898, + "fcm_dpo/delta": 0.0050959512591362, + "fcm_dpo/margin": 74.63172912597656, + "fcm_dpo/q_t": 0.40841954946517944, + "grad_norm": 9.771174430847168, + "learning_rate": 3.0973306224962437e-07, + "logits/chosen": -1.3904471397399902, + "logits/rejected": -0.8803360462188721, + "logps/chosen": -235.88450622558594, + "logps/ref_chosen": -100.61308288574219, + "logps/ref_rejected": -104.95370483398438, + "logps/rejected": -314.85687255859375, + "loss": 1.0881, + "margin_dpo/margin_mean": 74.63172912597656, + "margin_dpo/margin_std": 93.00950622558594, + "step": 329 + }, + { + "KL/chosen_KL_mean": -114.90520477294922, + "KL/mean": -154.475830078125, + "KL/rejected_KL_mean": -194.0464630126953, + "KL/std": 84.89775848388672, + "epoch": 0.4845814977973568, + "fcm_dpo/beta": 0.005290227476507425, + "fcm_dpo/delta": -0.01954089291393757, + "fcm_dpo/margin": 79.14125061035156, + "fcm_dpo/q_t": 0.3999236524105072, + "grad_norm": 8.447620391845703, + "learning_rate": 3.084861204504122e-07, + "logits/chosen": -1.7703384160995483, + "logits/rejected": -1.6913669109344482, + "logps/chosen": -191.8636474609375, + "logps/ref_chosen": -76.95843505859375, + "logps/ref_rejected": -92.81494140625, + "logps/rejected": -286.86138916015625, + "loss": 1.0504, + "margin_dpo/margin_mean": 79.14125061035156, + "margin_dpo/margin_std": 75.37956237792969, + "step": 330 + }, + { + "KL/chosen_KL_mean": -124.10118103027344, + "KL/mean": -168.7129669189453, + "KL/rejected_KL_mean": -213.32473754882812, + "KL/std": 83.27133178710938, + "epoch": 0.48604992657856094, + "fcm_dpo/beta": 0.005252513103187084, + "fcm_dpo/delta": -0.07201417535543442, + "fcm_dpo/margin": 89.22357940673828, + "fcm_dpo/q_t": 0.3895244896411896, + "grad_norm": 9.674551010131836, + "learning_rate": 3.072376374875335e-07, + "logits/chosen": -1.8069652318954468, + "logits/rejected": -1.5011857748031616, + "logps/chosen": -207.1605987548828, + "logps/ref_chosen": -83.05941772460938, + "logps/ref_rejected": -94.13113403320312, + "logps/rejected": -307.45587158203125, + "loss": 1.0183, + "margin_dpo/margin_mean": 89.22357940673828, + "margin_dpo/margin_std": 83.74002075195312, + "step": 331 + }, + { + "KL/chosen_KL_mean": -132.55140686035156, + "KL/mean": -163.17306518554688, + "KL/rejected_KL_mean": -193.79473876953125, + "KL/std": 91.64155578613281, + "epoch": 0.48751835535976507, + "fcm_dpo/beta": 0.00528405886143446, + "fcm_dpo/delta": 0.07880137115716934, + "fcm_dpo/margin": 61.24333953857422, + "fcm_dpo/q_t": 0.4242613911628723, + "grad_norm": 7.782529354095459, + "learning_rate": 3.059876462596758e-07, + "logits/chosen": -1.995274543762207, + "logits/rejected": -1.5595709085464478, + "logps/chosen": -214.4822998046875, + "logps/ref_chosen": -81.93089294433594, + "logps/ref_rejected": -82.84590148925781, + "logps/rejected": -276.640625, + "loss": 1.1426, + "margin_dpo/margin_mean": 61.24333953857422, + "margin_dpo/margin_std": 89.94986724853516, + "step": 332 + }, + { + "KL/chosen_KL_mean": -153.6798095703125, + "KL/mean": -196.93731689453125, + "KL/rejected_KL_mean": -240.19480895996094, + "KL/std": 96.05758666992188, + "epoch": 0.4889867841409692, + "fcm_dpo/beta": 0.0052257683128118515, + "fcm_dpo/delta": -0.055502861738204956, + "fcm_dpo/margin": 86.51498413085938, + "fcm_dpo/q_t": 0.3956648111343384, + "grad_norm": 8.08462142944336, + "learning_rate": 3.0473617970527015e-07, + "logits/chosen": -1.3471949100494385, + "logits/rejected": -0.9994245767593384, + "logps/chosen": -242.21240234375, + "logps/ref_chosen": -88.53257751464844, + "logps/ref_rejected": -102.46601867675781, + "logps/rejected": -342.66082763671875, + "loss": 1.055, + "margin_dpo/margin_mean": 86.51498413085938, + "margin_dpo/margin_std": 103.09303283691406, + "step": 333 + }, + { + "KL/chosen_KL_mean": -139.8485107421875, + "KL/mean": -177.95394897460938, + "KL/rejected_KL_mean": -216.0594024658203, + "KL/std": 90.49869537353516, + "epoch": 0.49045521292217326, + "fcm_dpo/beta": 0.005182007793337107, + "fcm_dpo/delta": -0.055397916585206985, + "fcm_dpo/margin": 76.21089935302734, + "fcm_dpo/q_t": 0.40680527687072754, + "grad_norm": 8.630681037902832, + "learning_rate": 3.034832708016243e-07, + "logits/chosen": -1.450178861618042, + "logits/rejected": -1.1560436487197876, + "logps/chosen": -237.60629272460938, + "logps/ref_chosen": -97.75778198242188, + "logps/ref_rejected": -100.0058364868164, + "logps/rejected": -316.06524658203125, + "loss": 1.0976, + "margin_dpo/margin_mean": 76.21089935302734, + "margin_dpo/margin_std": 100.74238586425781, + "step": 334 + }, + { + "KL/chosen_KL_mean": -138.5660400390625, + "KL/mean": -171.50950622558594, + "KL/rejected_KL_mean": -204.45297241210938, + "KL/std": 90.08235168457031, + "epoch": 0.4919236417033774, + "fcm_dpo/beta": 0.0052277748472988605, + "fcm_dpo/delta": 0.057526759803295135, + "fcm_dpo/margin": 65.88694763183594, + "fcm_dpo/q_t": 0.4200003743171692, + "grad_norm": 12.09694766998291, + "learning_rate": 3.022289525640531e-07, + "logits/chosen": -2.28955078125, + "logits/rejected": -1.8641899824142456, + "logps/chosen": -234.94207763671875, + "logps/ref_chosen": -96.37603759765625, + "logps/ref_rejected": -94.92401123046875, + "logps/rejected": -299.3769836425781, + "loss": 1.1348, + "margin_dpo/margin_mean": 65.8869400024414, + "margin_dpo/margin_std": 98.77403259277344, + "step": 335 + }, + { + "KL/chosen_KL_mean": -142.24081420898438, + "KL/mean": -184.5833282470703, + "KL/rejected_KL_mean": -226.9258270263672, + "KL/std": 92.70210266113281, + "epoch": 0.4933920704845815, + "fcm_dpo/beta": 0.005219377111643553, + "fcm_dpo/delta": -0.043948303908109665, + "fcm_dpo/margin": 84.68500518798828, + "fcm_dpo/q_t": 0.3972393870353699, + "grad_norm": 10.240348815917969, + "learning_rate": 3.009732580450086e-07, + "logits/chosen": -1.8144869804382324, + "logits/rejected": -1.6664538383483887, + "logps/chosen": -230.77035522460938, + "logps/ref_chosen": -88.52952575683594, + "logps/ref_rejected": -109.5557861328125, + "logps/rejected": -336.48162841796875, + "loss": 1.0528, + "margin_dpo/margin_mean": 84.68501281738281, + "margin_dpo/margin_std": 98.26302337646484, + "step": 336 + }, + { + "KL/chosen_KL_mean": -139.7945098876953, + "KL/mean": -180.14407348632812, + "KL/rejected_KL_mean": -220.4936065673828, + "KL/std": 93.29644775390625, + "epoch": 0.4948604992657856, + "fcm_dpo/beta": 0.005189026705920696, + "fcm_dpo/delta": -0.019588202238082886, + "fcm_dpo/margin": 80.6991195678711, + "fcm_dpo/q_t": 0.4038439095020294, + "grad_norm": 8.789505004882812, + "learning_rate": 2.9971622033320914e-07, + "logits/chosen": -2.2602341175079346, + "logits/rejected": -1.9733288288116455, + "logps/chosen": -237.10690307617188, + "logps/ref_chosen": -97.3123779296875, + "logps/ref_rejected": -110.13134765625, + "logps/rejected": -330.62493896484375, + "loss": 1.0863, + "margin_dpo/margin_mean": 80.69911193847656, + "margin_dpo/margin_std": 109.38632202148438, + "step": 337 + }, + { + "KL/chosen_KL_mean": -145.0054473876953, + "KL/mean": -181.9836883544922, + "KL/rejected_KL_mean": -218.96194458007812, + "KL/std": 94.61917114257812, + "epoch": 0.49632892804698975, + "fcm_dpo/beta": 0.005185029469430447, + "fcm_dpo/delta": 0.01719367876648903, + "fcm_dpo/margin": 73.95649719238281, + "fcm_dpo/q_t": 0.40975597500801086, + "grad_norm": 8.891013145446777, + "learning_rate": 2.984578725527675e-07, + "logits/chosen": -1.9223698377609253, + "logits/rejected": -1.7721664905548096, + "logps/chosen": -228.63555908203125, + "logps/ref_chosen": -83.63011169433594, + "logps/ref_rejected": -93.69235229492188, + "logps/rejected": -312.654296875, + "loss": 1.0946, + "margin_dpo/margin_mean": 73.95649719238281, + "margin_dpo/margin_std": 92.86002349853516, + "step": 338 + }, + { + "KL/chosen_KL_mean": -154.381591796875, + "KL/mean": -193.88868713378906, + "KL/rejected_KL_mean": -233.39578247070312, + "KL/std": 93.60277557373047, + "epoch": 0.4977973568281938, + "fcm_dpo/beta": 0.005157374776899815, + "fcm_dpo/delta": -0.07026338577270508, + "fcm_dpo/margin": 79.01419830322266, + "fcm_dpo/q_t": 0.40499579906463623, + "grad_norm": 7.875832557678223, + "learning_rate": 2.9719824786231796e-07, + "logits/chosen": -2.87162709236145, + "logits/rejected": -2.442060708999634, + "logps/chosen": -254.2333984375, + "logps/ref_chosen": -99.85182189941406, + "logps/ref_rejected": -109.0743637084961, + "logps/rejected": -342.47015380859375, + "loss": 1.0918, + "margin_dpo/margin_mean": 79.01419830322266, + "margin_dpo/margin_std": 104.06044006347656, + "step": 339 + }, + { + "KL/chosen_KL_mean": -166.36679077148438, + "KL/mean": -200.3663787841797, + "KL/rejected_KL_mean": -234.365966796875, + "KL/std": 85.84724426269531, + "epoch": 0.49926578560939794, + "fcm_dpo/beta": 0.0051616826094686985, + "fcm_dpo/delta": 0.05077475309371948, + "fcm_dpo/margin": 67.99917602539062, + "fcm_dpo/q_t": 0.4177219569683075, + "grad_norm": 9.743352890014648, + "learning_rate": 2.959373794541426e-07, + "logits/chosen": -1.3466472625732422, + "logits/rejected": -0.8567553758621216, + "logps/chosen": -251.34075927734375, + "logps/ref_chosen": -84.97396850585938, + "logps/ref_rejected": -78.80033874511719, + "logps/rejected": -313.16632080078125, + "loss": 1.1216, + "margin_dpo/margin_mean": 67.99917602539062, + "margin_dpo/margin_std": 94.52733612060547, + "step": 340 + }, + { + "KL/chosen_KL_mean": -173.53192138671875, + "KL/mean": -206.74200439453125, + "KL/rejected_KL_mean": -239.95208740234375, + "KL/std": 89.88375854492188, + "epoch": 0.5007342143906021, + "fcm_dpo/beta": 0.005205621011555195, + "fcm_dpo/delta": 0.05621008947491646, + "fcm_dpo/margin": 66.42013549804688, + "fcm_dpo/q_t": 0.4203876256942749, + "grad_norm": 9.62618637084961, + "learning_rate": 2.946753005532965e-07, + "logits/chosen": -1.385262131690979, + "logits/rejected": -1.1342041492462158, + "logps/chosen": -258.7958984375, + "logps/ref_chosen": -85.26399230957031, + "logps/ref_rejected": -106.08595275878906, + "logps/rejected": -346.03802490234375, + "loss": 1.1403, + "margin_dpo/margin_mean": 66.4201431274414, + "margin_dpo/margin_std": 103.22789001464844, + "step": 341 + }, + { + "KL/chosen_KL_mean": -138.84603881835938, + "KL/mean": -187.59945678710938, + "KL/rejected_KL_mean": -236.3529052734375, + "KL/std": 90.45417022705078, + "epoch": 0.5022026431718062, + "fcm_dpo/beta": 0.0051222434267401695, + "fcm_dpo/delta": -0.10586446523666382, + "fcm_dpo/margin": 97.50687408447266, + "fcm_dpo/q_t": 0.3817700743675232, + "grad_norm": 7.682787895202637, + "learning_rate": 2.934120444167326e-07, + "logits/chosen": -1.290515661239624, + "logits/rejected": -0.5167274475097656, + "logps/chosen": -239.88462829589844, + "logps/ref_chosen": -101.03860473632812, + "logps/ref_rejected": -98.89851379394531, + "logps/rejected": -335.25140380859375, + "loss": 1.0104, + "margin_dpo/margin_mean": 97.50688171386719, + "margin_dpo/margin_std": 96.39700317382812, + "step": 342 + }, + { + "KL/chosen_KL_mean": -151.55612182617188, + "KL/mean": -191.76034545898438, + "KL/rejected_KL_mean": -231.9645538330078, + "KL/std": 94.27001953125, + "epoch": 0.5036710719530103, + "fcm_dpo/beta": 0.005087699741125107, + "fcm_dpo/delta": -0.06798040866851807, + "fcm_dpo/margin": 80.40845489501953, + "fcm_dpo/q_t": 0.404390424489975, + "grad_norm": 9.761009216308594, + "learning_rate": 2.9214764433242476e-07, + "logits/chosen": -2.1991000175476074, + "logits/rejected": -2.0898547172546387, + "logps/chosen": -235.30348205566406, + "logps/ref_chosen": -83.74736022949219, + "logps/ref_rejected": -116.57908630371094, + "logps/rejected": -348.54364013671875, + "loss": 1.0825, + "margin_dpo/margin_mean": 80.408447265625, + "margin_dpo/margin_std": 98.9737548828125, + "step": 343 + }, + { + "KL/chosen_KL_mean": -150.92111206054688, + "KL/mean": -193.98861694335938, + "KL/rejected_KL_mean": -237.05615234375, + "KL/std": 99.92767333984375, + "epoch": 0.5051395007342144, + "fcm_dpo/beta": 0.005051136948168278, + "fcm_dpo/delta": -0.03701796382665634, + "fcm_dpo/margin": 86.13502502441406, + "fcm_dpo/q_t": 0.39953723549842834, + "grad_norm": 9.656281471252441, + "learning_rate": 2.9088213361849126e-07, + "logits/chosen": -1.4665610790252686, + "logits/rejected": -1.212233304977417, + "logps/chosen": -229.17507934570312, + "logps/ref_chosen": -78.25396728515625, + "logps/ref_rejected": -95.42870330810547, + "logps/rejected": -332.48486328125, + "loss": 1.0674, + "margin_dpo/margin_mean": 86.13502502441406, + "margin_dpo/margin_std": 105.87095642089844, + "step": 344 + }, + { + "KL/chosen_KL_mean": -155.3314208984375, + "KL/mean": -200.91249084472656, + "KL/rejected_KL_mean": -246.49356079101562, + "KL/std": 100.1898193359375, + "epoch": 0.5066079295154186, + "fcm_dpo/beta": 0.004966835491359234, + "fcm_dpo/delta": -0.055428341031074524, + "fcm_dpo/margin": 91.16216278076172, + "fcm_dpo/q_t": 0.39493459463119507, + "grad_norm": 9.278047561645508, + "learning_rate": 2.896155456223163e-07, + "logits/chosen": -1.5918264389038086, + "logits/rejected": -1.2987147569656372, + "logps/chosen": -253.41090393066406, + "logps/ref_chosen": -98.0794906616211, + "logps/ref_rejected": -104.03477478027344, + "logps/rejected": -350.5283203125, + "loss": 1.0457, + "margin_dpo/margin_mean": 91.16215515136719, + "margin_dpo/margin_std": 103.647216796875, + "step": 345 + }, + { + "KL/chosen_KL_mean": -172.59291076660156, + "KL/mean": -207.95062255859375, + "KL/rejected_KL_mean": -243.30833435058594, + "KL/std": 93.20742797851562, + "epoch": 0.5080763582966226, + "fcm_dpo/beta": 0.004995340947061777, + "fcm_dpo/delta": 0.04847484827041626, + "fcm_dpo/margin": 70.71542358398438, + "fcm_dpo/q_t": 0.4186241626739502, + "grad_norm": 7.907067775726318, + "learning_rate": 2.883479137196714e-07, + "logits/chosen": -1.9634160995483398, + "logits/rejected": -1.6597533226013184, + "logps/chosen": -253.27639770507812, + "logps/ref_chosen": -80.68348693847656, + "logps/ref_rejected": -85.89260864257812, + "logps/rejected": -329.200927734375, + "loss": 1.1313, + "margin_dpo/margin_mean": 70.71542358398438, + "margin_dpo/margin_std": 107.04525756835938, + "step": 346 + }, + { + "KL/chosen_KL_mean": -163.02511596679688, + "KL/mean": -196.81716918945312, + "KL/rejected_KL_mean": -230.60923767089844, + "KL/std": 92.2974853515625, + "epoch": 0.5095447870778267, + "fcm_dpo/beta": 0.005008503794670105, + "fcm_dpo/delta": -0.0007321417797356844, + "fcm_dpo/margin": 67.58413696289062, + "fcm_dpo/q_t": 0.42133665084838867, + "grad_norm": 8.228876113891602, + "learning_rate": 2.8707927131383614e-07, + "logits/chosen": -1.9228236675262451, + "logits/rejected": -1.6043643951416016, + "logps/chosen": -256.7044677734375, + "logps/ref_chosen": -93.67938232421875, + "logps/ref_rejected": -97.54251098632812, + "logps/rejected": -328.1517333984375, + "loss": 1.1363, + "margin_dpo/margin_mean": 67.58413696289062, + "margin_dpo/margin_std": 99.87893676757812, + "step": 347 + }, + { + "KL/chosen_KL_mean": -140.70156860351562, + "KL/mean": -179.17735290527344, + "KL/rejected_KL_mean": -217.65313720703125, + "KL/std": 86.80039978027344, + "epoch": 0.5110132158590308, + "fcm_dpo/beta": 0.004934805445373058, + "fcm_dpo/delta": -0.07375401258468628, + "fcm_dpo/margin": 76.95158386230469, + "fcm_dpo/q_t": 0.40964022278785706, + "grad_norm": 7.939823150634766, + "learning_rate": 2.858096518347179e-07, + "logits/chosen": -1.9222362041473389, + "logits/rejected": -1.6479917764663696, + "logps/chosen": -230.95379638671875, + "logps/ref_chosen": -90.25224304199219, + "logps/ref_rejected": -95.77027893066406, + "logps/rejected": -313.42340087890625, + "loss": 1.0937, + "margin_dpo/margin_mean": 76.95158386230469, + "margin_dpo/margin_std": 88.59011840820312, + "step": 348 + }, + { + "KL/chosen_KL_mean": -152.14627075195312, + "KL/mean": -187.83560180664062, + "KL/rejected_KL_mean": -223.52493286132812, + "KL/std": 95.50184631347656, + "epoch": 0.5124816446402349, + "fcm_dpo/beta": 0.004928614944219589, + "fcm_dpo/delta": -0.012560423463582993, + "fcm_dpo/margin": 71.37866973876953, + "fcm_dpo/q_t": 0.4180968403816223, + "grad_norm": 9.119296073913574, + "learning_rate": 2.845390887379706e-07, + "logits/chosen": -2.0517802238464355, + "logits/rejected": -1.826784372329712, + "logps/chosen": -244.20436096191406, + "logps/ref_chosen": -92.05809020996094, + "logps/ref_rejected": -103.96527099609375, + "logps/rejected": -327.4902038574219, + "loss": 1.1233, + "margin_dpo/margin_mean": 71.37867736816406, + "margin_dpo/margin_std": 99.10256958007812, + "step": 349 + }, + { + "KL/chosen_KL_mean": -160.86972045898438, + "KL/mean": -191.12863159179688, + "KL/rejected_KL_mean": -221.38754272460938, + "KL/std": 87.87226867675781, + "epoch": 0.5139500734214391, + "fcm_dpo/beta": 0.0049631791189312935, + "fcm_dpo/delta": 0.041226498782634735, + "fcm_dpo/margin": 60.51781463623047, + "fcm_dpo/q_t": 0.4296115040779114, + "grad_norm": 10.321072578430176, + "learning_rate": 2.8326761550411346e-07, + "logits/chosen": -1.853127121925354, + "logits/rejected": -1.6269001960754395, + "logps/chosen": -247.92391967773438, + "logps/ref_chosen": -87.05419921875, + "logps/ref_rejected": -95.72193145751953, + "logps/rejected": -317.1094665527344, + "loss": 1.1631, + "margin_dpo/margin_mean": 60.51781463623047, + "margin_dpo/margin_std": 95.85125732421875, + "step": 350 + }, + { + "KL/chosen_KL_mean": -160.72219848632812, + "KL/mean": -203.95358276367188, + "KL/rejected_KL_mean": -247.18499755859375, + "KL/std": 98.07379913330078, + "epoch": 0.5154185022026432, + "fcm_dpo/beta": 0.004950874485075474, + "fcm_dpo/delta": -0.02933877892792225, + "fcm_dpo/margin": 86.46279907226562, + "fcm_dpo/q_t": 0.40006253123283386, + "grad_norm": 8.126477241516113, + "learning_rate": 2.819952656376487e-07, + "logits/chosen": -1.7150453329086304, + "logits/rejected": -1.4142982959747314, + "logps/chosen": -249.8912353515625, + "logps/ref_chosen": -89.16903686523438, + "logps/ref_rejected": -108.29801940917969, + "logps/rejected": -355.4830322265625, + "loss": 1.0666, + "margin_dpo/margin_mean": 86.46279907226562, + "margin_dpo/margin_std": 104.249755859375, + "step": 351 + }, + { + "KL/chosen_KL_mean": -147.03170776367188, + "KL/mean": -187.869140625, + "KL/rejected_KL_mean": -228.70657348632812, + "KL/std": 96.88475036621094, + "epoch": 0.5168869309838473, + "fcm_dpo/beta": 0.004932219162583351, + "fcm_dpo/delta": -0.0029572767671197653, + "fcm_dpo/margin": 81.67485046386719, + "fcm_dpo/q_t": 0.4076157212257385, + "grad_norm": 8.16756820678711, + "learning_rate": 2.8072207266617854e-07, + "logits/chosen": -2.105515480041504, + "logits/rejected": -1.5747778415679932, + "logps/chosen": -243.84320068359375, + "logps/ref_chosen": -96.81147766113281, + "logps/ref_rejected": -84.40483093261719, + "logps/rejected": -313.11138916015625, + "loss": 1.0876, + "margin_dpo/margin_mean": 81.67485046386719, + "margin_dpo/margin_std": 104.96588897705078, + "step": 352 + }, + { + "KL/chosen_KL_mean": -168.7980194091797, + "KL/mean": -195.0948944091797, + "KL/rejected_KL_mean": -221.39175415039062, + "KL/std": 85.62964630126953, + "epoch": 0.5183553597650514, + "fcm_dpo/beta": 0.004969088826328516, + "fcm_dpo/delta": 0.03824581205844879, + "fcm_dpo/margin": 52.59373092651367, + "fcm_dpo/q_t": 0.43823564052581787, + "grad_norm": 9.168176651000977, + "learning_rate": 2.794480701395219e-07, + "logits/chosen": -2.0197463035583496, + "logits/rejected": -1.6257328987121582, + "logps/chosen": -254.16050720214844, + "logps/ref_chosen": -85.36248779296875, + "logps/ref_rejected": -87.36732482910156, + "logps/rejected": -308.75909423828125, + "loss": 1.1997, + "margin_dpo/margin_mean": 52.59373092651367, + "margin_dpo/margin_std": 96.77818298339844, + "step": 353 + }, + { + "KL/chosen_KL_mean": -147.91342163085938, + "KL/mean": -188.85592651367188, + "KL/rejected_KL_mean": -229.7984619140625, + "KL/std": 86.54969787597656, + "epoch": 0.5198237885462555, + "fcm_dpo/beta": 0.004973916336894035, + "fcm_dpo/delta": -0.00767357274889946, + "fcm_dpo/margin": 81.8850326538086, + "fcm_dpo/q_t": 0.4037133455276489, + "grad_norm": 7.6839118003845215, + "learning_rate": 2.781732916288303e-07, + "logits/chosen": -1.8105194568634033, + "logits/rejected": -1.3358511924743652, + "logps/chosen": -243.71458435058594, + "logps/ref_chosen": -95.8011703491211, + "logps/ref_rejected": -94.86614227294922, + "logps/rejected": -324.66461181640625, + "loss": 1.058, + "margin_dpo/margin_mean": 81.8850326538086, + "margin_dpo/margin_std": 80.45028686523438, + "step": 354 + }, + { + "KL/chosen_KL_mean": -146.56759643554688, + "KL/mean": -184.26931762695312, + "KL/rejected_KL_mean": -221.97100830078125, + "KL/std": 88.38886260986328, + "epoch": 0.5212922173274597, + "fcm_dpo/beta": 0.004964661784470081, + "fcm_dpo/delta": 0.026508353650569916, + "fcm_dpo/margin": 75.40342712402344, + "fcm_dpo/q_t": 0.4120803773403168, + "grad_norm": 9.448277473449707, + "learning_rate": 2.7689777072570284e-07, + "logits/chosen": -2.308893918991089, + "logits/rejected": -1.8136093616485596, + "logps/chosen": -232.75314331054688, + "logps/ref_chosen": -86.185546875, + "logps/ref_rejected": -88.4615707397461, + "logps/rejected": -310.4325866699219, + "loss": 1.1017, + "margin_dpo/margin_mean": 75.40342712402344, + "margin_dpo/margin_std": 94.25363159179688, + "step": 355 + }, + { + "KL/chosen_KL_mean": -172.5458984375, + "KL/mean": -202.02325439453125, + "KL/rejected_KL_mean": -231.50059509277344, + "KL/std": 86.2015609741211, + "epoch": 0.5227606461086637, + "fcm_dpo/beta": 0.005002472549676895, + "fcm_dpo/delta": 0.029246121644973755, + "fcm_dpo/margin": 58.95470428466797, + "fcm_dpo/q_t": 0.43088269233703613, + "grad_norm": 8.20483684539795, + "learning_rate": 2.7562154104130176e-07, + "logits/chosen": -1.788015604019165, + "logits/rejected": -1.3173956871032715, + "logps/chosen": -263.364013671875, + "logps/ref_chosen": -90.818115234375, + "logps/ref_rejected": -82.1273193359375, + "logps/rejected": -313.6279296875, + "loss": 1.1721, + "margin_dpo/margin_mean": 58.95470428466797, + "margin_dpo/margin_std": 98.95693969726562, + "step": 356 + }, + { + "KL/chosen_KL_mean": -145.1062774658203, + "KL/mean": -191.907958984375, + "KL/rejected_KL_mean": -238.70965576171875, + "KL/std": 94.88236999511719, + "epoch": 0.5242290748898678, + "fcm_dpo/beta": 0.004938648547977209, + "fcm_dpo/delta": -0.06639204919338226, + "fcm_dpo/margin": 93.60336303710938, + "fcm_dpo/q_t": 0.3908548951148987, + "grad_norm": 7.912759780883789, + "learning_rate": 2.7434463620546594e-07, + "logits/chosen": -1.7317296266555786, + "logits/rejected": -1.3505488634109497, + "logps/chosen": -232.61102294921875, + "logps/ref_chosen": -87.5047607421875, + "logps/ref_rejected": -92.24937438964844, + "logps/rejected": -330.9590148925781, + "loss": 1.0244, + "margin_dpo/margin_mean": 93.60336303710938, + "margin_dpo/margin_std": 88.33078002929688, + "step": 357 + }, + { + "KL/chosen_KL_mean": -155.79161071777344, + "KL/mean": -190.37872314453125, + "KL/rejected_KL_mean": -224.96585083007812, + "KL/std": 87.9743423461914, + "epoch": 0.5256975036710719, + "fcm_dpo/beta": 0.004991848021745682, + "fcm_dpo/delta": 0.05666042119264603, + "fcm_dpo/margin": 69.17424011230469, + "fcm_dpo/q_t": 0.42042723298072815, + "grad_norm": 8.212221145629883, + "learning_rate": 2.730670898658255e-07, + "logits/chosen": -1.6254724264144897, + "logits/rejected": -1.1543527841567993, + "logps/chosen": -250.84707641601562, + "logps/ref_chosen": -95.05546569824219, + "logps/ref_rejected": -96.47611999511719, + "logps/rejected": -321.44195556640625, + "loss": 1.1222, + "margin_dpo/margin_mean": 69.17424011230469, + "margin_dpo/margin_std": 94.25694274902344, + "step": 358 + }, + { + "KL/chosen_KL_mean": -175.09396362304688, + "KL/mean": -207.33108520507812, + "KL/rejected_KL_mean": -239.56820678710938, + "KL/std": 82.54708862304688, + "epoch": 0.527165932452276, + "fcm_dpo/beta": 0.004995310213416815, + "fcm_dpo/delta": -0.012033559381961823, + "fcm_dpo/margin": 64.47423553466797, + "fcm_dpo/q_t": 0.4248253107070923, + "grad_norm": 9.227970123291016, + "learning_rate": 2.717889356869146e-07, + "logits/chosen": -1.5680854320526123, + "logits/rejected": -0.9995710849761963, + "logps/chosen": -262.46728515625, + "logps/ref_chosen": -87.37332916259766, + "logps/ref_rejected": -88.77188110351562, + "logps/rejected": -328.340087890625, + "loss": 1.1479, + "margin_dpo/margin_mean": 64.47423553466797, + "margin_dpo/margin_std": 96.67312622070312, + "step": 359 + }, + { + "KL/chosen_KL_mean": -159.5872802734375, + "KL/mean": -200.59890747070312, + "KL/rejected_KL_mean": -241.6105499267578, + "KL/std": 83.31660461425781, + "epoch": 0.5286343612334802, + "fcm_dpo/beta": 0.004984111059457064, + "fcm_dpo/delta": -0.009212283417582512, + "fcm_dpo/margin": 82.02326965332031, + "fcm_dpo/q_t": 0.4031431972980499, + "grad_norm": 13.064961433410645, + "learning_rate": 2.7051020734928443e-07, + "logits/chosen": -1.4418290853500366, + "logits/rejected": -1.0884792804718018, + "logps/chosen": -234.81072998046875, + "logps/ref_chosen": -75.22344970703125, + "logps/ref_rejected": -76.09400177001953, + "logps/rejected": -317.70452880859375, + "loss": 1.0648, + "margin_dpo/margin_mean": 82.02326202392578, + "margin_dpo/margin_std": 88.16264343261719, + "step": 360 + }, + { + "KL/chosen_KL_mean": -162.84571838378906, + "KL/mean": -199.18038940429688, + "KL/rejected_KL_mean": -235.51507568359375, + "KL/std": 95.06013488769531, + "epoch": 0.5301027900146843, + "fcm_dpo/beta": 0.005016217939555645, + "fcm_dpo/delta": 0.0367913693189621, + "fcm_dpo/margin": 72.66934967041016, + "fcm_dpo/q_t": 0.4160331189632416, + "grad_norm": 9.651906967163086, + "learning_rate": 2.6923093854861593e-07, + "logits/chosen": -1.5508899688720703, + "logits/rejected": -1.30221688747406, + "logps/chosen": -244.79904174804688, + "logps/ref_chosen": -81.95333099365234, + "logps/ref_rejected": -96.67919921875, + "logps/rejected": -332.19427490234375, + "loss": 1.1157, + "margin_dpo/margin_mean": 72.66934204101562, + "margin_dpo/margin_std": 101.4786376953125, + "step": 361 + }, + { + "KL/chosen_KL_mean": -167.842041015625, + "KL/mean": -206.40011596679688, + "KL/rejected_KL_mean": -244.95822143554688, + "KL/std": 89.92999267578125, + "epoch": 0.5315712187958884, + "fcm_dpo/beta": 0.005022239871323109, + "fcm_dpo/delta": 0.013153315521776676, + "fcm_dpo/margin": 77.11618041992188, + "fcm_dpo/q_t": 0.4091334342956543, + "grad_norm": 8.851387023925781, + "learning_rate": 2.679511629948319e-07, + "logits/chosen": -1.782833218574524, + "logits/rejected": -1.5787432193756104, + "logps/chosen": -257.1817626953125, + "logps/ref_chosen": -89.33971405029297, + "logps/ref_rejected": -110.50196838378906, + "logps/rejected": -355.460205078125, + "loss": 1.0929, + "margin_dpo/margin_mean": 77.11618041992188, + "margin_dpo/margin_std": 96.0059814453125, + "step": 362 + }, + { + "KL/chosen_KL_mean": -157.31036376953125, + "KL/mean": -206.67315673828125, + "KL/rejected_KL_mean": -256.0359191894531, + "KL/std": 94.60166931152344, + "epoch": 0.5330396475770925, + "fcm_dpo/beta": 0.004984427243471146, + "fcm_dpo/delta": -0.09679950773715973, + "fcm_dpo/margin": 98.72555541992188, + "fcm_dpo/q_t": 0.3860166668891907, + "grad_norm": 11.03801441192627, + "learning_rate": 2.6667091441120816e-07, + "logits/chosen": -1.9064676761627197, + "logits/rejected": -1.480465292930603, + "logps/chosen": -238.0853729248047, + "logps/ref_chosen": -80.77499389648438, + "logps/ref_rejected": -79.43267822265625, + "logps/rejected": -335.4685974121094, + "loss": 1.0159, + "margin_dpo/margin_mean": 98.72555541992188, + "margin_dpo/margin_std": 103.05851745605469, + "step": 363 + }, + { + "KL/chosen_KL_mean": -155.9902801513672, + "KL/mean": -195.59634399414062, + "KL/rejected_KL_mean": -235.202392578125, + "KL/std": 87.65835571289062, + "epoch": 0.5345080763582967, + "fcm_dpo/beta": 0.004912791773676872, + "fcm_dpo/delta": -0.05458861589431763, + "fcm_dpo/margin": 79.21211242675781, + "fcm_dpo/q_t": 0.40737155079841614, + "grad_norm": 11.701790809631348, + "learning_rate": 2.6539022653348575e-07, + "logits/chosen": -2.0065109729766846, + "logits/rejected": -1.864483118057251, + "logps/chosen": -232.51988220214844, + "logps/ref_chosen": -76.52960205078125, + "logps/ref_rejected": -97.56173706054688, + "logps/rejected": -332.7641296386719, + "loss": 1.0787, + "margin_dpo/margin_mean": 79.21211242675781, + "margin_dpo/margin_std": 83.88143920898438, + "step": 364 + }, + { + "KL/chosen_KL_mean": -150.7875213623047, + "KL/mean": -194.66790771484375, + "KL/rejected_KL_mean": -238.54830932617188, + "KL/std": 94.74967956542969, + "epoch": 0.5359765051395007, + "fcm_dpo/beta": 0.004881038796156645, + "fcm_dpo/delta": -0.029769405722618103, + "fcm_dpo/margin": 87.76078033447266, + "fcm_dpo/q_t": 0.4015900790691376, + "grad_norm": 8.28507137298584, + "learning_rate": 2.641091331089811e-07, + "logits/chosen": -1.6776100397109985, + "logits/rejected": -1.5518178939819336, + "logps/chosen": -231.33241271972656, + "logps/ref_chosen": -80.54489135742188, + "logps/ref_rejected": -99.32316589355469, + "logps/rejected": -337.8714599609375, + "loss": 1.0689, + "margin_dpo/margin_mean": 87.76078033447266, + "margin_dpo/margin_std": 108.048095703125, + "step": 365 + }, + { + "KL/chosen_KL_mean": -152.82333374023438, + "KL/mean": -188.84579467773438, + "KL/rejected_KL_mean": -224.86822509765625, + "KL/std": 98.29963684082031, + "epoch": 0.5374449339207048, + "fcm_dpo/beta": 0.004892690107226372, + "fcm_dpo/delta": 0.04924327880144119, + "fcm_dpo/margin": 72.04490661621094, + "fcm_dpo/q_t": 0.41934844851493835, + "grad_norm": 8.162010192871094, + "learning_rate": 2.6282766789569736e-07, + "logits/chosen": -1.9759223461151123, + "logits/rejected": -2.008493185043335, + "logps/chosen": -225.0614013671875, + "logps/ref_chosen": -72.23806762695312, + "logps/ref_rejected": -91.67182922363281, + "logps/rejected": -316.5400695800781, + "loss": 1.1159, + "margin_dpo/margin_mean": 72.04490661621094, + "margin_dpo/margin_std": 96.59625244140625, + "step": 366 + }, + { + "KL/chosen_KL_mean": -138.40423583984375, + "KL/mean": -181.97520446777344, + "KL/rejected_KL_mean": -225.54617309570312, + "KL/std": 86.47904968261719, + "epoch": 0.5389133627019089, + "fcm_dpo/beta": 0.0048882076516747475, + "fcm_dpo/delta": -0.02711433917284012, + "fcm_dpo/margin": 87.14193725585938, + "fcm_dpo/q_t": 0.39972636103630066, + "grad_norm": 7.999646186828613, + "learning_rate": 2.615458646614349e-07, + "logits/chosen": -1.9296679496765137, + "logits/rejected": -1.590559482574463, + "logps/chosen": -223.82623291015625, + "logps/ref_chosen": -85.42201232910156, + "logps/ref_rejected": -83.51779174804688, + "logps/rejected": -309.06396484375, + "loss": 1.0498, + "margin_dpo/margin_mean": 87.14193725585938, + "margin_dpo/margin_std": 89.44941711425781, + "step": 367 + }, + { + "KL/chosen_KL_mean": -129.32821655273438, + "KL/mean": -177.3518829345703, + "KL/rejected_KL_mean": -225.37554931640625, + "KL/std": 95.51463317871094, + "epoch": 0.540381791483113, + "fcm_dpo/beta": 0.00484818872064352, + "fcm_dpo/delta": -0.06891346722841263, + "fcm_dpo/margin": 96.0473403930664, + "fcm_dpo/q_t": 0.3902171850204468, + "grad_norm": 9.052199363708496, + "learning_rate": 2.6026375718290083e-07, + "logits/chosen": -1.3902904987335205, + "logits/rejected": -1.3585437536239624, + "logps/chosen": -203.16934204101562, + "logps/ref_chosen": -73.84112548828125, + "logps/ref_rejected": -104.69528198242188, + "logps/rejected": -330.0708312988281, + "loss": 1.0287, + "margin_dpo/margin_mean": 96.0473403930664, + "margin_dpo/margin_std": 96.81156921386719, + "step": 368 + }, + { + "KL/chosen_KL_mean": -143.9194793701172, + "KL/mean": -188.78811645507812, + "KL/rejected_KL_mean": -233.65673828125, + "KL/std": 88.20640563964844, + "epoch": 0.5418502202643172, + "fcm_dpo/beta": 0.0047968486323952675, + "fcm_dpo/delta": -0.03182859346270561, + "fcm_dpo/margin": 89.73724365234375, + "fcm_dpo/q_t": 0.39866721630096436, + "grad_norm": 8.820496559143066, + "learning_rate": 2.589813792448196e-07, + "logits/chosen": -1.7988767623901367, + "logits/rejected": -1.4377846717834473, + "logps/chosen": -242.38143920898438, + "logps/ref_chosen": -98.46195983886719, + "logps/ref_rejected": -99.15672302246094, + "logps/rejected": -332.8134765625, + "loss": 1.0444, + "margin_dpo/margin_mean": 89.73724365234375, + "margin_dpo/margin_std": 88.95266723632812, + "step": 369 + }, + { + "KL/chosen_KL_mean": -159.89451599121094, + "KL/mean": -189.7515869140625, + "KL/rejected_KL_mean": -219.608642578125, + "KL/std": 84.88975524902344, + "epoch": 0.5433186490455213, + "fcm_dpo/beta": 0.004805641248822212, + "fcm_dpo/delta": 0.05085427314043045, + "fcm_dpo/margin": 59.71413040161133, + "fcm_dpo/q_t": 0.43212318420410156, + "grad_norm": 8.281012535095215, + "learning_rate": 2.5769876463904263e-07, + "logits/chosen": -1.9239224195480347, + "logits/rejected": -1.7259197235107422, + "logps/chosen": -256.6702575683594, + "logps/ref_chosen": -96.77574157714844, + "logps/ref_rejected": -102.74411010742188, + "logps/rejected": -322.3527526855469, + "loss": 1.1738, + "margin_dpo/margin_mean": 59.71412658691406, + "margin_dpo/margin_std": 98.31381225585938, + "step": 370 + }, + { + "KL/chosen_KL_mean": -152.7358856201172, + "KL/mean": -185.91757202148438, + "KL/rejected_KL_mean": -219.0992431640625, + "KL/std": 85.54147338867188, + "epoch": 0.5447870778267254, + "fcm_dpo/beta": 0.004836600739508867, + "fcm_dpo/delta": 0.013481990434229374, + "fcm_dpo/margin": 66.36337280273438, + "fcm_dpo/q_t": 0.4238986670970917, + "grad_norm": 10.920252799987793, + "learning_rate": 2.5641594716365744e-07, + "logits/chosen": -1.62270987033844, + "logits/rejected": -1.2429583072662354, + "logps/chosen": -255.72608947753906, + "logps/ref_chosen": -102.99020385742188, + "logps/ref_rejected": -105.35062408447266, + "logps/rejected": -324.44989013671875, + "loss": 1.14, + "margin_dpo/margin_mean": 66.36337280273438, + "margin_dpo/margin_std": 92.62242126464844, + "step": 371 + }, + { + "KL/chosen_KL_mean": -129.4332275390625, + "KL/mean": -179.0995330810547, + "KL/rejected_KL_mean": -228.76585388183594, + "KL/std": 99.67709350585938, + "epoch": 0.5462555066079295, + "fcm_dpo/beta": 0.004791527055203915, + "fcm_dpo/delta": -0.07968680560588837, + "fcm_dpo/margin": 99.33262634277344, + "fcm_dpo/q_t": 0.3892465829849243, + "grad_norm": 7.302544116973877, + "learning_rate": 2.551329606220976e-07, + "logits/chosen": -2.0579938888549805, + "logits/rejected": -1.4430837631225586, + "logps/chosen": -220.45945739746094, + "logps/ref_chosen": -91.02623748779297, + "logps/ref_rejected": -85.6102294921875, + "logps/rejected": -314.3760986328125, + "loss": 1.0223, + "margin_dpo/margin_mean": 99.33262634277344, + "margin_dpo/margin_std": 102.0501708984375, + "step": 372 + }, + { + "KL/chosen_KL_mean": -150.50437927246094, + "KL/mean": -187.273193359375, + "KL/rejected_KL_mean": -224.0419921875, + "KL/std": 95.267578125, + "epoch": 0.5477239353891337, + "fcm_dpo/beta": 0.004796641878783703, + "fcm_dpo/delta": 0.04902341216802597, + "fcm_dpo/margin": 73.53761291503906, + "fcm_dpo/q_t": 0.4177188277244568, + "grad_norm": 9.591846466064453, + "learning_rate": 2.538498388222517e-07, + "logits/chosen": -2.039701461791992, + "logits/rejected": -1.5097781419754028, + "logps/chosen": -250.57745361328125, + "logps/ref_chosen": -100.07307434082031, + "logps/ref_rejected": -91.11634826660156, + "logps/rejected": -315.1583251953125, + "loss": 1.1219, + "margin_dpo/margin_mean": 73.53761291503906, + "margin_dpo/margin_std": 102.26766967773438, + "step": 373 + }, + { + "KL/chosen_KL_mean": -130.49325561523438, + "KL/mean": -170.98703002929688, + "KL/rejected_KL_mean": -211.48077392578125, + "KL/std": 90.057373046875, + "epoch": 0.5491923641703378, + "fcm_dpo/beta": 0.004810908809304237, + "fcm_dpo/delta": 0.01072636991739273, + "fcm_dpo/margin": 80.98753356933594, + "fcm_dpo/q_t": 0.40843844413757324, + "grad_norm": 8.041906356811523, + "learning_rate": 2.525666155755725e-07, + "logits/chosen": -2.3030543327331543, + "logits/rejected": -1.9203490018844604, + "logps/chosen": -228.50559997558594, + "logps/ref_chosen": -98.01234436035156, + "logps/ref_rejected": -100.75224304199219, + "logps/rejected": -312.2330322265625, + "loss": 1.0759, + "margin_dpo/margin_mean": 80.98753356933594, + "margin_dpo/margin_std": 86.94792938232422, + "step": 374 + }, + { + "KL/chosen_KL_mean": -131.91326904296875, + "KL/mean": -178.53237915039062, + "KL/rejected_KL_mean": -225.1514892578125, + "KL/std": 87.07174682617188, + "epoch": 0.5506607929515418, + "fcm_dpo/beta": 0.004762294702231884, + "fcm_dpo/delta": -0.04704749584197998, + "fcm_dpo/margin": 93.23819732666016, + "fcm_dpo/q_t": 0.394857257604599, + "grad_norm": 8.932513236999512, + "learning_rate": 2.512833246961859e-07, + "logits/chosen": -1.653382658958435, + "logits/rejected": -1.4214141368865967, + "logps/chosen": -216.30955505371094, + "logps/ref_chosen": -84.39627838134766, + "logps/ref_rejected": -95.7188491821289, + "logps/rejected": -320.870361328125, + "loss": 1.0396, + "margin_dpo/margin_mean": 93.23819732666016, + "margin_dpo/margin_std": 92.36711120605469, + "step": 375 + }, + { + "KL/chosen_KL_mean": -127.81919860839844, + "KL/mean": -178.79754638671875, + "KL/rejected_KL_mean": -229.77593994140625, + "KL/std": 100.33113098144531, + "epoch": 0.5521292217327459, + "fcm_dpo/beta": 0.004714367911219597, + "fcm_dpo/delta": -0.08483125269412994, + "fcm_dpo/margin": 101.95670318603516, + "fcm_dpo/q_t": 0.3879043757915497, + "grad_norm": 8.880372047424316, + "learning_rate": 2.5e-07, + "logits/chosen": -1.8324880599975586, + "logits/rejected": -1.524383783340454, + "logps/chosen": -223.55690002441406, + "logps/ref_chosen": -95.73770141601562, + "logps/ref_rejected": -111.97331237792969, + "logps/rejected": -341.7492370605469, + "loss": 1.0218, + "margin_dpo/margin_mean": 101.95670318603516, + "margin_dpo/margin_std": 107.13682556152344, + "step": 376 + }, + { + "KL/chosen_KL_mean": -135.4739990234375, + "KL/mean": -188.46055603027344, + "KL/rejected_KL_mean": -241.44712829589844, + "KL/std": 100.69512939453125, + "epoch": 0.55359765051395, + "fcm_dpo/beta": 0.00461684912443161, + "fcm_dpo/delta": -0.09442128241062164, + "fcm_dpo/margin": 105.97311401367188, + "fcm_dpo/q_t": 0.3847930431365967, + "grad_norm": 8.08694839477539, + "learning_rate": 2.487166753038141e-07, + "logits/chosen": -1.624734878540039, + "logits/rejected": -1.510082721710205, + "logps/chosen": -214.4009552001953, + "logps/ref_chosen": -78.92694854736328, + "logps/ref_rejected": -105.89892578125, + "logps/rejected": -347.3460693359375, + "loss": 1.0068, + "margin_dpo/margin_mean": 105.97311401367188, + "margin_dpo/margin_std": 100.46519470214844, + "step": 377 + }, + { + "KL/chosen_KL_mean": -131.78115844726562, + "KL/mean": -173.58877563476562, + "KL/rejected_KL_mean": -215.39642333984375, + "KL/std": 95.47067260742188, + "epoch": 0.5550660792951542, + "fcm_dpo/beta": 0.004638203419744968, + "fcm_dpo/delta": 0.012258481234312057, + "fcm_dpo/margin": 83.61526489257812, + "fcm_dpo/q_t": 0.4080488383769989, + "grad_norm": 9.341957092285156, + "learning_rate": 2.4743338442442754e-07, + "logits/chosen": -2.2326016426086426, + "logits/rejected": -1.9615634679794312, + "logps/chosen": -207.153564453125, + "logps/ref_chosen": -75.37240600585938, + "logps/ref_rejected": -93.70409393310547, + "logps/rejected": -309.10052490234375, + "loss": 1.0875, + "margin_dpo/margin_mean": 83.61526489257812, + "margin_dpo/margin_std": 98.60076904296875, + "step": 378 + }, + { + "KL/chosen_KL_mean": -154.72335815429688, + "KL/mean": -191.8759765625, + "KL/rejected_KL_mean": -229.02859497070312, + "KL/std": 98.63135528564453, + "epoch": 0.5565345080763583, + "fcm_dpo/beta": 0.004657992161810398, + "fcm_dpo/delta": 0.0558420866727829, + "fcm_dpo/margin": 74.30522155761719, + "fcm_dpo/q_t": 0.4205613434314728, + "grad_norm": 8.567704200744629, + "learning_rate": 2.461501611777483e-07, + "logits/chosen": -1.7665473222732544, + "logits/rejected": -1.6926254034042358, + "logps/chosen": -243.57904052734375, + "logps/ref_chosen": -88.85568237304688, + "logps/ref_rejected": -121.38114166259766, + "logps/rejected": -350.40972900390625, + "loss": 1.1353, + "margin_dpo/margin_mean": 74.30522155761719, + "margin_dpo/margin_std": 112.70030212402344, + "step": 379 + }, + { + "KL/chosen_KL_mean": -169.372802734375, + "KL/mean": -214.88067626953125, + "KL/rejected_KL_mean": -260.3885803222656, + "KL/std": 101.7571792602539, + "epoch": 0.5580029368575624, + "fcm_dpo/beta": 0.004655890166759491, + "fcm_dpo/delta": -0.024813536554574966, + "fcm_dpo/margin": 91.0157470703125, + "fcm_dpo/q_t": 0.4028235375881195, + "grad_norm": 11.913869857788086, + "learning_rate": 2.4486703937790243e-07, + "logits/chosen": -1.3702871799468994, + "logits/rejected": -1.3439240455627441, + "logps/chosen": -249.20602416992188, + "logps/ref_chosen": -79.83321380615234, + "logps/ref_rejected": -110.7447509765625, + "logps/rejected": -371.1333312988281, + "loss": 1.0722, + "margin_dpo/margin_mean": 91.0157470703125, + "margin_dpo/margin_std": 114.99800109863281, + "step": 380 + }, + { + "KL/chosen_KL_mean": -149.1160125732422, + "KL/mean": -188.42755126953125, + "KL/rejected_KL_mean": -227.7390899658203, + "KL/std": 97.11800384521484, + "epoch": 0.5594713656387665, + "fcm_dpo/beta": 0.0046757897362113, + "fcm_dpo/delta": 0.03359299525618553, + "fcm_dpo/margin": 78.62306213378906, + "fcm_dpo/q_t": 0.41347819566726685, + "grad_norm": 9.305055618286133, + "learning_rate": 2.435840528363426e-07, + "logits/chosen": -1.821177363395691, + "logits/rejected": -1.2757813930511475, + "logps/chosen": -235.68284606933594, + "logps/ref_chosen": -86.56683349609375, + "logps/ref_rejected": -84.416748046875, + "logps/rejected": -312.15582275390625, + "loss": 1.105, + "margin_dpo/margin_mean": 78.62306213378906, + "margin_dpo/margin_std": 100.54057312011719, + "step": 381 + }, + { + "KL/chosen_KL_mean": -146.64369201660156, + "KL/mean": -191.88885498046875, + "KL/rejected_KL_mean": -237.13404846191406, + "KL/std": 95.49838256835938, + "epoch": 0.5609397944199707, + "fcm_dpo/beta": 0.0046416353434324265, + "fcm_dpo/delta": -0.021616537123918533, + "fcm_dpo/margin": 90.4903564453125, + "fcm_dpo/q_t": 0.4013921916484833, + "grad_norm": 8.945796966552734, + "learning_rate": 2.4230123536095745e-07, + "logits/chosen": -1.8326973915100098, + "logits/rejected": -1.5663193464279175, + "logps/chosen": -251.10549926757812, + "logps/ref_chosen": -104.46180725097656, + "logps/ref_rejected": -114.60223388671875, + "logps/rejected": -351.73626708984375, + "loss": 1.0611, + "margin_dpo/margin_mean": 90.4903564453125, + "margin_dpo/margin_std": 97.75071716308594, + "step": 382 + }, + { + "KL/chosen_KL_mean": -159.60577392578125, + "KL/mean": -197.79856872558594, + "KL/rejected_KL_mean": -235.9913787841797, + "KL/std": 98.73554229736328, + "epoch": 0.5624082232011748, + "fcm_dpo/beta": 0.004684256389737129, + "fcm_dpo/delta": 0.04377155005931854, + "fcm_dpo/margin": 76.38562774658203, + "fcm_dpo/q_t": 0.4160931408405304, + "grad_norm": 8.794547080993652, + "learning_rate": 2.4101862075518037e-07, + "logits/chosen": -1.4984644651412964, + "logits/rejected": -1.3735969066619873, + "logps/chosen": -241.72183227539062, + "logps/ref_chosen": -82.11607360839844, + "logps/ref_rejected": -98.77716064453125, + "logps/rejected": -334.7685546875, + "loss": 1.1107, + "margin_dpo/margin_mean": 76.38562774658203, + "margin_dpo/margin_std": 96.79356384277344, + "step": 383 + }, + { + "KL/chosen_KL_mean": -170.21539306640625, + "KL/mean": -208.12240600585938, + "KL/rejected_KL_mean": -246.0294189453125, + "KL/std": 102.38072204589844, + "epoch": 0.5638766519823789, + "fcm_dpo/beta": 0.004683772101998329, + "fcm_dpo/delta": -0.040233880281448364, + "fcm_dpo/margin": 75.81401062011719, + "fcm_dpo/q_t": 0.4166222810745239, + "grad_norm": 10.724540710449219, + "learning_rate": 2.397362428170992e-07, + "logits/chosen": -1.8102113008499146, + "logits/rejected": -1.5873998403549194, + "logps/chosen": -250.80064392089844, + "logps/ref_chosen": -80.58525085449219, + "logps/ref_rejected": -91.44789123535156, + "logps/rejected": -337.477294921875, + "loss": 1.1361, + "margin_dpo/margin_mean": 75.81401062011719, + "margin_dpo/margin_std": 113.7035140991211, + "step": 384 + }, + { + "KL/chosen_KL_mean": -142.04183959960938, + "KL/mean": -191.25686645507812, + "KL/rejected_KL_mean": -240.47186279296875, + "KL/std": 92.16252136230469, + "epoch": 0.5653450807635829, + "fcm_dpo/beta": 0.004603315144777298, + "fcm_dpo/delta": -0.056393664330244064, + "fcm_dpo/margin": 98.43002319335938, + "fcm_dpo/q_t": 0.3944080173969269, + "grad_norm": 9.644135475158691, + "learning_rate": 2.3845413533856514e-07, + "logits/chosen": -2.1324715614318848, + "logits/rejected": -1.5153433084487915, + "logps/chosen": -234.80201721191406, + "logps/ref_chosen": -92.76016235351562, + "logps/ref_rejected": -84.45997619628906, + "logps/rejected": -324.9318542480469, + "loss": 1.0414, + "margin_dpo/margin_mean": 98.43002319335938, + "margin_dpo/margin_std": 105.903076171875, + "step": 385 + }, + { + "KL/chosen_KL_mean": -150.625732421875, + "KL/mean": -197.98562622070312, + "KL/rejected_KL_mean": -245.3455352783203, + "KL/std": 94.32642364501953, + "epoch": 0.566813509544787, + "fcm_dpo/beta": 0.004587274976074696, + "fcm_dpo/delta": -0.03607851266860962, + "fcm_dpo/margin": 94.71981048583984, + "fcm_dpo/q_t": 0.39677664637565613, + "grad_norm": 7.512143135070801, + "learning_rate": 2.3717233210430254e-07, + "logits/chosen": -1.7768217325210571, + "logits/rejected": -1.4961553812026978, + "logps/chosen": -235.98468017578125, + "logps/ref_chosen": -85.35896301269531, + "logps/ref_rejected": -96.99065399169922, + "logps/rejected": -342.336181640625, + "loss": 1.0511, + "margin_dpo/margin_mean": 94.71980285644531, + "margin_dpo/margin_std": 102.27782440185547, + "step": 386 + }, + { + "KL/chosen_KL_mean": -175.24859619140625, + "KL/mean": -217.31689453125, + "KL/rejected_KL_mean": -259.38519287109375, + "KL/std": 91.85528564453125, + "epoch": 0.5682819383259912, + "fcm_dpo/beta": 0.004596139770001173, + "fcm_dpo/delta": 0.013757986016571522, + "fcm_dpo/margin": 84.1365966796875, + "fcm_dpo/q_t": 0.4088144898414612, + "grad_norm": 10.681920051574707, + "learning_rate": 2.3589086689101889e-07, + "logits/chosen": -2.393043041229248, + "logits/rejected": -1.7053593397140503, + "logps/chosen": -277.9367370605469, + "logps/ref_chosen": -102.68814086914062, + "logps/ref_rejected": -99.42858123779297, + "logps/rejected": -358.81378173828125, + "loss": 1.0813, + "margin_dpo/margin_mean": 84.13660430908203, + "margin_dpo/margin_std": 94.17853546142578, + "step": 387 + }, + { + "KL/chosen_KL_mean": -177.23260498046875, + "KL/mean": -223.73065185546875, + "KL/rejected_KL_mean": -270.22869873046875, + "KL/std": 101.81842041015625, + "epoch": 0.5697503671071953, + "fcm_dpo/beta": 0.004570071119815111, + "fcm_dpo/delta": -0.026149000972509384, + "fcm_dpo/margin": 92.99610900878906, + "fcm_dpo/q_t": 0.4001759886741638, + "grad_norm": 8.60345458984375, + "learning_rate": 2.3460977346651428e-07, + "logits/chosen": -1.6761045455932617, + "logits/rejected": -1.5581592321395874, + "logps/chosen": -260.8178405761719, + "logps/ref_chosen": -83.58523559570312, + "logps/ref_rejected": -110.10037231445312, + "logps/rejected": -380.3291015625, + "loss": 1.0607, + "margin_dpo/margin_mean": 92.99610900878906, + "margin_dpo/margin_std": 105.09429931640625, + "step": 388 + }, + { + "KL/chosen_KL_mean": -182.1714630126953, + "KL/mean": -221.9415283203125, + "KL/rejected_KL_mean": -261.7115783691406, + "KL/std": 103.77534484863281, + "epoch": 0.5712187958883994, + "fcm_dpo/beta": 0.00459069199860096, + "fcm_dpo/delta": 0.036189205944538116, + "fcm_dpo/margin": 79.54012298583984, + "fcm_dpo/q_t": 0.4151958227157593, + "grad_norm": 10.999826431274414, + "learning_rate": 2.3332908558879177e-07, + "logits/chosen": -2.4839155673980713, + "logits/rejected": -2.0433268547058105, + "logps/chosen": -271.99371337890625, + "logps/ref_chosen": -89.822265625, + "logps/ref_rejected": -84.85376739501953, + "logps/rejected": -346.5653381347656, + "loss": 1.1166, + "margin_dpo/margin_mean": 79.54012298583984, + "margin_dpo/margin_std": 112.048828125, + "step": 389 + }, + { + "KL/chosen_KL_mean": -182.55450439453125, + "KL/mean": -220.90225219726562, + "KL/rejected_KL_mean": -259.25, + "KL/std": 101.57498931884766, + "epoch": 0.5726872246696035, + "fcm_dpo/beta": 0.00457828538492322, + "fcm_dpo/delta": -0.025065837427973747, + "fcm_dpo/margin": 76.69551849365234, + "fcm_dpo/q_t": 0.4198220372200012, + "grad_norm": 8.391576766967773, + "learning_rate": 2.320488370051681e-07, + "logits/chosen": -1.4965027570724487, + "logits/rejected": -1.2872850894927979, + "logps/chosen": -260.0869445800781, + "logps/ref_chosen": -77.53244018554688, + "logps/ref_rejected": -92.58392333984375, + "logps/rejected": -351.83392333984375, + "loss": 1.1337, + "margin_dpo/margin_mean": 76.69551849365234, + "margin_dpo/margin_std": 116.51912689208984, + "step": 390 + }, + { + "KL/chosen_KL_mean": -164.91554260253906, + "KL/mean": -206.34451293945312, + "KL/rejected_KL_mean": -247.77349853515625, + "KL/std": 97.03406524658203, + "epoch": 0.5741556534508077, + "fcm_dpo/beta": 0.004579827189445496, + "fcm_dpo/delta": 0.021230699494481087, + "fcm_dpo/margin": 82.85796356201172, + "fcm_dpo/q_t": 0.4110923409461975, + "grad_norm": 8.695920944213867, + "learning_rate": 2.3076906145138405e-07, + "logits/chosen": -2.0394270420074463, + "logits/rejected": -1.5955464839935303, + "logps/chosen": -253.14846801757812, + "logps/ref_chosen": -88.2329330444336, + "logps/ref_rejected": -93.00756072998047, + "logps/rejected": -340.78106689453125, + "loss": 1.0954, + "margin_dpo/margin_mean": 82.85796356201172, + "margin_dpo/margin_std": 101.58324432373047, + "step": 391 + }, + { + "KL/chosen_KL_mean": -167.07406616210938, + "KL/mean": -217.9665069580078, + "KL/rejected_KL_mean": -268.85894775390625, + "KL/std": 109.32381439208984, + "epoch": 0.5756240822320118, + "fcm_dpo/beta": 0.004566463176161051, + "fcm_dpo/delta": -0.06795135885477066, + "fcm_dpo/margin": 101.78485870361328, + "fcm_dpo/q_t": 0.3917716145515442, + "grad_norm": 9.26330280303955, + "learning_rate": 2.294897926507156e-07, + "logits/chosen": -1.6375110149383545, + "logits/rejected": -1.5211455821990967, + "logps/chosen": -254.8184814453125, + "logps/ref_chosen": -87.74442291259766, + "logps/ref_rejected": -108.57071685791016, + "logps/rejected": -377.4296569824219, + "loss": 1.0424, + "margin_dpo/margin_mean": 101.78485870361328, + "margin_dpo/margin_std": 116.9224624633789, + "step": 392 + }, + { + "KL/chosen_KL_mean": -148.2543487548828, + "KL/mean": -193.87881469726562, + "KL/rejected_KL_mean": -239.5032958984375, + "KL/std": 102.76055908203125, + "epoch": 0.5770925110132159, + "fcm_dpo/beta": 0.004524551331996918, + "fcm_dpo/delta": -0.013424969278275967, + "fcm_dpo/margin": 91.24895477294922, + "fcm_dpo/q_t": 0.4031013548374176, + "grad_norm": 8.333599090576172, + "learning_rate": 2.2821106431308543e-07, + "logits/chosen": -1.726250410079956, + "logits/rejected": -1.40533447265625, + "logps/chosen": -224.8181610107422, + "logps/ref_chosen": -76.56381225585938, + "logps/ref_rejected": -79.72618103027344, + "logps/rejected": -319.2294921875, + "loss": 1.0645, + "margin_dpo/margin_mean": 91.24895477294922, + "margin_dpo/margin_std": 100.66075134277344, + "step": 393 + }, + { + "KL/chosen_KL_mean": -157.91297912597656, + "KL/mean": -207.65185546875, + "KL/rejected_KL_mean": -257.39068603515625, + "KL/std": 98.01484680175781, + "epoch": 0.57856093979442, + "fcm_dpo/beta": 0.004491634666919708, + "fcm_dpo/delta": -0.04899781942367554, + "fcm_dpo/margin": 99.47770690917969, + "fcm_dpo/q_t": 0.39410465955734253, + "grad_norm": 8.270844459533691, + "learning_rate": 2.2693291013417452e-07, + "logits/chosen": -1.4862959384918213, + "logits/rejected": -1.2529828548431396, + "logps/chosen": -242.6116485595703, + "logps/ref_chosen": -84.69866180419922, + "logps/ref_rejected": -97.457763671875, + "logps/rejected": -354.84844970703125, + "loss": 1.0328, + "margin_dpo/margin_mean": 99.47771453857422, + "margin_dpo/margin_std": 95.81720733642578, + "step": 394 + }, + { + "KL/chosen_KL_mean": -163.55706787109375, + "KL/mean": -201.77471923828125, + "KL/rejected_KL_mean": -239.99237060546875, + "KL/std": 102.40203094482422, + "epoch": 0.580029368575624, + "fcm_dpo/beta": 0.004466316197067499, + "fcm_dpo/delta": -0.014659581705927849, + "fcm_dpo/margin": 76.43531036376953, + "fcm_dpo/q_t": 0.4225883483886719, + "grad_norm": 8.785511016845703, + "learning_rate": 2.2565536379453404e-07, + "logits/chosen": -2.625396490097046, + "logits/rejected": -2.230534076690674, + "logps/chosen": -252.4447021484375, + "logps/ref_chosen": -88.88763427734375, + "logps/ref_rejected": -91.0455322265625, + "logps/rejected": -331.03790283203125, + "loss": 1.1646, + "margin_dpo/margin_mean": 76.43531036376953, + "margin_dpo/margin_std": 138.156005859375, + "step": 395 + }, + { + "KL/chosen_KL_mean": -172.32266235351562, + "KL/mean": -206.53025817871094, + "KL/rejected_KL_mean": -240.73785400390625, + "KL/std": 90.76689147949219, + "epoch": 0.5814977973568282, + "fcm_dpo/beta": 0.0045280614867806435, + "fcm_dpo/delta": 0.09311157464981079, + "fcm_dpo/margin": 68.41517639160156, + "fcm_dpo/q_t": 0.42777177691459656, + "grad_norm": 10.51520824432373, + "learning_rate": 2.2437845895869825e-07, + "logits/chosen": -1.9896714687347412, + "logits/rejected": -1.5045146942138672, + "logps/chosen": -274.79400634765625, + "logps/ref_chosen": -102.47132873535156, + "logps/ref_rejected": -93.23478698730469, + "logps/rejected": -333.9726257324219, + "loss": 1.156, + "margin_dpo/margin_mean": 68.4151611328125, + "margin_dpo/margin_std": 106.41094207763672, + "step": 396 + }, + { + "KL/chosen_KL_mean": -155.91302490234375, + "KL/mean": -216.68316650390625, + "KL/rejected_KL_mean": -277.45330810546875, + "KL/std": 105.60340881347656, + "epoch": 0.5829662261380323, + "fcm_dpo/beta": 0.004443579353392124, + "fcm_dpo/delta": -0.14838063716888428, + "fcm_dpo/margin": 121.54026794433594, + "fcm_dpo/q_t": 0.37464988231658936, + "grad_norm": 8.144950866699219, + "learning_rate": 2.2310222927429716e-07, + "logits/chosen": -2.2715883255004883, + "logits/rejected": -1.6745511293411255, + "logps/chosen": -256.95745849609375, + "logps/ref_chosen": -101.04441833496094, + "logps/ref_rejected": -108.8511734008789, + "logps/rejected": -386.3044738769531, + "loss": 0.9905, + "margin_dpo/margin_mean": 121.54027557373047, + "margin_dpo/margin_std": 126.0580825805664, + "step": 397 + }, + { + "KL/chosen_KL_mean": -164.30319213867188, + "KL/mean": -207.94981384277344, + "KL/rejected_KL_mean": -251.59645080566406, + "KL/std": 103.51817321777344, + "epoch": 0.5844346549192364, + "fcm_dpo/beta": 0.004421258345246315, + "fcm_dpo/delta": 0.01460410375148058, + "fcm_dpo/margin": 87.29324340820312, + "fcm_dpo/q_t": 0.4108482599258423, + "grad_norm": 7.92824125289917, + "learning_rate": 2.2182670837116972e-07, + "logits/chosen": -2.2637898921966553, + "logits/rejected": -1.9915108680725098, + "logps/chosen": -259.3524169921875, + "logps/ref_chosen": -95.04922485351562, + "logps/ref_rejected": -114.00563049316406, + "logps/rejected": -365.6020812988281, + "loss": 1.1065, + "margin_dpo/margin_mean": 87.29324340820312, + "margin_dpo/margin_std": 122.97125244140625, + "step": 398 + }, + { + "KL/chosen_KL_mean": -143.33665466308594, + "KL/mean": -199.37667846679688, + "KL/rejected_KL_mean": -255.41668701171875, + "KL/std": 93.90657043457031, + "epoch": 0.5859030837004405, + "fcm_dpo/beta": 0.0043755630031228065, + "fcm_dpo/delta": -0.09497334063053131, + "fcm_dpo/margin": 112.08001708984375, + "fcm_dpo/q_t": 0.3842849135398865, + "grad_norm": 8.265376091003418, + "learning_rate": 2.2055192986047804e-07, + "logits/chosen": -2.267026901245117, + "logits/rejected": -1.4912976026535034, + "logps/chosen": -239.97520446777344, + "logps/ref_chosen": -96.63854217529297, + "logps/ref_rejected": -83.53645324707031, + "logps/rejected": -338.953125, + "loss": 1.0014, + "margin_dpo/margin_mean": 112.08000946044922, + "margin_dpo/margin_std": 101.56687927246094, + "step": 399 + }, + { + "KL/chosen_KL_mean": -150.24603271484375, + "KL/mean": -207.30279541015625, + "KL/rejected_KL_mean": -264.3595275878906, + "KL/std": 103.20948791503906, + "epoch": 0.5873715124816447, + "fcm_dpo/beta": 0.004286617040634155, + "fcm_dpo/delta": -0.093747079372406, + "fcm_dpo/margin": 114.11351013183594, + "fcm_dpo/q_t": 0.3858647346496582, + "grad_norm": 12.003571510314941, + "learning_rate": 2.192779273338215e-07, + "logits/chosen": -2.319026470184326, + "logits/rejected": -1.847064733505249, + "logps/chosen": -255.54917907714844, + "logps/ref_chosen": -105.30314636230469, + "logps/ref_rejected": -108.6555404663086, + "logps/rejected": -373.01507568359375, + "loss": 1.0177, + "margin_dpo/margin_mean": 114.11351013183594, + "margin_dpo/margin_std": 118.26309967041016, + "step": 400 + }, + { + "KL/chosen_KL_mean": -165.57501220703125, + "KL/mean": -200.97299194335938, + "KL/rejected_KL_mean": -236.3709716796875, + "KL/std": 105.03636932373047, + "epoch": 0.5888399412628488, + "fcm_dpo/beta": 0.004283515270799398, + "fcm_dpo/delta": 0.02043536677956581, + "fcm_dpo/margin": 70.79595184326172, + "fcm_dpo/q_t": 0.4315440356731415, + "grad_norm": 10.821425437927246, + "learning_rate": 2.1800473436235136e-07, + "logits/chosen": -1.5078041553497314, + "logits/rejected": -1.2638230323791504, + "logps/chosen": -249.8641815185547, + "logps/ref_chosen": -84.28916931152344, + "logps/ref_rejected": -90.943115234375, + "logps/rejected": -327.3140869140625, + "loss": 1.1767, + "margin_dpo/margin_mean": 70.79595184326172, + "margin_dpo/margin_std": 128.64962768554688, + "step": 401 + }, + { + "KL/chosen_KL_mean": -146.51083374023438, + "KL/mean": -213.16024780273438, + "KL/rejected_KL_mean": -279.8096923828125, + "KL/std": 111.65316009521484, + "epoch": 0.5903083700440529, + "fcm_dpo/beta": 0.004186196718364954, + "fcm_dpo/delta": -0.1671200692653656, + "fcm_dpo/margin": 133.29884338378906, + "fcm_dpo/q_t": 0.36797064542770386, + "grad_norm": 7.594605922698975, + "learning_rate": 2.1673238449588665e-07, + "logits/chosen": -2.0720481872558594, + "logits/rejected": -1.617252230644226, + "logps/chosen": -230.1039581298828, + "logps/ref_chosen": -83.59312438964844, + "logps/ref_rejected": -87.81027221679688, + "logps/rejected": -367.61993408203125, + "loss": 0.9542, + "margin_dpo/margin_mean": 133.29884338378906, + "margin_dpo/margin_std": 109.2518081665039, + "step": 402 + }, + { + "KL/chosen_KL_mean": -149.97921752929688, + "KL/mean": -199.19219970703125, + "KL/rejected_KL_mean": -248.40518188476562, + "KL/std": 107.48170471191406, + "epoch": 0.591776798825257, + "fcm_dpo/beta": 0.004130107816308737, + "fcm_dpo/delta": -0.006883140653371811, + "fcm_dpo/margin": 98.42596435546875, + "fcm_dpo/q_t": 0.40466803312301636, + "grad_norm": 8.713815689086914, + "learning_rate": 2.154609112620295e-07, + "logits/chosen": -2.3627402782440186, + "logits/rejected": -1.9727840423583984, + "logps/chosen": -223.7322998046875, + "logps/ref_chosen": -73.75308227539062, + "logps/ref_rejected": -83.92012786865234, + "logps/rejected": -332.3253173828125, + "loss": 1.0755, + "margin_dpo/margin_mean": 98.42597198486328, + "margin_dpo/margin_std": 115.85055541992188, + "step": 403 + }, + { + "KL/chosen_KL_mean": -178.7392120361328, + "KL/mean": -220.9769287109375, + "KL/rejected_KL_mean": -263.21466064453125, + "KL/std": 103.2603988647461, + "epoch": 0.593245227606461, + "fcm_dpo/beta": 0.004159621428698301, + "fcm_dpo/delta": 0.050401441752910614, + "fcm_dpo/margin": 84.47544860839844, + "fcm_dpo/q_t": 0.41857510805130005, + "grad_norm": 11.859652519226074, + "learning_rate": 2.1419034816528218e-07, + "logits/chosen": -1.8403348922729492, + "logits/rejected": -1.4935059547424316, + "logps/chosen": -258.4154052734375, + "logps/ref_chosen": -79.67617797851562, + "logps/ref_rejected": -84.280517578125, + "logps/rejected": -347.49517822265625, + "loss": 1.1309, + "margin_dpo/margin_mean": 84.47544860839844, + "margin_dpo/margin_std": 126.14936828613281, + "step": 404 + }, + { + "KL/chosen_KL_mean": -182.48565673828125, + "KL/mean": -225.05123901367188, + "KL/rejected_KL_mean": -267.6167907714844, + "KL/std": 97.38519287109375, + "epoch": 0.5947136563876652, + "fcm_dpo/beta": 0.004169545602053404, + "fcm_dpo/delta": -0.022491540759801865, + "fcm_dpo/margin": 85.13111877441406, + "fcm_dpo/q_t": 0.4164101481437683, + "grad_norm": 7.930446624755859, + "learning_rate": 2.129207286861638e-07, + "logits/chosen": -1.6639671325683594, + "logits/rejected": -1.1961365938186646, + "logps/chosen": -278.9476318359375, + "logps/ref_chosen": -96.46195220947266, + "logps/ref_rejected": -92.87071228027344, + "logps/rejected": -360.48748779296875, + "loss": 1.108, + "margin_dpo/margin_mean": 85.13111877441406, + "margin_dpo/margin_std": 103.06645965576172, + "step": 405 + }, + { + "KL/chosen_KL_mean": -173.97332763671875, + "KL/mean": -212.91693115234375, + "KL/rejected_KL_mean": -251.86053466796875, + "KL/std": 104.26785278320312, + "epoch": 0.5961820851688693, + "fcm_dpo/beta": 0.004206824582070112, + "fcm_dpo/delta": 0.07479364424943924, + "fcm_dpo/margin": 77.88721466064453, + "fcm_dpo/q_t": 0.4238826036453247, + "grad_norm": 13.93974781036377, + "learning_rate": 2.1165208628032861e-07, + "logits/chosen": -1.925182819366455, + "logits/rejected": -1.758012294769287, + "logps/chosen": -252.10728454589844, + "logps/ref_chosen": -78.13396453857422, + "logps/ref_rejected": -98.28359985351562, + "logps/rejected": -350.1441345214844, + "loss": 1.1485, + "margin_dpo/margin_mean": 77.88722229003906, + "margin_dpo/margin_std": 122.2257308959961, + "step": 406 + }, + { + "KL/chosen_KL_mean": -180.55955505371094, + "KL/mean": -214.61465454101562, + "KL/rejected_KL_mean": -248.6697540283203, + "KL/std": 102.51031494140625, + "epoch": 0.5976505139500734, + "fcm_dpo/beta": 0.004268460441380739, + "fcm_dpo/delta": 0.05369499698281288, + "fcm_dpo/margin": 68.11019134521484, + "fcm_dpo/q_t": 0.4316493570804596, + "grad_norm": 9.310810089111328, + "learning_rate": 2.1038445437768375e-07, + "logits/chosen": -2.3327717781066895, + "logits/rejected": -1.8287901878356934, + "logps/chosen": -264.5723876953125, + "logps/ref_chosen": -84.01283264160156, + "logps/ref_rejected": -82.78103637695312, + "logps/rejected": -331.4508056640625, + "loss": 1.1686, + "margin_dpo/margin_mean": 68.11019134521484, + "margin_dpo/margin_std": 109.58731079101562, + "step": 407 + }, + { + "KL/chosen_KL_mean": -172.81137084960938, + "KL/mean": -207.1467742919922, + "KL/rejected_KL_mean": -241.48214721679688, + "KL/std": 97.80386352539062, + "epoch": 0.5991189427312775, + "fcm_dpo/beta": 0.004303076304495335, + "fcm_dpo/delta": 0.04038481041789055, + "fcm_dpo/margin": 68.6707763671875, + "fcm_dpo/q_t": 0.4315441846847534, + "grad_norm": 8.947710037231445, + "learning_rate": 2.0911786638150872e-07, + "logits/chosen": -2.3426196575164795, + "logits/rejected": -1.7310137748718262, + "logps/chosen": -277.27313232421875, + "logps/ref_chosen": -104.46174621582031, + "logps/ref_rejected": -96.37217712402344, + "logps/rejected": -337.8543395996094, + "loss": 1.1775, + "margin_dpo/margin_mean": 68.67078399658203, + "margin_dpo/margin_std": 121.33287048339844, + "step": 408 + }, + { + "KL/chosen_KL_mean": -184.01516723632812, + "KL/mean": -219.9274139404297, + "KL/rejected_KL_mean": -255.83966064453125, + "KL/std": 100.07394409179688, + "epoch": 0.6005873715124816, + "fcm_dpo/beta": 0.0043605901300907135, + "fcm_dpo/delta": 0.08961852639913559, + "fcm_dpo/margin": 71.82449340820312, + "fcm_dpo/q_t": 0.4256886839866638, + "grad_norm": 8.713200569152832, + "learning_rate": 2.0785235566757517e-07, + "logits/chosen": -1.6637320518493652, + "logits/rejected": -1.2354702949523926, + "logps/chosen": -281.6834716796875, + "logps/ref_chosen": -97.66830444335938, + "logps/ref_rejected": -90.04583740234375, + "logps/rejected": -345.885498046875, + "loss": 1.155, + "margin_dpo/margin_mean": 71.82449340820312, + "margin_dpo/margin_std": 110.18793487548828, + "step": 409 + }, + { + "KL/chosen_KL_mean": -178.03671264648438, + "KL/mean": -221.854248046875, + "KL/rejected_KL_mean": -265.6717529296875, + "KL/std": 106.2457275390625, + "epoch": 0.6020558002936858, + "fcm_dpo/beta": 0.004389758221805096, + "fcm_dpo/delta": 0.015918675810098648, + "fcm_dpo/margin": 87.63505554199219, + "fcm_dpo/q_t": 0.4089387357234955, + "grad_norm": 8.035231590270996, + "learning_rate": 2.065879555832674e-07, + "logits/chosen": -1.6794209480285645, + "logits/rejected": -1.5484282970428467, + "logps/chosen": -254.50595092773438, + "logps/ref_chosen": -76.46923828125, + "logps/ref_rejected": -88.64064025878906, + "logps/rejected": -354.3124084472656, + "loss": 1.0905, + "margin_dpo/margin_mean": 87.63505554199219, + "margin_dpo/margin_std": 106.79444885253906, + "step": 410 + }, + { + "KL/chosen_KL_mean": -191.19830322265625, + "KL/mean": -231.706298828125, + "KL/rejected_KL_mean": -272.21429443359375, + "KL/std": 98.99314880371094, + "epoch": 0.6035242290748899, + "fcm_dpo/beta": 0.004386193584650755, + "fcm_dpo/delta": -0.009879241697490215, + "fcm_dpo/margin": 81.01599884033203, + "fcm_dpo/q_t": 0.4174742102622986, + "grad_norm": 8.319835662841797, + "learning_rate": 2.0532469944670343e-07, + "logits/chosen": -2.338230848312378, + "logits/rejected": -1.934354543685913, + "logps/chosen": -278.3645935058594, + "logps/ref_chosen": -87.16630554199219, + "logps/ref_rejected": -87.09603118896484, + "logps/rejected": -359.3103332519531, + "loss": 1.1154, + "margin_dpo/margin_mean": 81.0159912109375, + "margin_dpo/margin_std": 106.55110168457031, + "step": 411 + }, + { + "KL/chosen_KL_mean": -181.22537231445312, + "KL/mean": -221.19668579101562, + "KL/rejected_KL_mean": -261.1679992675781, + "KL/std": 99.29476928710938, + "epoch": 0.604992657856094, + "fcm_dpo/beta": 0.004409436136484146, + "fcm_dpo/delta": 0.049248237162828445, + "fcm_dpo/margin": 79.94264221191406, + "fcm_dpo/q_t": 0.4195641279220581, + "grad_norm": 9.73497486114502, + "learning_rate": 2.0406262054585738e-07, + "logits/chosen": -2.3409171104431152, + "logits/rejected": -2.1250534057617188, + "logps/chosen": -260.1727294921875, + "logps/ref_chosen": -78.94734191894531, + "logps/ref_rejected": -106.10554504394531, + "logps/rejected": -367.2735595703125, + "loss": 1.1395, + "margin_dpo/margin_mean": 79.94264221191406, + "margin_dpo/margin_std": 127.5152816772461, + "step": 412 + }, + { + "KL/chosen_KL_mean": -193.44114685058594, + "KL/mean": -237.44496154785156, + "KL/rejected_KL_mean": -281.4487609863281, + "KL/std": 105.40950775146484, + "epoch": 0.6064610866372981, + "fcm_dpo/beta": 0.004423927515745163, + "fcm_dpo/delta": 0.010950472205877304, + "fcm_dpo/margin": 88.00759887695312, + "fcm_dpo/q_t": 0.4088040888309479, + "grad_norm": 9.673861503601074, + "learning_rate": 2.0280175213768205e-07, + "logits/chosen": -1.1915881633758545, + "logits/rejected": -0.9319154620170593, + "logps/chosen": -289.1358642578125, + "logps/ref_chosen": -95.69470977783203, + "logps/ref_rejected": -107.96085357666016, + "logps/rejected": -389.40960693359375, + "loss": 1.1071, + "margin_dpo/margin_mean": 88.00759887695312, + "margin_dpo/margin_std": 125.48699951171875, + "step": 413 + }, + { + "KL/chosen_KL_mean": -180.10107421875, + "KL/mean": -221.34173583984375, + "KL/rejected_KL_mean": -262.5823974609375, + "KL/std": 101.55104064941406, + "epoch": 0.6079295154185022, + "fcm_dpo/beta": 0.004467803984880447, + "fcm_dpo/delta": 0.03259623423218727, + "fcm_dpo/margin": 82.4813232421875, + "fcm_dpo/q_t": 0.4153996706008911, + "grad_norm": 12.718391418457031, + "learning_rate": 2.0154212744723247e-07, + "logits/chosen": -1.438899040222168, + "logits/rejected": -1.0733990669250488, + "logps/chosen": -268.37774658203125, + "logps/ref_chosen": -88.27667236328125, + "logps/ref_rejected": -92.87004089355469, + "logps/rejected": -355.45245361328125, + "loss": 1.1307, + "margin_dpo/margin_mean": 82.4813232421875, + "margin_dpo/margin_std": 130.08883666992188, + "step": 414 + }, + { + "KL/chosen_KL_mean": -194.8232421875, + "KL/mean": -229.5859832763672, + "KL/rejected_KL_mean": -264.34869384765625, + "KL/std": 93.02442932128906, + "epoch": 0.6093979441997063, + "fcm_dpo/beta": 0.004484872333705425, + "fcm_dpo/delta": 0.03682447224855423, + "fcm_dpo/margin": 69.52545166015625, + "fcm_dpo/q_t": 0.42746877670288086, + "grad_norm": 10.712226867675781, + "learning_rate": 2.002837796667909e-07, + "logits/chosen": -1.6358399391174316, + "logits/rejected": -1.2576943635940552, + "logps/chosen": -303.7391357421875, + "logps/ref_chosen": -108.91590118408203, + "logps/ref_rejected": -107.47135925292969, + "logps/rejected": -371.820068359375, + "loss": 1.1618, + "margin_dpo/margin_mean": 69.52545166015625, + "margin_dpo/margin_std": 114.09034729003906, + "step": 415 + }, + { + "KL/chosen_KL_mean": -194.02548217773438, + "KL/mean": -242.18931579589844, + "KL/rejected_KL_mean": -290.3531494140625, + "KL/std": 107.28054809570312, + "epoch": 0.6108663729809104, + "fcm_dpo/beta": 0.004463159013539553, + "fcm_dpo/delta": -0.0856553316116333, + "fcm_dpo/margin": 96.32766723632812, + "fcm_dpo/q_t": 0.39990150928497314, + "grad_norm": 8.888777732849121, + "learning_rate": 1.990267419549914e-07, + "logits/chosen": -1.8529837131500244, + "logits/rejected": -1.353346824645996, + "logps/chosen": -287.42437744140625, + "logps/ref_chosen": -93.39888000488281, + "logps/ref_rejected": -97.6729736328125, + "logps/rejected": -388.026123046875, + "loss": 1.0674, + "margin_dpo/margin_mean": 96.32767486572266, + "margin_dpo/margin_std": 113.21560668945312, + "step": 416 + }, + { + "KL/chosen_KL_mean": -179.7425537109375, + "KL/mean": -224.8054962158203, + "KL/rejected_KL_mean": -269.86846923828125, + "KL/std": 99.80001068115234, + "epoch": 0.6123348017621145, + "fcm_dpo/beta": 0.004430360160768032, + "fcm_dpo/delta": 0.0007128175348043442, + "fcm_dpo/margin": 90.12593078613281, + "fcm_dpo/q_t": 0.4061453938484192, + "grad_norm": 8.672005653381348, + "learning_rate": 1.9777104743594686e-07, + "logits/chosen": -2.450835704803467, + "logits/rejected": -1.5716674327850342, + "logps/chosen": -263.27789306640625, + "logps/ref_chosen": -83.53533172607422, + "logps/ref_rejected": -74.44184875488281, + "logps/rejected": -344.310302734375, + "loss": 1.0753, + "margin_dpo/margin_mean": 90.12593078613281, + "margin_dpo/margin_std": 102.06039428710938, + "step": 417 + }, + { + "KL/chosen_KL_mean": -171.91831970214844, + "KL/mean": -228.54393005371094, + "KL/rejected_KL_mean": -285.1695556640625, + "KL/std": 106.93478393554688, + "epoch": 0.6138032305433186, + "fcm_dpo/beta": 0.004396216943860054, + "fcm_dpo/delta": -0.10364756733179092, + "fcm_dpo/margin": 113.251220703125, + "fcm_dpo/q_t": 0.38388434052467346, + "grad_norm": 10.304213523864746, + "learning_rate": 1.965167291983757e-07, + "logits/chosen": -2.1975674629211426, + "logits/rejected": -1.543036699295044, + "logps/chosen": -280.13983154296875, + "logps/ref_chosen": -108.22152709960938, + "logps/ref_rejected": -111.8646469116211, + "logps/rejected": -397.0341796875, + "loss": 1.0065, + "margin_dpo/margin_mean": 113.251220703125, + "margin_dpo/margin_std": 105.12625122070312, + "step": 418 + }, + { + "KL/chosen_KL_mean": -179.26296997070312, + "KL/mean": -228.87451171875, + "KL/rejected_KL_mean": -278.48602294921875, + "KL/std": 95.41441345214844, + "epoch": 0.6152716593245228, + "fcm_dpo/beta": 0.004333334043622017, + "fcm_dpo/delta": -0.03156277537345886, + "fcm_dpo/margin": 99.2230453491211, + "fcm_dpo/q_t": 0.39901256561279297, + "grad_norm": 11.327917098999023, + "learning_rate": 1.9526382029472988e-07, + "logits/chosen": -1.536212682723999, + "logits/rejected": -1.0965124368667603, + "logps/chosen": -276.4462585449219, + "logps/ref_chosen": -97.18328094482422, + "logps/ref_rejected": -98.18531799316406, + "logps/rejected": -376.67132568359375, + "loss": 1.0518, + "margin_dpo/margin_mean": 99.22303771972656, + "margin_dpo/margin_std": 104.15109252929688, + "step": 419 + }, + { + "KL/chosen_KL_mean": -183.4001007080078, + "KL/mean": -227.8697509765625, + "KL/rejected_KL_mean": -272.3393859863281, + "KL/std": 100.98008728027344, + "epoch": 0.6167400881057269, + "fcm_dpo/beta": 0.004330971743911505, + "fcm_dpo/delta": 0.015145653858780861, + "fcm_dpo/margin": 88.93927001953125, + "fcm_dpo/q_t": 0.41050124168395996, + "grad_norm": 14.192485809326172, + "learning_rate": 1.9401235374032425e-07, + "logits/chosen": -2.2598860263824463, + "logits/rejected": -1.1676476001739502, + "logps/chosen": -297.7085876464844, + "logps/ref_chosen": -114.30848693847656, + "logps/ref_rejected": -75.68355560302734, + "logps/rejected": -348.02294921875, + "loss": 1.0985, + "margin_dpo/margin_mean": 88.93927001953125, + "margin_dpo/margin_std": 116.28192901611328, + "step": 420 + }, + { + "KL/chosen_KL_mean": -174.35287475585938, + "KL/mean": -223.04075622558594, + "KL/rejected_KL_mean": -271.7286376953125, + "KL/std": 103.98757934570312, + "epoch": 0.618208516886931, + "fcm_dpo/beta": 0.0043023196049034595, + "fcm_dpo/delta": -0.01982852630317211, + "fcm_dpo/margin": 97.37576293945312, + "fcm_dpo/q_t": 0.40179944038391113, + "grad_norm": 7.9784626960754395, + "learning_rate": 1.9276236251246653e-07, + "logits/chosen": -1.8924380540847778, + "logits/rejected": -1.5231058597564697, + "logps/chosen": -260.23272705078125, + "logps/ref_chosen": -85.87985229492188, + "logps/ref_rejected": -96.33648681640625, + "logps/rejected": -368.06512451171875, + "loss": 1.0689, + "margin_dpo/margin_mean": 97.37577056884766, + "margin_dpo/margin_std": 114.62490844726562, + "step": 421 + }, + { + "KL/chosen_KL_mean": -172.6793212890625, + "KL/mean": -219.54205322265625, + "KL/rejected_KL_mean": -266.40478515625, + "KL/std": 96.2136001586914, + "epoch": 0.6196769456681351, + "fcm_dpo/beta": 0.004305709153413773, + "fcm_dpo/delta": -0.0037238094955682755, + "fcm_dpo/margin": 93.72547912597656, + "fcm_dpo/q_t": 0.4056052267551422, + "grad_norm": 11.676365852355957, + "learning_rate": 1.9151387954958792e-07, + "logits/chosen": -2.4220902919769287, + "logits/rejected": -2.059406042098999, + "logps/chosen": -273.159912109375, + "logps/ref_chosen": -100.48060607910156, + "logps/ref_rejected": -94.40821838378906, + "logps/rejected": -360.8130187988281, + "loss": 1.0737, + "margin_dpo/margin_mean": 93.72547912597656, + "margin_dpo/margin_std": 107.88322448730469, + "step": 422 + }, + { + "KL/chosen_KL_mean": -159.6549072265625, + "KL/mean": -207.25613403320312, + "KL/rejected_KL_mean": -254.8573760986328, + "KL/std": 115.31645202636719, + "epoch": 0.6211453744493393, + "fcm_dpo/beta": 0.004309993702918291, + "fcm_dpo/delta": -0.011030737310647964, + "fcm_dpo/margin": 95.20246124267578, + "fcm_dpo/q_t": 0.40627768635749817, + "grad_norm": 9.54808235168457, + "learning_rate": 1.902669377503756e-07, + "logits/chosen": -1.8968216180801392, + "logits/rejected": -1.7785660028457642, + "logps/chosen": -238.10484313964844, + "logps/ref_chosen": -78.44993591308594, + "logps/ref_rejected": -92.04652404785156, + "logps/rejected": -346.9039001464844, + "loss": 1.0931, + "margin_dpo/margin_mean": 95.20245361328125, + "margin_dpo/margin_std": 130.82846069335938, + "step": 423 + }, + { + "KL/chosen_KL_mean": -179.8658447265625, + "KL/mean": -226.17735290527344, + "KL/rejected_KL_mean": -272.4888610839844, + "KL/std": 102.47991943359375, + "epoch": 0.6226138032305433, + "fcm_dpo/beta": 0.004290757235139608, + "fcm_dpo/delta": 0.0026815487071871758, + "fcm_dpo/margin": 92.62301635742188, + "fcm_dpo/q_t": 0.4062444865703583, + "grad_norm": 8.440460205078125, + "learning_rate": 1.890215699729057e-07, + "logits/chosen": -1.7731688022613525, + "logits/rejected": -1.1821210384368896, + "logps/chosen": -267.5081787109375, + "logps/ref_chosen": -87.64236450195312, + "logps/ref_rejected": -72.36566162109375, + "logps/rejected": -344.8545227050781, + "loss": 1.0821, + "margin_dpo/margin_mean": 92.62301635742188, + "margin_dpo/margin_std": 110.81394958496094, + "step": 424 + }, + { + "KL/chosen_KL_mean": -186.88282775878906, + "KL/mean": -218.54039001464844, + "KL/rejected_KL_mean": -250.1979522705078, + "KL/std": 106.28575134277344, + "epoch": 0.6240822320117474, + "fcm_dpo/beta": 0.004330229014158249, + "fcm_dpo/delta": 0.04308256134390831, + "fcm_dpo/margin": 63.31511688232422, + "fcm_dpo/q_t": 0.4383828341960907, + "grad_norm": 10.14073371887207, + "learning_rate": 1.8777780903377732e-07, + "logits/chosen": -1.6557912826538086, + "logits/rejected": -1.5911717414855957, + "logps/chosen": -265.4026184082031, + "logps/ref_chosen": -78.51979064941406, + "logps/ref_rejected": -102.74864196777344, + "logps/rejected": -352.94659423828125, + "loss": 1.2085, + "margin_dpo/margin_mean": 63.31511688232422, + "margin_dpo/margin_std": 132.601318359375, + "step": 425 + }, + { + "KL/chosen_KL_mean": -159.47183227539062, + "KL/mean": -205.91732788085938, + "KL/rejected_KL_mean": -252.36279296875, + "KL/std": 97.26757049560547, + "epoch": 0.6255506607929515, + "fcm_dpo/beta": 0.004338660277426243, + "fcm_dpo/delta": -0.0032616257667541504, + "fcm_dpo/margin": 92.89096069335938, + "fcm_dpo/q_t": 0.40564587712287903, + "grad_norm": 10.055276870727539, + "learning_rate": 1.8653568770724803e-07, + "logits/chosen": -1.6178650856018066, + "logits/rejected": -0.8660005331039429, + "logps/chosen": -267.9776611328125, + "logps/ref_chosen": -108.50582885742188, + "logps/ref_rejected": -88.300048828125, + "logps/rejected": -340.662841796875, + "loss": 1.0816, + "margin_dpo/margin_mean": 92.89096069335938, + "margin_dpo/margin_std": 113.41606140136719, + "step": 426 + }, + { + "KL/chosen_KL_mean": -131.5442352294922, + "KL/mean": -174.1750946044922, + "KL/rejected_KL_mean": -216.8059539794922, + "KL/std": 92.49711608886719, + "epoch": 0.6270190895741556, + "fcm_dpo/beta": 0.0043511101976037025, + "fcm_dpo/delta": 0.03008575737476349, + "fcm_dpo/margin": 85.26173400878906, + "fcm_dpo/q_t": 0.4125143885612488, + "grad_norm": 8.564836502075195, + "learning_rate": 1.8529523872436977e-07, + "logits/chosen": -2.5279462337493896, + "logits/rejected": -1.9070940017700195, + "logps/chosen": -230.66470336914062, + "logps/ref_chosen": -99.12046813964844, + "logps/ref_rejected": -85.724609375, + "logps/rejected": -302.53057861328125, + "loss": 1.0942, + "margin_dpo/margin_mean": 85.26173400878906, + "margin_dpo/margin_std": 99.58175659179688, + "step": 427 + }, + { + "KL/chosen_KL_mean": -188.04348754882812, + "KL/mean": -224.72398376464844, + "KL/rejected_KL_mean": -261.40447998046875, + "KL/std": 104.38056182861328, + "epoch": 0.6284875183553598, + "fcm_dpo/beta": 0.004351767245680094, + "fcm_dpo/delta": -0.002016030717641115, + "fcm_dpo/margin": 73.36100006103516, + "fcm_dpo/q_t": 0.4255887269973755, + "grad_norm": 9.564895629882812, + "learning_rate": 1.8405649477212697e-07, + "logits/chosen": -1.875936508178711, + "logits/rejected": -1.6235854625701904, + "logps/chosen": -294.01275634765625, + "logps/ref_chosen": -105.96925354003906, + "logps/ref_rejected": -109.10218048095703, + "logps/rejected": -370.50665283203125, + "loss": 1.1544, + "margin_dpo/margin_mean": 73.36100769042969, + "margin_dpo/margin_std": 116.09918975830078, + "step": 428 + }, + { + "KL/chosen_KL_mean": -172.08187866210938, + "KL/mean": -218.90805053710938, + "KL/rejected_KL_mean": -265.73419189453125, + "KL/std": 109.05897521972656, + "epoch": 0.6299559471365639, + "fcm_dpo/beta": 0.004321882501244545, + "fcm_dpo/delta": -0.06914899498224258, + "fcm_dpo/margin": 93.65232849121094, + "fcm_dpo/q_t": 0.4064915180206299, + "grad_norm": 8.121362686157227, + "learning_rate": 1.828194884925749e-07, + "logits/chosen": -1.9417109489440918, + "logits/rejected": -1.2245090007781982, + "logps/chosen": -285.62677001953125, + "logps/ref_chosen": -113.54487609863281, + "logps/ref_rejected": -98.24201965332031, + "logps/rejected": -363.9762268066406, + "loss": 1.0937, + "margin_dpo/margin_mean": 93.65232849121094, + "margin_dpo/margin_std": 124.04888916015625, + "step": 429 + }, + { + "KL/chosen_KL_mean": -163.50238037109375, + "KL/mean": -199.7823486328125, + "KL/rejected_KL_mean": -236.06234741210938, + "KL/std": 101.12538146972656, + "epoch": 0.631424375917768, + "fcm_dpo/beta": 0.004299278371036053, + "fcm_dpo/delta": 0.01693517528474331, + "fcm_dpo/margin": 72.5599365234375, + "fcm_dpo/q_t": 0.4275030493736267, + "grad_norm": 8.765769958496094, + "learning_rate": 1.8158425248197928e-07, + "logits/chosen": -1.235914707183838, + "logits/rejected": -1.1994497776031494, + "logps/chosen": -254.82174682617188, + "logps/ref_chosen": -91.31936645507812, + "logps/ref_rejected": -110.1096420288086, + "logps/rejected": -346.1719970703125, + "loss": 1.1596, + "margin_dpo/margin_mean": 72.5599365234375, + "margin_dpo/margin_std": 117.38682556152344, + "step": 430 + }, + { + "KL/chosen_KL_mean": -165.48614501953125, + "KL/mean": -208.19583129882812, + "KL/rejected_KL_mean": -250.905517578125, + "KL/std": 103.6623306274414, + "epoch": 0.6328928046989721, + "fcm_dpo/beta": 0.004321317188441753, + "fcm_dpo/delta": 0.03206493332982063, + "fcm_dpo/margin": 85.41938781738281, + "fcm_dpo/q_t": 0.41387611627578735, + "grad_norm": 9.80251407623291, + "learning_rate": 1.8035081928995788e-07, + "logits/chosen": -2.067448377609253, + "logits/rejected": -1.6924456357955933, + "logps/chosen": -258.6673583984375, + "logps/ref_chosen": -93.18122100830078, + "logps/ref_rejected": -98.13226318359375, + "logps/rejected": -349.03778076171875, + "loss": 1.113, + "margin_dpo/margin_mean": 85.41938781738281, + "margin_dpo/margin_std": 117.9261703491211, + "step": 431 + }, + { + "KL/chosen_KL_mean": -144.88470458984375, + "KL/mean": -196.82659912109375, + "KL/rejected_KL_mean": -248.76849365234375, + "KL/std": 107.77914428710938, + "epoch": 0.6343612334801763, + "fcm_dpo/beta": 0.004315475933253765, + "fcm_dpo/delta": -0.05062083154916763, + "fcm_dpo/margin": 103.88379669189453, + "fcm_dpo/q_t": 0.3944205641746521, + "grad_norm": 8.59517765045166, + "learning_rate": 1.791192214186223e-07, + "logits/chosen": -1.3122472763061523, + "logits/rejected": -0.7425343990325928, + "logps/chosen": -249.31947326660156, + "logps/ref_chosen": -104.43478393554688, + "logps/ref_rejected": -105.08955383300781, + "logps/rejected": -353.8580322265625, + "loss": 1.0382, + "margin_dpo/margin_mean": 103.88379669189453, + "margin_dpo/margin_std": 103.88294219970703, + "step": 432 + }, + { + "KL/chosen_KL_mean": -172.14376831054688, + "KL/mean": -212.92405700683594, + "KL/rejected_KL_mean": -253.70433044433594, + "KL/std": 101.35973358154297, + "epoch": 0.6358296622613803, + "fcm_dpo/beta": 0.004284270573407412, + "fcm_dpo/delta": -0.014788562431931496, + "fcm_dpo/margin": 81.5605697631836, + "fcm_dpo/q_t": 0.4182785153388977, + "grad_norm": 8.573363304138184, + "learning_rate": 1.7788949132172193e-07, + "logits/chosen": -1.595853328704834, + "logits/rejected": -1.2731595039367676, + "logps/chosen": -261.98699951171875, + "logps/ref_chosen": -89.84322357177734, + "logps/ref_rejected": -101.73345947265625, + "logps/rejected": -355.43780517578125, + "loss": 1.1263, + "margin_dpo/margin_mean": 81.56056213378906, + "margin_dpo/margin_std": 114.823974609375, + "step": 433 + }, + { + "KL/chosen_KL_mean": -149.53402709960938, + "KL/mean": -195.58148193359375, + "KL/rejected_KL_mean": -241.62890625, + "KL/std": 112.89448547363281, + "epoch": 0.6372980910425844, + "fcm_dpo/beta": 0.004284962080419064, + "fcm_dpo/delta": 0.005579883232712746, + "fcm_dpo/margin": 92.09489440917969, + "fcm_dpo/q_t": 0.41094207763671875, + "grad_norm": 11.747979164123535, + "learning_rate": 1.7666166140378853e-07, + "logits/chosen": -2.36212158203125, + "logits/rejected": -1.6805628538131714, + "logps/chosen": -247.2265625, + "logps/ref_chosen": -97.6925277709961, + "logps/ref_rejected": -84.09129333496094, + "logps/rejected": -325.72021484375, + "loss": 1.1017, + "margin_dpo/margin_mean": 92.09489440917969, + "margin_dpo/margin_std": 131.29226684570312, + "step": 434 + }, + { + "KL/chosen_KL_mean": -147.0272979736328, + "KL/mean": -194.65455627441406, + "KL/rejected_KL_mean": -242.28182983398438, + "KL/std": 104.88186645507812, + "epoch": 0.6387665198237885, + "fcm_dpo/beta": 0.004283071495592594, + "fcm_dpo/delta": -0.00834620464593172, + "fcm_dpo/margin": 95.2545166015625, + "fcm_dpo/q_t": 0.4055781662464142, + "grad_norm": 9.863697052001953, + "learning_rate": 1.7543576401928218e-07, + "logits/chosen": -2.391676187515259, + "logits/rejected": -1.94583261013031, + "logps/chosen": -233.19921875, + "logps/ref_chosen": -86.17192077636719, + "logps/ref_rejected": -93.751708984375, + "logps/rejected": -336.03350830078125, + "loss": 1.0779, + "margin_dpo/margin_mean": 95.2545166015625, + "margin_dpo/margin_std": 116.85872650146484, + "step": 435 + }, + { + "KL/chosen_KL_mean": -122.24266815185547, + "KL/mean": -171.98390197753906, + "KL/rejected_KL_mean": -221.72512817382812, + "KL/std": 88.58323669433594, + "epoch": 0.6402349486049926, + "fcm_dpo/beta": 0.0042562964372336864, + "fcm_dpo/delta": -0.02449946664273739, + "fcm_dpo/margin": 99.48246765136719, + "fcm_dpo/q_t": 0.3992065191268921, + "grad_norm": 9.942447662353516, + "learning_rate": 1.742118314717391e-07, + "logits/chosen": -1.8173494338989258, + "logits/rejected": -0.9608921408653259, + "logps/chosen": -228.02976989746094, + "logps/ref_chosen": -105.78710174560547, + "logps/ref_rejected": -88.62471008300781, + "logps/rejected": -310.349853515625, + "loss": 1.0434, + "margin_dpo/margin_mean": 99.48246765136719, + "margin_dpo/margin_std": 91.65057373046875, + "step": 436 + }, + { + "KL/chosen_KL_mean": -143.17430114746094, + "KL/mean": -183.54501342773438, + "KL/rejected_KL_mean": -223.9157257080078, + "KL/std": 95.59382629394531, + "epoch": 0.6417033773861968, + "fcm_dpo/beta": 0.0042947446927428246, + "fcm_dpo/delta": 0.05505699664354324, + "fcm_dpo/margin": 80.74140930175781, + "fcm_dpo/q_t": 0.41714122891426086, + "grad_norm": 10.052477836608887, + "learning_rate": 1.7298989601292036e-07, + "logits/chosen": -1.827838659286499, + "logits/rejected": -1.155079960823059, + "logps/chosen": -239.23635864257812, + "logps/ref_chosen": -96.06204223632812, + "logps/ref_rejected": -89.01220703125, + "logps/rejected": -312.92791748046875, + "loss": 1.1091, + "margin_dpo/margin_mean": 80.74140930175781, + "margin_dpo/margin_std": 94.30926513671875, + "step": 437 + }, + { + "KL/chosen_KL_mean": -148.0892333984375, + "KL/mean": -188.77459716796875, + "KL/rejected_KL_mean": -229.45993041992188, + "KL/std": 96.01412963867188, + "epoch": 0.6431718061674009, + "fcm_dpo/beta": 0.0042906044982373714, + "fcm_dpo/delta": -0.013005274347960949, + "fcm_dpo/margin": 81.37069702148438, + "fcm_dpo/q_t": 0.4172900915145874, + "grad_norm": 9.351896286010742, + "learning_rate": 1.7176998984196144e-07, + "logits/chosen": -2.1286299228668213, + "logits/rejected": -1.3172246217727661, + "logps/chosen": -249.94461059570312, + "logps/ref_chosen": -101.85537719726562, + "logps/ref_rejected": -89.44766235351562, + "logps/rejected": -318.9075927734375, + "loss": 1.1159, + "margin_dpo/margin_mean": 81.37069702148438, + "margin_dpo/margin_std": 103.24181365966797, + "step": 438 + }, + { + "KL/chosen_KL_mean": -158.1829833984375, + "KL/mean": -199.0579376220703, + "KL/rejected_KL_mean": -239.93287658691406, + "KL/std": 104.05538940429688, + "epoch": 0.644640234948605, + "fcm_dpo/beta": 0.004260015208274126, + "fcm_dpo/delta": -0.035774268209934235, + "fcm_dpo/margin": 81.7499008178711, + "fcm_dpo/q_t": 0.4178205728530884, + "grad_norm": 9.368892669677734, + "learning_rate": 1.7055214510452458e-07, + "logits/chosen": -1.7336612939834595, + "logits/rejected": -1.5019488334655762, + "logps/chosen": -239.93861389160156, + "logps/ref_chosen": -81.75563049316406, + "logps/ref_rejected": -90.58636474609375, + "logps/rejected": -330.51922607421875, + "loss": 1.1161, + "margin_dpo/margin_mean": 81.7499008178711, + "margin_dpo/margin_std": 101.83357238769531, + "step": 439 + }, + { + "KL/chosen_KL_mean": -141.52011108398438, + "KL/mean": -192.32632446289062, + "KL/rejected_KL_mean": -243.1325225830078, + "KL/std": 105.81680297851562, + "epoch": 0.6461086637298091, + "fcm_dpo/beta": 0.004220467060804367, + "fcm_dpo/delta": -0.030752703547477722, + "fcm_dpo/margin": 101.61243438720703, + "fcm_dpo/q_t": 0.40036624670028687, + "grad_norm": 7.317651748657227, + "learning_rate": 1.6933639389195134e-07, + "logits/chosen": -2.4706578254699707, + "logits/rejected": -1.9770441055297852, + "logps/chosen": -247.1611785888672, + "logps/ref_chosen": -105.64108276367188, + "logps/ref_rejected": -103.4010009765625, + "logps/rejected": -346.5335388183594, + "loss": 1.0622, + "margin_dpo/margin_mean": 101.61243438720703, + "margin_dpo/margin_std": 117.44085693359375, + "step": 440 + }, + { + "KL/chosen_KL_mean": -182.84115600585938, + "KL/mean": -225.94801330566406, + "KL/rejected_KL_mean": -269.0549011230469, + "KL/std": 100.33851623535156, + "epoch": 0.6475770925110133, + "fcm_dpo/beta": 0.004250752739608288, + "fcm_dpo/delta": 0.03481417894363403, + "fcm_dpo/margin": 86.21375274658203, + "fcm_dpo/q_t": 0.4153675436973572, + "grad_norm": 8.287605285644531, + "learning_rate": 1.681227682404166e-07, + "logits/chosen": -1.7635142803192139, + "logits/rejected": -1.4059213399887085, + "logps/chosen": -274.37066650390625, + "logps/ref_chosen": -91.529541015625, + "logps/ref_rejected": -103.619384765625, + "logps/rejected": -372.67425537109375, + "loss": 1.1031, + "margin_dpo/margin_mean": 86.2137451171875, + "margin_dpo/margin_std": 109.54653930664062, + "step": 441 + }, + { + "KL/chosen_KL_mean": -178.48822021484375, + "KL/mean": -232.74468994140625, + "KL/rejected_KL_mean": -287.00115966796875, + "KL/std": 105.9219970703125, + "epoch": 0.6490455212922174, + "fcm_dpo/beta": 0.004226325079798698, + "fcm_dpo/delta": -0.061407655477523804, + "fcm_dpo/margin": 108.51292419433594, + "fcm_dpo/q_t": 0.3918910026550293, + "grad_norm": 10.600053787231445, + "learning_rate": 1.669113001300851e-07, + "logits/chosen": -1.7542593479156494, + "logits/rejected": -1.2927069664001465, + "logps/chosen": -263.2657775878906, + "logps/ref_chosen": -84.77755737304688, + "logps/ref_rejected": -83.82415771484375, + "logps/rejected": -370.8253173828125, + "loss": 1.0273, + "margin_dpo/margin_mean": 108.51292419433594, + "margin_dpo/margin_std": 106.8646240234375, + "step": 442 + }, + { + "KL/chosen_KL_mean": -174.8520965576172, + "KL/mean": -208.32086181640625, + "KL/rejected_KL_mean": -241.78961181640625, + "KL/std": 103.3300552368164, + "epoch": 0.6505139500734214, + "fcm_dpo/beta": 0.004232651088386774, + "fcm_dpo/delta": 0.025125082582235336, + "fcm_dpo/margin": 66.9375, + "fcm_dpo/q_t": 0.43301093578338623, + "grad_norm": 9.790900230407715, + "learning_rate": 1.6570202148426815e-07, + "logits/chosen": -1.9333646297454834, + "logits/rejected": -1.384993076324463, + "logps/chosen": -277.5013732910156, + "logps/ref_chosen": -102.64927673339844, + "logps/ref_rejected": -93.03807067871094, + "logps/rejected": -334.82769775390625, + "loss": 1.1682, + "margin_dpo/margin_mean": 66.93750762939453, + "margin_dpo/margin_std": 100.76521301269531, + "step": 443 + }, + { + "KL/chosen_KL_mean": -178.84686279296875, + "KL/mean": -218.85072326660156, + "KL/rejected_KL_mean": -258.85455322265625, + "KL/std": 100.89628601074219, + "epoch": 0.6519823788546255, + "fcm_dpo/beta": 0.004274386912584305, + "fcm_dpo/delta": 0.06000368297100067, + "fcm_dpo/margin": 80.0076904296875, + "fcm_dpo/q_t": 0.4206140637397766, + "grad_norm": 10.3822660446167, + "learning_rate": 1.6449496416858282e-07, + "logits/chosen": -1.6811553239822388, + "logits/rejected": -1.516911268234253, + "logps/chosen": -266.7666015625, + "logps/ref_chosen": -87.91971588134766, + "logps/ref_rejected": -103.32345581054688, + "logps/rejected": -362.17803955078125, + "loss": 1.1325, + "margin_dpo/margin_mean": 80.0076904296875, + "margin_dpo/margin_std": 116.80731201171875, + "step": 444 + }, + { + "KL/chosen_KL_mean": -157.44281005859375, + "KL/mean": -203.68724060058594, + "KL/rejected_KL_mean": -249.9316864013672, + "KL/std": 98.18214416503906, + "epoch": 0.6534508076358296, + "fcm_dpo/beta": 0.0042739138007164, + "fcm_dpo/delta": 0.004733650013804436, + "fcm_dpo/margin": 92.4888916015625, + "fcm_dpo/q_t": 0.40683087706565857, + "grad_norm": 8.112725257873535, + "learning_rate": 1.6329015999011182e-07, + "logits/chosen": -1.9232900142669678, + "logits/rejected": -1.5068702697753906, + "logps/chosen": -258.8436584472656, + "logps/ref_chosen": -101.40086364746094, + "logps/ref_rejected": -99.03790283203125, + "logps/rejected": -348.9696044921875, + "loss": 1.0771, + "margin_dpo/margin_mean": 92.48888397216797, + "margin_dpo/margin_std": 103.79328155517578, + "step": 445 + }, + { + "KL/chosen_KL_mean": -150.9923095703125, + "KL/mean": -192.4937744140625, + "KL/rejected_KL_mean": -233.99525451660156, + "KL/std": 103.28872680664062, + "epoch": 0.6549192364170338, + "fcm_dpo/beta": 0.004322713706642389, + "fcm_dpo/delta": 0.04259772598743439, + "fcm_dpo/margin": 83.00293731689453, + "fcm_dpo/q_t": 0.41559872031211853, + "grad_norm": 12.37559700012207, + "learning_rate": 1.6208764069656578e-07, + "logits/chosen": -1.9989771842956543, + "logits/rejected": -1.861940860748291, + "logps/chosen": -238.41464233398438, + "logps/ref_chosen": -87.4223403930664, + "logps/ref_rejected": -106.70075988769531, + "logps/rejected": -340.6960144042969, + "loss": 1.1038, + "margin_dpo/margin_mean": 83.00294494628906, + "margin_dpo/margin_std": 99.38538360595703, + "step": 446 + }, + { + "KL/chosen_KL_mean": -148.73526000976562, + "KL/mean": -203.46554565429688, + "KL/rejected_KL_mean": -258.19586181640625, + "KL/std": 107.6642074584961, + "epoch": 0.6563876651982379, + "fcm_dpo/beta": 0.004278257489204407, + "fcm_dpo/delta": -0.07166311144828796, + "fcm_dpo/margin": 109.4605941772461, + "fcm_dpo/q_t": 0.39114248752593994, + "grad_norm": 8.289060592651367, + "learning_rate": 1.608874379754465e-07, + "logits/chosen": -2.037332773208618, + "logits/rejected": -1.9929592609405518, + "logps/chosen": -232.3505096435547, + "logps/ref_chosen": -83.6152572631836, + "logps/ref_rejected": -104.91239929199219, + "logps/rejected": -363.1082763671875, + "loss": 1.0381, + "margin_dpo/margin_mean": 109.46060180664062, + "margin_dpo/margin_std": 123.45143127441406, + "step": 447 + }, + { + "KL/chosen_KL_mean": -161.17611694335938, + "KL/mean": -213.86087036132812, + "KL/rejected_KL_mean": -266.5456237792969, + "KL/std": 112.47804260253906, + "epoch": 0.657856093979442, + "fcm_dpo/beta": 0.004249397665262222, + "fcm_dpo/delta": -0.050152819603681564, + "fcm_dpo/margin": 105.36949157714844, + "fcm_dpo/q_t": 0.3967461884021759, + "grad_norm": 9.608505249023438, + "learning_rate": 1.5968958345321177e-07, + "logits/chosen": -1.7243329286575317, + "logits/rejected": -1.4556059837341309, + "logps/chosen": -253.7519073486328, + "logps/ref_chosen": -92.5757827758789, + "logps/ref_rejected": -107.68977355957031, + "logps/rejected": -374.2353820800781, + "loss": 1.0638, + "margin_dpo/margin_mean": 105.36949157714844, + "margin_dpo/margin_std": 131.62281799316406, + "step": 448 + }, + { + "KL/chosen_KL_mean": -158.67913818359375, + "KL/mean": -213.29476928710938, + "KL/rejected_KL_mean": -267.9104309082031, + "KL/std": 105.8449478149414, + "epoch": 0.6593245227606461, + "fcm_dpo/beta": 0.004182309843599796, + "fcm_dpo/delta": -0.059573158621788025, + "fcm_dpo/margin": 109.23129272460938, + "fcm_dpo/q_t": 0.39313215017318726, + "grad_norm": 8.536856651306152, + "learning_rate": 1.584941086944423e-07, + "logits/chosen": -1.708990216255188, + "logits/rejected": -1.2822885513305664, + "logps/chosen": -261.07806396484375, + "logps/ref_chosen": -102.39893341064453, + "logps/ref_rejected": -95.14886474609375, + "logps/rejected": -363.0592956542969, + "loss": 1.0413, + "margin_dpo/margin_mean": 109.23128509521484, + "margin_dpo/margin_std": 121.24258422851562, + "step": 449 + }, + { + "KL/chosen_KL_mean": -141.81605529785156, + "KL/mean": -192.88380432128906, + "KL/rejected_KL_mean": -243.95156860351562, + "KL/std": 106.66605377197266, + "epoch": 0.6607929515418502, + "fcm_dpo/beta": 0.004151148721575737, + "fcm_dpo/delta": -0.025080326944589615, + "fcm_dpo/margin": 102.135498046875, + "fcm_dpo/q_t": 0.400248646736145, + "grad_norm": 9.05582332611084, + "learning_rate": 1.573010452010098e-07, + "logits/chosen": -2.3348100185394287, + "logits/rejected": -2.1421189308166504, + "logps/chosen": -228.8089141845703, + "logps/ref_chosen": -86.99285888671875, + "logps/ref_rejected": -108.53203582763672, + "logps/rejected": -352.48358154296875, + "loss": 1.0617, + "margin_dpo/margin_mean": 102.135498046875, + "margin_dpo/margin_std": 116.01069641113281, + "step": 450 + }, + { + "KL/chosen_KL_mean": -160.21636962890625, + "KL/mean": -200.87149047851562, + "KL/rejected_KL_mean": -241.526611328125, + "KL/std": 105.99018859863281, + "epoch": 0.6622613803230544, + "fcm_dpo/beta": 0.004193603992462158, + "fcm_dpo/delta": 0.06096739321947098, + "fcm_dpo/margin": 81.31025695800781, + "fcm_dpo/q_t": 0.4217182397842407, + "grad_norm": 13.223312377929688, + "learning_rate": 1.5611042441124687e-07, + "logits/chosen": -2.2851643562316895, + "logits/rejected": -1.7675621509552002, + "logps/chosen": -247.02764892578125, + "logps/ref_chosen": -86.811279296875, + "logps/ref_rejected": -79.8555908203125, + "logps/rejected": -321.3822021484375, + "loss": 1.1477, + "margin_dpo/margin_mean": 81.31025695800781, + "margin_dpo/margin_std": 132.5671844482422, + "step": 451 + }, + { + "KL/chosen_KL_mean": -139.62159729003906, + "KL/mean": -185.6328125, + "KL/rejected_KL_mean": -231.64402770996094, + "KL/std": 100.92298889160156, + "epoch": 0.6637298091042585, + "fcm_dpo/beta": 0.004152514040470123, + "fcm_dpo/delta": -0.057407230138778687, + "fcm_dpo/margin": 92.02242279052734, + "fcm_dpo/q_t": 0.4089251756668091, + "grad_norm": 8.04272747039795, + "learning_rate": 1.549222776991186e-07, + "logits/chosen": -1.9601504802703857, + "logits/rejected": -1.903310775756836, + "logps/chosen": -219.00123596191406, + "logps/ref_chosen": -79.379638671875, + "logps/ref_rejected": -103.71539306640625, + "logps/rejected": -335.35943603515625, + "loss": 1.081, + "margin_dpo/margin_mean": 92.02242279052734, + "margin_dpo/margin_std": 93.621337890625, + "step": 452 + }, + { + "KL/chosen_KL_mean": -169.2482147216797, + "KL/mean": -225.22848510742188, + "KL/rejected_KL_mean": -281.2087707519531, + "KL/std": 103.44989013671875, + "epoch": 0.6651982378854625, + "fcm_dpo/beta": 0.004117329604923725, + "fcm_dpo/delta": -0.06388219445943832, + "fcm_dpo/margin": 111.96056365966797, + "fcm_dpo/q_t": 0.39218634366989136, + "grad_norm": 12.239067077636719, + "learning_rate": 1.5373663637339584e-07, + "logits/chosen": -1.9556211233139038, + "logits/rejected": -1.4861172437667847, + "logps/chosen": -256.94342041015625, + "logps/ref_chosen": -87.6951904296875, + "logps/ref_rejected": -90.0582275390625, + "logps/rejected": -371.2669982910156, + "loss": 1.0273, + "margin_dpo/margin_mean": 111.96056365966797, + "margin_dpo/margin_std": 110.98213958740234, + "step": 453 + }, + { + "KL/chosen_KL_mean": -177.6344757080078, + "KL/mean": -222.7556610107422, + "KL/rejected_KL_mean": -267.8768615722656, + "KL/std": 106.05061340332031, + "epoch": 0.6666666666666666, + "fcm_dpo/beta": 0.004087763838469982, + "fcm_dpo/delta": -0.029437892138957977, + "fcm_dpo/margin": 90.24238586425781, + "fcm_dpo/q_t": 0.4147986173629761, + "grad_norm": 9.041144371032715, + "learning_rate": 1.5255353167683017e-07, + "logits/chosen": -2.0324559211730957, + "logits/rejected": -1.5890324115753174, + "logps/chosen": -267.2007141113281, + "logps/ref_chosen": -89.56623840332031, + "logps/ref_rejected": -92.92105102539062, + "logps/rejected": -360.79791259765625, + "loss": 1.1282, + "margin_dpo/margin_mean": 90.24238586425781, + "margin_dpo/margin_std": 137.99774169921875, + "step": 454 + }, + { + "KL/chosen_KL_mean": -166.79348754882812, + "KL/mean": -209.1966552734375, + "KL/rejected_KL_mean": -251.59982299804688, + "KL/std": 103.95687866210938, + "epoch": 0.6681350954478708, + "fcm_dpo/beta": 0.00407512579113245, + "fcm_dpo/delta": -0.001483845291659236, + "fcm_dpo/margin": 84.80633544921875, + "fcm_dpo/q_t": 0.4188273847103119, + "grad_norm": 9.120612144470215, + "learning_rate": 1.5137299478533064e-07, + "logits/chosen": -2.023879051208496, + "logits/rejected": -1.9858670234680176, + "logps/chosen": -244.42347717285156, + "logps/ref_chosen": -77.62998962402344, + "logps/ref_rejected": -118.97795104980469, + "logps/rejected": -370.5777893066406, + "loss": 1.1278, + "margin_dpo/margin_mean": 84.80635070800781, + "margin_dpo/margin_std": 119.13844299316406, + "step": 455 + }, + { + "KL/chosen_KL_mean": -168.79946899414062, + "KL/mean": -211.44793701171875, + "KL/rejected_KL_mean": -254.0963897705078, + "KL/std": 105.70134735107422, + "epoch": 0.6696035242290749, + "fcm_dpo/beta": 0.0041126469150185585, + "fcm_dpo/delta": 0.05084359273314476, + "fcm_dpo/margin": 85.29690551757812, + "fcm_dpo/q_t": 0.41757047176361084, + "grad_norm": 11.163372993469238, + "learning_rate": 1.5019505680714232e-07, + "logits/chosen": -1.9497323036193848, + "logits/rejected": -1.8175973892211914, + "logps/chosen": -258.4163513183594, + "logps/ref_chosen": -89.61686706542969, + "logps/ref_rejected": -109.5597152709961, + "logps/rejected": -363.6560974121094, + "loss": 1.1129, + "margin_dpo/margin_mean": 85.29690551757812, + "margin_dpo/margin_std": 107.62725830078125, + "step": 456 + }, + { + "KL/chosen_KL_mean": -137.1720733642578, + "KL/mean": -193.56689453125, + "KL/rejected_KL_mean": -249.96173095703125, + "KL/std": 105.61326599121094, + "epoch": 0.671071953010279, + "fcm_dpo/beta": 0.0040517039597034454, + "fcm_dpo/delta": -0.061049215495586395, + "fcm_dpo/margin": 112.78965759277344, + "fcm_dpo/q_t": 0.39104849100112915, + "grad_norm": 9.477270126342773, + "learning_rate": 1.4901974878202627e-07, + "logits/chosen": -2.048971176147461, + "logits/rejected": -1.7304811477661133, + "logps/chosen": -224.49375915527344, + "logps/ref_chosen": -87.32168579101562, + "logps/ref_rejected": -90.76660919189453, + "logps/rejected": -340.72833251953125, + "loss": 1.0192, + "margin_dpo/margin_mean": 112.78965759277344, + "margin_dpo/margin_std": 94.38288879394531, + "step": 457 + }, + { + "KL/chosen_KL_mean": -160.56723022460938, + "KL/mean": -201.14279174804688, + "KL/rejected_KL_mean": -241.71835327148438, + "KL/std": 105.17810821533203, + "epoch": 0.6725403817914831, + "fcm_dpo/beta": 0.004072901792824268, + "fcm_dpo/delta": 0.01642300933599472, + "fcm_dpo/margin": 81.15113830566406, + "fcm_dpo/q_t": 0.4237171709537506, + "grad_norm": 9.095541000366211, + "learning_rate": 1.4784710168044212e-07, + "logits/chosen": -1.5677192211151123, + "logits/rejected": -1.3271763324737549, + "logps/chosen": -254.08767700195312, + "logps/ref_chosen": -93.52044677734375, + "logps/ref_rejected": -103.36898803710938, + "logps/rejected": -345.08734130859375, + "loss": 1.1415, + "margin_dpo/margin_mean": 81.15113067626953, + "margin_dpo/margin_std": 122.51338958740234, + "step": 458 + }, + { + "KL/chosen_KL_mean": -152.54318237304688, + "KL/mean": -204.0995635986328, + "KL/rejected_KL_mean": -255.65594482421875, + "KL/std": 110.07746887207031, + "epoch": 0.6740088105726872, + "fcm_dpo/beta": 0.004050368443131447, + "fcm_dpo/delta": -0.018888771533966064, + "fcm_dpo/margin": 103.11275482177734, + "fcm_dpo/q_t": 0.4028448760509491, + "grad_norm": 6.887370586395264, + "learning_rate": 1.466771464027316e-07, + "logits/chosen": -1.6962974071502686, + "logits/rejected": -1.4479608535766602, + "logps/chosen": -228.2313995361328, + "logps/ref_chosen": -75.68820190429688, + "logps/ref_rejected": -92.17048645019531, + "logps/rejected": -347.826416015625, + "loss": 1.0673, + "margin_dpo/margin_mean": 103.11275482177734, + "margin_dpo/margin_std": 118.2739028930664, + "step": 459 + }, + { + "KL/chosen_KL_mean": -156.66201782226562, + "KL/mean": -204.0841064453125, + "KL/rejected_KL_mean": -251.50619506835938, + "KL/std": 106.85321044921875, + "epoch": 0.6754772393538914, + "fcm_dpo/beta": 0.0040590595453977585, + "fcm_dpo/delta": 0.015417791903018951, + "fcm_dpo/margin": 94.84416198730469, + "fcm_dpo/q_t": 0.4102768301963806, + "grad_norm": 10.083240509033203, + "learning_rate": 1.4550991377830423e-07, + "logits/chosen": -2.106811285018921, + "logits/rejected": -2.1325631141662598, + "logps/chosen": -237.7799072265625, + "logps/ref_chosen": -81.11788940429688, + "logps/ref_rejected": -110.31238555908203, + "logps/rejected": -361.8185729980469, + "loss": 1.0982, + "margin_dpo/margin_mean": 94.84416198730469, + "margin_dpo/margin_std": 122.52616882324219, + "step": 460 + }, + { + "KL/chosen_KL_mean": -192.66212463378906, + "KL/mean": -231.67709350585938, + "KL/rejected_KL_mean": -270.6921081542969, + "KL/std": 93.80046081542969, + "epoch": 0.6769456681350955, + "fcm_dpo/beta": 0.004091168288141489, + "fcm_dpo/delta": 0.01763981208205223, + "fcm_dpo/margin": 78.02999114990234, + "fcm_dpo/q_t": 0.4252292513847351, + "grad_norm": 8.205183982849121, + "learning_rate": 1.4434543456482518e-07, + "logits/chosen": -1.396677851676941, + "logits/rejected": -1.1992204189300537, + "logps/chosen": -274.24566650390625, + "logps/ref_chosen": -81.58352661132812, + "logps/ref_rejected": -93.87710571289062, + "logps/rejected": -364.5692138671875, + "loss": 1.1446, + "margin_dpo/margin_mean": 78.02998352050781, + "margin_dpo/margin_std": 113.22684478759766, + "step": 461 + }, + { + "KL/chosen_KL_mean": -146.56976318359375, + "KL/mean": -195.79876708984375, + "KL/rejected_KL_mean": -245.02777099609375, + "KL/std": 100.84786987304688, + "epoch": 0.6784140969162996, + "fcm_dpo/beta": 0.004094731993973255, + "fcm_dpo/delta": -0.003325372003018856, + "fcm_dpo/margin": 98.45801544189453, + "fcm_dpo/q_t": 0.40532612800598145, + "grad_norm": 8.431817054748535, + "learning_rate": 1.4318373944740484e-07, + "logits/chosen": -2.306246757507324, + "logits/rejected": -1.8081347942352295, + "logps/chosen": -240.768310546875, + "logps/ref_chosen": -94.19854736328125, + "logps/ref_rejected": -85.63162994384766, + "logps/rejected": -330.659423828125, + "loss": 1.0702, + "margin_dpo/margin_mean": 98.45801544189453, + "margin_dpo/margin_std": 108.60719299316406, + "step": 462 + }, + { + "KL/chosen_KL_mean": -142.71499633789062, + "KL/mean": -201.4064178466797, + "KL/rejected_KL_mean": -260.09783935546875, + "KL/std": 108.75080871582031, + "epoch": 0.6798825256975036, + "fcm_dpo/beta": 0.004063591826707125, + "fcm_dpo/delta": -0.08109010756015778, + "fcm_dpo/margin": 117.3828353881836, + "fcm_dpo/q_t": 0.38752663135528564, + "grad_norm": 13.197741508483887, + "learning_rate": 1.4202485903778976e-07, + "logits/chosen": -1.8157784938812256, + "logits/rejected": -1.3875007629394531, + "logps/chosen": -228.63973999023438, + "logps/ref_chosen": -85.92474365234375, + "logps/ref_rejected": -96.90184020996094, + "logps/rejected": -356.99969482421875, + "loss": 1.0233, + "margin_dpo/margin_mean": 117.38284301757812, + "margin_dpo/margin_std": 115.56355285644531, + "step": 463 + }, + { + "KL/chosen_KL_mean": -175.63992309570312, + "KL/mean": -223.54788208007812, + "KL/rejected_KL_mean": -271.45587158203125, + "KL/std": 108.31089782714844, + "epoch": 0.6813509544787077, + "fcm_dpo/beta": 0.004023114684969187, + "fcm_dpo/delta": 0.015042667277157307, + "fcm_dpo/margin": 95.8159408569336, + "fcm_dpo/q_t": 0.4102011024951935, + "grad_norm": 11.25102710723877, + "learning_rate": 1.4086882387355658e-07, + "logits/chosen": -1.8083809614181519, + "logits/rejected": -1.8889503479003906, + "logps/chosen": -255.32913208007812, + "logps/ref_chosen": -79.68919372558594, + "logps/ref_rejected": -107.29232025146484, + "logps/rejected": -378.7481689453125, + "loss": 1.0962, + "margin_dpo/margin_mean": 95.8159408569336, + "margin_dpo/margin_std": 123.82083129882812, + "step": 464 + }, + { + "KL/chosen_KL_mean": -166.99349975585938, + "KL/mean": -223.34901428222656, + "KL/rejected_KL_mean": -279.70452880859375, + "KL/std": 117.02505493164062, + "epoch": 0.6828193832599119, + "fcm_dpo/beta": 0.00399835966527462, + "fcm_dpo/delta": -0.05313951522111893, + "fcm_dpo/margin": 112.71102905273438, + "fcm_dpo/q_t": 0.39359086751937866, + "grad_norm": 8.355056762695312, + "learning_rate": 1.3971566441730714e-07, + "logits/chosen": -1.6797990798950195, + "logits/rejected": -1.3802953958511353, + "logps/chosen": -258.8537292480469, + "logps/ref_chosen": -91.8602294921875, + "logps/ref_rejected": -118.71001434326172, + "logps/rejected": -398.41455078125, + "loss": 1.032, + "margin_dpo/margin_mean": 112.7110366821289, + "margin_dpo/margin_std": 110.83982849121094, + "step": 465 + }, + { + "KL/chosen_KL_mean": -148.77413940429688, + "KL/mean": -206.149658203125, + "KL/rejected_KL_mean": -263.525146484375, + "KL/std": 111.46836853027344, + "epoch": 0.684287812041116, + "fcm_dpo/beta": 0.0039554983377456665, + "fcm_dpo/delta": -0.05651950463652611, + "fcm_dpo/margin": 114.75102233886719, + "fcm_dpo/q_t": 0.3933257758617401, + "grad_norm": 6.835327625274658, + "learning_rate": 1.3856541105586545e-07, + "logits/chosen": -1.3796275854110718, + "logits/rejected": -1.0510189533233643, + "logps/chosen": -233.4755401611328, + "logps/ref_chosen": -84.70140838623047, + "logps/ref_rejected": -96.05084228515625, + "logps/rejected": -359.57598876953125, + "loss": 1.0339, + "margin_dpo/margin_mean": 114.75102233886719, + "margin_dpo/margin_std": 117.52569580078125, + "step": 466 + }, + { + "KL/chosen_KL_mean": -190.7152557373047, + "KL/mean": -238.3769073486328, + "KL/rejected_KL_mean": -286.0385437011719, + "KL/std": 116.53121185302734, + "epoch": 0.6857562408223201, + "fcm_dpo/beta": 0.003923690877854824, + "fcm_dpo/delta": -0.02995622530579567, + "fcm_dpo/margin": 95.32328033447266, + "fcm_dpo/q_t": 0.41228967905044556, + "grad_norm": 12.012237548828125, + "learning_rate": 1.3741809409947729e-07, + "logits/chosen": -2.1684374809265137, + "logits/rejected": -1.5761277675628662, + "logps/chosen": -300.0135803222656, + "logps/ref_chosen": -109.29832458496094, + "logps/ref_rejected": -108.8436508178711, + "logps/rejected": -394.8822021484375, + "loss": 1.112, + "margin_dpo/margin_mean": 95.32328796386719, + "margin_dpo/margin_std": 131.83474731445312, + "step": 467 + }, + { + "KL/chosen_KL_mean": -147.77432250976562, + "KL/mean": -192.05670166015625, + "KL/rejected_KL_mean": -236.33908081054688, + "KL/std": 114.13752746582031, + "epoch": 0.6872246696035242, + "fcm_dpo/beta": 0.003950462676584721, + "fcm_dpo/delta": 0.051976777613162994, + "fcm_dpo/margin": 88.56475067138672, + "fcm_dpo/q_t": 0.4197770953178406, + "grad_norm": 8.753684043884277, + "learning_rate": 1.362737437810114e-07, + "logits/chosen": -1.8183279037475586, + "logits/rejected": -1.61104416847229, + "logps/chosen": -246.09597778320312, + "logps/ref_chosen": -98.32164764404297, + "logps/ref_rejected": -106.68048095703125, + "logps/rejected": -343.01953125, + "loss": 1.1304, + "margin_dpo/margin_mean": 88.56475067138672, + "margin_dpo/margin_std": 131.6105194091797, + "step": 468 + }, + { + "KL/chosen_KL_mean": -181.4677276611328, + "KL/mean": -235.90357971191406, + "KL/rejected_KL_mean": -290.33941650390625, + "KL/std": 113.92741394042969, + "epoch": 0.6886930983847284, + "fcm_dpo/beta": 0.0039282143115997314, + "fcm_dpo/delta": -0.02953207865357399, + "fcm_dpo/margin": 108.87169647216797, + "fcm_dpo/q_t": 0.4000743627548218, + "grad_norm": 9.85301685333252, + "learning_rate": 1.351323902551631e-07, + "logits/chosen": -2.242103099822998, + "logits/rejected": -1.823288917541504, + "logps/chosen": -278.23193359375, + "logps/ref_chosen": -96.76420593261719, + "logps/ref_rejected": -109.59500885009766, + "logps/rejected": -399.9344482421875, + "loss": 1.0614, + "margin_dpo/margin_mean": 108.87168884277344, + "margin_dpo/margin_std": 123.83409118652344, + "step": 469 + }, + { + "KL/chosen_KL_mean": -166.46792602539062, + "KL/mean": -216.46983337402344, + "KL/rejected_KL_mean": -266.47174072265625, + "KL/std": 105.8537826538086, + "epoch": 0.6901615271659325, + "fcm_dpo/beta": 0.0039438409730792046, + "fcm_dpo/delta": 0.005830895621329546, + "fcm_dpo/margin": 100.00384521484375, + "fcm_dpo/q_t": 0.4067780375480652, + "grad_norm": 8.820821762084961, + "learning_rate": 1.339940635976592e-07, + "logits/chosen": -1.6768912076950073, + "logits/rejected": -1.2582461833953857, + "logps/chosen": -249.96456909179688, + "logps/ref_chosen": -83.49665832519531, + "logps/ref_rejected": -88.48578643798828, + "logps/rejected": -354.95751953125, + "loss": 1.0717, + "margin_dpo/margin_mean": 100.00384521484375, + "margin_dpo/margin_std": 105.56761169433594, + "step": 470 + }, + { + "KL/chosen_KL_mean": -191.78350830078125, + "KL/mean": -239.0438232421875, + "KL/rejected_KL_mean": -286.30419921875, + "KL/std": 105.09850311279297, + "epoch": 0.6916299559471366, + "fcm_dpo/beta": 0.003960807342082262, + "fcm_dpo/delta": 0.026623040437698364, + "fcm_dpo/margin": 94.52067565917969, + "fcm_dpo/q_t": 0.4116513133049011, + "grad_norm": 8.525957107543945, + "learning_rate": 1.3285879380446563e-07, + "logits/chosen": -1.6159522533416748, + "logits/rejected": -1.201261281967163, + "logps/chosen": -280.2578125, + "logps/ref_chosen": -88.47430419921875, + "logps/ref_rejected": -90.48171997070312, + "logps/rejected": -376.785888671875, + "loss": 1.0886, + "margin_dpo/margin_mean": 94.52067565917969, + "margin_dpo/margin_std": 106.35237121582031, + "step": 471 + }, + { + "KL/chosen_KL_mean": -180.53919982910156, + "KL/mean": -236.33090209960938, + "KL/rejected_KL_mean": -292.1225891113281, + "KL/std": 105.68905639648438, + "epoch": 0.6930983847283406, + "fcm_dpo/beta": 0.003933281637728214, + "fcm_dpo/delta": -0.040896937251091, + "fcm_dpo/margin": 111.58338928222656, + "fcm_dpo/q_t": 0.397246778011322, + "grad_norm": 7.867846965789795, + "learning_rate": 1.317266107909975e-07, + "logits/chosen": -2.1931662559509277, + "logits/rejected": -1.498880386352539, + "logps/chosen": -305.77288818359375, + "logps/ref_chosen": -125.23370361328125, + "logps/ref_rejected": -121.05349731445312, + "logps/rejected": -413.17608642578125, + "loss": 1.0417, + "margin_dpo/margin_mean": 111.58338928222656, + "margin_dpo/margin_std": 112.47453308105469, + "step": 472 + }, + { + "KL/chosen_KL_mean": -204.4556121826172, + "KL/mean": -239.10324096679688, + "KL/rejected_KL_mean": -273.7508544921875, + "KL/std": 106.93110656738281, + "epoch": 0.6945668135095447, + "fcm_dpo/beta": 0.003934693988412619, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 69.29525756835938, + "fcm_dpo/q_t": 0.4367903470993042, + "grad_norm": 9.614045143127441, + "learning_rate": 1.3059754439133002e-07, + "logits/chosen": -1.9692649841308594, + "logits/rejected": -1.4926143884658813, + "logps/chosen": -300.0669860839844, + "logps/ref_chosen": -95.61137390136719, + "logps/ref_rejected": -88.15115356445312, + "logps/rejected": -361.9020080566406, + "loss": 1.1894, + "margin_dpo/margin_mean": 69.29525756835938, + "margin_dpo/margin_std": 125.0532455444336, + "step": 473 + }, + { + "KL/chosen_KL_mean": -190.392333984375, + "KL/mean": -226.40716552734375, + "KL/rejected_KL_mean": -262.4219970703125, + "KL/std": 101.19680786132812, + "epoch": 0.6960352422907489, + "fcm_dpo/beta": 0.003933422267436981, + "fcm_dpo/delta": -0.003233524737879634, + "fcm_dpo/margin": 72.0296630859375, + "fcm_dpo/q_t": 0.4336917996406555, + "grad_norm": 13.4172945022583, + "learning_rate": 1.2947162435741277e-07, + "logits/chosen": -1.468416452407837, + "logits/rejected": -1.277721881866455, + "logps/chosen": -271.8720703125, + "logps/ref_chosen": -81.47975158691406, + "logps/ref_rejected": -96.46562957763672, + "logps/rejected": -358.88763427734375, + "loss": 1.185, + "margin_dpo/margin_mean": 72.0296630859375, + "margin_dpo/margin_std": 123.54573059082031, + "step": 474 + }, + { + "KL/chosen_KL_mean": -144.3023681640625, + "KL/mean": -199.78546142578125, + "KL/rejected_KL_mean": -255.26856994628906, + "KL/std": 108.2242431640625, + "epoch": 0.697503671071953, + "fcm_dpo/beta": 0.0039138151332736015, + "fcm_dpo/delta": -0.03584384173154831, + "fcm_dpo/margin": 110.9662094116211, + "fcm_dpo/q_t": 0.3987465500831604, + "grad_norm": 7.769962787628174, + "learning_rate": 1.2834888035828596e-07, + "logits/chosen": -2.192134380340576, + "logits/rejected": -2.140284538269043, + "logps/chosen": -218.49835205078125, + "logps/ref_chosen": -74.19598388671875, + "logps/ref_rejected": -94.69242095947266, + "logps/rejected": -349.96099853515625, + "loss": 1.0534, + "margin_dpo/margin_mean": 110.96621704101562, + "margin_dpo/margin_std": 123.66749572753906, + "step": 475 + }, + { + "KL/chosen_KL_mean": -168.52392578125, + "KL/mean": -218.12692260742188, + "KL/rejected_KL_mean": -267.72991943359375, + "KL/std": 110.69952392578125, + "epoch": 0.6989720998531571, + "fcm_dpo/beta": 0.003901264164596796, + "fcm_dpo/delta": 0.013379184529185295, + "fcm_dpo/margin": 99.20600891113281, + "fcm_dpo/q_t": 0.41008588671684265, + "grad_norm": 9.65160083770752, + "learning_rate": 1.2722934197929802e-07, + "logits/chosen": -2.1841890811920166, + "logits/rejected": -1.8196310997009277, + "logps/chosen": -240.49502563476562, + "logps/ref_chosen": -71.97109985351562, + "logps/ref_rejected": -80.26224517822266, + "logps/rejected": -347.9921875, + "loss": 1.0885, + "margin_dpo/margin_mean": 99.20600891113281, + "margin_dpo/margin_std": 119.98284912109375, + "step": 476 + }, + { + "KL/chosen_KL_mean": -169.69662475585938, + "KL/mean": -218.6262969970703, + "KL/rejected_KL_mean": -267.55596923828125, + "KL/std": 97.36880493164062, + "epoch": 0.7004405286343612, + "fcm_dpo/beta": 0.003927314188331366, + "fcm_dpo/delta": 0.016274182125926018, + "fcm_dpo/margin": 97.8593521118164, + "fcm_dpo/q_t": 0.40946266055107117, + "grad_norm": 8.574677467346191, + "learning_rate": 1.2611303872132631e-07, + "logits/chosen": -2.130237579345703, + "logits/rejected": -1.2065376043319702, + "logps/chosen": -274.7021789550781, + "logps/ref_chosen": -105.00555419921875, + "logps/ref_rejected": -81.87843322753906, + "logps/rejected": -349.43438720703125, + "loss": 1.0925, + "margin_dpo/margin_mean": 97.85935974121094, + "margin_dpo/margin_std": 121.09370422363281, + "step": 477 + }, + { + "KL/chosen_KL_mean": -164.22198486328125, + "KL/mean": -215.223388671875, + "KL/rejected_KL_mean": -266.22479248046875, + "KL/std": 105.84947967529297, + "epoch": 0.7019089574155654, + "fcm_dpo/beta": 0.003927084617316723, + "fcm_dpo/delta": -0.0005975192179903388, + "fcm_dpo/margin": 102.00279235839844, + "fcm_dpo/q_t": 0.40583863854408264, + "grad_norm": 8.494955062866211, + "learning_rate": 1.2500000000000005e-07, + "logits/chosen": -1.4578558206558228, + "logits/rejected": -1.2777230739593506, + "logps/chosen": -241.01019287109375, + "logps/ref_chosen": -76.7882080078125, + "logps/ref_rejected": -90.43994140625, + "logps/rejected": -356.66473388671875, + "loss": 1.076, + "margin_dpo/margin_mean": 102.00279235839844, + "margin_dpo/margin_std": 117.43643188476562, + "step": 478 + }, + { + "KL/chosen_KL_mean": -166.7762451171875, + "KL/mean": -218.04080200195312, + "KL/rejected_KL_mean": -269.30535888671875, + "KL/std": 107.05646514892578, + "epoch": 0.7033773861967695, + "fcm_dpo/beta": 0.003938902169466019, + "fcm_dpo/delta": -0.004278495907783508, + "fcm_dpo/margin": 102.52912902832031, + "fcm_dpo/q_t": 0.4045076072216034, + "grad_norm": 12.294120788574219, + "learning_rate": 1.2389025514492456e-07, + "logits/chosen": -1.37725031375885, + "logits/rejected": -1.234118938446045, + "logps/chosen": -248.1385498046875, + "logps/ref_chosen": -81.3623046875, + "logps/ref_rejected": -101.09114074707031, + "logps/rejected": -370.3965148925781, + "loss": 1.0703, + "margin_dpo/margin_mean": 102.52912902832031, + "margin_dpo/margin_std": 111.4662857055664, + "step": 479 + }, + { + "KL/chosen_KL_mean": -194.56231689453125, + "KL/mean": -234.51068115234375, + "KL/rejected_KL_mean": -274.458984375, + "KL/std": 101.8110122680664, + "epoch": 0.7048458149779736, + "fcm_dpo/beta": 0.003974815830588341, + "fcm_dpo/delta": 0.085101418197155, + "fcm_dpo/margin": 79.89668273925781, + "fcm_dpo/q_t": 0.4255372881889343, + "grad_norm": 9.53321361541748, + "learning_rate": 1.227838333989088e-07, + "logits/chosen": -1.8773475885391235, + "logits/rejected": -1.212838053703308, + "logps/chosen": -291.33624267578125, + "logps/ref_chosen": -96.7739028930664, + "logps/ref_rejected": -86.40473937988281, + "logps/rejected": -360.8637390136719, + "loss": 1.1416, + "margin_dpo/margin_mean": 79.89667510986328, + "margin_dpo/margin_std": 112.32437133789062, + "step": 480 + }, + { + "KL/chosen_KL_mean": -189.54202270507812, + "KL/mean": -236.66067504882812, + "KL/rejected_KL_mean": -283.7793273925781, + "KL/std": 106.41290283203125, + "epoch": 0.7063142437591777, + "fcm_dpo/beta": 0.0040034460835158825, + "fcm_dpo/delta": 0.02362295240163803, + "fcm_dpo/margin": 94.23728942871094, + "fcm_dpo/q_t": 0.4113905429840088, + "grad_norm": 11.179119110107422, + "learning_rate": 1.2168076391719489e-07, + "logits/chosen": -1.7786369323730469, + "logits/rejected": -1.332186222076416, + "logps/chosen": -281.2121887207031, + "logps/ref_chosen": -91.670166015625, + "logps/ref_rejected": -98.69490051269531, + "logps/rejected": -382.4742431640625, + "loss": 1.0969, + "margin_dpo/margin_mean": 94.23728942871094, + "margin_dpo/margin_std": 117.689453125, + "step": 481 + }, + { + "KL/chosen_KL_mean": -182.7600860595703, + "KL/mean": -229.0103759765625, + "KL/rejected_KL_mean": -275.2606506347656, + "KL/std": 105.2197494506836, + "epoch": 0.7077826725403817, + "fcm_dpo/beta": 0.004017159342765808, + "fcm_dpo/delta": 0.029450150206685066, + "fcm_dpo/margin": 92.50057220458984, + "fcm_dpo/q_t": 0.4123062491416931, + "grad_norm": 12.130163192749023, + "learning_rate": 1.2058107576668938e-07, + "logits/chosen": -1.5374221801757812, + "logits/rejected": -1.1778550148010254, + "logps/chosen": -281.28021240234375, + "logps/ref_chosen": -98.52011108398438, + "logps/ref_rejected": -94.82945251464844, + "logps/rejected": -370.090087890625, + "loss": 1.1029, + "margin_dpo/margin_mean": 92.50057220458984, + "margin_dpo/margin_std": 117.57794189453125, + "step": 482 + }, + { + "KL/chosen_KL_mean": -189.22280883789062, + "KL/mean": -238.18719482421875, + "KL/rejected_KL_mean": -287.151611328125, + "KL/std": 110.83888244628906, + "epoch": 0.7092511013215859, + "fcm_dpo/beta": 0.004012971185147762, + "fcm_dpo/delta": -0.05088834464550018, + "fcm_dpo/margin": 97.92879486083984, + "fcm_dpo/q_t": 0.40683647990226746, + "grad_norm": 7.473385810852051, + "learning_rate": 1.194847979251979e-07, + "logits/chosen": -1.9428081512451172, + "logits/rejected": -1.2895302772521973, + "logps/chosen": -296.3414306640625, + "logps/ref_chosen": -107.11860656738281, + "logps/ref_rejected": -101.11499786376953, + "logps/rejected": -388.2666015625, + "loss": 1.0809, + "margin_dpo/margin_mean": 97.92880249023438, + "margin_dpo/margin_std": 110.62600708007812, + "step": 483 + }, + { + "KL/chosen_KL_mean": -172.4029998779297, + "KL/mean": -227.1865234375, + "KL/rejected_KL_mean": -281.97003173828125, + "KL/std": 112.21696472167969, + "epoch": 0.71071953010279, + "fcm_dpo/beta": 0.003981337416917086, + "fcm_dpo/delta": -0.03793150186538696, + "fcm_dpo/margin": 109.56703186035156, + "fcm_dpo/q_t": 0.397506445646286, + "grad_norm": 8.627911567687988, + "learning_rate": 1.1839195928066101e-07, + "logits/chosen": -2.453545331954956, + "logits/rejected": -1.9802464246749878, + "logps/chosen": -259.3829345703125, + "logps/ref_chosen": -86.97991943359375, + "logps/ref_rejected": -90.72367095947266, + "logps/rejected": -372.6936950683594, + "loss": 1.0542, + "margin_dpo/margin_mean": 109.56703186035156, + "margin_dpo/margin_std": 122.67124938964844, + "step": 484 + }, + { + "KL/chosen_KL_mean": -164.11558532714844, + "KL/mean": -223.28280639648438, + "KL/rejected_KL_mean": -282.45001220703125, + "KL/std": 117.07911682128906, + "epoch": 0.7121879588839941, + "fcm_dpo/beta": 0.003924118354916573, + "fcm_dpo/delta": -0.06747239083051682, + "fcm_dpo/margin": 118.33447265625, + "fcm_dpo/q_t": 0.3911016583442688, + "grad_norm": 8.484262466430664, + "learning_rate": 1.1730258863039347e-07, + "logits/chosen": -1.7410882711410522, + "logits/rejected": -1.3221330642700195, + "logps/chosen": -258.17431640625, + "logps/ref_chosen": -94.05874633789062, + "logps/ref_rejected": -108.56298828125, + "logps/rejected": -391.01300048828125, + "loss": 1.026, + "margin_dpo/margin_mean": 118.33447265625, + "margin_dpo/margin_std": 118.95500183105469, + "step": 485 + }, + { + "KL/chosen_KL_mean": -188.41806030273438, + "KL/mean": -240.04917907714844, + "KL/rejected_KL_mean": -291.6802978515625, + "KL/std": 112.5208740234375, + "epoch": 0.7136563876651982, + "fcm_dpo/beta": 0.003916551824659109, + "fcm_dpo/delta": -0.004740707576274872, + "fcm_dpo/margin": 103.26220703125, + "fcm_dpo/q_t": 0.40490737557411194, + "grad_norm": 10.068588256835938, + "learning_rate": 1.1621671468032493e-07, + "logits/chosen": -1.889577865600586, + "logits/rejected": -1.1691875457763672, + "logps/chosen": -282.1639404296875, + "logps/ref_chosen": -93.74588012695312, + "logps/ref_rejected": -98.07064819335938, + "logps/rejected": -389.7509460449219, + "loss": 1.0762, + "margin_dpo/margin_mean": 103.26220703125, + "margin_dpo/margin_std": 120.4369125366211, + "step": 486 + }, + { + "KL/chosen_KL_mean": -182.69142150878906, + "KL/mean": -229.45248413085938, + "KL/rejected_KL_mean": -276.2135314941406, + "KL/std": 107.39639282226562, + "epoch": 0.7151248164464024, + "fcm_dpo/beta": 0.0039243861101567745, + "fcm_dpo/delta": 0.0342482291162014, + "fcm_dpo/margin": 93.52210235595703, + "fcm_dpo/q_t": 0.4137478470802307, + "grad_norm": 9.828581809997559, + "learning_rate": 1.1513436604424378e-07, + "logits/chosen": -2.1258187294006348, + "logits/rejected": -1.8154586553573608, + "logps/chosen": -270.7249755859375, + "logps/ref_chosen": -88.0335693359375, + "logps/ref_rejected": -98.47209930419922, + "logps/rejected": -374.68560791015625, + "loss": 1.1012, + "margin_dpo/margin_mean": 93.52210998535156, + "margin_dpo/margin_std": 114.90782165527344, + "step": 487 + }, + { + "KL/chosen_KL_mean": -163.84619140625, + "KL/mean": -216.0897979736328, + "KL/rejected_KL_mean": -268.3333740234375, + "KL/std": 113.79476165771484, + "epoch": 0.7165932452276065, + "fcm_dpo/beta": 0.003930740989744663, + "fcm_dpo/delta": -0.011188503354787827, + "fcm_dpo/margin": 104.48719024658203, + "fcm_dpo/q_t": 0.40240877866744995, + "grad_norm": 8.870756149291992, + "learning_rate": 1.1405557124304335e-07, + "logits/chosen": -1.7000606060028076, + "logits/rejected": -1.4414727687835693, + "logps/chosen": -248.63583374023438, + "logps/ref_chosen": -84.78964233398438, + "logps/ref_rejected": -90.2734603881836, + "logps/rejected": -358.60687255859375, + "loss": 1.0571, + "margin_dpo/margin_mean": 104.48719024658203, + "margin_dpo/margin_std": 101.5090560913086, + "step": 488 + }, + { + "KL/chosen_KL_mean": -185.60153198242188, + "KL/mean": -233.37515258789062, + "KL/rejected_KL_mean": -281.14874267578125, + "KL/std": 104.13800048828125, + "epoch": 0.7180616740088106, + "fcm_dpo/beta": 0.003939471673220396, + "fcm_dpo/delta": 0.024505067616701126, + "fcm_dpo/margin": 95.54719543457031, + "fcm_dpo/q_t": 0.4118320345878601, + "grad_norm": 9.545340538024902, + "learning_rate": 1.1298035870396985e-07, + "logits/chosen": -1.6859453916549683, + "logits/rejected": -1.199491024017334, + "logps/chosen": -276.0708312988281, + "logps/ref_chosen": -90.46929931640625, + "logps/ref_rejected": -86.39761352539062, + "logps/rejected": -367.5463562011719, + "loss": 1.0943, + "margin_dpo/margin_mean": 95.54719543457031, + "margin_dpo/margin_std": 115.8781509399414, + "step": 489 + }, + { + "KL/chosen_KL_mean": -186.06907653808594, + "KL/mean": -230.87518310546875, + "KL/rejected_KL_mean": -275.6812744140625, + "KL/std": 112.82815551757812, + "epoch": 0.7195301027900147, + "fcm_dpo/beta": 0.003927274141460657, + "fcm_dpo/delta": -0.03976406529545784, + "fcm_dpo/margin": 89.61219024658203, + "fcm_dpo/q_t": 0.41919487714767456, + "grad_norm": 8.340740203857422, + "learning_rate": 1.1190875675987355e-07, + "logits/chosen": -2.165680408477783, + "logits/rejected": -2.245126724243164, + "logps/chosen": -271.38922119140625, + "logps/ref_chosen": -85.32012939453125, + "logps/ref_rejected": -115.99385070800781, + "logps/rejected": -391.67510986328125, + "loss": 1.1496, + "margin_dpo/margin_mean": 89.61219787597656, + "margin_dpo/margin_std": 148.9447021484375, + "step": 490 + }, + { + "KL/chosen_KL_mean": -189.2417449951172, + "KL/mean": -230.4969940185547, + "KL/rejected_KL_mean": -271.752197265625, + "KL/std": 114.90867614746094, + "epoch": 0.7209985315712188, + "fcm_dpo/beta": 0.003955123946070671, + "fcm_dpo/delta": 0.07616802304983139, + "fcm_dpo/margin": 82.51049041748047, + "fcm_dpo/q_t": 0.42336469888687134, + "grad_norm": 9.988592147827148, + "learning_rate": 1.1084079364846241e-07, + "logits/chosen": -1.8470979928970337, + "logits/rejected": -1.3571316003799438, + "logps/chosen": -275.38525390625, + "logps/ref_chosen": -86.14352416992188, + "logps/ref_rejected": -80.67945861816406, + "logps/rejected": -352.43170166015625, + "loss": 1.1373, + "margin_dpo/margin_mean": 82.51048278808594, + "margin_dpo/margin_std": 116.42083740234375, + "step": 491 + }, + { + "KL/chosen_KL_mean": -178.52621459960938, + "KL/mean": -211.72604370117188, + "KL/rejected_KL_mean": -244.92587280273438, + "KL/std": 113.08814239501953, + "epoch": 0.7224669603524229, + "fcm_dpo/beta": 0.0039717028848826885, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 66.39967346191406, + "fcm_dpo/q_t": 0.4386589527130127, + "grad_norm": 12.74101734161377, + "learning_rate": 1.097764975115576e-07, + "logits/chosen": -2.1324310302734375, + "logits/rejected": -1.7540669441223145, + "logps/chosen": -259.6337890625, + "logps/ref_chosen": -81.10757446289062, + "logps/ref_rejected": -80.75199890136719, + "logps/rejected": -325.6778564453125, + "loss": 1.2144, + "margin_dpo/margin_mean": 66.39967346191406, + "margin_dpo/margin_std": 140.76681518554688, + "step": 492 + }, + { + "KL/chosen_KL_mean": -175.70169067382812, + "KL/mean": -220.04055786132812, + "KL/rejected_KL_mean": -264.37945556640625, + "KL/std": 101.97767639160156, + "epoch": 0.723935389133627, + "fcm_dpo/beta": 0.0039587621577084064, + "fcm_dpo/delta": -0.016317540779709816, + "fcm_dpo/margin": 88.67771911621094, + "fcm_dpo/q_t": 0.4165477752685547, + "grad_norm": 14.901061058044434, + "learning_rate": 1.0871589639435203e-07, + "logits/chosen": -2.492276906967163, + "logits/rejected": -1.7722182273864746, + "logps/chosen": -287.9090270996094, + "logps/ref_chosen": -112.20733642578125, + "logps/ref_rejected": -93.60719299316406, + "logps/rejected": -357.98663330078125, + "loss": 1.1124, + "margin_dpo/margin_mean": 88.67771911621094, + "margin_dpo/margin_std": 109.20489501953125, + "step": 493 + }, + { + "KL/chosen_KL_mean": -170.80015563964844, + "KL/mean": -228.9921112060547, + "KL/rejected_KL_mean": -287.18408203125, + "KL/std": 113.13653564453125, + "epoch": 0.7254038179148311, + "fcm_dpo/beta": 0.003905025077983737, + "fcm_dpo/delta": -0.05789130926132202, + "fcm_dpo/margin": 116.38390350341797, + "fcm_dpo/q_t": 0.39332085847854614, + "grad_norm": 9.406898498535156, + "learning_rate": 1.0765901824467166e-07, + "logits/chosen": -1.3142294883728027, + "logits/rejected": -1.2379295825958252, + "logps/chosen": -243.91505432128906, + "logps/ref_chosen": -73.11489868164062, + "logps/ref_rejected": -92.16300201416016, + "logps/rejected": -379.3470764160156, + "loss": 1.0394, + "margin_dpo/margin_mean": 116.38390350341797, + "margin_dpo/margin_std": 123.26473999023438, + "step": 494 + }, + { + "KL/chosen_KL_mean": -176.23895263671875, + "KL/mean": -226.0841827392578, + "KL/rejected_KL_mean": -275.9294128417969, + "KL/std": 107.29304504394531, + "epoch": 0.7268722466960352, + "fcm_dpo/beta": 0.003916910383850336, + "fcm_dpo/delta": 0.009907922707498074, + "fcm_dpo/margin": 99.69044494628906, + "fcm_dpo/q_t": 0.40909022092819214, + "grad_norm": 10.464844703674316, + "learning_rate": 1.0660589091223854e-07, + "logits/chosen": -2.159003257751465, + "logits/rejected": -1.688874363899231, + "logps/chosen": -275.75927734375, + "logps/ref_chosen": -99.52031707763672, + "logps/ref_rejected": -97.93089294433594, + "logps/rejected": -373.86029052734375, + "loss": 1.0938, + "margin_dpo/margin_mean": 99.69044494628906, + "margin_dpo/margin_std": 129.49729919433594, + "step": 495 + }, + { + "KL/chosen_KL_mean": -195.28173828125, + "KL/mean": -238.1721649169922, + "KL/rejected_KL_mean": -281.0625915527344, + "KL/std": 107.84174346923828, + "epoch": 0.7283406754772394, + "fcm_dpo/beta": 0.00392390601336956, + "fcm_dpo/delta": 0.007526843808591366, + "fcm_dpo/margin": 85.78083801269531, + "fcm_dpo/q_t": 0.4199199676513672, + "grad_norm": 8.699403762817383, + "learning_rate": 1.0555654214793722e-07, + "logits/chosen": -2.1817100048065186, + "logits/rejected": -1.5394623279571533, + "logps/chosen": -303.13848876953125, + "logps/ref_chosen": -107.85675048828125, + "logps/ref_rejected": -92.77056121826172, + "logps/rejected": -373.8331298828125, + "loss": 1.1233, + "margin_dpo/margin_mean": 85.78083801269531, + "margin_dpo/margin_std": 109.09320831298828, + "step": 496 + }, + { + "KL/chosen_KL_mean": -208.52532958984375, + "KL/mean": -236.30435180664062, + "KL/rejected_KL_mean": -264.0833435058594, + "KL/std": 109.271728515625, + "epoch": 0.7298091042584435, + "fcm_dpo/beta": 0.003926859702914953, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 55.558013916015625, + "fcm_dpo/q_t": 0.4486031234264374, + "grad_norm": 8.046308517456055, + "learning_rate": 1.0451099960308374e-07, + "logits/chosen": -1.6113924980163574, + "logits/rejected": -1.1524041891098022, + "logps/chosen": -300.60858154296875, + "logps/ref_chosen": -92.08322143554688, + "logps/ref_rejected": -81.79503631591797, + "logps/rejected": -345.87835693359375, + "loss": 1.2321, + "margin_dpo/margin_mean": 55.558013916015625, + "margin_dpo/margin_std": 116.76002502441406, + "step": 497 + }, + { + "KL/chosen_KL_mean": -177.03164672851562, + "KL/mean": -223.6384735107422, + "KL/rejected_KL_mean": -270.24530029296875, + "KL/std": 107.69236755371094, + "epoch": 0.7312775330396476, + "fcm_dpo/beta": 0.0039444491267204285, + "fcm_dpo/delta": 0.03356942534446716, + "fcm_dpo/margin": 93.21366882324219, + "fcm_dpo/q_t": 0.4133025109767914, + "grad_norm": 10.81260871887207, + "learning_rate": 1.0346929082869641e-07, + "logits/chosen": -1.829862117767334, + "logits/rejected": -1.333831787109375, + "logps/chosen": -275.22601318359375, + "logps/ref_chosen": -98.19437408447266, + "logps/ref_rejected": -90.68745422363281, + "logps/rejected": -360.9327392578125, + "loss": 1.1048, + "margin_dpo/margin_mean": 93.21366882324219, + "margin_dpo/margin_std": 119.22862243652344, + "step": 498 + }, + { + "KL/chosen_KL_mean": -185.80911254882812, + "KL/mean": -234.31475830078125, + "KL/rejected_KL_mean": -282.82037353515625, + "KL/std": 115.52024841308594, + "epoch": 0.7327459618208517, + "fcm_dpo/beta": 0.003966744989156723, + "fcm_dpo/delta": 0.01574990153312683, + "fcm_dpo/margin": 97.01126861572266, + "fcm_dpo/q_t": 0.4101901650428772, + "grad_norm": 9.266758918762207, + "learning_rate": 1.0243144327477013e-07, + "logits/chosen": -2.2828783988952637, + "logits/rejected": -2.2341904640197754, + "logps/chosen": -266.8490295410156, + "logps/ref_chosen": -81.0399169921875, + "logps/ref_rejected": -106.92170715332031, + "logps/rejected": -389.7420654296875, + "loss": 1.0946, + "margin_dpo/margin_mean": 97.01127624511719, + "margin_dpo/margin_std": 123.3255615234375, + "step": 499 + }, + { + "KL/chosen_KL_mean": -205.74810791015625, + "KL/mean": -247.64076232910156, + "KL/rejected_KL_mean": -289.53338623046875, + "KL/std": 111.3338623046875, + "epoch": 0.7342143906020558, + "fcm_dpo/beta": 0.0040001594461500645, + "fcm_dpo/delta": 0.06714823842048645, + "fcm_dpo/margin": 83.78529357910156, + "fcm_dpo/q_t": 0.4230141043663025, + "grad_norm": 11.359566688537598, + "learning_rate": 1.0139748428955333e-07, + "logits/chosen": -1.4569916725158691, + "logits/rejected": -1.28076171875, + "logps/chosen": -294.99615478515625, + "logps/ref_chosen": -89.248046875, + "logps/ref_rejected": -100.41021728515625, + "logps/rejected": -389.943603515625, + "loss": 1.1401, + "margin_dpo/margin_mean": 83.7853012084961, + "margin_dpo/margin_std": 126.78570556640625, + "step": 500 + }, + { + "KL/chosen_KL_mean": -179.64697265625, + "KL/mean": -226.18873596191406, + "KL/rejected_KL_mean": -272.73052978515625, + "KL/std": 120.93177795410156, + "epoch": 0.73568281938326, + "fcm_dpo/beta": 0.003997047431766987, + "fcm_dpo/delta": -0.05593404918909073, + "fcm_dpo/margin": 93.08353424072266, + "fcm_dpo/q_t": 0.4137415289878845, + "grad_norm": 8.646526336669922, + "learning_rate": 1.0036744111882672e-07, + "logits/chosen": -1.61490797996521, + "logits/rejected": -1.1807053089141846, + "logps/chosen": -270.243896484375, + "logps/ref_chosen": -90.596923828125, + "logps/ref_rejected": -85.2972640991211, + "logps/rejected": -358.02777099609375, + "loss": 1.1226, + "margin_dpo/margin_mean": 93.08353424072266, + "margin_dpo/margin_std": 135.6177978515625, + "step": 501 + }, + { + "KL/chosen_KL_mean": -180.65415954589844, + "KL/mean": -222.42820739746094, + "KL/rejected_KL_mean": -264.2022705078125, + "KL/std": 100.51045227050781, + "epoch": 0.737151248164464, + "fcm_dpo/beta": 0.003974404186010361, + "fcm_dpo/delta": -0.0007198267849162221, + "fcm_dpo/margin": 83.54811096191406, + "fcm_dpo/q_t": 0.42157667875289917, + "grad_norm": 9.545127868652344, + "learning_rate": 9.934134090518592e-08, + "logits/chosen": -1.9886888265609741, + "logits/rejected": -1.2634838819503784, + "logps/chosen": -284.1248779296875, + "logps/ref_chosen": -103.47073364257812, + "logps/ref_rejected": -89.04415893554688, + "logps/rejected": -353.2464294433594, + "loss": 1.1409, + "margin_dpo/margin_mean": 83.54811096191406, + "margin_dpo/margin_std": 123.66584777832031, + "step": 502 + }, + { + "KL/chosen_KL_mean": -179.72842407226562, + "KL/mean": -226.67210388183594, + "KL/rejected_KL_mean": -273.61578369140625, + "KL/std": 107.78350830078125, + "epoch": 0.7386196769456681, + "fcm_dpo/beta": 0.003964018076658249, + "fcm_dpo/delta": -0.02547951601445675, + "fcm_dpo/margin": 93.88733673095703, + "fcm_dpo/q_t": 0.4123993217945099, + "grad_norm": 8.895367622375488, + "learning_rate": 9.831921068732571e-08, + "logits/chosen": -1.7202941179275513, + "logits/rejected": -1.2193692922592163, + "logps/chosen": -273.45843505859375, + "logps/ref_chosen": -93.72998809814453, + "logps/ref_rejected": -87.60896301269531, + "logps/rejected": -361.2247314453125, + "loss": 1.1028, + "margin_dpo/margin_mean": 93.88733673095703, + "margin_dpo/margin_std": 118.7103271484375, + "step": 503 + }, + { + "KL/chosen_KL_mean": -182.45199584960938, + "KL/mean": -242.7974395751953, + "KL/rejected_KL_mean": -303.14288330078125, + "KL/std": 112.29166412353516, + "epoch": 0.7400881057268722, + "fcm_dpo/beta": 0.0039139739237725735, + "fcm_dpo/delta": -0.07591018080711365, + "fcm_dpo/margin": 120.69091796875, + "fcm_dpo/q_t": 0.388175904750824, + "grad_norm": 9.44067668914795, + "learning_rate": 9.730107739932805e-08, + "logits/chosen": -2.0505452156066895, + "logits/rejected": -1.717842936515808, + "logps/chosen": -274.7438659667969, + "logps/ref_chosen": -92.2918701171875, + "logps/ref_rejected": -109.03765106201172, + "logps/rejected": -412.1805419921875, + "loss": 1.0156, + "margin_dpo/margin_mean": 120.69091796875, + "margin_dpo/margin_std": 113.42808532714844, + "step": 504 + }, + { + "KL/chosen_KL_mean": -200.13525390625, + "KL/mean": -243.42214965820312, + "KL/rejected_KL_mean": -286.7090148925781, + "KL/std": 111.30079650878906, + "epoch": 0.7415565345080763, + "fcm_dpo/beta": 0.00392896868288517, + "fcm_dpo/delta": 0.061983369290828705, + "fcm_dpo/margin": 86.57379150390625, + "fcm_dpo/q_t": 0.4198564887046814, + "grad_norm": 8.148636817932129, + "learning_rate": 9.628696786995188e-08, + "logits/chosen": -2.3748321533203125, + "logits/rejected": -1.816833734512329, + "logps/chosen": -306.41217041015625, + "logps/ref_chosen": -106.27693176269531, + "logps/ref_rejected": -94.20611572265625, + "logps/rejected": -380.9151611328125, + "loss": 1.1185, + "margin_dpo/margin_mean": 86.57379150390625, + "margin_dpo/margin_std": 107.85874938964844, + "step": 505 + }, + { + "KL/chosen_KL_mean": -193.41326904296875, + "KL/mean": -242.438720703125, + "KL/rejected_KL_mean": -291.46417236328125, + "KL/std": 105.14840698242188, + "epoch": 0.7430249632892805, + "fcm_dpo/beta": 0.003937737084925175, + "fcm_dpo/delta": 0.014265947043895721, + "fcm_dpo/margin": 98.05091857910156, + "fcm_dpo/q_t": 0.40805986523628235, + "grad_norm": 8.797200202941895, + "learning_rate": 9.527690882192635e-08, + "logits/chosen": -1.9347736835479736, + "logits/rejected": -1.5731432437896729, + "logps/chosen": -274.8541259765625, + "logps/ref_chosen": -81.44086456298828, + "logps/ref_rejected": -85.66439819335938, + "logps/rejected": -377.1285705566406, + "loss": 1.0831, + "margin_dpo/margin_mean": 98.05091857910156, + "margin_dpo/margin_std": 109.73526000976562, + "step": 506 + }, + { + "KL/chosen_KL_mean": -202.03317260742188, + "KL/mean": -251.17965698242188, + "KL/rejected_KL_mean": -300.32611083984375, + "KL/std": 109.95832824707031, + "epoch": 0.7444933920704846, + "fcm_dpo/beta": 0.003960990346968174, + "fcm_dpo/delta": 0.011093826964497566, + "fcm_dpo/margin": 98.29291534423828, + "fcm_dpo/q_t": 0.4095962643623352, + "grad_norm": 8.922673225402832, + "learning_rate": 9.427092687124691e-08, + "logits/chosen": -2.125525951385498, + "logits/rejected": -1.6822545528411865, + "logps/chosen": -301.46014404296875, + "logps/ref_chosen": -99.42694854736328, + "logps/ref_rejected": -102.25296020507812, + "logps/rejected": -402.5790710449219, + "loss": 1.0984, + "margin_dpo/margin_mean": 98.29291534423828, + "margin_dpo/margin_std": 129.6722869873047, + "step": 507 + }, + { + "KL/chosen_KL_mean": -194.7322540283203, + "KL/mean": -235.767578125, + "KL/rejected_KL_mean": -276.8028869628906, + "KL/std": 118.11274719238281, + "epoch": 0.7459618208516887, + "fcm_dpo/beta": 0.004006821662187576, + "fcm_dpo/delta": 0.07357528060674667, + "fcm_dpo/margin": 82.0706558227539, + "fcm_dpo/q_t": 0.4246722459793091, + "grad_norm": 15.261529922485352, + "learning_rate": 9.326904852647344e-08, + "logits/chosen": -2.052635669708252, + "logits/rejected": -1.5721744298934937, + "logps/chosen": -292.64788818359375, + "logps/ref_chosen": -97.9156494140625, + "logps/ref_rejected": -103.79782104492188, + "logps/rejected": -380.6007080078125, + "loss": 1.1593, + "margin_dpo/margin_mean": 82.07064819335938, + "margin_dpo/margin_std": 137.97544860839844, + "step": 508 + }, + { + "KL/chosen_KL_mean": -181.87240600585938, + "KL/mean": -220.01315307617188, + "KL/rejected_KL_mean": -258.15386962890625, + "KL/std": 92.80377960205078, + "epoch": 0.7474302496328928, + "fcm_dpo/beta": 0.004035960882902145, + "fcm_dpo/delta": 0.03529799357056618, + "fcm_dpo/margin": 76.28148651123047, + "fcm_dpo/q_t": 0.42670390009880066, + "grad_norm": 14.2485990524292, + "learning_rate": 9.227130018803195e-08, + "logits/chosen": -2.026393413543701, + "logits/rejected": -1.3643516302108765, + "logps/chosen": -282.6022644042969, + "logps/ref_chosen": -100.72984313964844, + "logps/ref_rejected": -86.943359375, + "logps/rejected": -345.09722900390625, + "loss": 1.1619, + "margin_dpo/margin_mean": 76.281494140625, + "margin_dpo/margin_std": 122.8749008178711, + "step": 509 + }, + { + "KL/chosen_KL_mean": -215.20278930664062, + "KL/mean": -261.70635986328125, + "KL/rejected_KL_mean": -308.2099304199219, + "KL/std": 111.25944519042969, + "epoch": 0.748898678414097, + "fcm_dpo/beta": 0.0040697501972317696, + "fcm_dpo/delta": 0.02225029282271862, + "fcm_dpo/margin": 93.00714874267578, + "fcm_dpo/q_t": 0.41105180978775024, + "grad_norm": 11.217575073242188, + "learning_rate": 9.127770814751932e-08, + "logits/chosen": -1.532074213027954, + "logits/rejected": -1.3468949794769287, + "logps/chosen": -304.3721618652344, + "logps/ref_chosen": -89.16938781738281, + "logps/ref_rejected": -107.91939544677734, + "logps/rejected": -416.12933349609375, + "loss": 1.0943, + "margin_dpo/margin_mean": 93.00714874267578, + "margin_dpo/margin_std": 111.80921173095703, + "step": 510 + }, + { + "KL/chosen_KL_mean": -196.1717987060547, + "KL/mean": -246.56007385253906, + "KL/rejected_KL_mean": -296.9483642578125, + "KL/std": 110.18350219726562, + "epoch": 0.750367107195301, + "fcm_dpo/beta": 0.004070533439517021, + "fcm_dpo/delta": -0.010722624137997627, + "fcm_dpo/margin": 100.77655792236328, + "fcm_dpo/q_t": 0.40484482049942017, + "grad_norm": 10.913744926452637, + "learning_rate": 9.028829858700973e-08, + "logits/chosen": -2.133849859237671, + "logits/rejected": -1.566208839416504, + "logps/chosen": -285.92156982421875, + "logps/ref_chosen": -89.74979400634766, + "logps/ref_rejected": -97.73127746582031, + "logps/rejected": -394.67962646484375, + "loss": 1.0772, + "margin_dpo/margin_mean": 100.77656555175781, + "margin_dpo/margin_std": 123.26954650878906, + "step": 511 + }, + { + "KL/chosen_KL_mean": -185.60037231445312, + "KL/mean": -234.26329040527344, + "KL/rejected_KL_mean": -282.92620849609375, + "KL/std": 107.42489624023438, + "epoch": 0.7518355359765051, + "fcm_dpo/beta": 0.004080546088516712, + "fcm_dpo/delta": 0.0024984925985336304, + "fcm_dpo/margin": 97.3258285522461, + "fcm_dpo/q_t": 0.40711161494255066, + "grad_norm": 13.457165718078613, + "learning_rate": 8.930309757836516e-08, + "logits/chosen": -1.9062421321868896, + "logits/rejected": -1.4717152118682861, + "logps/chosen": -271.33551025390625, + "logps/ref_chosen": -85.73515319824219, + "logps/ref_rejected": -87.6248779296875, + "logps/rejected": -370.55108642578125, + "loss": 1.0941, + "margin_dpo/margin_mean": 97.32583618164062, + "margin_dpo/margin_std": 126.97872924804688, + "step": 512 + }, + { + "KL/chosen_KL_mean": -197.99903869628906, + "KL/mean": -238.67803955078125, + "KL/rejected_KL_mean": -279.3570556640625, + "KL/std": 102.75556182861328, + "epoch": 0.7533039647577092, + "fcm_dpo/beta": 0.00405617943033576, + "fcm_dpo/delta": -0.006659490987658501, + "fcm_dpo/margin": 81.35800170898438, + "fcm_dpo/q_t": 0.42128869891166687, + "grad_norm": 10.68964672088623, + "learning_rate": 8.832213108254863e-08, + "logits/chosen": -2.3747973442077637, + "logits/rejected": -1.6879041194915771, + "logps/chosen": -294.1895446777344, + "logps/ref_chosen": -96.19050598144531, + "logps/ref_rejected": -82.91773986816406, + "logps/rejected": -362.2747802734375, + "loss": 1.1396, + "margin_dpo/margin_mean": 81.35800170898438, + "margin_dpo/margin_std": 116.92046356201172, + "step": 513 + }, + { + "KL/chosen_KL_mean": -192.1576690673828, + "KL/mean": -236.41180419921875, + "KL/rejected_KL_mean": -280.6659851074219, + "KL/std": 103.27780151367188, + "epoch": 0.7547723935389133, + "fcm_dpo/beta": 0.004082049708813429, + "fcm_dpo/delta": 0.04013773053884506, + "fcm_dpo/margin": 88.50830078125, + "fcm_dpo/q_t": 0.41640210151672363, + "grad_norm": 10.681291580200195, + "learning_rate": 8.734542494893954e-08, + "logits/chosen": -1.6041903495788574, + "logits/rejected": -1.0222392082214355, + "logps/chosen": -289.9813232421875, + "logps/ref_chosen": -97.82363891601562, + "logps/ref_rejected": -85.36982727050781, + "logps/rejected": -366.03582763671875, + "loss": 1.116, + "margin_dpo/margin_mean": 88.50830078125, + "margin_dpo/margin_std": 122.02279663085938, + "step": 514 + }, + { + "KL/chosen_KL_mean": -183.2168731689453, + "KL/mean": -226.84437561035156, + "KL/rejected_KL_mean": -270.4718933105469, + "KL/std": 107.51303100585938, + "epoch": 0.7562408223201175, + "fcm_dpo/beta": 0.004114504437893629, + "fcm_dpo/delta": 0.04251883551478386, + "fcm_dpo/margin": 87.25503540039062, + "fcm_dpo/q_t": 0.4160889983177185, + "grad_norm": 11.353057861328125, + "learning_rate": 8.637300491465272e-08, + "logits/chosen": -1.786031723022461, + "logits/rejected": -1.5551931858062744, + "logps/chosen": -265.1504211425781, + "logps/ref_chosen": -81.93355560302734, + "logps/ref_rejected": -93.57656860351562, + "logps/rejected": -364.0484619140625, + "loss": 1.117, + "margin_dpo/margin_mean": 87.25503540039062, + "margin_dpo/margin_std": 118.18733215332031, + "step": 515 + }, + { + "KL/chosen_KL_mean": -189.4136505126953, + "KL/mean": -231.18515014648438, + "KL/rejected_KL_mean": -272.95660400390625, + "KL/std": 123.27816009521484, + "epoch": 0.7577092511013216, + "fcm_dpo/beta": 0.004163405857980251, + "fcm_dpo/delta": 0.05392815172672272, + "fcm_dpo/margin": 83.54295349121094, + "fcm_dpo/q_t": 0.42119747400283813, + "grad_norm": 12.090835571289062, + "learning_rate": 8.540489660386064e-08, + "logits/chosen": -2.4288816452026367, + "logits/rejected": -2.25951886177063, + "logps/chosen": -286.50531005859375, + "logps/ref_chosen": -97.09166717529297, + "logps/ref_rejected": -118.55667114257812, + "logps/rejected": -391.5133056640625, + "loss": 1.1555, + "margin_dpo/margin_mean": 83.54295349121094, + "margin_dpo/margin_std": 146.1177978515625, + "step": 516 + }, + { + "KL/chosen_KL_mean": -201.92050170898438, + "KL/mean": -253.25572204589844, + "KL/rejected_KL_mean": -304.5909423828125, + "KL/std": 110.46018981933594, + "epoch": 0.7591776798825257, + "fcm_dpo/beta": 0.004105024505406618, + "fcm_dpo/delta": -0.07673335075378418, + "fcm_dpo/margin": 102.67044067382812, + "fcm_dpo/q_t": 0.4017278850078583, + "grad_norm": 8.936898231506348, + "learning_rate": 8.444112552711752e-08, + "logits/chosen": -2.0397822856903076, + "logits/rejected": -1.625511646270752, + "logps/chosen": -295.1070556640625, + "logps/ref_chosen": -93.18656921386719, + "logps/ref_rejected": -95.5927963256836, + "logps/rejected": -400.1837463378906, + "loss": 1.0765, + "margin_dpo/margin_mean": 102.67042541503906, + "margin_dpo/margin_std": 126.3651123046875, + "step": 517 + }, + { + "KL/chosen_KL_mean": -210.923583984375, + "KL/mean": -248.91769409179688, + "KL/rejected_KL_mean": -286.9118347167969, + "KL/std": 104.56220245361328, + "epoch": 0.7606461086637298, + "fcm_dpo/beta": 0.004156820941716433, + "fcm_dpo/delta": 0.08690877258777618, + "fcm_dpo/margin": 75.9882583618164, + "fcm_dpo/q_t": 0.4275084137916565, + "grad_norm": 9.773492813110352, + "learning_rate": 8.348171708068747e-08, + "logits/chosen": -1.7810637950897217, + "logits/rejected": -1.446563959121704, + "logps/chosen": -297.9449462890625, + "logps/ref_chosen": -87.0213623046875, + "logps/ref_rejected": -91.24456024169922, + "logps/rejected": -378.1564025878906, + "loss": 1.1642, + "margin_dpo/margin_mean": 75.98825073242188, + "margin_dpo/margin_std": 129.84288024902344, + "step": 518 + }, + { + "KL/chosen_KL_mean": -194.88018798828125, + "KL/mean": -236.708740234375, + "KL/rejected_KL_mean": -278.5372619628906, + "KL/std": 97.46993255615234, + "epoch": 0.762114537444934, + "fcm_dpo/beta": 0.0042000580579042435, + "fcm_dpo/delta": 0.05042509362101555, + "fcm_dpo/margin": 83.65708923339844, + "fcm_dpo/q_t": 0.4178110361099243, + "grad_norm": 10.447071075439453, + "learning_rate": 8.25266965458755e-08, + "logits/chosen": -2.0044147968292236, + "logits/rejected": -1.6359965801239014, + "logps/chosen": -295.5677490234375, + "logps/ref_chosen": -100.68755340576172, + "logps/ref_rejected": -109.03042602539062, + "logps/rejected": -387.56768798828125, + "loss": 1.1099, + "margin_dpo/margin_mean": 83.65708923339844, + "margin_dpo/margin_std": 102.42189025878906, + "step": 519 + }, + { + "KL/chosen_KL_mean": -181.1160888671875, + "KL/mean": -231.02084350585938, + "KL/rejected_KL_mean": -280.925537109375, + "KL/std": 109.89045715332031, + "epoch": 0.7635829662261381, + "fcm_dpo/beta": 0.004199155606329441, + "fcm_dpo/delta": -0.020054802298545837, + "fcm_dpo/margin": 99.8094711303711, + "fcm_dpo/q_t": 0.4011247456073761, + "grad_norm": 9.208995819091797, + "learning_rate": 8.15760890883607e-08, + "logits/chosen": -1.6201674938201904, + "logits/rejected": -1.1856615543365479, + "logps/chosen": -279.89373779296875, + "logps/ref_chosen": -98.77762603759766, + "logps/ref_rejected": -106.9457015991211, + "logps/rejected": -387.87127685546875, + "loss": 1.0594, + "margin_dpo/margin_mean": 99.80947875976562, + "margin_dpo/margin_std": 107.50337982177734, + "step": 520 + }, + { + "KL/chosen_KL_mean": -182.4607391357422, + "KL/mean": -233.66900634765625, + "KL/rejected_KL_mean": -284.8772888183594, + "KL/std": 102.87747192382812, + "epoch": 0.7650513950073421, + "fcm_dpo/beta": 0.004200609866529703, + "fcm_dpo/delta": -0.0318087637424469, + "fcm_dpo/margin": 102.41654968261719, + "fcm_dpo/q_t": 0.3987743854522705, + "grad_norm": 10.099702835083008, + "learning_rate": 8.062991975753378e-08, + "logits/chosen": -1.9827301502227783, + "logits/rejected": -1.6475906372070312, + "logps/chosen": -268.81182861328125, + "logps/ref_chosen": -86.35108947753906, + "logps/ref_rejected": -90.39317321777344, + "logps/rejected": -375.27044677734375, + "loss": 1.0492, + "margin_dpo/margin_mean": 102.41654968261719, + "margin_dpo/margin_std": 104.80770874023438, + "step": 521 + }, + { + "KL/chosen_KL_mean": -190.17807006835938, + "KL/mean": -235.4043426513672, + "KL/rejected_KL_mean": -280.630615234375, + "KL/std": 112.65061950683594, + "epoch": 0.7665198237885462, + "fcm_dpo/beta": 0.0041961874812841415, + "fcm_dpo/delta": 0.021154247224330902, + "fcm_dpo/margin": 90.45255279541016, + "fcm_dpo/q_t": 0.41140180826187134, + "grad_norm": 9.924357414245605, + "learning_rate": 7.968821348583643e-08, + "logits/chosen": -1.5827698707580566, + "logits/rejected": -1.208116054534912, + "logps/chosen": -266.1861267089844, + "logps/ref_chosen": -76.008056640625, + "logps/ref_rejected": -74.44125366210938, + "logps/rejected": -355.0718688964844, + "loss": 1.0999, + "margin_dpo/margin_mean": 90.45254516601562, + "margin_dpo/margin_std": 117.11664581298828, + "step": 522 + }, + { + "KL/chosen_KL_mean": -211.0486297607422, + "KL/mean": -244.14881896972656, + "KL/rejected_KL_mean": -277.2490234375, + "KL/std": 110.52507019042969, + "epoch": 0.7679882525697503, + "fcm_dpo/beta": 0.00423425855115056, + "fcm_dpo/delta": 0.04835720360279083, + "fcm_dpo/margin": 66.20036315917969, + "fcm_dpo/q_t": 0.43549644947052, + "grad_norm": 11.738451957702637, + "learning_rate": 7.875099508810484e-08, + "logits/chosen": -2.0724406242370605, + "logits/rejected": -1.381223440170288, + "logps/chosen": -307.544921875, + "logps/ref_chosen": -96.49627685546875, + "logps/ref_rejected": -91.0152816772461, + "logps/rejected": -368.2642822265625, + "loss": 1.1965, + "margin_dpo/margin_mean": 66.20036315917969, + "margin_dpo/margin_std": 129.156005859375, + "step": 523 + }, + { + "KL/chosen_KL_mean": -192.61854553222656, + "KL/mean": -239.88951110839844, + "KL/rejected_KL_mean": -287.1604919433594, + "KL/std": 110.71673583984375, + "epoch": 0.7694566813509545, + "fcm_dpo/beta": 0.0042311521247029305, + "fcm_dpo/delta": -3.3204443752765656e-05, + "fcm_dpo/margin": 94.54196166992188, + "fcm_dpo/q_t": 0.4073396325111389, + "grad_norm": 12.265958786010742, + "learning_rate": 7.781828926091535e-08, + "logits/chosen": -2.0376334190368652, + "logits/rejected": -1.3248038291931152, + "logps/chosen": -296.0240478515625, + "logps/ref_chosen": -103.40550994873047, + "logps/ref_rejected": -87.91015625, + "logps/rejected": -375.0706481933594, + "loss": 1.0842, + "margin_dpo/margin_mean": 94.54196166992188, + "margin_dpo/margin_std": 117.80670928955078, + "step": 524 + }, + { + "KL/chosen_KL_mean": -181.10916137695312, + "KL/mean": -234.8898162841797, + "KL/rejected_KL_mean": -288.67047119140625, + "KL/std": 111.06787109375, + "epoch": 0.7709251101321586, + "fcm_dpo/beta": 0.004136846866458654, + "fcm_dpo/delta": -0.1163390725851059, + "fcm_dpo/margin": 107.56129455566406, + "fcm_dpo/q_t": 0.39781540632247925, + "grad_norm": 10.726316452026367, + "learning_rate": 7.689012058193384e-08, + "logits/chosen": -1.6314878463745117, + "logits/rejected": -1.5452654361724854, + "logps/chosen": -265.315673828125, + "logps/ref_chosen": -84.20648193359375, + "logps/ref_rejected": -106.26158905029297, + "logps/rejected": -394.93206787109375, + "loss": 1.0624, + "margin_dpo/margin_mean": 107.5613021850586, + "margin_dpo/margin_std": 128.64280700683594, + "step": 525 + }, + { + "KL/chosen_KL_mean": -186.1490936279297, + "KL/mean": -234.78330993652344, + "KL/rejected_KL_mean": -283.41754150390625, + "KL/std": 105.11842346191406, + "epoch": 0.7723935389133627, + "fcm_dpo/beta": 0.004136267118155956, + "fcm_dpo/delta": -0.0024262825027108192, + "fcm_dpo/margin": 97.2684326171875, + "fcm_dpo/q_t": 0.406277060508728, + "grad_norm": 8.273813247680664, + "learning_rate": 7.596651350926836e-08, + "logits/chosen": -1.7758026123046875, + "logits/rejected": -1.1259726285934448, + "logps/chosen": -289.0032043457031, + "logps/ref_chosen": -102.85411071777344, + "logps/ref_rejected": -93.65145874023438, + "logps/rejected": -377.0689697265625, + "loss": 1.0862, + "margin_dpo/margin_mean": 97.2684326171875, + "margin_dpo/margin_std": 123.83148193359375, + "step": 526 + }, + { + "KL/chosen_KL_mean": -188.56680297851562, + "KL/mean": -237.99942016601562, + "KL/rejected_KL_mean": -287.43206787109375, + "KL/std": 110.31617736816406, + "epoch": 0.7738619676945668, + "fcm_dpo/beta": 0.004128835164010525, + "fcm_dpo/delta": -0.008549420163035393, + "fcm_dpo/margin": 98.86524963378906, + "fcm_dpo/q_t": 0.4029432535171509, + "grad_norm": 10.541111946105957, + "learning_rate": 7.504749238082414e-08, + "logits/chosen": -3.2081172466278076, + "logits/rejected": -2.4103713035583496, + "logps/chosen": -285.3322448730469, + "logps/ref_chosen": -96.76543426513672, + "logps/ref_rejected": -85.39518737792969, + "logps/rejected": -372.8272399902344, + "loss": 1.0571, + "margin_dpo/margin_mean": 98.86524963378906, + "margin_dpo/margin_std": 96.53399658203125, + "step": 527 + }, + { + "KL/chosen_KL_mean": -176.10191345214844, + "KL/mean": -224.69422912597656, + "KL/rejected_KL_mean": -273.28656005859375, + "KL/std": 111.78994750976562, + "epoch": 0.775330396475771, + "fcm_dpo/beta": 0.004133075010031462, + "fcm_dpo/delta": -0.0017875507473945618, + "fcm_dpo/margin": 97.18464660644531, + "fcm_dpo/q_t": 0.40661191940307617, + "grad_norm": 9.107051849365234, + "learning_rate": 7.413308141366254e-08, + "logits/chosen": -2.2358603477478027, + "logits/rejected": -1.7314858436584473, + "logps/chosen": -270.266845703125, + "logps/ref_chosen": -94.1649169921875, + "logps/ref_rejected": -100.372314453125, + "logps/rejected": -373.65887451171875, + "loss": 1.0895, + "margin_dpo/margin_mean": 97.18464660644531, + "margin_dpo/margin_std": 126.81500244140625, + "step": 528 + }, + { + "KL/chosen_KL_mean": -204.88844299316406, + "KL/mean": -249.43350219726562, + "KL/rejected_KL_mean": -293.9785461425781, + "KL/std": 110.73883056640625, + "epoch": 0.7767988252569751, + "fcm_dpo/beta": 0.0041540986858308315, + "fcm_dpo/delta": 0.03090469352900982, + "fcm_dpo/margin": 89.09011840820312, + "fcm_dpo/q_t": 0.4150165319442749, + "grad_norm": 10.644501686096191, + "learning_rate": 7.322330470336313e-08, + "logits/chosen": -1.8178691864013672, + "logits/rejected": -1.642407774925232, + "logps/chosen": -285.66522216796875, + "logps/ref_chosen": -80.77680206298828, + "logps/ref_rejected": -97.191162109375, + "logps/rejected": -391.1697082519531, + "loss": 1.1208, + "margin_dpo/margin_mean": 89.09011840820312, + "margin_dpo/margin_std": 131.6557159423828, + "step": 529 + }, + { + "KL/chosen_KL_mean": -188.54885864257812, + "KL/mean": -232.77499389648438, + "KL/rejected_KL_mean": -277.0010986328125, + "KL/std": 106.88856506347656, + "epoch": 0.7782672540381792, + "fcm_dpo/beta": 0.0041601043194532394, + "fcm_dpo/delta": 0.03317118063569069, + "fcm_dpo/margin": 88.45223236083984, + "fcm_dpo/q_t": 0.4139227569103241, + "grad_norm": 9.741031646728516, + "learning_rate": 7.231818622338822e-08, + "logits/chosen": -1.556344985961914, + "logits/rejected": -1.2364283800125122, + "logps/chosen": -268.3245849609375, + "logps/ref_chosen": -79.7757339477539, + "logps/ref_rejected": -92.05390167236328, + "logps/rejected": -369.05499267578125, + "loss": 1.096, + "margin_dpo/margin_mean": 88.45223236083984, + "margin_dpo/margin_std": 103.24551391601562, + "step": 530 + }, + { + "KL/chosen_KL_mean": -181.57249450683594, + "KL/mean": -236.80592346191406, + "KL/rejected_KL_mean": -292.03936767578125, + "KL/std": 112.72777557373047, + "epoch": 0.7797356828193832, + "fcm_dpo/beta": 0.004138609394431114, + "fcm_dpo/delta": -0.05999011546373367, + "fcm_dpo/margin": 110.46685028076172, + "fcm_dpo/q_t": 0.39188629388809204, + "grad_norm": 11.031170845031738, + "learning_rate": 7.141774982445147e-08, + "logits/chosen": -1.9628932476043701, + "logits/rejected": -1.3742516040802002, + "logps/chosen": -260.8916931152344, + "logps/ref_chosen": -79.31919860839844, + "logps/ref_rejected": -78.10832214355469, + "logps/rejected": -370.1476745605469, + "loss": 1.0313, + "margin_dpo/margin_mean": 110.46684265136719, + "margin_dpo/margin_std": 112.10696411132812, + "step": 531 + }, + { + "KL/chosen_KL_mean": -194.917236328125, + "KL/mean": -255.138427734375, + "KL/rejected_KL_mean": -315.359619140625, + "KL/std": 107.52310180664062, + "epoch": 0.7812041116005873, + "fcm_dpo/beta": 0.00408159289509058, + "fcm_dpo/delta": -0.09622061997652054, + "fcm_dpo/margin": 120.44237518310547, + "fcm_dpo/q_t": 0.38491082191467285, + "grad_norm": 11.192266464233398, + "learning_rate": 7.052201923388953e-08, + "logits/chosen": -1.684445858001709, + "logits/rejected": -1.059739351272583, + "logps/chosen": -296.55413818359375, + "logps/ref_chosen": -101.63691711425781, + "logps/ref_rejected": -93.34539031982422, + "logps/rejected": -408.70501708984375, + "loss": 1.0114, + "margin_dpo/margin_mean": 120.4423828125, + "margin_dpo/margin_std": 118.99960327148438, + "step": 532 + }, + { + "KL/chosen_KL_mean": -189.69781494140625, + "KL/mean": -224.699951171875, + "KL/rejected_KL_mean": -259.7020568847656, + "KL/std": 113.68994140625, + "epoch": 0.7826725403817915, + "fcm_dpo/beta": 0.004050960764288902, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 70.00425720214844, + "fcm_dpo/q_t": 0.43451547622680664, + "grad_norm": 11.54209041595459, + "learning_rate": 6.963101805503646e-08, + "logits/chosen": -2.474362850189209, + "logits/rejected": -1.880601167678833, + "logps/chosen": -280.4444580078125, + "logps/ref_chosen": -90.74664306640625, + "logps/ref_rejected": -82.88848876953125, + "logps/rejected": -342.590576171875, + "loss": 1.1807, + "margin_dpo/margin_mean": 70.00425720214844, + "margin_dpo/margin_std": 121.96046447753906, + "step": 533 + }, + { + "KL/chosen_KL_mean": -168.35586547851562, + "KL/mean": -220.06646728515625, + "KL/rejected_KL_mean": -271.777099609375, + "KL/std": 106.7122802734375, + "epoch": 0.7841409691629956, + "fcm_dpo/beta": 0.004044422414153814, + "fcm_dpo/delta": -0.019084319472312927, + "fcm_dpo/margin": 103.42121887207031, + "fcm_dpo/q_t": 0.40096551179885864, + "grad_norm": 8.90018367767334, + "learning_rate": 6.874476976660184e-08, + "logits/chosen": -1.9054774045944214, + "logits/rejected": -1.505403757095337, + "logps/chosen": -254.9866943359375, + "logps/ref_chosen": -86.6308364868164, + "logps/ref_rejected": -84.87196350097656, + "logps/rejected": -356.6490478515625, + "loss": 1.0545, + "margin_dpo/margin_mean": 103.42121887207031, + "margin_dpo/margin_std": 105.16766357421875, + "step": 534 + }, + { + "KL/chosen_KL_mean": -170.619384765625, + "KL/mean": -225.118408203125, + "KL/rejected_KL_mean": -279.6174621582031, + "KL/std": 107.64976501464844, + "epoch": 0.7856093979441997, + "fcm_dpo/beta": 0.004028125200420618, + "fcm_dpo/delta": -0.04104717820882797, + "fcm_dpo/margin": 108.9980697631836, + "fcm_dpo/q_t": 0.3977544903755188, + "grad_norm": 7.785837650299072, + "learning_rate": 6.786329772205246e-08, + "logits/chosen": -1.609428882598877, + "logits/rejected": -1.42098069190979, + "logps/chosen": -260.96478271484375, + "logps/ref_chosen": -90.34539031982422, + "logps/ref_rejected": -100.7779769897461, + "logps/rejected": -380.39544677734375, + "loss": 1.0538, + "margin_dpo/margin_mean": 108.9980697631836, + "margin_dpo/margin_std": 122.8719482421875, + "step": 535 + }, + { + "KL/chosen_KL_mean": -168.3565673828125, + "KL/mean": -222.52008056640625, + "KL/rejected_KL_mean": -276.68359375, + "KL/std": 108.9873046875, + "epoch": 0.7870778267254038, + "fcm_dpo/beta": 0.003975989297032356, + "fcm_dpo/delta": -0.03221966326236725, + "fcm_dpo/margin": 108.32705688476562, + "fcm_dpo/q_t": 0.39833328127861023, + "grad_norm": 8.551009178161621, + "learning_rate": 6.698662514899638e-08, + "logits/chosen": -1.905432939529419, + "logits/rejected": -1.6565274000167847, + "logps/chosen": -247.31613159179688, + "logps/ref_chosen": -78.95956420898438, + "logps/ref_rejected": -93.08779907226562, + "logps/rejected": -369.77142333984375, + "loss": 1.0505, + "margin_dpo/margin_mean": 108.32705688476562, + "margin_dpo/margin_std": 114.12397766113281, + "step": 536 + }, + { + "KL/chosen_KL_mean": -148.21780395507812, + "KL/mean": -204.68319702148438, + "KL/rejected_KL_mean": -261.14862060546875, + "KL/std": 111.81452941894531, + "epoch": 0.788546255506608, + "fcm_dpo/beta": 0.003969781566411257, + "fcm_dpo/delta": -0.050967130810022354, + "fcm_dpo/margin": 112.93083190917969, + "fcm_dpo/q_t": 0.39411282539367676, + "grad_norm": 10.106608390808105, + "learning_rate": 6.611477514857114e-08, + "logits/chosen": -1.6083147525787354, + "logits/rejected": -0.9229831695556641, + "logps/chosen": -234.92718505859375, + "logps/ref_chosen": -86.70939636230469, + "logps/ref_rejected": -78.90766906738281, + "logps/rejected": -340.0563049316406, + "loss": 1.0358, + "margin_dpo/margin_mean": 112.93082427978516, + "margin_dpo/margin_std": 109.3245849609375, + "step": 537 + }, + { + "KL/chosen_KL_mean": -202.8919677734375, + "KL/mean": -248.98818969726562, + "KL/rejected_KL_mean": -295.08441162109375, + "KL/std": 100.3636474609375, + "epoch": 0.7900146842878121, + "fcm_dpo/beta": 0.003958378918468952, + "fcm_dpo/delta": 0.036391228437423706, + "fcm_dpo/margin": 92.19239044189453, + "fcm_dpo/q_t": 0.4139641523361206, + "grad_norm": 8.873498916625977, + "learning_rate": 6.524777069483525e-08, + "logits/chosen": -1.7490143775939941, + "logits/rejected": -1.2038695812225342, + "logps/chosen": -301.3341064453125, + "logps/ref_chosen": -98.44214630126953, + "logps/ref_rejected": -91.09465026855469, + "logps/rejected": -386.1790466308594, + "loss": 1.0979, + "margin_dpo/margin_mean": 92.19239807128906, + "margin_dpo/margin_std": 107.98262023925781, + "step": 538 + }, + { + "KL/chosen_KL_mean": -176.15225219726562, + "KL/mean": -224.4956817626953, + "KL/rejected_KL_mean": -272.839111328125, + "KL/std": 109.3220443725586, + "epoch": 0.7914831130690162, + "fcm_dpo/beta": 0.003976786509156227, + "fcm_dpo/delta": 0.016103433445096016, + "fcm_dpo/margin": 96.68684387207031, + "fcm_dpo/q_t": 0.4100765585899353, + "grad_norm": 8.289037704467773, + "learning_rate": 6.438563463416221e-08, + "logits/chosen": -2.3903229236602783, + "logits/rejected": -1.9081597328186035, + "logps/chosen": -267.0843505859375, + "logps/ref_chosen": -90.93211364746094, + "logps/ref_rejected": -97.44546508789062, + "logps/rejected": -370.2845458984375, + "loss": 1.0969, + "margin_dpo/margin_mean": 96.68684387207031, + "margin_dpo/margin_std": 125.0517349243164, + "step": 539 + }, + { + "KL/chosen_KL_mean": -188.3211669921875, + "KL/mean": -247.48739624023438, + "KL/rejected_KL_mean": -306.65362548828125, + "KL/std": 120.37994384765625, + "epoch": 0.7929515418502202, + "fcm_dpo/beta": 0.00395231181755662, + "fcm_dpo/delta": -0.07105652987957001, + "fcm_dpo/margin": 118.33247375488281, + "fcm_dpo/q_t": 0.39300185441970825, + "grad_norm": 10.326759338378906, + "learning_rate": 6.352838968463919e-08, + "logits/chosen": -1.9810630083084106, + "logits/rejected": -1.7435001134872437, + "logps/chosen": -296.1910705566406, + "logps/ref_chosen": -107.86990356445312, + "logps/ref_rejected": -121.39350128173828, + "logps/rejected": -428.047119140625, + "loss": 1.0455, + "margin_dpo/margin_mean": 118.33245849609375, + "margin_dpo/margin_std": 141.44961547851562, + "step": 540 + }, + { + "KL/chosen_KL_mean": -208.34991455078125, + "KL/mean": -245.27102661132812, + "KL/rejected_KL_mean": -282.1921081542969, + "KL/std": 99.07437133789062, + "epoch": 0.7944199706314243, + "fcm_dpo/beta": 0.003920359071344137, + "fcm_dpo/delta": -0.001973494188860059, + "fcm_dpo/margin": 73.84220886230469, + "fcm_dpo/q_t": 0.4318574070930481, + "grad_norm": 9.59099292755127, + "learning_rate": 6.267605843546767e-08, + "logits/chosen": -2.4143664836883545, + "logits/rejected": -1.957679271697998, + "logps/chosen": -314.91741943359375, + "logps/ref_chosen": -106.56748962402344, + "logps/ref_rejected": -109.06633758544922, + "logps/rejected": -391.2584533691406, + "loss": 1.1651, + "margin_dpo/margin_mean": 73.84220886230469, + "margin_dpo/margin_std": 107.66023254394531, + "step": 541 + }, + { + "KL/chosen_KL_mean": -176.13027954101562, + "KL/mean": -233.16513061523438, + "KL/rejected_KL_mean": -290.199951171875, + "KL/std": 112.2148666381836, + "epoch": 0.7958883994126285, + "fcm_dpo/beta": 0.003893241286277771, + "fcm_dpo/delta": -0.046153679490089417, + "fcm_dpo/margin": 114.06965637207031, + "fcm_dpo/q_t": 0.3952111601829529, + "grad_norm": 10.262336730957031, + "learning_rate": 6.182866334636888e-08, + "logits/chosen": -2.1802868843078613, + "logits/rejected": -2.055865526199341, + "logps/chosen": -263.49957275390625, + "logps/ref_chosen": -87.36929321289062, + "logps/ref_rejected": -102.82034301757812, + "logps/rejected": -393.02032470703125, + "loss": 1.0438, + "margin_dpo/margin_mean": 114.06965637207031, + "margin_dpo/margin_std": 121.2616958618164, + "step": 542 + }, + { + "KL/chosen_KL_mean": -203.35427856445312, + "KL/mean": -233.46575927734375, + "KL/rejected_KL_mean": -263.5772399902344, + "KL/std": 118.56103515625, + "epoch": 0.7973568281938326, + "fcm_dpo/beta": 0.003922195639461279, + "fcm_dpo/delta": 0.04849550127983093, + "fcm_dpo/margin": 60.22294235229492, + "fcm_dpo/q_t": 0.44569286704063416, + "grad_norm": 11.878764152526855, + "learning_rate": 6.098622674699147e-08, + "logits/chosen": -1.6643638610839844, + "logits/rejected": -1.7139687538146973, + "logps/chosen": -290.5068664550781, + "logps/ref_chosen": -87.152587890625, + "logps/ref_rejected": -111.53172302246094, + "logps/rejected": -375.10894775390625, + "loss": 1.2417, + "margin_dpo/margin_mean": 60.22294616699219, + "margin_dpo/margin_std": 144.78778076171875, + "step": 543 + }, + { + "KL/chosen_KL_mean": -212.8037109375, + "KL/mean": -251.02288818359375, + "KL/rejected_KL_mean": -289.2420959472656, + "KL/std": 111.28211212158203, + "epoch": 0.7988252569750367, + "fcm_dpo/beta": 0.003956153057515621, + "fcm_dpo/delta": 0.043102238327264786, + "fcm_dpo/margin": 76.43841552734375, + "fcm_dpo/q_t": 0.4294566512107849, + "grad_norm": 9.049397468566895, + "learning_rate": 6.01487708363232e-08, + "logits/chosen": -1.7759820222854614, + "logits/rejected": -1.6834551095962524, + "logps/chosen": -298.95062255859375, + "logps/ref_chosen": -86.14691162109375, + "logps/ref_rejected": -106.3939208984375, + "logps/rejected": -395.6360168457031, + "loss": 1.1571, + "margin_dpo/margin_mean": 76.43841552734375, + "margin_dpo/margin_std": 117.00006866455078, + "step": 544 + }, + { + "KL/chosen_KL_mean": -188.0811767578125, + "KL/mean": -235.30410766601562, + "KL/rejected_KL_mean": -282.52703857421875, + "KL/std": 110.29922485351562, + "epoch": 0.8002936857562408, + "fcm_dpo/beta": 0.003976769745349884, + "fcm_dpo/delta": 0.02528545819222927, + "fcm_dpo/margin": 94.44583892822266, + "fcm_dpo/q_t": 0.4127165675163269, + "grad_norm": 8.903777122497559, + "learning_rate": 5.9316317682106294e-08, + "logits/chosen": -1.6127533912658691, + "logits/rejected": -1.1913080215454102, + "logps/chosen": -284.50543212890625, + "logps/ref_chosen": -96.42424011230469, + "logps/ref_rejected": -100.73405456542969, + "logps/rejected": -383.26104736328125, + "loss": 1.1161, + "margin_dpo/margin_mean": 94.44583129882812, + "margin_dpo/margin_std": 136.73757934570312, + "step": 545 + }, + { + "KL/chosen_KL_mean": -194.41531372070312, + "KL/mean": -236.50909423828125, + "KL/rejected_KL_mean": -278.6029052734375, + "KL/std": 111.8950424194336, + "epoch": 0.801762114537445, + "fcm_dpo/beta": 0.004007780458778143, + "fcm_dpo/delta": 0.06483136117458344, + "fcm_dpo/margin": 84.18757629394531, + "fcm_dpo/q_t": 0.4204888939857483, + "grad_norm": 8.961593627929688, + "learning_rate": 5.848888922025552e-08, + "logits/chosen": -1.5761886835098267, + "logits/rejected": -1.2249510288238525, + "logps/chosen": -280.4364318847656, + "logps/ref_chosen": -86.02112579345703, + "logps/ref_rejected": -87.37263488769531, + "logps/rejected": -365.97552490234375, + "loss": 1.1359, + "margin_dpo/margin_mean": 84.18758392333984, + "margin_dpo/margin_std": 123.34027862548828, + "step": 546 + }, + { + "KL/chosen_KL_mean": -186.7025146484375, + "KL/mean": -237.83062744140625, + "KL/rejected_KL_mean": -288.958740234375, + "KL/std": 109.60289764404297, + "epoch": 0.8032305433186491, + "fcm_dpo/beta": 0.004029825329780579, + "fcm_dpo/delta": -0.012673644348978996, + "fcm_dpo/margin": 102.25625610351562, + "fcm_dpo/q_t": 0.40199214220046997, + "grad_norm": 9.157228469848633, + "learning_rate": 5.7666507254280265e-08, + "logits/chosen": -1.8417205810546875, + "logits/rejected": -1.378598690032959, + "logps/chosen": -287.353515625, + "logps/ref_chosen": -100.6510009765625, + "logps/ref_rejected": -97.9117431640625, + "logps/rejected": -386.8704833984375, + "loss": 1.0538, + "margin_dpo/margin_mean": 102.25625610351562, + "margin_dpo/margin_std": 97.4776611328125, + "step": 547 + }, + { + "KL/chosen_KL_mean": -183.82083129882812, + "KL/mean": -225.64892578125, + "KL/rejected_KL_mean": -267.47698974609375, + "KL/std": 111.69984436035156, + "epoch": 0.8046989720998532, + "fcm_dpo/beta": 0.0040570092387497425, + "fcm_dpo/delta": 0.06271268427371979, + "fcm_dpo/margin": 83.6561279296875, + "fcm_dpo/q_t": 0.4214785397052765, + "grad_norm": 10.480459213256836, + "learning_rate": 5.684919345471029e-08, + "logits/chosen": -2.3335888385772705, + "logits/rejected": -1.980445146560669, + "logps/chosen": -279.8509521484375, + "logps/ref_chosen": -96.03010559082031, + "logps/ref_rejected": -98.41322326660156, + "logps/rejected": -365.89019775390625, + "loss": 1.1525, + "margin_dpo/margin_mean": 83.6561279296875, + "margin_dpo/margin_std": 139.59446716308594, + "step": 548 + }, + { + "KL/chosen_KL_mean": -173.96072387695312, + "KL/mean": -216.42337036132812, + "KL/rejected_KL_mean": -258.885986328125, + "KL/std": 105.89651489257812, + "epoch": 0.8061674008810573, + "fcm_dpo/beta": 0.0040628137066960335, + "fcm_dpo/delta": -0.014284750446677208, + "fcm_dpo/margin": 84.92527770996094, + "fcm_dpo/q_t": 0.41891396045684814, + "grad_norm": 10.847293853759766, + "learning_rate": 5.603696935852426e-08, + "logits/chosen": -1.8971278667449951, + "logits/rejected": -1.4823839664459229, + "logps/chosen": -257.41168212890625, + "logps/ref_chosen": -83.45095825195312, + "logps/ref_rejected": -81.38023376464844, + "logps/rejected": -340.2662353515625, + "loss": 1.1409, + "margin_dpo/margin_mean": 84.92527770996094, + "margin_dpo/margin_std": 130.41908264160156, + "step": 549 + }, + { + "KL/chosen_KL_mean": -193.0083770751953, + "KL/mean": -234.84326171875, + "KL/rejected_KL_mean": -276.67816162109375, + "KL/std": 100.83824157714844, + "epoch": 0.8076358296622613, + "fcm_dpo/beta": 0.004099378362298012, + "fcm_dpo/delta": 0.05888357013463974, + "fcm_dpo/margin": 83.66975402832031, + "fcm_dpo/q_t": 0.4198307693004608, + "grad_norm": 9.087468147277832, + "learning_rate": 5.5229856368582376e-08, + "logits/chosen": -1.5271849632263184, + "logits/rejected": -1.3121836185455322, + "logps/chosen": -282.60272216796875, + "logps/ref_chosen": -89.59434509277344, + "logps/ref_rejected": -101.31549835205078, + "logps/rejected": -377.99365234375, + "loss": 1.1223, + "margin_dpo/margin_mean": 83.66976165771484, + "margin_dpo/margin_std": 109.50350952148438, + "step": 550 + }, + { + "KL/chosen_KL_mean": -171.8982696533203, + "KL/mean": -223.13900756835938, + "KL/rejected_KL_mean": -274.3797607421875, + "KL/std": 115.19548797607422, + "epoch": 0.8091042584434655, + "fcm_dpo/beta": 0.004074958153069019, + "fcm_dpo/delta": -0.018885452300310135, + "fcm_dpo/margin": 102.48149108886719, + "fcm_dpo/q_t": 0.40341877937316895, + "grad_norm": 10.372711181640625, + "learning_rate": 5.4427875753062734e-08, + "logits/chosen": -1.7168750762939453, + "logits/rejected": -1.5784611701965332, + "logps/chosen": -266.98040771484375, + "logps/ref_chosen": -95.08216094970703, + "logps/ref_rejected": -117.7192611694336, + "logps/rejected": -392.0990295410156, + "loss": 1.0663, + "margin_dpo/margin_mean": 102.48149108886719, + "margin_dpo/margin_std": 117.40731811523438, + "step": 551 + }, + { + "KL/chosen_KL_mean": -162.25375366210938, + "KL/mean": -235.99691772460938, + "KL/rejected_KL_mean": -309.7400817871094, + "KL/std": 124.73002624511719, + "epoch": 0.8105726872246696, + "fcm_dpo/beta": 0.003944192081689835, + "fcm_dpo/delta": -0.19540630280971527, + "fcm_dpo/margin": 147.48631286621094, + "fcm_dpo/q_t": 0.3634408116340637, + "grad_norm": 11.11823558807373, + "learning_rate": 5.363104864490034e-08, + "logits/chosen": -2.0533602237701416, + "logits/rejected": -1.691412091255188, + "logps/chosen": -263.6629638671875, + "logps/ref_chosen": -101.40918731689453, + "logps/ref_rejected": -110.83843994140625, + "logps/rejected": -420.5785217285156, + "loss": 0.9501, + "margin_dpo/margin_mean": 147.48631286621094, + "margin_dpo/margin_std": 128.76419067382812, + "step": 552 + }, + { + "KL/chosen_KL_mean": -177.9302978515625, + "KL/mean": -221.57574462890625, + "KL/rejected_KL_mean": -265.22119140625, + "KL/std": 115.44457244873047, + "epoch": 0.8120411160058737, + "fcm_dpo/beta": 0.003971524070948362, + "fcm_dpo/delta": 0.05510055273771286, + "fcm_dpo/margin": 87.29090881347656, + "fcm_dpo/q_t": 0.4185052514076233, + "grad_norm": 9.109249114990234, + "learning_rate": 5.2839396041230415e-08, + "logits/chosen": -1.3791086673736572, + "logits/rejected": -1.0422030687332153, + "logps/chosen": -275.8941650390625, + "logps/ref_chosen": -97.96388244628906, + "logps/ref_rejected": -103.57719421386719, + "logps/rejected": -368.79840087890625, + "loss": 1.1283, + "margin_dpo/margin_mean": 87.29090881347656, + "margin_dpo/margin_std": 124.11241149902344, + "step": 553 + }, + { + "KL/chosen_KL_mean": -186.0308837890625, + "KL/mean": -234.49024963378906, + "KL/rejected_KL_mean": -282.9495849609375, + "KL/std": 116.15403747558594, + "epoch": 0.8135095447870778, + "fcm_dpo/beta": 0.0039992001838982105, + "fcm_dpo/delta": 0.012554768472909927, + "fcm_dpo/margin": 96.91871643066406, + "fcm_dpo/q_t": 0.41062331199645996, + "grad_norm": 8.550159454345703, + "learning_rate": 5.205293880283551e-08, + "logits/chosen": -1.8397386074066162, + "logits/rejected": -1.057518482208252, + "logps/chosen": -289.7176818847656, + "logps/ref_chosen": -103.68680572509766, + "logps/ref_rejected": -92.99210357666016, + "logps/rejected": -375.94171142578125, + "loss": 1.1214, + "margin_dpo/margin_mean": 96.91871643066406, + "margin_dpo/margin_std": 149.53387451171875, + "step": 554 + }, + { + "KL/chosen_KL_mean": -194.73667907714844, + "KL/mean": -239.07911682128906, + "KL/rejected_KL_mean": -283.42156982421875, + "KL/std": 119.47474670410156, + "epoch": 0.8149779735682819, + "fcm_dpo/beta": 0.0040138796903193, + "fcm_dpo/delta": 0.0456521213054657, + "fcm_dpo/margin": 88.68490600585938, + "fcm_dpo/q_t": 0.4184246063232422, + "grad_norm": 9.095036506652832, + "learning_rate": 5.127169765359515e-08, + "logits/chosen": -1.6609153747558594, + "logits/rejected": -1.5697379112243652, + "logps/chosen": -280.9403076171875, + "logps/ref_chosen": -86.20362854003906, + "logps/ref_rejected": -114.54719543457031, + "logps/rejected": -397.96875, + "loss": 1.1339, + "margin_dpo/margin_mean": 88.68489837646484, + "margin_dpo/margin_std": 137.96524047851562, + "step": 555 + }, + { + "KL/chosen_KL_mean": -175.3978271484375, + "KL/mean": -223.60971069335938, + "KL/rejected_KL_mean": -271.82159423828125, + "KL/std": 106.0242919921875, + "epoch": 0.8164464023494861, + "fcm_dpo/beta": 0.004019307438284159, + "fcm_dpo/delta": 0.01281764730811119, + "fcm_dpo/margin": 96.42373657226562, + "fcm_dpo/q_t": 0.4092983603477478, + "grad_norm": 10.368645668029785, + "learning_rate": 5.049569317994012e-08, + "logits/chosen": -1.487354040145874, + "logits/rejected": -1.1843937635421753, + "logps/chosen": -263.80133056640625, + "logps/ref_chosen": -88.40348815917969, + "logps/ref_rejected": -106.63911437988281, + "logps/rejected": -378.460693359375, + "loss": 1.0921, + "margin_dpo/margin_mean": 96.42373657226562, + "margin_dpo/margin_std": 120.55360412597656, + "step": 556 + }, + { + "KL/chosen_KL_mean": -187.70565795898438, + "KL/mean": -242.84906005859375, + "KL/rejected_KL_mean": -297.9924621582031, + "KL/std": 122.87167358398438, + "epoch": 0.8179148311306902, + "fcm_dpo/beta": 0.003991108387708664, + "fcm_dpo/delta": -0.04255310073494911, + "fcm_dpo/margin": 110.28684997558594, + "fcm_dpo/q_t": 0.398430198431015, + "grad_norm": 8.112653732299805, + "learning_rate": 4.9724945830310144e-08, + "logits/chosen": -1.9215428829193115, + "logits/rejected": -1.748028039932251, + "logps/chosen": -286.5408935546875, + "logps/ref_chosen": -98.83523559570312, + "logps/ref_rejected": -115.64888763427734, + "logps/rejected": -413.641357421875, + "loss": 1.0617, + "margin_dpo/margin_mean": 110.28684997558594, + "margin_dpo/margin_std": 133.552978515625, + "step": 557 + }, + { + "KL/chosen_KL_mean": -183.21401977539062, + "KL/mean": -242.38037109375, + "KL/rejected_KL_mean": -301.5467224121094, + "KL/std": 106.28973388671875, + "epoch": 0.8193832599118943, + "fcm_dpo/beta": 0.0039487662725150585, + "fcm_dpo/delta": -0.07081079483032227, + "fcm_dpo/margin": 118.33269500732422, + "fcm_dpo/q_t": 0.3897389769554138, + "grad_norm": 10.605459213256836, + "learning_rate": 4.8959475914614554e-08, + "logits/chosen": -2.327383041381836, + "logits/rejected": -1.714231252670288, + "logps/chosen": -289.2258605957031, + "logps/ref_chosen": -106.01183319091797, + "logps/ref_rejected": -108.39131927490234, + "logps/rejected": -409.93804931640625, + "loss": 1.0187, + "margin_dpo/margin_mean": 118.33269500732422, + "margin_dpo/margin_std": 110.4193115234375, + "step": 558 + }, + { + "KL/chosen_KL_mean": -201.95263671875, + "KL/mean": -248.3455810546875, + "KL/rejected_KL_mean": -294.738525390625, + "KL/std": 108.78970336914062, + "epoch": 0.8208516886930984, + "fcm_dpo/beta": 0.0039668213576078415, + "fcm_dpo/delta": 0.03309304639697075, + "fcm_dpo/margin": 92.785888671875, + "fcm_dpo/q_t": 0.4132770895957947, + "grad_norm": 10.393129348754883, + "learning_rate": 4.8199303603697614e-08, + "logits/chosen": -2.2402353286743164, + "logits/rejected": -1.7538548707962036, + "logps/chosen": -302.37738037109375, + "logps/ref_chosen": -100.42474365234375, + "logps/ref_rejected": -100.41270446777344, + "logps/rejected": -395.1512451171875, + "loss": 1.11, + "margin_dpo/margin_mean": 92.785888671875, + "margin_dpo/margin_std": 123.98502349853516, + "step": 559 + }, + { + "KL/chosen_KL_mean": -189.90618896484375, + "KL/mean": -236.27590942382812, + "KL/rejected_KL_mean": -282.6456298828125, + "KL/std": 106.77798461914062, + "epoch": 0.8223201174743024, + "fcm_dpo/beta": 0.003984309732913971, + "fcm_dpo/delta": 0.03167944774031639, + "fcm_dpo/margin": 92.73944091796875, + "fcm_dpo/q_t": 0.4136253595352173, + "grad_norm": 11.9866304397583, + "learning_rate": 4.7444448928806615e-08, + "logits/chosen": -1.2863881587982178, + "logits/rejected": -0.8358875513076782, + "logps/chosen": -277.5417175292969, + "logps/ref_chosen": -87.63551330566406, + "logps/ref_rejected": -87.37985229492188, + "logps/rejected": -370.0255126953125, + "loss": 1.1077, + "margin_dpo/margin_mean": 92.73944091796875, + "margin_dpo/margin_std": 122.90707397460938, + "step": 560 + }, + { + "KL/chosen_KL_mean": -199.8811798095703, + "KL/mean": -242.90728759765625, + "KL/rejected_KL_mean": -285.93341064453125, + "KL/std": 105.08248901367188, + "epoch": 0.8237885462555066, + "fcm_dpo/beta": 0.004034621641039848, + "fcm_dpo/delta": 0.05453299731016159, + "fcm_dpo/margin": 86.05224609375, + "fcm_dpo/q_t": 0.4181227385997772, + "grad_norm": 8.237642288208008, + "learning_rate": 4.669493178106432e-08, + "logits/chosen": -1.7476694583892822, + "logits/rejected": -1.721436858177185, + "logps/chosen": -274.7625732421875, + "logps/ref_chosen": -74.88140106201172, + "logps/ref_rejected": -106.04796600341797, + "logps/rejected": -391.98138427734375, + "loss": 1.1089, + "margin_dpo/margin_mean": 86.05224609375, + "margin_dpo/margin_std": 101.30625915527344, + "step": 561 + }, + { + "KL/chosen_KL_mean": -181.40444946289062, + "KL/mean": -224.77960205078125, + "KL/rejected_KL_mean": -268.1547546386719, + "KL/std": 106.30143737792969, + "epoch": 0.8252569750367107, + "fcm_dpo/beta": 0.0040319073013961315, + "fcm_dpo/delta": -0.007692549843341112, + "fcm_dpo/margin": 86.75032043457031, + "fcm_dpo/q_t": 0.418799489736557, + "grad_norm": 13.258758544921875, + "learning_rate": 4.5950771910944596e-08, + "logits/chosen": -1.9952281713485718, + "logits/rejected": -1.5297197103500366, + "logps/chosen": -282.06768798828125, + "logps/ref_chosen": -100.66322326660156, + "logps/ref_rejected": -102.24087524414062, + "logps/rejected": -370.3956298828125, + "loss": 1.1345, + "margin_dpo/margin_mean": 86.75031280517578, + "margin_dpo/margin_std": 132.64466857910156, + "step": 562 + }, + { + "KL/chosen_KL_mean": -191.42404174804688, + "KL/mean": -227.6953582763672, + "KL/rejected_KL_mean": -263.9666748046875, + "KL/std": 107.09082794189453, + "epoch": 0.8267254038179148, + "fcm_dpo/beta": 0.004042648710310459, + "fcm_dpo/delta": 0.026570217683911324, + "fcm_dpo/margin": 72.5426254272461, + "fcm_dpo/q_t": 0.4308703541755676, + "grad_norm": 10.151522636413574, + "learning_rate": 4.521198892775202e-08, + "logits/chosen": -1.7156529426574707, + "logits/rejected": -1.3790748119354248, + "logps/chosen": -287.6832580566406, + "logps/ref_chosen": -96.25921630859375, + "logps/ref_rejected": -101.20503997802734, + "logps/rejected": -365.17169189453125, + "loss": 1.1658, + "margin_dpo/margin_mean": 72.5426254272461, + "margin_dpo/margin_std": 113.81546020507812, + "step": 563 + }, + { + "KL/chosen_KL_mean": -186.30447387695312, + "KL/mean": -227.33822631835938, + "KL/rejected_KL_mean": -268.3719787597656, + "KL/std": 107.06916809082031, + "epoch": 0.8281938325991189, + "fcm_dpo/beta": 0.00410117581486702, + "fcm_dpo/delta": 0.06542927771806717, + "fcm_dpo/margin": 82.0675048828125, + "fcm_dpo/q_t": 0.42022332549095154, + "grad_norm": 11.273946762084961, + "learning_rate": 4.447860229910544e-08, + "logits/chosen": -2.2707040309906006, + "logits/rejected": -1.692732334136963, + "logps/chosen": -299.264892578125, + "logps/ref_chosen": -112.96040344238281, + "logps/ref_rejected": -99.52803802490234, + "logps/rejected": -367.9000244140625, + "loss": 1.1177, + "margin_dpo/margin_mean": 82.0675048828125, + "margin_dpo/margin_std": 98.18555450439453, + "step": 564 + }, + { + "KL/chosen_KL_mean": -185.06207275390625, + "KL/mean": -235.034912109375, + "KL/rejected_KL_mean": -285.00775146484375, + "KL/std": 116.40463256835938, + "epoch": 0.8296622613803231, + "fcm_dpo/beta": 0.004095645155757666, + "fcm_dpo/delta": -0.009785487316548824, + "fcm_dpo/margin": 99.94569396972656, + "fcm_dpo/q_t": 0.40508097410202026, + "grad_norm": 9.245997428894043, + "learning_rate": 4.375063135042445e-08, + "logits/chosen": -1.679760456085205, + "logits/rejected": -1.174295425415039, + "logps/chosen": -280.42767333984375, + "logps/ref_chosen": -95.36558532714844, + "logps/ref_rejected": -92.21737670898438, + "logps/rejected": -377.22515869140625, + "loss": 1.0827, + "margin_dpo/margin_mean": 99.94569396972656, + "margin_dpo/margin_std": 127.49440002441406, + "step": 565 + }, + { + "KL/chosen_KL_mean": -192.6268768310547, + "KL/mean": -239.96502685546875, + "KL/rejected_KL_mean": -287.30316162109375, + "KL/std": 118.42991638183594, + "epoch": 0.8311306901615272, + "fcm_dpo/beta": 0.004114923067390919, + "fcm_dpo/delta": 0.010699309408664703, + "fcm_dpo/margin": 94.67634582519531, + "fcm_dpo/q_t": 0.4082973003387451, + "grad_norm": 8.766554832458496, + "learning_rate": 4.3028095264420525e-08, + "logits/chosen": -1.9097862243652344, + "logits/rejected": -1.4794607162475586, + "logps/chosen": -291.60009765625, + "logps/ref_chosen": -98.97320556640625, + "logps/ref_rejected": -110.82090759277344, + "logps/rejected": -398.12408447265625, + "loss": 1.0905, + "margin_dpo/margin_mean": 94.67633819580078, + "margin_dpo/margin_std": 117.06986999511719, + "step": 566 + }, + { + "KL/chosen_KL_mean": -186.6658935546875, + "KL/mean": -228.60211181640625, + "KL/rejected_KL_mean": -270.538330078125, + "KL/std": 107.04507446289062, + "epoch": 0.8325991189427313, + "fcm_dpo/beta": 0.00409280089661479, + "fcm_dpo/delta": -0.017962271347641945, + "fcm_dpo/margin": 83.87245178222656, + "fcm_dpo/q_t": 0.41831058263778687, + "grad_norm": 9.604767799377441, + "learning_rate": 4.231101308059165e-08, + "logits/chosen": -2.2468090057373047, + "logits/rejected": -1.5719454288482666, + "logps/chosen": -279.3695068359375, + "logps/ref_chosen": -92.7035903930664, + "logps/ref_rejected": -91.22431945800781, + "logps/rejected": -361.7626647949219, + "loss": 1.1267, + "margin_dpo/margin_mean": 83.87245178222656, + "margin_dpo/margin_std": 113.43426513671875, + "step": 567 + }, + { + "KL/chosen_KL_mean": -176.72360229492188, + "KL/mean": -232.8488006591797, + "KL/rejected_KL_mean": -288.9739990234375, + "KL/std": 112.92816162109375, + "epoch": 0.8340675477239354, + "fcm_dpo/beta": 0.00404569273814559, + "fcm_dpo/delta": -0.05707617476582527, + "fcm_dpo/margin": 112.25041198730469, + "fcm_dpo/q_t": 0.3934527039527893, + "grad_norm": 8.518280982971191, + "learning_rate": 4.1599403694720145e-08, + "logits/chosen": -1.4681239128112793, + "logits/rejected": -1.3411650657653809, + "logps/chosen": -249.85586547851562, + "logps/ref_chosen": -73.13226318359375, + "logps/ref_rejected": -94.95265197753906, + "logps/rejected": -383.9266662597656, + "loss": 1.0337, + "margin_dpo/margin_mean": 112.25041961669922, + "margin_dpo/margin_std": 114.32151794433594, + "step": 568 + }, + { + "KL/chosen_KL_mean": -187.18194580078125, + "KL/mean": -233.02822875976562, + "KL/rejected_KL_mean": -278.87451171875, + "KL/std": 114.64080810546875, + "epoch": 0.8355359765051396, + "fcm_dpo/beta": 0.004068131558597088, + "fcm_dpo/delta": 0.02796497754752636, + "fcm_dpo/margin": 91.69258117675781, + "fcm_dpo/q_t": 0.412933886051178, + "grad_norm": 12.933995246887207, + "learning_rate": 4.089328585837512e-08, + "logits/chosen": -2.0733516216278076, + "logits/rejected": -1.703305959701538, + "logps/chosen": -276.3990478515625, + "logps/ref_chosen": -89.21708679199219, + "logps/ref_rejected": -86.41105651855469, + "logps/rejected": -365.28558349609375, + "loss": 1.1232, + "margin_dpo/margin_mean": 91.69258117675781, + "margin_dpo/margin_std": 137.6352996826172, + "step": 569 + }, + { + "KL/chosen_KL_mean": -167.8793487548828, + "KL/mean": -226.79827880859375, + "KL/rejected_KL_mean": -285.7171936035156, + "KL/std": 108.667236328125, + "epoch": 0.8370044052863436, + "fcm_dpo/beta": 0.004019296728074551, + "fcm_dpo/delta": -0.0773601233959198, + "fcm_dpo/margin": 117.83782958984375, + "fcm_dpo/q_t": 0.3878633379936218, + "grad_norm": 8.910812377929688, + "learning_rate": 4.019267817841834e-08, + "logits/chosen": -2.4476561546325684, + "logits/rejected": -1.7825052738189697, + "logps/chosen": -262.6421203613281, + "logps/ref_chosen": -94.76277160644531, + "logps/ref_rejected": -88.846923828125, + "logps/rejected": -374.5641174316406, + "loss": 1.0175, + "margin_dpo/margin_mean": 117.83782958984375, + "margin_dpo/margin_std": 113.94918823242188, + "step": 570 + }, + { + "KL/chosen_KL_mean": -175.6620330810547, + "KL/mean": -227.6282501220703, + "KL/rejected_KL_mean": -279.594482421875, + "KL/std": 106.80409240722656, + "epoch": 0.8384728340675477, + "fcm_dpo/beta": 0.004005197435617447, + "fcm_dpo/delta": -0.01703745126724243, + "fcm_dpo/margin": 103.93246459960938, + "fcm_dpo/q_t": 0.40318673849105835, + "grad_norm": 12.429949760437012, + "learning_rate": 3.9497599116513705e-08, + "logits/chosen": -1.5648592710494995, + "logits/rejected": -1.4034879207611084, + "logps/chosen": -256.4833068847656, + "logps/ref_chosen": -80.8212890625, + "logps/ref_rejected": -97.01528930664062, + "logps/rejected": -376.6097717285156, + "loss": 1.0765, + "margin_dpo/margin_mean": 103.93246459960938, + "margin_dpo/margin_std": 129.15676879882812, + "step": 571 + }, + { + "KL/chosen_KL_mean": -179.8299102783203, + "KL/mean": -232.33876037597656, + "KL/rejected_KL_mean": -284.8476257324219, + "KL/std": 113.78169250488281, + "epoch": 0.8399412628487518, + "fcm_dpo/beta": 0.003977552987635136, + "fcm_dpo/delta": -0.01853088289499283, + "fcm_dpo/margin": 105.01771545410156, + "fcm_dpo/q_t": 0.40315932035446167, + "grad_norm": 8.510725021362305, + "learning_rate": 3.880806698864086e-08, + "logits/chosen": -1.8613808155059814, + "logits/rejected": -1.6963881254196167, + "logps/chosen": -257.2098388671875, + "logps/ref_chosen": -77.37992858886719, + "logps/ref_rejected": -90.56597900390625, + "logps/rejected": -375.4136047363281, + "loss": 1.0667, + "margin_dpo/margin_mean": 105.01771545410156, + "margin_dpo/margin_std": 121.76112365722656, + "step": 572 + }, + { + "KL/chosen_KL_mean": -175.18084716796875, + "KL/mean": -223.95648193359375, + "KL/rejected_KL_mean": -272.7320861816406, + "KL/std": 113.06573486328125, + "epoch": 0.8414096916299559, + "fcm_dpo/beta": 0.003957781475037336, + "fcm_dpo/delta": -0.051512010395526886, + "fcm_dpo/margin": 97.55123138427734, + "fcm_dpo/q_t": 0.4086564779281616, + "grad_norm": 8.708194732666016, + "learning_rate": 3.812409996461275e-08, + "logits/chosen": -2.1375205516815186, + "logits/rejected": -1.9066476821899414, + "logps/chosen": -258.8869323730469, + "logps/ref_chosen": -83.7060775756836, + "logps/ref_rejected": -90.69746398925781, + "logps/rejected": -363.4295654296875, + "loss": 1.0937, + "margin_dpo/margin_mean": 97.55123901367188, + "margin_dpo/margin_std": 119.69587707519531, + "step": 573 + }, + { + "KL/chosen_KL_mean": -184.34471130371094, + "KL/mean": -237.38047790527344, + "KL/rejected_KL_mean": -290.41619873046875, + "KL/std": 112.2585678100586, + "epoch": 0.8428781204111601, + "fcm_dpo/beta": 0.003938412293791771, + "fcm_dpo/delta": -0.018658364191651344, + "fcm_dpo/margin": 106.0715103149414, + "fcm_dpo/q_t": 0.4022049307823181, + "grad_norm": 9.27223014831543, + "learning_rate": 3.74457160675965e-08, + "logits/chosen": -2.2239112854003906, + "logits/rejected": -1.9022197723388672, + "logps/chosen": -269.47418212890625, + "logps/ref_chosen": -85.12948608398438, + "logps/ref_rejected": -99.05403137207031, + "logps/rejected": -389.4702453613281, + "loss": 1.0714, + "margin_dpo/margin_mean": 106.07151794433594, + "margin_dpo/margin_std": 126.59791564941406, + "step": 574 + }, + { + "KL/chosen_KL_mean": -190.53359985351562, + "KL/mean": -239.09487915039062, + "KL/rejected_KL_mean": -287.65618896484375, + "KL/std": 114.81494140625, + "epoch": 0.8443465491923642, + "fcm_dpo/beta": 0.003933538682758808, + "fcm_dpo/delta": 0.0186786912381649, + "fcm_dpo/margin": 97.12259674072266, + "fcm_dpo/q_t": 0.4112199544906616, + "grad_norm": 10.575162887573242, + "learning_rate": 3.677293317363864e-08, + "logits/chosen": -1.2434958219528198, + "logits/rejected": -0.7674872279167175, + "logps/chosen": -296.304443359375, + "logps/ref_chosen": -105.77084350585938, + "logps/ref_rejected": -100.94944763183594, + "logps/rejected": -388.60565185546875, + "loss": 1.1055, + "margin_dpo/margin_mean": 97.12259674072266, + "margin_dpo/margin_std": 133.65907287597656, + "step": 575 + }, + { + "KL/chosen_KL_mean": -188.86343383789062, + "KL/mean": -235.95697021484375, + "KL/rejected_KL_mean": -283.05047607421875, + "KL/std": 106.71722412109375, + "epoch": 0.8458149779735683, + "fcm_dpo/beta": 0.003959815017879009, + "fcm_dpo/delta": 0.02799609862267971, + "fcm_dpo/margin": 94.18707275390625, + "fcm_dpo/q_t": 0.4126693904399872, + "grad_norm": 10.981925010681152, + "learning_rate": 3.6105769011194224e-08, + "logits/chosen": -2.0085506439208984, + "logits/rejected": -1.948533296585083, + "logps/chosen": -270.0679626464844, + "logps/ref_chosen": -81.20452880859375, + "logps/ref_rejected": -105.40940856933594, + "logps/rejected": -388.45989990234375, + "loss": 1.1099, + "margin_dpo/margin_mean": 94.18707275390625, + "margin_dpo/margin_std": 128.36178588867188, + "step": 576 + }, + { + "KL/chosen_KL_mean": -170.01434326171875, + "KL/mean": -228.88552856445312, + "KL/rejected_KL_mean": -287.7567138671875, + "KL/std": 115.63882446289062, + "epoch": 0.8472834067547724, + "fcm_dpo/beta": 0.003909149672836065, + "fcm_dpo/delta": -0.06362038105726242, + "fcm_dpo/margin": 117.74236297607422, + "fcm_dpo/q_t": 0.39235496520996094, + "grad_norm": 8.003883361816406, + "learning_rate": 3.5444241160659304e-08, + "logits/chosen": -2.1418039798736572, + "logits/rejected": -1.4616312980651855, + "logps/chosen": -271.4785461425781, + "logps/ref_chosen": -101.46419525146484, + "logps/ref_rejected": -89.32637023925781, + "logps/rejected": -377.08306884765625, + "loss": 1.0412, + "margin_dpo/margin_mean": 117.74236297607422, + "margin_dpo/margin_std": 131.4029541015625, + "step": 577 + }, + { + "KL/chosen_KL_mean": -202.08888244628906, + "KL/mean": -245.14999389648438, + "KL/rejected_KL_mean": -288.21112060546875, + "KL/std": 110.09112548828125, + "epoch": 0.8487518355359766, + "fcm_dpo/beta": 0.003944209311157465, + "fcm_dpo/delta": 0.06246161460876465, + "fcm_dpo/margin": 86.12225341796875, + "fcm_dpo/q_t": 0.4206528663635254, + "grad_norm": 13.475691795349121, + "learning_rate": 3.478836705390808e-08, + "logits/chosen": -1.2917630672454834, + "logits/rejected": -1.0502583980560303, + "logps/chosen": -279.38128662109375, + "logps/ref_chosen": -77.29241943359375, + "logps/ref_rejected": -88.24701690673828, + "logps/rejected": -376.4581298828125, + "loss": 1.1342, + "margin_dpo/margin_mean": 86.12226104736328, + "margin_dpo/margin_std": 126.02249145507812, + "step": 578 + }, + { + "KL/chosen_KL_mean": -181.1404266357422, + "KL/mean": -225.11721801757812, + "KL/rejected_KL_mean": -269.09405517578125, + "KL/std": 109.78843688964844, + "epoch": 0.8502202643171806, + "fcm_dpo/beta": 0.00394777674227953, + "fcm_dpo/delta": -0.013691302388906479, + "fcm_dpo/margin": 87.95361328125, + "fcm_dpo/q_t": 0.41843438148498535, + "grad_norm": 7.778433799743652, + "learning_rate": 3.41381639738331e-08, + "logits/chosen": -1.8752148151397705, + "logits/rejected": -1.5858491659164429, + "logps/chosen": -268.48345947265625, + "logps/ref_chosen": -87.34305572509766, + "logps/ref_rejected": -102.12100982666016, + "logps/rejected": -371.2150573730469, + "loss": 1.1196, + "margin_dpo/margin_mean": 87.95361328125, + "margin_dpo/margin_std": 111.86346435546875, + "step": 579 + }, + { + "KL/chosen_KL_mean": -173.6749267578125, + "KL/mean": -226.8583526611328, + "KL/rejected_KL_mean": -280.04180908203125, + "KL/std": 109.77399444580078, + "epoch": 0.8516886930983847, + "fcm_dpo/beta": 0.0039232028648257256, + "fcm_dpo/delta": -0.018368642777204514, + "fcm_dpo/margin": 106.36687469482422, + "fcm_dpo/q_t": 0.4037545323371887, + "grad_norm": 10.025492668151855, + "learning_rate": 3.349364905389032e-08, + "logits/chosen": -1.6152920722961426, + "logits/rejected": -1.3063846826553345, + "logps/chosen": -262.60369873046875, + "logps/ref_chosen": -88.92879486083984, + "logps/ref_rejected": -88.69661712646484, + "logps/rejected": -368.7384033203125, + "loss": 1.0767, + "margin_dpo/margin_mean": 106.36686706542969, + "margin_dpo/margin_std": 133.18289184570312, + "step": 580 + }, + { + "KL/chosen_KL_mean": -183.19503784179688, + "KL/mean": -232.03536987304688, + "KL/rejected_KL_mean": -280.875732421875, + "KL/std": 110.61803436279297, + "epoch": 0.8531571218795888, + "fcm_dpo/beta": 0.003946345765143633, + "fcm_dpo/delta": 0.01506105437874794, + "fcm_dpo/margin": 97.68067932128906, + "fcm_dpo/q_t": 0.41012513637542725, + "grad_norm": 10.578690528869629, + "learning_rate": 3.285483927764726e-08, + "logits/chosen": -1.9257314205169678, + "logits/rejected": -1.5755860805511475, + "logps/chosen": -275.41827392578125, + "logps/ref_chosen": -92.22323608398438, + "logps/ref_rejected": -99.38943481445312, + "logps/rejected": -380.26513671875, + "loss": 1.1085, + "margin_dpo/margin_mean": 97.68067932128906, + "margin_dpo/margin_std": 138.07290649414062, + "step": 581 + }, + { + "KL/chosen_KL_mean": -162.97067260742188, + "KL/mean": -206.9716796875, + "KL/rejected_KL_mean": -250.9727020263672, + "KL/std": 115.35728454589844, + "epoch": 0.8546255506607929, + "fcm_dpo/beta": 0.003923641983419657, + "fcm_dpo/delta": -0.054857250303030014, + "fcm_dpo/margin": 88.00202941894531, + "fcm_dpo/q_t": 0.42033272981643677, + "grad_norm": 9.29529094696045, + "learning_rate": 3.222175147833556e-08, + "logits/chosen": -2.0911173820495605, + "logits/rejected": -2.0835142135620117, + "logps/chosen": -250.30630493164062, + "logps/ref_chosen": -87.33561706542969, + "logps/ref_rejected": -115.1544189453125, + "logps/rejected": -366.12713623046875, + "loss": 1.1347, + "margin_dpo/margin_mean": 88.00202941894531, + "margin_dpo/margin_std": 127.08179473876953, + "step": 582 + }, + { + "KL/chosen_KL_mean": -195.3851318359375, + "KL/mean": -244.30120849609375, + "KL/rejected_KL_mean": -293.2173156738281, + "KL/std": 109.18648529052734, + "epoch": 0.856093979441997, + "fcm_dpo/beta": 0.0038992201443761587, + "fcm_dpo/delta": 0.019070586189627647, + "fcm_dpo/margin": 97.83218383789062, + "fcm_dpo/q_t": 0.41160455346107483, + "grad_norm": 11.060173988342285, + "learning_rate": 3.159440233840763e-08, + "logits/chosen": -1.4794248342514038, + "logits/rejected": -1.2447502613067627, + "logps/chosen": -279.1022033691406, + "logps/ref_chosen": -83.71707153320312, + "logps/ref_rejected": -94.58465576171875, + "logps/rejected": -387.8019714355469, + "loss": 1.1016, + "margin_dpo/margin_mean": 97.83219146728516, + "margin_dpo/margin_std": 129.11358642578125, + "step": 583 + }, + { + "KL/chosen_KL_mean": -173.88180541992188, + "KL/mean": -225.32388305664062, + "KL/rejected_KL_mean": -276.7659606933594, + "KL/std": 112.76244354248047, + "epoch": 0.8575624082232012, + "fcm_dpo/beta": 0.00389112439006567, + "fcm_dpo/delta": -0.06657686084508896, + "fcm_dpo/margin": 102.8841552734375, + "fcm_dpo/q_t": 0.40561097860336304, + "grad_norm": 7.687736988067627, + "learning_rate": 3.0972808389096635e-08, + "logits/chosen": -1.858134150505066, + "logits/rejected": -1.3956893682479858, + "logps/chosen": -268.141357421875, + "logps/ref_chosen": -94.25955963134766, + "logps/ref_rejected": -104.48269653320312, + "logps/rejected": -381.2486572265625, + "loss": 1.0889, + "margin_dpo/margin_mean": 102.8841552734375, + "margin_dpo/margin_std": 127.93247985839844, + "step": 584 + }, + { + "KL/chosen_KL_mean": -186.596435546875, + "KL/mean": -243.97555541992188, + "KL/rejected_KL_mean": -301.35467529296875, + "KL/std": 128.8909149169922, + "epoch": 0.8590308370044053, + "fcm_dpo/beta": 0.0038514507468789816, + "fcm_dpo/delta": -0.043976567685604095, + "fcm_dpo/margin": 114.75820922851562, + "fcm_dpo/q_t": 0.39856576919555664, + "grad_norm": 8.535517692565918, + "learning_rate": 3.035698600998121e-08, + "logits/chosen": -2.046078681945801, + "logits/rejected": -1.7791507244110107, + "logps/chosen": -272.23321533203125, + "logps/ref_chosen": -85.63676452636719, + "logps/ref_rejected": -90.28547668457031, + "logps/rejected": -391.64013671875, + "loss": 1.0731, + "margin_dpo/margin_mean": 114.75820922851562, + "margin_dpo/margin_std": 152.0629425048828, + "step": 585 + }, + { + "KL/chosen_KL_mean": -206.1322784423828, + "KL/mean": -244.49789428710938, + "KL/rejected_KL_mean": -282.863525390625, + "KL/std": 112.7052993774414, + "epoch": 0.8604992657856094, + "fcm_dpo/beta": 0.0038691554218530655, + "fcm_dpo/delta": 0.04906633868813515, + "fcm_dpo/margin": 76.73126220703125, + "fcm_dpo/q_t": 0.42960894107818604, + "grad_norm": 9.505632400512695, + "learning_rate": 2.974695142855388e-08, + "logits/chosen": -1.964722752571106, + "logits/rejected": -1.7065095901489258, + "logps/chosen": -289.81341552734375, + "logps/ref_chosen": -83.68114471435547, + "logps/ref_rejected": -98.35916137695312, + "logps/rejected": -381.22271728515625, + "loss": 1.1537, + "margin_dpo/margin_mean": 76.73126983642578, + "margin_dpo/margin_std": 107.5704345703125, + "step": 586 + }, + { + "KL/chosen_KL_mean": -166.3143310546875, + "KL/mean": -208.69918823242188, + "KL/rejected_KL_mean": -251.08404541015625, + "KL/std": 109.4501953125, + "epoch": 0.8619676945668135, + "fcm_dpo/beta": 0.0039144158363342285, + "fcm_dpo/delta": 0.07041102647781372, + "fcm_dpo/margin": 84.76972198486328, + "fcm_dpo/q_t": 0.42214369773864746, + "grad_norm": 10.67780590057373, + "learning_rate": 2.9142720719793122e-08, + "logits/chosen": -2.2781388759613037, + "logits/rejected": -1.9919813871383667, + "logps/chosen": -243.891845703125, + "logps/ref_chosen": -77.57750701904297, + "logps/ref_rejected": -89.32032775878906, + "logps/rejected": -340.4043884277344, + "loss": 1.1289, + "margin_dpo/margin_mean": 84.76971435546875, + "margin_dpo/margin_std": 113.54153442382812, + "step": 587 + }, + { + "KL/chosen_KL_mean": -181.08746337890625, + "KL/mean": -220.97515869140625, + "KL/rejected_KL_mean": -260.86285400390625, + "KL/std": 110.893798828125, + "epoch": 0.8634361233480177, + "fcm_dpo/beta": 0.0039445785805583, + "fcm_dpo/delta": 0.02611854113638401, + "fcm_dpo/margin": 79.77537536621094, + "fcm_dpo/q_t": 0.4274389147758484, + "grad_norm": 13.267123222351074, + "learning_rate": 2.8544309805740018e-08, + "logits/chosen": -1.8999654054641724, + "logits/rejected": -1.863416314125061, + "logps/chosen": -264.8917236328125, + "logps/ref_chosen": -83.80426025390625, + "logps/ref_rejected": -113.11943054199219, + "logps/rejected": -373.9822998046875, + "loss": 1.1622, + "margin_dpo/margin_mean": 79.77537536621094, + "margin_dpo/margin_std": 133.38803100585938, + "step": 588 + }, + { + "KL/chosen_KL_mean": -193.58914184570312, + "KL/mean": -245.51968383789062, + "KL/rejected_KL_mean": -297.4501953125, + "KL/std": 109.301513671875, + "epoch": 0.8649045521292217, + "fcm_dpo/beta": 0.003940122202038765, + "fcm_dpo/delta": -0.009617887437343597, + "fcm_dpo/margin": 103.86106872558594, + "fcm_dpo/q_t": 0.40294596552848816, + "grad_norm": 8.448515892028809, + "learning_rate": 2.7951734455078786e-08, + "logits/chosen": -1.565617561340332, + "logits/rejected": -1.16941499710083, + "logps/chosen": -290.0541687011719, + "logps/ref_chosen": -96.46501922607422, + "logps/ref_rejected": -116.03719329833984, + "logps/rejected": -413.4873962402344, + "loss": 1.0591, + "margin_dpo/margin_mean": 103.86106872558594, + "margin_dpo/margin_std": 104.7183837890625, + "step": 589 + }, + { + "KL/chosen_KL_mean": -177.35427856445312, + "KL/mean": -229.47930908203125, + "KL/rejected_KL_mean": -281.6043395996094, + "KL/std": 101.55866241455078, + "epoch": 0.8663729809104258, + "fcm_dpo/beta": 0.0039449273608624935, + "fcm_dpo/delta": -0.01198958232998848, + "fcm_dpo/margin": 104.25007629394531, + "fcm_dpo/q_t": 0.4031516909599304, + "grad_norm": 11.420241355895996, + "learning_rate": 2.736501028272095e-08, + "logits/chosen": -2.03340482711792, + "logits/rejected": -1.8912010192871094, + "logps/chosen": -265.7821350097656, + "logps/ref_chosen": -88.4278564453125, + "logps/ref_rejected": -110.19876861572266, + "logps/rejected": -391.8031005859375, + "loss": 1.0592, + "margin_dpo/margin_mean": 104.25007629394531, + "margin_dpo/margin_std": 106.25035095214844, + "step": 590 + }, + { + "KL/chosen_KL_mean": -170.3348388671875, + "KL/mean": -225.35415649414062, + "KL/rejected_KL_mean": -280.37347412109375, + "KL/std": 109.25648498535156, + "epoch": 0.8678414096916299, + "fcm_dpo/beta": 0.003901613410562277, + "fcm_dpo/delta": -0.030792508274316788, + "fcm_dpo/margin": 110.03866577148438, + "fcm_dpo/q_t": 0.39846694469451904, + "grad_norm": 7.1532769203186035, + "learning_rate": 2.678415274939408e-08, + "logits/chosen": -1.966714859008789, + "logits/rejected": -1.3548333644866943, + "logps/chosen": -258.18450927734375, + "logps/ref_chosen": -87.84968566894531, + "logps/ref_rejected": -89.57919311523438, + "logps/rejected": -369.9526672363281, + "loss": 1.0438, + "margin_dpo/margin_mean": 110.03866577148438, + "margin_dpo/margin_std": 106.27468872070312, + "step": 591 + }, + { + "KL/chosen_KL_mean": -200.9137725830078, + "KL/mean": -243.66827392578125, + "KL/rejected_KL_mean": -286.42279052734375, + "KL/std": 106.69731903076172, + "epoch": 0.869309838472834, + "fcm_dpo/beta": 0.003942975774407387, + "fcm_dpo/delta": 0.06500554829835892, + "fcm_dpo/margin": 85.50900268554688, + "fcm_dpo/q_t": 0.4212019145488739, + "grad_norm": 12.713777542114258, + "learning_rate": 2.6209177161234442e-08, + "logits/chosen": -1.8829023838043213, + "logits/rejected": -1.558463215827942, + "logps/chosen": -275.8895263671875, + "logps/ref_chosen": -74.97573852539062, + "logps/ref_rejected": -81.55262756347656, + "logps/rejected": -367.97540283203125, + "loss": 1.1438, + "margin_dpo/margin_mean": 85.50900268554688, + "margin_dpo/margin_std": 133.84201049804688, + "step": 592 + }, + { + "KL/chosen_KL_mean": -183.4608612060547, + "KL/mean": -220.23834228515625, + "KL/rejected_KL_mean": -257.01580810546875, + "KL/std": 111.59335327148438, + "epoch": 0.8707782672540382, + "fcm_dpo/beta": 0.003973287530243397, + "fcm_dpo/delta": 0.023653369396924973, + "fcm_dpo/margin": 73.55494689941406, + "fcm_dpo/q_t": 0.43189066648483276, + "grad_norm": 13.498023986816406, + "learning_rate": 2.564009866938349e-08, + "logits/chosen": -0.8610674142837524, + "logits/rejected": -0.5588955283164978, + "logps/chosen": -255.69989013671875, + "logps/ref_chosen": -72.23902893066406, + "logps/ref_rejected": -70.09652709960938, + "logps/rejected": -327.11236572265625, + "loss": 1.1736, + "margin_dpo/margin_mean": 73.55494689941406, + "margin_dpo/margin_std": 123.48003387451172, + "step": 593 + }, + { + "KL/chosen_KL_mean": -164.04953002929688, + "KL/mean": -220.51300048828125, + "KL/rejected_KL_mean": -276.9764709472656, + "KL/std": 108.49273681640625, + "epoch": 0.8722466960352423, + "fcm_dpo/beta": 0.003957782872021198, + "fcm_dpo/delta": -0.049205463379621506, + "fcm_dpo/margin": 112.92694091796875, + "fcm_dpo/q_t": 0.3944060802459717, + "grad_norm": 8.890219688415527, + "learning_rate": 2.5076932269588708e-08, + "logits/chosen": -1.8701916933059692, + "logits/rejected": -1.3391462564468384, + "logps/chosen": -254.48724365234375, + "logps/ref_chosen": -90.43771362304688, + "logps/ref_rejected": -92.267578125, + "logps/rejected": -369.2440490722656, + "loss": 1.0308, + "margin_dpo/margin_mean": 112.92694091796875, + "margin_dpo/margin_std": 105.21086120605469, + "step": 594 + }, + { + "KL/chosen_KL_mean": -177.05056762695312, + "KL/mean": -222.8773193359375, + "KL/rejected_KL_mean": -268.7041015625, + "KL/std": 108.85829162597656, + "epoch": 0.8737151248164464, + "fcm_dpo/beta": 0.00396577175706625, + "fcm_dpo/delta": 0.03770260140299797, + "fcm_dpo/margin": 91.65351867675781, + "fcm_dpo/q_t": 0.4157801866531372, + "grad_norm": 9.045574188232422, + "learning_rate": 2.451969280180849e-08, + "logits/chosen": -2.0839295387268066, + "logits/rejected": -1.6958184242248535, + "logps/chosen": -258.56536865234375, + "logps/ref_chosen": -81.51480102539062, + "logps/ref_rejected": -86.52047729492188, + "logps/rejected": -355.22454833984375, + "loss": 1.1135, + "margin_dpo/margin_mean": 91.65351867675781, + "margin_dpo/margin_std": 124.27123260498047, + "step": 595 + }, + { + "KL/chosen_KL_mean": -184.35015869140625, + "KL/mean": -225.3868408203125, + "KL/rejected_KL_mean": -266.42352294921875, + "KL/std": 107.36467742919922, + "epoch": 0.8751835535976505, + "fcm_dpo/beta": 0.0040108636021614075, + "fcm_dpo/delta": 0.07315412908792496, + "fcm_dpo/margin": 82.0733642578125, + "fcm_dpo/q_t": 0.4241868257522583, + "grad_norm": 11.727372169494629, + "learning_rate": 2.396839494982103e-08, + "logits/chosen": -1.6635912656784058, + "logits/rejected": -1.0872061252593994, + "logps/chosen": -272.5582275390625, + "logps/ref_chosen": -88.20805358886719, + "logps/ref_rejected": -86.09375, + "logps/rejected": -352.51727294921875, + "loss": 1.1558, + "margin_dpo/margin_mean": 82.0733642578125, + "margin_dpo/margin_std": 136.48329162597656, + "step": 596 + }, + { + "KL/chosen_KL_mean": -178.85769653320312, + "KL/mean": -234.02279663085938, + "KL/rejected_KL_mean": -289.18792724609375, + "KL/std": 115.64497375488281, + "epoch": 0.8766519823788547, + "fcm_dpo/beta": 0.003988821990787983, + "fcm_dpo/delta": -0.04211900755763054, + "fcm_dpo/margin": 110.33018493652344, + "fcm_dpo/q_t": 0.3975440263748169, + "grad_norm": 9.088208198547363, + "learning_rate": 2.3423053240837514e-08, + "logits/chosen": -1.4629243612289429, + "logits/rejected": -1.348548173904419, + "logps/chosen": -262.4476623535156, + "logps/ref_chosen": -83.58998107910156, + "logps/ref_rejected": -106.94439697265625, + "logps/rejected": -396.13232421875, + "loss": 1.0526, + "margin_dpo/margin_mean": 110.33019256591797, + "margin_dpo/margin_std": 125.17523193359375, + "step": 597 + }, + { + "KL/chosen_KL_mean": -170.55517578125, + "KL/mean": -217.14370727539062, + "KL/rejected_KL_mean": -263.73223876953125, + "KL/std": 108.18994140625, + "epoch": 0.8781204111600588, + "fcm_dpo/beta": 0.003991022706031799, + "fcm_dpo/delta": 0.029031306505203247, + "fcm_dpo/margin": 93.17707061767578, + "fcm_dpo/q_t": 0.4123697876930237, + "grad_norm": 9.256612777709961, + "learning_rate": 2.2883682045119062e-08, + "logits/chosen": -1.9533220529556274, + "logits/rejected": -1.641535997390747, + "logps/chosen": -260.73779296875, + "logps/ref_chosen": -90.18262481689453, + "logps/ref_rejected": -94.79856872558594, + "logps/rejected": -358.53082275390625, + "loss": 1.1081, + "margin_dpo/margin_mean": 93.17707824707031, + "margin_dpo/margin_std": 122.35414123535156, + "step": 598 + }, + { + "KL/chosen_KL_mean": -172.3333740234375, + "KL/mean": -213.53013610839844, + "KL/rejected_KL_mean": -254.72691345214844, + "KL/std": 112.74430847167969, + "epoch": 0.8795888399412628, + "fcm_dpo/beta": 0.004003605339676142, + "fcm_dpo/delta": -0.010556099005043507, + "fcm_dpo/margin": 82.39353942871094, + "fcm_dpo/q_t": 0.422207236289978, + "grad_norm": 8.678912162780762, + "learning_rate": 2.2350295575598367e-08, + "logits/chosen": -1.4942137002944946, + "logits/rejected": -1.3687318563461304, + "logps/chosen": -250.1756591796875, + "logps/ref_chosen": -77.84226989746094, + "logps/ref_rejected": -89.1976318359375, + "logps/rejected": -343.924560546875, + "loss": 1.1451, + "margin_dpo/margin_mean": 82.39353942871094, + "margin_dpo/margin_std": 124.41415405273438, + "step": 599 + }, + { + "KL/chosen_KL_mean": -193.49203491210938, + "KL/mean": -239.5679473876953, + "KL/rejected_KL_mean": -285.64385986328125, + "KL/std": 106.37391662597656, + "epoch": 0.8810572687224669, + "fcm_dpo/beta": 0.004024241119623184, + "fcm_dpo/delta": 0.030254486948251724, + "fcm_dpo/margin": 92.15185546875, + "fcm_dpo/q_t": 0.41345566511154175, + "grad_norm": 8.820852279663086, + "learning_rate": 2.1822907887504932e-08, + "logits/chosen": -2.5194473266601562, + "logits/rejected": -2.183413505554199, + "logps/chosen": -285.6798400878906, + "logps/ref_chosen": -92.18781280517578, + "logps/ref_rejected": -92.95429992675781, + "logps/rejected": -378.5981750488281, + "loss": 1.1022, + "margin_dpo/margin_mean": 92.15185546875, + "margin_dpo/margin_std": 115.8713607788086, + "step": 600 + }, + { + "KL/chosen_KL_mean": -195.01119995117188, + "KL/mean": -244.45657348632812, + "KL/rejected_KL_mean": -293.9019775390625, + "KL/std": 108.30038452148438, + "epoch": 0.882525697503671, + "fcm_dpo/beta": 0.004017959348857403, + "fcm_dpo/delta": 0.002610735595226288, + "fcm_dpo/margin": 98.89080047607422, + "fcm_dpo/q_t": 0.4062908887863159, + "grad_norm": 9.566874504089355, + "learning_rate": 2.1301532877994742e-08, + "logits/chosen": -1.8796908855438232, + "logits/rejected": -1.5611186027526855, + "logps/chosen": -284.22735595703125, + "logps/ref_chosen": -89.21614074707031, + "logps/ref_rejected": -100.17054748535156, + "logps/rejected": -394.0725402832031, + "loss": 1.0699, + "margin_dpo/margin_mean": 98.89079284667969, + "margin_dpo/margin_std": 103.63615417480469, + "step": 601 + }, + { + "KL/chosen_KL_mean": -155.3017578125, + "KL/mean": -219.16592407226562, + "KL/rejected_KL_mean": -283.03009033203125, + "KL/std": 118.75299072265625, + "epoch": 0.8839941262848752, + "fcm_dpo/beta": 0.003964175470173359, + "fcm_dpo/delta": -0.11199070513248444, + "fcm_dpo/margin": 127.72828674316406, + "fcm_dpo/q_t": 0.3826666474342346, + "grad_norm": 9.319535255432129, + "learning_rate": 2.0786184285784298e-08, + "logits/chosen": -2.0947747230529785, + "logits/rejected": -1.8340649604797363, + "logps/chosen": -235.359375, + "logps/ref_chosen": -80.05760192871094, + "logps/ref_rejected": -93.197509765625, + "logps/rejected": -376.22760009765625, + "loss": 1.0097, + "margin_dpo/margin_mean": 127.72828674316406, + "margin_dpo/margin_std": 135.908447265625, + "step": 602 + }, + { + "KL/chosen_KL_mean": -165.49957275390625, + "KL/mean": -218.2401123046875, + "KL/rejected_KL_mean": -270.9806213378906, + "KL/std": 106.63116455078125, + "epoch": 0.8854625550660793, + "fcm_dpo/beta": 0.003934741020202637, + "fcm_dpo/delta": -0.015692541375756264, + "fcm_dpo/margin": 105.48103332519531, + "fcm_dpo/q_t": 0.40293973684310913, + "grad_norm": 8.120116233825684, + "learning_rate": 2.0276875690788204e-08, + "logits/chosen": -2.1011710166931152, + "logits/rejected": -1.5195586681365967, + "logps/chosen": -267.80914306640625, + "logps/ref_chosen": -102.30958557128906, + "logps/ref_rejected": -108.06884765625, + "logps/rejected": -379.0494689941406, + "loss": 1.0677, + "margin_dpo/margin_mean": 105.48103332519531, + "margin_dpo/margin_std": 122.36427307128906, + "step": 603 + }, + { + "KL/chosen_KL_mean": -172.99131774902344, + "KL/mean": -227.8787841796875, + "KL/rejected_KL_mean": -282.7662353515625, + "KL/std": 112.15409851074219, + "epoch": 0.8869309838472834, + "fcm_dpo/beta": 0.0038999803364276886, + "fcm_dpo/delta": -0.029628120362758636, + "fcm_dpo/margin": 109.77490234375, + "fcm_dpo/q_t": 0.39813992381095886, + "grad_norm": 8.693439483642578, + "learning_rate": 1.977362051376158e-08, + "logits/chosen": -1.487715721130371, + "logits/rejected": -1.3304961919784546, + "logps/chosen": -251.1654052734375, + "logps/ref_chosen": -78.17408752441406, + "logps/ref_rejected": -99.49610900878906, + "logps/rejected": -382.2623291015625, + "loss": 1.0522, + "margin_dpo/margin_mean": 109.77490234375, + "margin_dpo/margin_std": 115.15666198730469, + "step": 604 + }, + { + "KL/chosen_KL_mean": -188.95111083984375, + "KL/mean": -234.07000732421875, + "KL/rejected_KL_mean": -279.1888732910156, + "KL/std": 100.14305114746094, + "epoch": 0.8883994126284875, + "fcm_dpo/beta": 0.0039304220117628574, + "fcm_dpo/delta": 0.04701454937458038, + "fcm_dpo/margin": 90.23776245117188, + "fcm_dpo/q_t": 0.4168788194656372, + "grad_norm": 9.627431869506836, + "learning_rate": 1.9276432015946446e-08, + "logits/chosen": -1.7562696933746338, + "logits/rejected": -1.254887342453003, + "logps/chosen": -283.72442626953125, + "logps/ref_chosen": -94.77332305908203, + "logps/ref_rejected": -107.30490112304688, + "logps/rejected": -386.4937744140625, + "loss": 1.1097, + "margin_dpo/margin_mean": 90.23775482177734, + "margin_dpo/margin_std": 112.81745910644531, + "step": 605 + }, + { + "KL/chosen_KL_mean": -182.39306640625, + "KL/mean": -228.991943359375, + "KL/rejected_KL_mean": -275.5907897949219, + "KL/std": 109.53794860839844, + "epoch": 0.8898678414096917, + "fcm_dpo/beta": 0.003929854836314917, + "fcm_dpo/delta": -0.031113799661397934, + "fcm_dpo/margin": 93.19770812988281, + "fcm_dpo/q_t": 0.4130397140979767, + "grad_norm": 8.124847412109375, + "learning_rate": 1.8785323298722093e-08, + "logits/chosen": -1.795448899269104, + "logits/rejected": -1.406607747077942, + "logps/chosen": -270.146484375, + "logps/ref_chosen": -87.7533950805664, + "logps/ref_rejected": -104.2422103881836, + "logps/rejected": -379.8330078125, + "loss": 1.1057, + "margin_dpo/margin_mean": 93.19770812988281, + "margin_dpo/margin_std": 116.86910247802734, + "step": 606 + }, + { + "KL/chosen_KL_mean": -176.47531127929688, + "KL/mean": -229.19363403320312, + "KL/rejected_KL_mean": -281.9119873046875, + "KL/std": 117.06456756591797, + "epoch": 0.8913362701908958, + "fcm_dpo/beta": 0.003912745974957943, + "fcm_dpo/delta": -0.013087640516459942, + "fcm_dpo/margin": 105.43665313720703, + "fcm_dpo/q_t": 0.4045717716217041, + "grad_norm": 10.934760093688965, + "learning_rate": 1.8300307303259904e-08, + "logits/chosen": -1.6783382892608643, + "logits/rejected": -1.1429007053375244, + "logps/chosen": -264.8043518066406, + "logps/ref_chosen": -88.32904052734375, + "logps/ref_rejected": -86.76810455322266, + "logps/rejected": -368.6800537109375, + "loss": 1.091, + "margin_dpo/margin_mean": 105.43665313720703, + "margin_dpo/margin_std": 144.213134765625, + "step": 607 + }, + { + "KL/chosen_KL_mean": -161.14767456054688, + "KL/mean": -216.14566040039062, + "KL/rejected_KL_mean": -271.1436462402344, + "KL/std": 115.4263687133789, + "epoch": 0.8928046989720999, + "fcm_dpo/beta": 0.0038679465651512146, + "fcm_dpo/delta": -0.027507182210683823, + "fcm_dpo/margin": 109.99598693847656, + "fcm_dpo/q_t": 0.3995211720466614, + "grad_norm": 8.010201454162598, + "learning_rate": 1.7821396810182437e-08, + "logits/chosen": -1.8765852451324463, + "logits/rejected": -1.5803370475769043, + "logps/chosen": -246.9170379638672, + "logps/ref_chosen": -85.76937866210938, + "logps/ref_rejected": -100.23281860351562, + "logps/rejected": -371.37646484375, + "loss": 1.0661, + "margin_dpo/margin_mean": 109.99598693847656, + "margin_dpo/margin_std": 125.82890319824219, + "step": 608 + }, + { + "KL/chosen_KL_mean": -180.2808380126953, + "KL/mean": -231.44041442871094, + "KL/rejected_KL_mean": -282.5999755859375, + "KL/std": 116.58121490478516, + "epoch": 0.8942731277533039, + "fcm_dpo/beta": 0.0038539059460163116, + "fcm_dpo/delta": -0.041400469839572906, + "fcm_dpo/margin": 102.319091796875, + "fcm_dpo/q_t": 0.40778660774230957, + "grad_norm": 10.630335807800293, + "learning_rate": 1.7348604439226617e-08, + "logits/chosen": -2.297642707824707, + "logits/rejected": -1.882304072380066, + "logps/chosen": -273.2474365234375, + "logps/ref_chosen": -92.9665756225586, + "logps/ref_rejected": -95.91818237304688, + "logps/rejected": -378.51812744140625, + "loss": 1.0897, + "margin_dpo/margin_mean": 102.319091796875, + "margin_dpo/margin_std": 124.1063232421875, + "step": 609 + }, + { + "KL/chosen_KL_mean": -164.69947814941406, + "KL/mean": -219.94285583496094, + "KL/rejected_KL_mean": -275.186279296875, + "KL/std": 110.60639190673828, + "epoch": 0.895741556534508, + "fcm_dpo/beta": 0.0038321493193507195, + "fcm_dpo/delta": -0.024567697197198868, + "fcm_dpo/margin": 110.48677825927734, + "fcm_dpo/q_t": 0.39954763650894165, + "grad_norm": 8.435887336730957, + "learning_rate": 1.6881942648911074e-08, + "logits/chosen": -1.8060765266418457, + "logits/rejected": -1.2700965404510498, + "logps/chosen": -259.3997802734375, + "logps/ref_chosen": -94.70028686523438, + "logps/ref_rejected": -89.68739318847656, + "logps/rejected": -364.8736572265625, + "loss": 1.0555, + "margin_dpo/margin_mean": 110.48677825927734, + "margin_dpo/margin_std": 117.20846557617188, + "step": 610 + }, + { + "KL/chosen_KL_mean": -174.323974609375, + "KL/mean": -228.17726135253906, + "KL/rejected_KL_mean": -282.030517578125, + "KL/std": 114.42023468017578, + "epoch": 0.8972099853157122, + "fcm_dpo/beta": 0.003843258135020733, + "fcm_dpo/delta": -0.014867845922708511, + "fcm_dpo/margin": 107.70655059814453, + "fcm_dpo/q_t": 0.4030778408050537, + "grad_norm": 9.592172622680664, + "learning_rate": 1.6421423736208e-08, + "logits/chosen": -2.1820597648620605, + "logits/rejected": -1.8455564975738525, + "logps/chosen": -261.1072998046875, + "logps/ref_chosen": -86.78334045410156, + "logps/ref_rejected": -89.84307861328125, + "logps/rejected": -371.87359619140625, + "loss": 1.0634, + "margin_dpo/margin_mean": 107.70654296875, + "margin_dpo/margin_std": 116.05413818359375, + "step": 611 + }, + { + "KL/chosen_KL_mean": -191.6583251953125, + "KL/mean": -247.10220336914062, + "KL/rejected_KL_mean": -302.54608154296875, + "KL/std": 108.65845489501953, + "epoch": 0.8986784140969163, + "fcm_dpo/beta": 0.0038062939420342445, + "fcm_dpo/delta": -0.023094428703188896, + "fcm_dpo/margin": 110.88774108886719, + "fcm_dpo/q_t": 0.4003736674785614, + "grad_norm": 8.83055591583252, + "learning_rate": 1.5967059836219042e-08, + "logits/chosen": -2.038139820098877, + "logits/rejected": -1.1902828216552734, + "logps/chosen": -292.678466796875, + "logps/ref_chosen": -101.02015686035156, + "logps/ref_rejected": -93.78302764892578, + "logps/rejected": -396.3291015625, + "loss": 1.052, + "margin_dpo/margin_mean": 110.88774108886719, + "margin_dpo/margin_std": 113.39448547363281, + "step": 612 + }, + { + "KL/chosen_KL_mean": -153.7947998046875, + "KL/mean": -216.18380737304688, + "KL/rejected_KL_mean": -278.5728454589844, + "KL/std": 111.1944351196289, + "epoch": 0.9001468428781204, + "fcm_dpo/beta": 0.003759459126740694, + "fcm_dpo/delta": -0.07265369594097137, + "fcm_dpo/margin": 124.77806091308594, + "fcm_dpo/q_t": 0.3885994255542755, + "grad_norm": 8.025390625, + "learning_rate": 1.551886292185553e-08, + "logits/chosen": -1.8691550493240356, + "logits/rejected": -1.6710984706878662, + "logps/chosen": -242.78341674804688, + "logps/ref_chosen": -88.98861694335938, + "logps/ref_rejected": -109.99551391601562, + "logps/rejected": -388.568359375, + "loss": 1.0107, + "margin_dpo/margin_mean": 124.77806091308594, + "margin_dpo/margin_std": 108.96006774902344, + "step": 613 + }, + { + "KL/chosen_KL_mean": -192.46804809570312, + "KL/mean": -243.35745239257812, + "KL/rejected_KL_mean": -294.246826171875, + "KL/std": 105.44712829589844, + "epoch": 0.9016152716593245, + "fcm_dpo/beta": 0.003757667727768421, + "fcm_dpo/delta": 0.018245378509163857, + "fcm_dpo/margin": 101.77880859375, + "fcm_dpo/q_t": 0.4098703861236572, + "grad_norm": 9.768272399902344, + "learning_rate": 1.507684480352292e-08, + "logits/chosen": -1.640364170074463, + "logits/rejected": -1.6304938793182373, + "logps/chosen": -272.6680908203125, + "logps/ref_chosen": -80.20005798339844, + "logps/ref_rejected": -109.86239624023438, + "logps/rejected": -404.1092529296875, + "loss": 1.0972, + "margin_dpo/margin_mean": 101.77880859375, + "margin_dpo/margin_std": 129.19949340820312, + "step": 614 + }, + { + "KL/chosen_KL_mean": -180.46514892578125, + "KL/mean": -221.78871154785156, + "KL/rejected_KL_mean": -263.1122741699219, + "KL/std": 106.55547332763672, + "epoch": 0.9030837004405287, + "fcm_dpo/beta": 0.003759893123060465, + "fcm_dpo/delta": -0.01309838518500328, + "fcm_dpo/margin": 82.64714813232422, + "fcm_dpo/q_t": 0.4255291819572449, + "grad_norm": 8.882323265075684, + "learning_rate": 1.4641017128809801e-08, + "logits/chosen": -1.9366450309753418, + "logits/rejected": -1.547764778137207, + "logps/chosen": -280.900390625, + "logps/ref_chosen": -100.43525695800781, + "logps/ref_rejected": -101.1800537109375, + "logps/rejected": -364.2923583984375, + "loss": 1.1383, + "margin_dpo/margin_mean": 82.64714813232422, + "margin_dpo/margin_std": 101.86264038085938, + "step": 615 + }, + { + "KL/chosen_KL_mean": -182.88592529296875, + "KL/mean": -239.62872314453125, + "KL/rejected_KL_mean": -296.3714904785156, + "KL/std": 106.27088165283203, + "epoch": 0.9045521292217328, + "fcm_dpo/beta": 0.003753126133233309, + "fcm_dpo/delta": -0.02724025398492813, + "fcm_dpo/margin": 113.48554229736328, + "fcm_dpo/q_t": 0.39982104301452637, + "grad_norm": 8.79699420928955, + "learning_rate": 1.4211391382180637e-08, + "logits/chosen": -1.8284904956817627, + "logits/rejected": -1.3341734409332275, + "logps/chosen": -275.37884521484375, + "logps/ref_chosen": -92.49292755126953, + "logps/ref_rejected": -82.06065368652344, + "logps/rejected": -378.43212890625, + "loss": 1.0574, + "margin_dpo/margin_mean": 113.48554229736328, + "margin_dpo/margin_std": 124.32967376708984, + "step": 616 + }, + { + "KL/chosen_KL_mean": -189.74118041992188, + "KL/mean": -234.5286102294922, + "KL/rejected_KL_mean": -279.3160400390625, + "KL/std": 103.72169494628906, + "epoch": 0.9060205580029369, + "fcm_dpo/beta": 0.0037694016937166452, + "fcm_dpo/delta": 0.06455539166927338, + "fcm_dpo/margin": 89.57487487792969, + "fcm_dpo/q_t": 0.420167475938797, + "grad_norm": 9.129219055175781, + "learning_rate": 1.378797888467345e-08, + "logits/chosen": -1.6300756931304932, + "logits/rejected": -0.995729923248291, + "logps/chosen": -280.83819580078125, + "logps/ref_chosen": -91.09699249267578, + "logps/ref_rejected": -70.41004943847656, + "logps/rejected": -349.7261047363281, + "loss": 1.124, + "margin_dpo/margin_mean": 89.57487487792969, + "margin_dpo/margin_std": 116.30337524414062, + "step": 617 + }, + { + "KL/chosen_KL_mean": -191.69598388671875, + "KL/mean": -242.4030303955078, + "KL/rejected_KL_mean": -293.110107421875, + "KL/std": 115.41868591308594, + "epoch": 0.9074889867841409, + "fcm_dpo/beta": 0.0037880889140069485, + "fcm_dpo/delta": 0.016459230333566666, + "fcm_dpo/margin": 101.41413116455078, + "fcm_dpo/q_t": 0.4089798927307129, + "grad_norm": 7.9046854972839355, + "learning_rate": 1.3370790793601371e-08, + "logits/chosen": -2.1167428493499756, + "logits/rejected": -1.760259985923767, + "logps/chosen": -293.7165832519531, + "logps/ref_chosen": -102.02059936523438, + "logps/ref_rejected": -99.80119323730469, + "logps/rejected": -392.9112854003906, + "loss": 1.0951, + "margin_dpo/margin_mean": 101.41413879394531, + "margin_dpo/margin_std": 127.3936538696289, + "step": 618 + }, + { + "KL/chosen_KL_mean": -185.44357299804688, + "KL/mean": -229.78912353515625, + "KL/rejected_KL_mean": -274.1346435546875, + "KL/std": 113.30085754394531, + "epoch": 0.908957415565345, + "fcm_dpo/beta": 0.0038301139138638973, + "fcm_dpo/delta": 0.06243818253278732, + "fcm_dpo/margin": 88.69110870361328, + "fcm_dpo/q_t": 0.422113835811615, + "grad_norm": 8.41944694519043, + "learning_rate": 1.2959838102258535e-08, + "logits/chosen": -1.458711862564087, + "logits/rejected": -1.092092514038086, + "logps/chosen": -275.1849365234375, + "logps/ref_chosen": -89.74136352539062, + "logps/ref_rejected": -99.90138244628906, + "logps/rejected": -374.0360412597656, + "loss": 1.1391, + "margin_dpo/margin_mean": 88.69111633300781, + "margin_dpo/margin_std": 136.2492218017578, + "step": 619 + }, + { + "KL/chosen_KL_mean": -176.85470581054688, + "KL/mean": -226.03201293945312, + "KL/rejected_KL_mean": -275.20928955078125, + "KL/std": 118.94052124023438, + "epoch": 0.9104258443465492, + "fcm_dpo/beta": 0.003787559922784567, + "fcm_dpo/delta": -0.07300166040658951, + "fcm_dpo/margin": 98.35460662841797, + "fcm_dpo/q_t": 0.4120763838291168, + "grad_norm": 9.32966136932373, + "learning_rate": 1.2555131639630567e-08, + "logits/chosen": -2.3457906246185303, + "logits/rejected": -1.8342537879943848, + "logps/chosen": -261.97900390625, + "logps/ref_chosen": -85.12431335449219, + "logps/ref_rejected": -85.41253662109375, + "logps/rejected": -360.621826171875, + "loss": 1.11, + "margin_dpo/margin_mean": 98.35459899902344, + "margin_dpo/margin_std": 127.18399047851562, + "step": 620 + }, + { + "KL/chosen_KL_mean": -181.07896423339844, + "KL/mean": -229.0821533203125, + "KL/rejected_KL_mean": -277.0853271484375, + "KL/std": 115.03120422363281, + "epoch": 0.9118942731277533, + "fcm_dpo/beta": 0.0037645839620381594, + "fcm_dpo/delta": -0.061032526195049286, + "fcm_dpo/margin": 96.00635528564453, + "fcm_dpo/q_t": 0.4152405858039856, + "grad_norm": 8.988188743591309, + "learning_rate": 1.2156682070109086e-08, + "logits/chosen": -1.9616132974624634, + "logits/rejected": -1.6998029947280884, + "logps/chosen": -270.327392578125, + "logps/ref_chosen": -89.24842071533203, + "logps/ref_rejected": -95.46463775634766, + "logps/rejected": -372.5499572753906, + "loss": 1.1253, + "margin_dpo/margin_mean": 96.00636291503906, + "margin_dpo/margin_std": 134.9449462890625, + "step": 621 + }, + { + "KL/chosen_KL_mean": -178.47454833984375, + "KL/mean": -229.943115234375, + "KL/rejected_KL_mean": -281.41168212890625, + "KL/std": 102.43006896972656, + "epoch": 0.9133627019089574, + "fcm_dpo/beta": 0.003743324428796768, + "fcm_dpo/delta": 0.01522154826670885, + "fcm_dpo/margin": 102.93711853027344, + "fcm_dpo/q_t": 0.4086582064628601, + "grad_norm": 9.605335235595703, + "learning_rate": 1.1764499893210878e-08, + "logits/chosen": -1.3714749813079834, + "logits/rejected": -0.7045179605484009, + "logps/chosen": -278.2686767578125, + "logps/ref_chosen": -99.79412841796875, + "logps/ref_rejected": -90.82821655273438, + "logps/rejected": -372.2398986816406, + "loss": 1.0798, + "margin_dpo/margin_mean": 102.93711853027344, + "margin_dpo/margin_std": 111.34817504882812, + "step": 622 + }, + { + "KL/chosen_KL_mean": -185.93972778320312, + "KL/mean": -222.5531005859375, + "KL/rejected_KL_mean": -259.1664733886719, + "KL/std": 111.14846801757812, + "epoch": 0.9148311306901615, + "fcm_dpo/beta": 0.0037793661467731, + "fcm_dpo/delta": 0.034983083605766296, + "fcm_dpo/margin": 73.22674560546875, + "fcm_dpo/q_t": 0.43571192026138306, + "grad_norm": 8.993992805480957, + "learning_rate": 1.1378595443300998e-08, + "logits/chosen": -2.2459628582000732, + "logits/rejected": -1.8093518018722534, + "logps/chosen": -276.395263671875, + "logps/ref_chosen": -90.45555114746094, + "logps/ref_rejected": -91.32276153564453, + "logps/rejected": -350.4892272949219, + "loss": 1.1926, + "margin_dpo/margin_mean": 73.22674560546875, + "margin_dpo/margin_std": 136.1776123046875, + "step": 623 + }, + { + "KL/chosen_KL_mean": -192.54164123535156, + "KL/mean": -250.57403564453125, + "KL/rejected_KL_mean": -308.6064453125, + "KL/std": 110.08511352539062, + "epoch": 0.9162995594713657, + "fcm_dpo/beta": 0.0037574414163827896, + "fcm_dpo/delta": -0.0377613827586174, + "fcm_dpo/margin": 116.06480407714844, + "fcm_dpo/q_t": 0.3977317810058594, + "grad_norm": 9.43755054473877, + "learning_rate": 1.0998978889320582e-08, + "logits/chosen": -2.629471778869629, + "logits/rejected": -1.7353299856185913, + "logps/chosen": -302.4168701171875, + "logps/ref_chosen": -109.87522888183594, + "logps/ref_rejected": -104.77320861816406, + "logps/rejected": -413.379638671875, + "loss": 1.0505, + "margin_dpo/margin_mean": 116.06480407714844, + "margin_dpo/margin_std": 126.56492614746094, + "step": 624 + }, + { + "KL/chosen_KL_mean": -200.04615783691406, + "KL/mean": -241.7044219970703, + "KL/rejected_KL_mean": -283.3627014160156, + "KL/std": 108.86798095703125, + "epoch": 0.9177679882525698, + "fcm_dpo/beta": 0.003760346444323659, + "fcm_dpo/delta": 0.02503892406821251, + "fcm_dpo/margin": 83.31654357910156, + "fcm_dpo/q_t": 0.426126629114151, + "grad_norm": 8.50216007232666, + "learning_rate": 1.0625660234518913e-08, + "logits/chosen": -1.551748514175415, + "logits/rejected": -1.1699554920196533, + "logps/chosen": -287.2143249511719, + "logps/ref_chosen": -87.16815948486328, + "logps/ref_rejected": -91.86148071289062, + "logps/rejected": -375.22418212890625, + "loss": 1.1435, + "margin_dpo/margin_mean": 83.31653594970703, + "margin_dpo/margin_std": 116.60951232910156, + "step": 625 + }, + { + "KL/chosen_KL_mean": -187.85755920410156, + "KL/mean": -229.61895751953125, + "KL/rejected_KL_mean": -271.38037109375, + "KL/std": 120.573486328125, + "epoch": 0.9192364170337739, + "fcm_dpo/beta": 0.0037786171305924654, + "fcm_dpo/delta": 0.02343526855111122, + "fcm_dpo/margin": 83.5228042602539, + "fcm_dpo/q_t": 0.42650681734085083, + "grad_norm": 10.359896659851074, + "learning_rate": 1.0258649316189721e-08, + "logits/chosen": -1.428344488143921, + "logits/rejected": -0.9952055215835571, + "logps/chosen": -292.081787109375, + "logps/ref_chosen": -104.22421264648438, + "logps/ref_rejected": -104.1774673461914, + "logps/rejected": -375.557861328125, + "loss": 1.1545, + "margin_dpo/margin_mean": 83.52279663085938, + "margin_dpo/margin_std": 131.68812561035156, + "step": 626 + }, + { + "KL/chosen_KL_mean": -166.05178833007812, + "KL/mean": -219.41494750976562, + "KL/rejected_KL_mean": -272.778076171875, + "KL/std": 123.51414489746094, + "epoch": 0.920704845814978, + "fcm_dpo/beta": 0.0037863804027438164, + "fcm_dpo/delta": -0.0042802682146430016, + "fcm_dpo/margin": 106.72628784179688, + "fcm_dpo/q_t": 0.4067443609237671, + "grad_norm": 6.791172981262207, + "learning_rate": 9.897955805412e-09, + "logits/chosen": -1.3849577903747559, + "logits/rejected": -1.5382847785949707, + "logps/chosen": -240.9864044189453, + "logps/ref_chosen": -74.93461608886719, + "logps/ref_rejected": -112.57289123535156, + "logps/rejected": -385.3509521484375, + "loss": 1.0843, + "margin_dpo/margin_mean": 106.72628784179688, + "margin_dpo/margin_std": 136.26800537109375, + "step": 627 + }, + { + "KL/chosen_KL_mean": -177.37405395507812, + "KL/mean": -231.8850555419922, + "KL/rejected_KL_mean": -286.39605712890625, + "KL/std": 111.60227966308594, + "epoch": 0.922173274596182, + "fcm_dpo/beta": 0.0037779156118631363, + "fcm_dpo/delta": -0.012384535744786263, + "fcm_dpo/margin": 109.02200317382812, + "fcm_dpo/q_t": 0.4034252166748047, + "grad_norm": 9.962944030761719, + "learning_rate": 9.543589206795238e-09, + "logits/chosen": -2.058600425720215, + "logits/rejected": -1.7986319065093994, + "logps/chosen": -271.06512451171875, + "logps/ref_chosen": -93.69107818603516, + "logps/ref_rejected": -107.34395599365234, + "logps/rejected": -393.739990234375, + "loss": 1.0691, + "margin_dpo/margin_mean": 109.02200317382812, + "margin_dpo/margin_std": 124.29576873779297, + "step": 628 + }, + { + "KL/chosen_KL_mean": -190.33973693847656, + "KL/mean": -239.97738647460938, + "KL/rejected_KL_mean": -289.61505126953125, + "KL/std": 112.42868041992188, + "epoch": 0.9236417033773862, + "fcm_dpo/beta": 0.0037791533395648003, + "fcm_dpo/delta": 0.025711238384246826, + "fcm_dpo/margin": 99.27531433105469, + "fcm_dpo/q_t": 0.4111293852329254, + "grad_norm": 9.445576667785645, + "learning_rate": 9.19555885822887e-09, + "logits/chosen": -2.0779647827148438, + "logits/rejected": -1.646055817604065, + "logps/chosen": -293.57012939453125, + "logps/ref_chosen": -103.23037719726562, + "logps/ref_rejected": -97.16841888427734, + "logps/rejected": -386.78350830078125, + "loss": 1.0958, + "margin_dpo/margin_mean": 99.27531433105469, + "margin_dpo/margin_std": 119.48643493652344, + "step": 629 + }, + { + "KL/chosen_KL_mean": -173.012451171875, + "KL/mean": -204.70458984375, + "KL/rejected_KL_mean": -236.39675903320312, + "KL/std": 104.33016967773438, + "epoch": 0.9251101321585903, + "fcm_dpo/beta": 0.003814146388322115, + "fcm_dpo/delta": 0.05195235460996628, + "fcm_dpo/margin": 63.384307861328125, + "fcm_dpo/q_t": 0.4427708685398102, + "grad_norm": 10.90400505065918, + "learning_rate": 8.85387393063622e-09, + "logits/chosen": -2.097539186477661, + "logits/rejected": -1.5802279710769653, + "logps/chosen": -266.90997314453125, + "logps/ref_chosen": -93.89755249023438, + "logps/ref_rejected": -89.3743896484375, + "logps/rejected": -325.7711486816406, + "loss": 1.2088, + "margin_dpo/margin_mean": 63.384307861328125, + "margin_dpo/margin_std": 117.64893341064453, + "step": 630 + }, + { + "KL/chosen_KL_mean": -170.91452026367188, + "KL/mean": -218.89349365234375, + "KL/rejected_KL_mean": -266.8724670410156, + "KL/std": 104.21376037597656, + "epoch": 0.9265785609397944, + "fcm_dpo/beta": 0.0038488118443638086, + "fcm_dpo/delta": 0.03186452388763428, + "fcm_dpo/margin": 95.95794677734375, + "fcm_dpo/q_t": 0.412794828414917, + "grad_norm": 8.763140678405762, + "learning_rate": 8.518543427732949e-09, + "logits/chosen": -2.3164591789245605, + "logits/rejected": -1.8770625591278076, + "logps/chosen": -258.6853332519531, + "logps/ref_chosen": -87.77082061767578, + "logps/ref_rejected": -88.68241882324219, + "logps/rejected": -355.55487060546875, + "loss": 1.1013, + "margin_dpo/margin_mean": 95.95794677734375, + "margin_dpo/margin_std": 119.26992797851562, + "step": 631 + }, + { + "KL/chosen_KL_mean": -166.23529052734375, + "KL/mean": -218.92901611328125, + "KL/rejected_KL_mean": -271.6227722167969, + "KL/std": 109.13834381103516, + "epoch": 0.9280469897209985, + "fcm_dpo/beta": 0.003838640870526433, + "fcm_dpo/delta": -0.00515325739979744, + "fcm_dpo/margin": 105.38748931884766, + "fcm_dpo/q_t": 0.4058246612548828, + "grad_norm": 10.527085304260254, + "learning_rate": 8.189576185789637e-09, + "logits/chosen": -2.1889047622680664, + "logits/rejected": -1.7760957479476929, + "logps/chosen": -254.86181640625, + "logps/ref_chosen": -88.62652587890625, + "logps/ref_rejected": -91.45091247558594, + "logps/rejected": -363.07366943359375, + "loss": 1.0877, + "margin_dpo/margin_mean": 105.38748931884766, + "margin_dpo/margin_std": 135.35980224609375, + "step": 632 + }, + { + "KL/chosen_KL_mean": -208.9979705810547, + "KL/mean": -239.45794677734375, + "KL/rejected_KL_mean": -269.9179382324219, + "KL/std": 110.80070495605469, + "epoch": 0.9295154185022027, + "fcm_dpo/beta": 0.0038688089698553085, + "fcm_dpo/delta": 0.0369894914329052, + "fcm_dpo/margin": 60.91996383666992, + "fcm_dpo/q_t": 0.44631657004356384, + "grad_norm": 9.852951049804688, + "learning_rate": 7.866980873399015e-09, + "logits/chosen": -2.2323689460754395, + "logits/rejected": -2.044524669647217, + "logps/chosen": -290.37237548828125, + "logps/ref_chosen": -81.37442016601562, + "logps/ref_rejected": -98.62571716308594, + "logps/rejected": -368.5436706542969, + "loss": 1.245, + "margin_dpo/margin_mean": 60.91996383666992, + "margin_dpo/margin_std": 149.2567138671875, + "step": 633 + }, + { + "KL/chosen_KL_mean": -192.2566680908203, + "KL/mean": -232.26593017578125, + "KL/rejected_KL_mean": -272.27520751953125, + "KL/std": 100.35428619384766, + "epoch": 0.9309838472834068, + "fcm_dpo/beta": 0.0038733449764549732, + "fcm_dpo/delta": -0.025234822183847427, + "fcm_dpo/margin": 80.0185317993164, + "fcm_dpo/q_t": 0.4262515902519226, + "grad_norm": 10.22681713104248, + "learning_rate": 7.550765991247654e-09, + "logits/chosen": -1.4065275192260742, + "logits/rejected": -1.2347018718719482, + "logps/chosen": -288.3795166015625, + "logps/ref_chosen": -96.12284851074219, + "logps/ref_rejected": -112.84780883789062, + "logps/rejected": -385.1230163574219, + "loss": 1.1424, + "margin_dpo/margin_mean": 80.0185317993164, + "margin_dpo/margin_std": 102.03579711914062, + "step": 634 + }, + { + "KL/chosen_KL_mean": -200.042236328125, + "KL/mean": -230.71713256835938, + "KL/rejected_KL_mean": -261.39202880859375, + "KL/std": 109.91458129882812, + "epoch": 0.9324522760646109, + "fcm_dpo/beta": 0.003879968076944351, + "fcm_dpo/delta": 0.0422615110874176, + "fcm_dpo/margin": 61.34981918334961, + "fcm_dpo/q_t": 0.44473838806152344, + "grad_norm": 11.883227348327637, + "learning_rate": 7.240939871891699e-09, + "logits/chosen": -1.7093758583068848, + "logits/rejected": -1.1638599634170532, + "logps/chosen": -298.7263488769531, + "logps/ref_chosen": -98.68411254882812, + "logps/ref_rejected": -89.89920043945312, + "logps/rejected": -351.2912292480469, + "loss": 1.2266, + "margin_dpo/margin_mean": 61.34981918334961, + "margin_dpo/margin_std": 132.36192321777344, + "step": 635 + }, + { + "KL/chosen_KL_mean": -178.34039306640625, + "KL/mean": -231.24098205566406, + "KL/rejected_KL_mean": -284.14154052734375, + "KL/std": 112.56794738769531, + "epoch": 0.933920704845815, + "fcm_dpo/beta": 0.0038938471116125584, + "fcm_dpo/delta": -0.012502969242632389, + "fcm_dpo/margin": 105.8011703491211, + "fcm_dpo/q_t": 0.40304362773895264, + "grad_norm": 8.933439254760742, + "learning_rate": 6.937510679537628e-09, + "logits/chosen": -1.5890717506408691, + "logits/rejected": -1.1134685277938843, + "logps/chosen": -268.75836181640625, + "logps/ref_chosen": -90.41796112060547, + "logps/ref_rejected": -87.70687866210938, + "logps/rejected": -371.84844970703125, + "loss": 1.0618, + "margin_dpo/margin_mean": 105.8011703491211, + "margin_dpo/margin_std": 112.42809295654297, + "step": 636 + }, + { + "KL/chosen_KL_mean": -179.61044311523438, + "KL/mean": -236.06961059570312, + "KL/rejected_KL_mean": -292.52880859375, + "KL/std": 124.97705078125, + "epoch": 0.9353891336270191, + "fcm_dpo/beta": 0.003869942855089903, + "fcm_dpo/delta": -0.03866468742489815, + "fcm_dpo/margin": 112.91836547851562, + "fcm_dpo/q_t": 0.39703550934791565, + "grad_norm": 8.080221176147461, + "learning_rate": 6.640486409826785e-09, + "logits/chosen": -1.9098118543624878, + "logits/rejected": -1.8062106370925903, + "logps/chosen": -262.0601501464844, + "logps/ref_chosen": -82.44971466064453, + "logps/ref_rejected": -104.02860260009766, + "logps/rejected": -396.557373046875, + "loss": 1.0472, + "margin_dpo/margin_mean": 112.91836547851562, + "margin_dpo/margin_std": 118.51007080078125, + "step": 637 + }, + { + "KL/chosen_KL_mean": -168.32565307617188, + "KL/mean": -223.0592041015625, + "KL/rejected_KL_mean": -277.79278564453125, + "KL/std": 111.24354553222656, + "epoch": 0.9368575624082232, + "fcm_dpo/beta": 0.0038613637443631887, + "fcm_dpo/delta": -0.024044867604970932, + "fcm_dpo/margin": 109.46715545654297, + "fcm_dpo/q_t": 0.3997325301170349, + "grad_norm": 8.261194229125977, + "learning_rate": 6.349874889624962e-09, + "logits/chosen": -1.5519225597381592, + "logits/rejected": -0.8704928159713745, + "logps/chosen": -260.2506103515625, + "logps/ref_chosen": -91.92498779296875, + "logps/ref_rejected": -86.28703308105469, + "logps/rejected": -364.079833984375, + "loss": 1.0489, + "margin_dpo/margin_mean": 109.46715545654297, + "margin_dpo/margin_std": 103.84730529785156, + "step": 638 + }, + { + "KL/chosen_KL_mean": -223.8411865234375, + "KL/mean": -249.5852508544922, + "KL/rejected_KL_mean": -275.329345703125, + "KL/std": 102.98002624511719, + "epoch": 0.9383259911894273, + "fcm_dpo/beta": 0.0038381926715373993, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 51.488128662109375, + "fcm_dpo/q_t": 0.452871710062027, + "grad_norm": 9.00898551940918, + "learning_rate": 6.065683776815933e-09, + "logits/chosen": -1.3313965797424316, + "logits/rejected": -0.5628180503845215, + "logps/chosen": -328.3687438964844, + "logps/ref_chosen": -104.52755737304688, + "logps/ref_rejected": -81.4803466796875, + "logps/rejected": -356.8096923828125, + "loss": 1.2578, + "margin_dpo/margin_mean": 51.48813247680664, + "margin_dpo/margin_std": 128.21505737304688, + "step": 639 + }, + { + "KL/chosen_KL_mean": -190.1381072998047, + "KL/mean": -239.64845275878906, + "KL/rejected_KL_mean": -289.1588134765625, + "KL/std": 111.77400207519531, + "epoch": 0.9397944199706314, + "fcm_dpo/beta": 0.0038121978286653757, + "fcm_dpo/delta": -0.06818970292806625, + "fcm_dpo/margin": 99.02069854736328, + "fcm_dpo/q_t": 0.41030749678611755, + "grad_norm": 11.37141227722168, + "learning_rate": 5.7879205600998296e-09, + "logits/chosen": -1.7587177753448486, + "logits/rejected": -1.2360551357269287, + "logps/chosen": -288.02337646484375, + "logps/ref_chosen": -97.88526916503906, + "logps/ref_rejected": -112.70502471923828, + "logps/rejected": -401.86383056640625, + "loss": 1.0948, + "margin_dpo/margin_mean": 99.02069854736328, + "margin_dpo/margin_std": 112.4079360961914, + "step": 640 + }, + { + "KL/chosen_KL_mean": -173.28643798828125, + "KL/mean": -224.64605712890625, + "KL/rejected_KL_mean": -276.00567626953125, + "KL/std": 129.56117248535156, + "epoch": 0.9412628487518355, + "fcm_dpo/beta": 0.003794408403337002, + "fcm_dpo/delta": 0.010643563233315945, + "fcm_dpo/margin": 102.71925354003906, + "fcm_dpo/q_t": 0.41164323687553406, + "grad_norm": 8.372820854187012, + "learning_rate": 5.516592558795746e-09, + "logits/chosen": -2.2098498344421387, + "logits/rejected": -1.6607255935668945, + "logps/chosen": -269.73211669921875, + "logps/ref_chosen": -96.4456787109375, + "logps/ref_rejected": -95.13568878173828, + "logps/rejected": -371.141357421875, + "loss": 1.1016, + "margin_dpo/margin_mean": 102.71925354003906, + "margin_dpo/margin_std": 143.9680938720703, + "step": 641 + }, + { + "KL/chosen_KL_mean": -192.73406982421875, + "KL/mean": -241.2641143798828, + "KL/rejected_KL_mean": -289.7941589355469, + "KL/std": 111.4757080078125, + "epoch": 0.9427312775330396, + "fcm_dpo/beta": 0.003820315934717655, + "fcm_dpo/delta": 0.030141085386276245, + "fcm_dpo/margin": 97.06008911132812, + "fcm_dpo/q_t": 0.41512829065322876, + "grad_norm": 10.769035339355469, + "learning_rate": 5.251706922648868e-09, + "logits/chosen": -1.6352812051773071, + "logits/rejected": -1.3077454566955566, + "logps/chosen": -293.4939270019531, + "logps/ref_chosen": -100.75984954833984, + "logps/ref_rejected": -114.7076416015625, + "logps/rejected": -404.5018005371094, + "loss": 1.1146, + "margin_dpo/margin_mean": 97.06008911132812, + "margin_dpo/margin_std": 138.93240356445312, + "step": 642 + }, + { + "KL/chosen_KL_mean": -180.5733642578125, + "KL/mean": -226.9500732421875, + "KL/rejected_KL_mean": -273.32684326171875, + "KL/std": 103.77207946777344, + "epoch": 0.9441997063142438, + "fcm_dpo/beta": 0.0037874109111726284, + "fcm_dpo/delta": -0.03918962553143501, + "fcm_dpo/margin": 92.75345611572266, + "fcm_dpo/q_t": 0.4163810908794403, + "grad_norm": 11.389684677124023, + "learning_rate": 4.993270631642038e-09, + "logits/chosen": -2.1419332027435303, + "logits/rejected": -1.7664439678192139, + "logps/chosen": -265.3170166015625, + "logps/ref_chosen": -84.74365997314453, + "logps/ref_rejected": -94.31842041015625, + "logps/rejected": -367.645263671875, + "loss": 1.1186, + "margin_dpo/margin_mean": 92.75345611572266, + "margin_dpo/margin_std": 119.85426330566406, + "step": 643 + }, + { + "KL/chosen_KL_mean": -186.76336669921875, + "KL/mean": -225.10137939453125, + "KL/rejected_KL_mean": -263.43939208984375, + "KL/std": 104.70063781738281, + "epoch": 0.9456681350954479, + "fcm_dpo/beta": 0.0037874109111726284, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 76.67604064941406, + "fcm_dpo/q_t": 0.4309181869029999, + "grad_norm": 10.11521053314209, + "learning_rate": 4.741290495811873e-09, + "logits/chosen": -1.7592540979385376, + "logits/rejected": -1.4889845848083496, + "logps/chosen": -272.08612060546875, + "logps/ref_chosen": -85.32274627685547, + "logps/ref_rejected": -94.60861206054688, + "logps/rejected": -358.04803466796875, + "loss": 1.1615, + "margin_dpo/margin_mean": 76.67604064941406, + "margin_dpo/margin_std": 113.71969604492188, + "step": 644 + }, + { + "KL/chosen_KL_mean": -199.71237182617188, + "KL/mean": -224.39837646484375, + "KL/rejected_KL_mean": -249.08436584472656, + "KL/std": 113.5405502319336, + "epoch": 0.947136563876652, + "fcm_dpo/beta": 0.0037874109111726284, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 49.371978759765625, + "fcm_dpo/q_t": 0.4559004604816437, + "grad_norm": 9.404606819152832, + "learning_rate": 4.495773155069299e-09, + "logits/chosen": -1.4555082321166992, + "logits/rejected": -1.2892038822174072, + "logps/chosen": -282.3026123046875, + "logps/ref_chosen": -82.59024047851562, + "logps/ref_rejected": -103.00375366210938, + "logps/rejected": -352.088134765625, + "loss": 1.2669, + "margin_dpo/margin_mean": 49.371978759765625, + "margin_dpo/margin_std": 129.0969696044922, + "step": 645 + }, + { + "KL/chosen_KL_mean": -175.08935546875, + "KL/mean": -224.29525756835938, + "KL/rejected_KL_mean": -273.50115966796875, + "KL/std": 99.33821868896484, + "epoch": 0.9486049926578561, + "fcm_dpo/beta": 0.0037891233805567026, + "fcm_dpo/delta": 0.027975942939519882, + "fcm_dpo/margin": 98.41181945800781, + "fcm_dpo/q_t": 0.41160231828689575, + "grad_norm": 8.314691543579102, + "learning_rate": 4.256725079024553e-09, + "logits/chosen": -1.9570796489715576, + "logits/rejected": -1.3679468631744385, + "logps/chosen": -266.2813720703125, + "logps/ref_chosen": -91.1920394897461, + "logps/ref_rejected": -83.77833557128906, + "logps/rejected": -357.27947998046875, + "loss": 1.0891, + "margin_dpo/margin_mean": 98.41181945800781, + "margin_dpo/margin_std": 107.44400024414062, + "step": 646 + }, + { + "KL/chosen_KL_mean": -190.22341918945312, + "KL/mean": -233.48648071289062, + "KL/rejected_KL_mean": -276.74957275390625, + "KL/std": 119.07595825195312, + "epoch": 0.9500734214390602, + "fcm_dpo/beta": 0.0038478460628539324, + "fcm_dpo/delta": 0.06937824934720993, + "fcm_dpo/margin": 86.52615356445312, + "fcm_dpo/q_t": 0.42297041416168213, + "grad_norm": 9.62779426574707, + "learning_rate": 4.024152566816791e-09, + "logits/chosen": -1.6432267427444458, + "logits/rejected": -1.5103105306625366, + "logps/chosen": -279.06787109375, + "logps/ref_chosen": -88.8444595336914, + "logps/ref_rejected": -99.49833679199219, + "logps/rejected": -376.2478942871094, + "loss": 1.1417, + "margin_dpo/margin_mean": 86.52615356445312, + "margin_dpo/margin_std": 131.43270874023438, + "step": 647 + }, + { + "KL/chosen_KL_mean": -176.59988403320312, + "KL/mean": -236.47048950195312, + "KL/rejected_KL_mean": -296.3410949707031, + "KL/std": 122.33714294433594, + "epoch": 0.9515418502202643, + "fcm_dpo/beta": 0.0038349798414856195, + "fcm_dpo/delta": -0.062018170952796936, + "fcm_dpo/margin": 119.74122619628906, + "fcm_dpo/q_t": 0.3923817574977875, + "grad_norm": 9.694408416748047, + "learning_rate": 3.798061746947995e-09, + "logits/chosen": -1.862655520439148, + "logits/rejected": -1.790158748626709, + "logps/chosen": -264.447998046875, + "logps/ref_chosen": -87.84810638427734, + "logps/ref_rejected": -104.67005920410156, + "logps/rejected": -401.01116943359375, + "loss": 1.0378, + "margin_dpo/margin_mean": 119.74121856689453, + "margin_dpo/margin_std": 129.68258666992188, + "step": 648 + }, + { + "KL/chosen_KL_mean": -162.93673706054688, + "KL/mean": -212.46551513671875, + "KL/rejected_KL_mean": -261.9942932128906, + "KL/std": 106.67803192138672, + "epoch": 0.9530102790014684, + "fcm_dpo/beta": 0.0037901827599853277, + "fcm_dpo/delta": -0.031678035855293274, + "fcm_dpo/margin": 99.05757141113281, + "fcm_dpo/q_t": 0.410619854927063, + "grad_norm": 10.537824630737305, + "learning_rate": 3.5784585771215235e-09, + "logits/chosen": -2.418762683868408, + "logits/rejected": -2.0164315700531006, + "logps/chosen": -252.62924194335938, + "logps/ref_chosen": -89.6925048828125, + "logps/ref_rejected": -88.70658111572266, + "logps/rejected": -350.70086669921875, + "loss": 1.0856, + "margin_dpo/margin_mean": 99.05757141113281, + "margin_dpo/margin_std": 104.78544616699219, + "step": 649 + }, + { + "KL/chosen_KL_mean": -192.8601531982422, + "KL/mean": -239.61412048339844, + "KL/rejected_KL_mean": -286.36810302734375, + "KL/std": 115.449462890625, + "epoch": 0.9544787077826725, + "fcm_dpo/beta": 0.0038184935692697763, + "fcm_dpo/delta": 0.04448123648762703, + "fcm_dpo/margin": 93.5079345703125, + "fcm_dpo/q_t": 0.4167550206184387, + "grad_norm": 9.346925735473633, + "learning_rate": 3.3653488440851253e-09, + "logits/chosen": -1.2074940204620361, + "logits/rejected": -1.006216287612915, + "logps/chosen": -282.790771484375, + "logps/ref_chosen": -89.93060302734375, + "logps/ref_rejected": -102.61282348632812, + "logps/rejected": -388.98089599609375, + "loss": 1.1248, + "margin_dpo/margin_mean": 93.5079345703125, + "margin_dpo/margin_std": 136.7960205078125, + "step": 650 + }, + { + "KL/chosen_KL_mean": -159.58489990234375, + "KL/mean": -217.26043701171875, + "KL/rejected_KL_mean": -274.93597412109375, + "KL/std": 111.26205444335938, + "epoch": 0.9559471365638766, + "fcm_dpo/beta": 0.0037994948215782642, + "fcm_dpo/delta": -0.0400543138384819, + "fcm_dpo/margin": 115.35110473632812, + "fcm_dpo/q_t": 0.3957407474517822, + "grad_norm": 8.717374801635742, + "learning_rate": 3.158738163478475e-09, + "logits/chosen": -2.1445353031158447, + "logits/rejected": -1.9700628519058228, + "logps/chosen": -238.772216796875, + "logps/ref_chosen": -79.18731689453125, + "logps/ref_rejected": -105.93333435058594, + "logps/rejected": -380.86932373046875, + "loss": 1.0371, + "margin_dpo/margin_mean": 115.35110473632812, + "margin_dpo/margin_std": 109.70521545410156, + "step": 651 + }, + { + "KL/chosen_KL_mean": -167.8321533203125, + "KL/mean": -215.88595581054688, + "KL/rejected_KL_mean": -263.9397277832031, + "KL/std": 103.7525634765625, + "epoch": 0.9574155653450808, + "fcm_dpo/beta": 0.0038213147781789303, + "fcm_dpo/delta": 0.03381127491593361, + "fcm_dpo/margin": 96.10755157470703, + "fcm_dpo/q_t": 0.41279900074005127, + "grad_norm": 8.846878051757812, + "learning_rate": 2.9586319796851555e-09, + "logits/chosen": -2.650205135345459, + "logits/rejected": -2.3624677658081055, + "logps/chosen": -269.62237548828125, + "logps/ref_chosen": -101.79022216796875, + "logps/ref_rejected": -116.3245849609375, + "logps/rejected": -380.2643127441406, + "loss": 1.0867, + "margin_dpo/margin_mean": 96.10755157470703, + "margin_dpo/margin_std": 98.927978515625, + "step": 652 + }, + { + "KL/chosen_KL_mean": -191.9810791015625, + "KL/mean": -238.82293701171875, + "KL/rejected_KL_mean": -285.664794921875, + "KL/std": 113.55856323242188, + "epoch": 0.9588839941262849, + "fcm_dpo/beta": 0.0038483156822621822, + "fcm_dpo/delta": 0.04084649309515953, + "fcm_dpo/margin": 93.68368530273438, + "fcm_dpo/q_t": 0.413957417011261, + "grad_norm": 9.661733627319336, + "learning_rate": 2.7650355656892166e-09, + "logits/chosen": -2.088801383972168, + "logits/rejected": -1.8333606719970703, + "logps/chosen": -285.33465576171875, + "logps/ref_chosen": -93.35359191894531, + "logps/ref_rejected": -109.12324523925781, + "logps/rejected": -394.78802490234375, + "loss": 1.1029, + "margin_dpo/margin_mean": 93.68368530273438, + "margin_dpo/margin_std": 111.59762573242188, + "step": 653 + }, + { + "KL/chosen_KL_mean": -192.82717895507812, + "KL/mean": -233.19229125976562, + "KL/rejected_KL_mean": -273.55743408203125, + "KL/std": 108.76021575927734, + "epoch": 0.960352422907489, + "fcm_dpo/beta": 0.0038835632149130106, + "fcm_dpo/delta": 0.0426095575094223, + "fcm_dpo/margin": 80.73028564453125, + "fcm_dpo/q_t": 0.42822974920272827, + "grad_norm": 11.199869155883789, + "learning_rate": 2.577954022936174e-09, + "logits/chosen": -2.3049092292785645, + "logits/rejected": -2.1248888969421387, + "logps/chosen": -281.94268798828125, + "logps/ref_chosen": -89.11553955078125, + "logps/ref_rejected": -104.91995239257812, + "logps/rejected": -378.4773864746094, + "loss": 1.164, + "margin_dpo/margin_mean": 80.73028564453125, + "margin_dpo/margin_std": 137.359375, + "step": 654 + }, + { + "KL/chosen_KL_mean": -175.066162109375, + "KL/mean": -223.59652709960938, + "KL/rejected_KL_mean": -272.12689208984375, + "KL/std": 118.29328155517578, + "epoch": 0.9618208516886931, + "fcm_dpo/beta": 0.0039000471588224173, + "fcm_dpo/delta": 0.022265257313847542, + "fcm_dpo/margin": 97.06072998046875, + "fcm_dpo/q_t": 0.4109921455383301, + "grad_norm": 8.485185623168945, + "learning_rate": 2.397392281198729e-09, + "logits/chosen": -2.1425085067749023, + "logits/rejected": -2.1433873176574707, + "logps/chosen": -256.1022644042969, + "logps/ref_chosen": -81.03610229492188, + "logps/ref_rejected": -102.80233764648438, + "logps/rejected": -374.9292297363281, + "loss": 1.1021, + "margin_dpo/margin_mean": 97.06072998046875, + "margin_dpo/margin_std": 126.66535186767578, + "step": 655 + }, + { + "KL/chosen_KL_mean": -177.36502075195312, + "KL/mean": -239.14007568359375, + "KL/rejected_KL_mean": -300.9151306152344, + "KL/std": 118.62073516845703, + "epoch": 0.9632892804698973, + "fcm_dpo/beta": 0.0038512481842190027, + "fcm_dpo/delta": -0.07970499992370605, + "fcm_dpo/margin": 123.55011749267578, + "fcm_dpo/q_t": 0.38876470923423767, + "grad_norm": 9.504347801208496, + "learning_rate": 2.223355098446622e-09, + "logits/chosen": -1.5670292377471924, + "logits/rejected": -1.635893702507019, + "logps/chosen": -262.69036865234375, + "logps/ref_chosen": -85.32534790039062, + "logps/ref_rejected": -118.33866119384766, + "logps/rejected": -419.2537841796875, + "loss": 1.0174, + "margin_dpo/margin_mean": 123.55012512207031, + "margin_dpo/margin_std": 122.35933685302734, + "step": 656 + }, + { + "KL/chosen_KL_mean": -194.00511169433594, + "KL/mean": -239.27474975585938, + "KL/rejected_KL_mean": -284.54437255859375, + "KL/std": 109.36349487304688, + "epoch": 0.9647577092511013, + "fcm_dpo/beta": 0.003825712949037552, + "fcm_dpo/delta": -0.017601385712623596, + "fcm_dpo/margin": 90.53927612304688, + "fcm_dpo/q_t": 0.4194551706314087, + "grad_norm": 10.166887283325195, + "learning_rate": 2.055847060721566e-09, + "logits/chosen": -1.9024608135223389, + "logits/rejected": -1.7356362342834473, + "logps/chosen": -274.20281982421875, + "logps/ref_chosen": -80.19772338867188, + "logps/ref_rejected": -102.581298828125, + "logps/rejected": -387.12567138671875, + "loss": 1.1321, + "margin_dpo/margin_mean": 90.53927612304688, + "margin_dpo/margin_std": 131.91270446777344, + "step": 657 + }, + { + "KL/chosen_KL_mean": -189.87538146972656, + "KL/mean": -232.2349395751953, + "KL/rejected_KL_mean": -274.5945129394531, + "KL/std": 103.32640075683594, + "epoch": 0.9662261380323054, + "fcm_dpo/beta": 0.0038302855100482702, + "fcm_dpo/delta": 0.011937890201807022, + "fcm_dpo/margin": 84.71913146972656, + "fcm_dpo/q_t": 0.42307817935943604, + "grad_norm": 8.205497741699219, + "learning_rate": 1.8948725820160662e-09, + "logits/chosen": -2.1717429161071777, + "logits/rejected": -1.7159290313720703, + "logps/chosen": -284.5098876953125, + "logps/ref_chosen": -94.634521484375, + "logps/ref_rejected": -101.6316146850586, + "logps/rejected": -376.22613525390625, + "loss": 1.1324, + "margin_dpo/margin_mean": 84.71913146972656, + "margin_dpo/margin_std": 110.98983001708984, + "step": 658 + }, + { + "KL/chosen_KL_mean": -172.4925537109375, + "KL/mean": -223.4534912109375, + "KL/rejected_KL_mean": -274.4144287109375, + "KL/std": 111.18952941894531, + "epoch": 0.9676945668135095, + "fcm_dpo/beta": 0.0038269883953034878, + "fcm_dpo/delta": 0.010125715285539627, + "fcm_dpo/margin": 101.92189025878906, + "fcm_dpo/q_t": 0.40981197357177734, + "grad_norm": 7.7141618728637695, + "learning_rate": 1.7404359041573723e-09, + "logits/chosen": -2.16481876373291, + "logits/rejected": -1.2066476345062256, + "logps/chosen": -285.0484313964844, + "logps/ref_chosen": -112.55587005615234, + "logps/ref_rejected": -93.9216079711914, + "logps/rejected": -368.3360595703125, + "loss": 1.1091, + "margin_dpo/margin_mean": 101.921875, + "margin_dpo/margin_std": 144.13241577148438, + "step": 659 + }, + { + "KL/chosen_KL_mean": -180.93466186523438, + "KL/mean": -233.36708068847656, + "KL/rejected_KL_mean": -285.79949951171875, + "KL/std": 108.1478271484375, + "epoch": 0.9691629955947136, + "fcm_dpo/beta": 0.00384066766127944, + "fcm_dpo/delta": -0.0028668222948908806, + "fcm_dpo/margin": 104.86483764648438, + "fcm_dpo/q_t": 0.4054628312587738, + "grad_norm": 11.359251022338867, + "learning_rate": 1.592541096695571e-09, + "logits/chosen": -1.7729545831680298, + "logits/rejected": -1.140892505645752, + "logps/chosen": -274.31207275390625, + "logps/ref_chosen": -93.37742614746094, + "logps/ref_rejected": -81.39482116699219, + "logps/rejected": -367.1943054199219, + "loss": 1.0727, + "margin_dpo/margin_mean": 104.86483764648438, + "margin_dpo/margin_std": 119.114501953125, + "step": 660 + }, + { + "KL/chosen_KL_mean": -179.06536865234375, + "KL/mean": -238.27749633789062, + "KL/rejected_KL_mean": -297.4896240234375, + "KL/std": 121.45243835449219, + "epoch": 0.9706314243759178, + "fcm_dpo/beta": 0.003816891461610794, + "fcm_dpo/delta": -0.05444643646478653, + "fcm_dpo/margin": 118.42427062988281, + "fcm_dpo/q_t": 0.3950696587562561, + "grad_norm": 9.491766929626465, + "learning_rate": 1.4511920567963908e-09, + "logits/chosen": -1.7265509366989136, + "logits/rejected": -1.2062363624572754, + "logps/chosen": -266.9205322265625, + "logps/ref_chosen": -87.85516357421875, + "logps/ref_rejected": -92.40330505371094, + "logps/rejected": -389.8929443359375, + "loss": 1.0481, + "margin_dpo/margin_mean": 118.42427062988281, + "margin_dpo/margin_std": 134.95252990722656, + "step": 661 + }, + { + "KL/chosen_KL_mean": -174.8328857421875, + "KL/mean": -229.270751953125, + "KL/rejected_KL_mean": -283.7086181640625, + "KL/std": 105.77983093261719, + "epoch": 0.9720998531571219, + "fcm_dpo/beta": 0.003797510638833046, + "fcm_dpo/delta": -0.014074645936489105, + "fcm_dpo/margin": 108.87571716308594, + "fcm_dpo/q_t": 0.402312695980072, + "grad_norm": 9.805207252502441, + "learning_rate": 1.3163925091384532e-09, + "logits/chosen": -1.816255807876587, + "logits/rejected": -1.2520594596862793, + "logps/chosen": -277.6126708984375, + "logps/ref_chosen": -102.77980041503906, + "logps/ref_rejected": -95.22531127929688, + "logps/rejected": -378.9339294433594, + "loss": 1.059, + "margin_dpo/margin_mean": 108.87571716308594, + "margin_dpo/margin_std": 111.89689636230469, + "step": 662 + }, + { + "KL/chosen_KL_mean": -176.68136596679688, + "KL/mean": -225.45603942871094, + "KL/rejected_KL_mean": -274.230712890625, + "KL/std": 109.37639617919922, + "epoch": 0.973568281938326, + "fcm_dpo/beta": 0.00381136080250144, + "fcm_dpo/delta": 0.029179099947214127, + "fcm_dpo/margin": 97.54936218261719, + "fcm_dpo/q_t": 0.41302502155303955, + "grad_norm": 11.187478065490723, + "learning_rate": 1.1881460058152382e-09, + "logits/chosen": -2.202484607696533, + "logits/rejected": -1.9862010478973389, + "logps/chosen": -273.0279541015625, + "logps/ref_chosen": -96.34658813476562, + "logps/ref_rejected": -120.52645874023438, + "logps/rejected": -394.7571716308594, + "loss": 1.1053, + "margin_dpo/margin_mean": 97.54936218261719, + "margin_dpo/margin_std": 127.24519348144531, + "step": 663 + }, + { + "KL/chosen_KL_mean": -196.04647827148438, + "KL/mean": -240.283203125, + "KL/rejected_KL_mean": -284.5198974609375, + "KL/std": 112.3060302734375, + "epoch": 0.9750367107195301, + "fcm_dpo/beta": 0.003852805122733116, + "fcm_dpo/delta": 0.06101434677839279, + "fcm_dpo/margin": 88.47343444824219, + "fcm_dpo/q_t": 0.4196123480796814, + "grad_norm": 8.336725234985352, + "learning_rate": 1.066455926241383e-09, + "logits/chosen": -1.7888169288635254, + "logits/rejected": -1.6194840669631958, + "logps/chosen": -287.888916015625, + "logps/ref_chosen": -91.84242248535156, + "logps/ref_rejected": -111.8366928100586, + "logps/rejected": -396.35662841796875, + "loss": 1.1192, + "margin_dpo/margin_mean": 88.47342681884766, + "margin_dpo/margin_std": 110.52880859375, + "step": 664 + }, + { + "KL/chosen_KL_mean": -167.45315551757812, + "KL/mean": -216.00076293945312, + "KL/rejected_KL_mean": -264.54840087890625, + "KL/std": 108.16895294189453, + "epoch": 0.9765051395007343, + "fcm_dpo/beta": 0.0038790619000792503, + "fcm_dpo/delta": 0.024137815460562706, + "fcm_dpo/margin": 97.09523010253906, + "fcm_dpo/q_t": 0.41057807207107544, + "grad_norm": 7.814632892608643, + "learning_rate": 9.513254770636137e-10, + "logits/chosen": -2.469727039337158, + "logits/rejected": -2.0466561317443848, + "logps/chosen": -255.6393280029297, + "logps/ref_chosen": -88.18618774414062, + "logps/ref_rejected": -91.9120101928711, + "logps/rejected": -356.46038818359375, + "loss": 1.0899, + "margin_dpo/margin_mean": 97.09523010253906, + "margin_dpo/margin_std": 111.64039611816406, + "step": 665 + }, + { + "KL/chosen_KL_mean": -183.67776489257812, + "KL/mean": -233.32681274414062, + "KL/rejected_KL_mean": -282.9758605957031, + "KL/std": 106.06681823730469, + "epoch": 0.9779735682819384, + "fcm_dpo/beta": 0.003875638358294964, + "fcm_dpo/delta": 0.015685006976127625, + "fcm_dpo/margin": 99.298095703125, + "fcm_dpo/q_t": 0.41040295362472534, + "grad_norm": 10.720526695251465, + "learning_rate": 8.427576920763956e-10, + "logits/chosen": -1.6271196603775024, + "logits/rejected": -1.2489068508148193, + "logps/chosen": -284.65234375, + "logps/ref_chosen": -100.97459411621094, + "logps/ref_rejected": -101.24992370605469, + "logps/rejected": -384.22576904296875, + "loss": 1.0965, + "margin_dpo/margin_mean": 99.29811096191406, + "margin_dpo/margin_std": 126.03630065917969, + "step": 666 + }, + { + "KL/chosen_KL_mean": -202.2061767578125, + "KL/mean": -252.56402587890625, + "KL/rejected_KL_mean": -302.921875, + "KL/std": 110.61636352539062, + "epoch": 0.9794419970631424, + "fcm_dpo/beta": 0.0038884032983332872, + "fcm_dpo/delta": 0.008699965663254261, + "fcm_dpo/margin": 100.71569061279297, + "fcm_dpo/q_t": 0.40710946917533875, + "grad_norm": 8.844096183776855, + "learning_rate": 7.407554321417764e-10, + "logits/chosen": -1.4849872589111328, + "logits/rejected": -0.8287320733070374, + "logps/chosen": -299.77734375, + "logps/ref_chosen": -97.5711669921875, + "logps/ref_rejected": -93.58476257324219, + "logps/rejected": -396.50665283203125, + "loss": 1.0742, + "margin_dpo/margin_mean": 100.71569061279297, + "margin_dpo/margin_std": 107.05015563964844, + "step": 667 + }, + { + "KL/chosen_KL_mean": -177.9842529296875, + "KL/mean": -237.2086181640625, + "KL/rejected_KL_mean": -296.4330139160156, + "KL/std": 118.19255828857422, + "epoch": 0.9809104258443465, + "fcm_dpo/beta": 0.003863723948597908, + "fcm_dpo/delta": -0.06037880480289459, + "fcm_dpo/margin": 118.44877624511719, + "fcm_dpo/q_t": 0.3940025866031647, + "grad_norm": 9.988701820373535, + "learning_rate": 6.453213851142225e-10, + "logits/chosen": -2.1105642318725586, + "logits/rejected": -1.574021577835083, + "logps/chosen": -280.559326171875, + "logps/ref_chosen": -102.5750503540039, + "logps/ref_rejected": -108.81768798828125, + "logps/rejected": -405.2507019042969, + "loss": 1.0379, + "margin_dpo/margin_mean": 118.44877624511719, + "margin_dpo/margin_std": 129.19381713867188, + "step": 668 + }, + { + "KL/chosen_KL_mean": -162.16372680664062, + "KL/mean": -218.8978729248047, + "KL/rejected_KL_mean": -275.63201904296875, + "KL/std": 114.9798583984375, + "epoch": 0.9823788546255506, + "fcm_dpo/beta": 0.003828328335657716, + "fcm_dpo/delta": -0.03594854846596718, + "fcm_dpo/margin": 113.46829223632812, + "fcm_dpo/q_t": 0.39817070960998535, + "grad_norm": 9.285100936889648, + "learning_rate": 5.564580657695939e-10, + "logits/chosen": -1.6900508403778076, + "logits/rejected": -1.2936570644378662, + "logps/chosen": -251.65850830078125, + "logps/ref_chosen": -89.49478149414062, + "logps/ref_rejected": -82.51950073242188, + "logps/rejected": -358.1515197753906, + "loss": 1.0509, + "margin_dpo/margin_mean": 113.46829223632812, + "margin_dpo/margin_std": 123.40389251708984, + "step": 669 + }, + { + "KL/chosen_KL_mean": -162.95864868164062, + "KL/mean": -225.34881591796875, + "KL/rejected_KL_mean": -287.739013671875, + "KL/std": 117.26303100585938, + "epoch": 0.9838472834067548, + "fcm_dpo/beta": 0.003783291671425104, + "fcm_dpo/delta": -0.07558497786521912, + "fcm_dpo/margin": 124.78034973144531, + "fcm_dpo/q_t": 0.38797491788864136, + "grad_norm": 7.269352436065674, + "learning_rate": 4.741678157389739e-10, + "logits/chosen": -1.7142994403839111, + "logits/rejected": -1.174939513206482, + "logps/chosen": -258.4132385253906, + "logps/ref_chosen": -95.45459747314453, + "logps/ref_rejected": -101.53292846679688, + "logps/rejected": -389.2719421386719, + "loss": 1.0206, + "margin_dpo/margin_mean": 124.78034973144531, + "margin_dpo/margin_std": 122.63932800292969, + "step": 670 + }, + { + "KL/chosen_KL_mean": -198.68067932128906, + "KL/mean": -241.49562072753906, + "KL/rejected_KL_mean": -284.310546875, + "KL/std": 108.82093048095703, + "epoch": 0.9853157121879589, + "fcm_dpo/beta": 0.003759223036468029, + "fcm_dpo/delta": -0.010290359146893024, + "fcm_dpo/margin": 85.6298828125, + "fcm_dpo/q_t": 0.4244227707386017, + "grad_norm": 8.976449012756348, + "learning_rate": 3.9845280344705245e-10, + "logits/chosen": -1.5819687843322754, + "logits/rejected": -1.3541139364242554, + "logps/chosen": -280.8038024902344, + "logps/ref_chosen": -82.12312316894531, + "logps/ref_rejected": -90.21969604492188, + "logps/rejected": -374.5302734375, + "loss": 1.1417, + "margin_dpo/margin_mean": 85.6298828125, + "margin_dpo/margin_std": 120.84451293945312, + "step": 671 + }, + { + "KL/chosen_KL_mean": -198.71026611328125, + "KL/mean": -234.968017578125, + "KL/rejected_KL_mean": -271.2257995605469, + "KL/std": 104.13059997558594, + "epoch": 0.986784140969163, + "fcm_dpo/beta": 0.003793198149651289, + "fcm_dpo/delta": 0.05013387277722359, + "fcm_dpo/margin": 72.51554870605469, + "fcm_dpo/q_t": 0.43499308824539185, + "grad_norm": 12.089276313781738, + "learning_rate": 3.293150240547549e-10, + "logits/chosen": -2.1074886322021484, + "logits/rejected": -1.7845215797424316, + "logps/chosen": -288.77215576171875, + "logps/ref_chosen": -90.0619125366211, + "logps/ref_rejected": -100.45323181152344, + "logps/rejected": -371.67901611328125, + "loss": 1.1744, + "margin_dpo/margin_mean": 72.51554107666016, + "margin_dpo/margin_std": 109.86177062988281, + "step": 672 + }, + { + "KL/chosen_KL_mean": -177.21229553222656, + "KL/mean": -218.13320922851562, + "KL/rejected_KL_mean": -259.0541076660156, + "KL/std": 107.51701354980469, + "epoch": 0.9882525697503671, + "fcm_dpo/beta": 0.0038026715628802776, + "fcm_dpo/delta": 0.024912934750318527, + "fcm_dpo/margin": 81.84182739257812, + "fcm_dpo/q_t": 0.4263882637023926, + "grad_norm": 10.61828327178955, + "learning_rate": 2.6675629940689504e-10, + "logits/chosen": -1.853633999824524, + "logits/rejected": -1.4833993911743164, + "logps/chosen": -256.4754333496094, + "logps/ref_chosen": -79.26315307617188, + "logps/ref_rejected": -91.34925079345703, + "logps/rejected": -350.40338134765625, + "loss": 1.1409, + "margin_dpo/margin_mean": 81.84182739257812, + "margin_dpo/margin_std": 109.56309509277344, + "step": 673 + }, + { + "KL/chosen_KL_mean": -166.49276733398438, + "KL/mean": -225.67752075195312, + "KL/rejected_KL_mean": -284.8623046875, + "KL/std": 107.59810638427734, + "epoch": 0.9897209985315712, + "fcm_dpo/beta": 0.003781170817092061, + "fcm_dpo/delta": -0.04985257610678673, + "fcm_dpo/margin": 118.36953735351562, + "fcm_dpo/q_t": 0.3950941562652588, + "grad_norm": 7.943295955657959, + "learning_rate": 2.1077827798404725e-10, + "logits/chosen": -1.6309033632278442, + "logits/rejected": -1.2305643558502197, + "logps/chosen": -241.95108032226562, + "logps/ref_chosen": -75.45831298828125, + "logps/ref_rejected": -76.20362854003906, + "logps/rejected": -361.06591796875, + "loss": 1.0343, + "margin_dpo/margin_mean": 118.36953735351562, + "margin_dpo/margin_std": 118.20276641845703, + "step": 674 + }, + { + "KL/chosen_KL_mean": -182.51034545898438, + "KL/mean": -235.08111572265625, + "KL/rejected_KL_mean": -287.65191650390625, + "KL/std": 116.70008087158203, + "epoch": 0.9911894273127754, + "fcm_dpo/beta": 0.003734186990186572, + "fcm_dpo/delta": -0.05345737189054489, + "fcm_dpo/margin": 105.14154052734375, + "fcm_dpo/q_t": 0.40766555070877075, + "grad_norm": 10.928277969360352, + "learning_rate": 1.6138243485910863e-10, + "logits/chosen": -1.285999059677124, + "logits/rejected": -0.934632420539856, + "logps/chosen": -262.4198913574219, + "logps/ref_chosen": -79.90953063964844, + "logps/ref_rejected": -81.21824645996094, + "logps/rejected": -368.8701477050781, + "loss": 1.1019, + "margin_dpo/margin_mean": 105.14154815673828, + "margin_dpo/margin_std": 143.1363525390625, + "step": 675 + }, + { + "KL/chosen_KL_mean": -196.07440185546875, + "KL/mean": -247.11036682128906, + "KL/rejected_KL_mean": -298.14630126953125, + "KL/std": 111.57682800292969, + "epoch": 0.9926578560939795, + "fcm_dpo/beta": 0.003737333696335554, + "fcm_dpo/delta": 0.019213024526834488, + "fcm_dpo/margin": 102.07192993164062, + "fcm_dpo/q_t": 0.41006457805633545, + "grad_norm": 8.219503402709961, + "learning_rate": 1.1857007165852472e-10, + "logits/chosen": -1.7843862771987915, + "logits/rejected": -1.1825398206710815, + "logps/chosen": -294.2455139160156, + "logps/ref_chosen": -98.17111206054688, + "logps/ref_rejected": -95.024658203125, + "logps/rejected": -393.1709899902344, + "loss": 1.0951, + "margin_dpo/margin_mean": 102.07192993164062, + "margin_dpo/margin_std": 125.75214385986328, + "step": 676 + }, + { + "KL/chosen_KL_mean": -176.21795654296875, + "KL/mean": -228.08029174804688, + "KL/rejected_KL_mean": -279.942626953125, + "KL/std": 110.58978271484375, + "epoch": 0.9941262848751835, + "fcm_dpo/beta": 0.00374723132699728, + "fcm_dpo/delta": 0.011711678467690945, + "fcm_dpo/margin": 103.72468566894531, + "fcm_dpo/q_t": 0.4079986810684204, + "grad_norm": 7.956014633178711, + "learning_rate": 8.23423165278725e-11, + "logits/chosen": -1.5618531703948975, + "logits/rejected": -0.8732852935791016, + "logps/chosen": -267.59722900390625, + "logps/ref_chosen": -91.37928771972656, + "logps/ref_rejected": -82.87776947021484, + "logps/rejected": -362.8204040527344, + "loss": 1.0759, + "margin_dpo/margin_mean": 103.72468566894531, + "margin_dpo/margin_std": 108.58404541015625, + "step": 677 + }, + { + "KL/chosen_KL_mean": -166.2670440673828, + "KL/mean": -227.56236267089844, + "KL/rejected_KL_mean": -288.857666015625, + "KL/std": 118.49006652832031, + "epoch": 0.9955947136563876, + "fcm_dpo/beta": 0.0037383511662483215, + "fcm_dpo/delta": -0.0611700601875782, + "fcm_dpo/margin": 122.59062957763672, + "fcm_dpo/q_t": 0.3919582962989807, + "grad_norm": 9.8098783493042, + "learning_rate": 5.270012410216185e-11, + "logits/chosen": -1.7469590902328491, + "logits/rejected": -1.4247102737426758, + "logps/chosen": -241.91290283203125, + "logps/ref_chosen": -75.6458740234375, + "logps/ref_rejected": -86.96611022949219, + "logps/rejected": -375.82379150390625, + "loss": 1.0294, + "margin_dpo/margin_mean": 122.59063720703125, + "margin_dpo/margin_std": 120.68907165527344, + "step": 678 + }, + { + "KL/chosen_KL_mean": -192.85543823242188, + "KL/mean": -243.40980529785156, + "KL/rejected_KL_mean": -293.96417236328125, + "KL/std": 98.76133728027344, + "epoch": 0.9970631424375918, + "fcm_dpo/beta": 0.0037227459251880646, + "fcm_dpo/delta": 0.024527989327907562, + "fcm_dpo/margin": 101.1087646484375, + "fcm_dpo/q_t": 0.41120100021362305, + "grad_norm": 8.339025497436523, + "learning_rate": 2.9644275480772416e-11, + "logits/chosen": -2.2584304809570312, + "logits/rejected": -1.7472279071807861, + "logps/chosen": -273.6288757324219, + "logps/ref_chosen": -80.77344512939453, + "logps/ref_rejected": -82.87850189208984, + "logps/rejected": -376.8426818847656, + "loss": 1.0813, + "margin_dpo/margin_mean": 101.1087646484375, + "margin_dpo/margin_std": 105.755126953125, + "step": 679 + }, + { + "KL/chosen_KL_mean": -193.2791748046875, + "KL/mean": -248.47329711914062, + "KL/rejected_KL_mean": -303.66741943359375, + "KL/std": 117.02078247070312, + "epoch": 0.9985315712187959, + "fcm_dpo/beta": 0.0037055439315736294, + "fcm_dpo/delta": -0.06578289717435837, + "fcm_dpo/margin": 110.38827514648438, + "fcm_dpo/q_t": 0.40424782037734985, + "grad_norm": 7.081179141998291, + "learning_rate": 1.31753782067201e-11, + "logits/chosen": -1.9725468158721924, + "logits/rejected": -1.643679141998291, + "logps/chosen": -300.96209716796875, + "logps/ref_chosen": -107.68292999267578, + "logps/ref_rejected": -116.0948715209961, + "logps/rejected": -419.7623291015625, + "loss": 1.0748, + "margin_dpo/margin_mean": 110.38825988769531, + "margin_dpo/margin_std": 127.57222747802734, + "step": 680 + }, + { + "KL/chosen_KL_mean": -176.45477294921875, + "KL/mean": -224.28012084960938, + "KL/rejected_KL_mean": -272.10546875, + "KL/std": 110.61670684814453, + "epoch": 1.0, + "fcm_dpo/beta": 0.003713160753250122, + "fcm_dpo/delta": 0.04634173586964607, + "fcm_dpo/margin": 95.65071868896484, + "fcm_dpo/q_t": 0.416808545589447, + "grad_norm": 9.360736846923828, + "learning_rate": 3.2938662507808745e-12, + "logits/chosen": -2.351592540740967, + "logits/rejected": -2.0445010662078857, + "logps/chosen": -269.4658203125, + "logps/ref_chosen": -93.01106262207031, + "logps/ref_rejected": -94.8221664428711, + "logps/rejected": -366.92767333984375, + "loss": 1.111, + "margin_dpo/margin_mean": 95.65072631835938, + "margin_dpo/margin_std": 120.97123718261719, + "step": 681 + }, + { + "epoch": 1.0, + "step": 681, + "total_flos": 0.0, + "train_loss": 1.1086520982383807, + "train_runtime": 1981.9264, + "train_samples_per_second": 21.998, + "train_steps_per_second": 0.344 + } + ], + "logging_steps": 1, + "max_steps": 681, + "num_input_tokens_seen": 0, + "num_train_epochs": 1, + "save_steps": 50, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": false, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 0.0, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +}