diff --git "a/trainer_state.json" "b/trainer_state.json" new file mode 100644--- /dev/null +++ "b/trainer_state.json" @@ -0,0 +1,15706 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.0, + "eval_steps": 100, + "global_step": 681, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "KL/chosen_KL_mean": 0.05419158935546875, + "KL/mean": 0.023573115468025208, + "KL/rejected_KL_mean": -0.007045745849609375, + "KL/std": 0.33097243309020996, + "epoch": 0.0014684287812041115, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.06123492121696472, + "fcm_dpo/q_t": 0.4984707534313202, + "grad_norm": 38.8179931640625, + "learning_rate": 0.0, + "logits/chosen": 0.2677389979362488, + "logits/rejected": 0.6191319227218628, + "logps/chosen": -85.37664031982422, + "logps/ref_chosen": -85.43083190917969, + "logps/ref_rejected": -79.90458679199219, + "logps/rejected": -79.91163635253906, + "loss": 1.3807, + "margin_dpo/margin_mean": 0.061235010623931885, + "margin_dpo/margin_std": 0.4424324035644531, + "step": 1 + }, + { + "KL/chosen_KL_mean": -0.044902801513671875, + "KL/mean": 0.017303764820098877, + "KL/rejected_KL_mean": 0.07951736450195312, + "KL/std": 0.2933448553085327, + "epoch": 0.002936857562408223, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.12442219257354736, + "fcm_dpo/q_t": 0.5031089186668396, + "grad_norm": 29.757017135620117, + "learning_rate": 7.246376811594203e-09, + "logits/chosen": 0.47771453857421875, + "logits/rejected": 0.5908545851707458, + "logps/chosen": -82.11383056640625, + "logps/ref_chosen": -82.06893157958984, + "logps/ref_rejected": -81.65457916259766, + "logps/rejected": -81.57505798339844, + "loss": 1.3992, + "margin_dpo/margin_mean": -0.12442246079444885, + "margin_dpo/margin_std": 0.41358208656311035, + "step": 2 + }, + { + "KL/chosen_KL_mean": 0.024188995361328125, + "KL/mean": 0.006642557680606842, + "KL/rejected_KL_mean": -0.010898590087890625, + "KL/std": 0.3097880482673645, + "epoch": 0.004405286343612335, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.03509153425693512, + "fcm_dpo/q_t": 0.4991232752799988, + "grad_norm": 26.000652313232422, + "learning_rate": 1.4492753623188406e-08, + "logits/chosen": 0.42709434032440186, + "logits/rejected": 0.650235652923584, + "logps/chosen": -93.78679656982422, + "logps/ref_chosen": -93.81098937988281, + "logps/ref_rejected": -74.22950744628906, + "logps/rejected": -74.24040222167969, + "loss": 1.3834, + "margin_dpo/margin_mean": 0.03509168326854706, + "margin_dpo/margin_std": 0.48292967677116394, + "step": 3 + }, + { + "KL/chosen_KL_mean": -0.07156753540039062, + "KL/mean": -0.0424887016415596, + "KL/rejected_KL_mean": -0.013408660888671875, + "KL/std": 0.33623313903808594, + "epoch": 0.005873715124816446, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.05815790593624115, + "fcm_dpo/q_t": 0.5014516115188599, + "grad_norm": 32.81612777709961, + "learning_rate": 2.1739130434782606e-08, + "logits/chosen": 0.6472669839859009, + "logits/rejected": 0.7786778211593628, + "logps/chosen": -87.3640365600586, + "logps/ref_chosen": -87.29246520996094, + "logps/ref_rejected": -93.82425689697266, + "logps/rejected": -93.83766174316406, + "loss": 1.3926, + "margin_dpo/margin_mean": -0.058158084750175476, + "margin_dpo/margin_std": 0.43809592723846436, + "step": 4 + }, + { + "KL/chosen_KL_mean": 0.0205230712890625, + "KL/mean": -0.0029112771153450012, + "KL/rejected_KL_mean": -0.026348114013671875, + "KL/std": 0.3122915029525757, + "epoch": 0.007342143906020558, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.046868130564689636, + "fcm_dpo/q_t": 0.4988270699977875, + "grad_norm": 35.3472900390625, + "learning_rate": 2.898550724637681e-08, + "logits/chosen": 0.38373515009880066, + "logits/rejected": 0.5904780030250549, + "logps/chosen": -89.31623840332031, + "logps/ref_chosen": -89.33676147460938, + "logps/ref_rejected": -88.74783325195312, + "logps/rejected": -88.77418518066406, + "loss": 1.3822, + "margin_dpo/margin_mean": 0.046868354082107544, + "margin_dpo/margin_std": 0.4555869698524475, + "step": 5 + }, + { + "KL/chosen_KL_mean": -0.04218292236328125, + "KL/mean": -0.04054884612560272, + "KL/rejected_KL_mean": -0.03891754150390625, + "KL/std": 0.30800819396972656, + "epoch": 0.00881057268722467, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.003269016742706299, + "fcm_dpo/q_t": 0.5000814199447632, + "grad_norm": 36.064876556396484, + "learning_rate": 3.6231884057971014e-08, + "logits/chosen": 0.4867173433303833, + "logits/rejected": 0.8018932342529297, + "logps/chosen": -97.36366271972656, + "logps/ref_chosen": -97.32147216796875, + "logps/ref_rejected": -97.88345336914062, + "logps/rejected": -97.92236328125, + "loss": 1.3871, + "margin_dpo/margin_mean": -0.0032696127891540527, + "margin_dpo/margin_std": 0.4106166958808899, + "step": 6 + }, + { + "KL/chosen_KL_mean": -0.018283843994140625, + "KL/mean": -0.032152727246284485, + "KL/rejected_KL_mean": -0.04602813720703125, + "KL/std": 0.31660744547843933, + "epoch": 0.010279001468428781, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.027742892503738403, + "fcm_dpo/q_t": 0.4993075728416443, + "grad_norm": 37.842254638671875, + "learning_rate": 4.347826086956521e-08, + "logits/chosen": 0.48803120851516724, + "logits/rejected": 0.5360611081123352, + "logps/chosen": -86.66681671142578, + "logps/ref_chosen": -86.64852905273438, + "logps/ref_rejected": -109.61618041992188, + "logps/rejected": -109.6622085571289, + "loss": 1.384, + "margin_dpo/margin_mean": 0.02774253487586975, + "margin_dpo/margin_std": 0.42086201906204224, + "step": 7 + }, + { + "KL/chosen_KL_mean": -0.0385284423828125, + "KL/mean": -0.018659211695194244, + "KL/rejected_KL_mean": 0.00121307373046875, + "KL/std": 0.34195733070373535, + "epoch": 0.011747430249632892, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.03973837196826935, + "fcm_dpo/q_t": 0.5009934902191162, + "grad_norm": 32.732177734375, + "learning_rate": 5.0724637681159424e-08, + "logits/chosen": 0.4075879454612732, + "logits/rejected": 0.719414234161377, + "logps/chosen": -89.96221160888672, + "logps/ref_chosen": -89.9236831665039, + "logps/ref_rejected": -86.22804260253906, + "logps/rejected": -86.22682189941406, + "loss": 1.3908, + "margin_dpo/margin_mean": -0.03973822295665741, + "margin_dpo/margin_std": 0.46964970231056213, + "step": 8 + }, + { + "KL/chosen_KL_mean": 0.0368194580078125, + "KL/mean": -0.006641313433647156, + "KL/rejected_KL_mean": -0.0500946044921875, + "KL/std": 0.36104485392570496, + "epoch": 0.013215859030837005, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.08690676093101501, + "fcm_dpo/q_t": 0.49782997369766235, + "grad_norm": 36.08580780029297, + "learning_rate": 5.797101449275362e-08, + "logits/chosen": 0.719907820224762, + "logits/rejected": 0.903343677520752, + "logps/chosen": -103.8203125, + "logps/ref_chosen": -103.85713195800781, + "logps/ref_rejected": -104.31932067871094, + "logps/rejected": -104.36941528320312, + "loss": 1.3783, + "margin_dpo/margin_mean": 0.08690647780895233, + "margin_dpo/margin_std": 0.4995291531085968, + "step": 9 + }, + { + "KL/chosen_KL_mean": -0.005512237548828125, + "KL/mean": 0.04146455228328705, + "KL/rejected_KL_mean": 0.08843612670898438, + "KL/std": 0.26931285858154297, + "epoch": 0.014684287812041116, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.09395936131477356, + "fcm_dpo/q_t": 0.5023483037948608, + "grad_norm": 33.644287109375, + "learning_rate": 6.521739130434782e-08, + "logits/chosen": 0.5238694548606873, + "logits/rejected": 0.7369919419288635, + "logps/chosen": -76.20987701416016, + "logps/ref_chosen": -76.20436096191406, + "logps/ref_rejected": -87.152099609375, + "logps/rejected": -87.06366729736328, + "loss": 1.396, + "margin_dpo/margin_mean": -0.09395924210548401, + "margin_dpo/margin_std": 0.3398798406124115, + "step": 10 + }, + { + "KL/chosen_KL_mean": 0.0365447998046875, + "KL/mean": 0.018243655562400818, + "KL/rejected_KL_mean": -5.340576171875e-05, + "KL/std": 0.306723952293396, + "epoch": 0.016152716593245228, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.03659513592720032, + "fcm_dpo/q_t": 0.499085009098053, + "grad_norm": 37.03773880004883, + "learning_rate": 7.246376811594203e-08, + "logits/chosen": 0.3368200361728668, + "logits/rejected": 0.555517315864563, + "logps/chosen": -82.3299560546875, + "logps/ref_chosen": -82.36650085449219, + "logps/ref_rejected": -94.26461791992188, + "logps/rejected": -94.26467895507812, + "loss": 1.3831, + "margin_dpo/margin_mean": 0.03659576177597046, + "margin_dpo/margin_std": 0.42015379667282104, + "step": 11 + }, + { + "KL/chosen_KL_mean": 0.016056060791015625, + "KL/mean": -0.013466224074363708, + "KL/rejected_KL_mean": -0.042987823486328125, + "KL/std": 0.36298543214797974, + "epoch": 0.01762114537444934, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.05904069542884827, + "fcm_dpo/q_t": 0.4985245168209076, + "grad_norm": 40.121910095214844, + "learning_rate": 7.971014492753623e-08, + "logits/chosen": 0.5381792783737183, + "logits/rejected": 0.670903205871582, + "logps/chosen": -99.08944702148438, + "logps/ref_chosen": -99.10549926757812, + "logps/ref_rejected": -110.27140808105469, + "logps/rejected": -110.31439971923828, + "loss": 1.381, + "margin_dpo/margin_mean": 0.05904114246368408, + "margin_dpo/margin_std": 0.5055064558982849, + "step": 12 + }, + { + "KL/chosen_KL_mean": 0.023166656494140625, + "KL/mean": 0.023870334029197693, + "KL/rejected_KL_mean": 0.02457427978515625, + "KL/std": 0.29374513030052185, + "epoch": 0.01908957415565345, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.0014038383960723877, + "fcm_dpo/q_t": 0.5000355243682861, + "grad_norm": 41.807979583740234, + "learning_rate": 8.695652173913042e-08, + "logits/chosen": 0.362546443939209, + "logits/rejected": 0.6090709567070007, + "logps/chosen": -90.53656005859375, + "logps/ref_chosen": -90.55972290039062, + "logps/ref_rejected": -93.69110107421875, + "logps/rejected": -93.66651916503906, + "loss": 1.3869, + "margin_dpo/margin_mean": -0.0014039874076843262, + "margin_dpo/margin_std": 0.4222813546657562, + "step": 13 + }, + { + "KL/chosen_KL_mean": -0.068084716796875, + "KL/mean": -0.022164784371852875, + "KL/rejected_KL_mean": 0.0237579345703125, + "KL/std": 0.34052860736846924, + "epoch": 0.020558002936857563, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.09185294806957245, + "fcm_dpo/q_t": 0.5022954940795898, + "grad_norm": 36.01750183105469, + "learning_rate": 9.420289855072464e-08, + "logits/chosen": 0.9169459939002991, + "logits/rejected": 1.0052374601364136, + "logps/chosen": -99.895263671875, + "logps/ref_chosen": -99.82717895507812, + "logps/ref_rejected": -108.94200134277344, + "logps/rejected": -108.91824340820312, + "loss": 1.396, + "margin_dpo/margin_mean": -0.09185254573822021, + "margin_dpo/margin_std": 0.45488160848617554, + "step": 14 + }, + { + "KL/chosen_KL_mean": -0.009136199951171875, + "KL/mean": -0.028981618583202362, + "KL/rejected_KL_mean": -0.048828125, + "KL/std": 0.3243560492992401, + "epoch": 0.022026431718061675, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.03969092667102814, + "fcm_dpo/q_t": 0.49900883436203003, + "grad_norm": 31.644763946533203, + "learning_rate": 1.0144927536231885e-07, + "logits/chosen": 0.3826875388622284, + "logits/rejected": 0.5501496195793152, + "logps/chosen": -78.91910552978516, + "logps/ref_chosen": -78.90997314453125, + "logps/ref_rejected": -90.06234741210938, + "logps/rejected": -90.11117553710938, + "loss": 1.3829, + "margin_dpo/margin_mean": 0.03969104588031769, + "margin_dpo/margin_std": 0.4570927619934082, + "step": 15 + }, + { + "KL/chosen_KL_mean": 0.02811431884765625, + "KL/mean": 0.016169600188732147, + "KL/rejected_KL_mean": 0.00421905517578125, + "KL/std": 0.3299415111541748, + "epoch": 0.023494860499265784, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.023894861340522766, + "fcm_dpo/q_t": 0.49940139055252075, + "grad_norm": 34.61058807373047, + "learning_rate": 1.0869565217391303e-07, + "logits/chosen": 0.5568860769271851, + "logits/rejected": 0.8136681318283081, + "logps/chosen": -97.3951644897461, + "logps/ref_chosen": -97.42327880859375, + "logps/ref_rejected": -90.59945678710938, + "logps/rejected": -90.59524536132812, + "loss": 1.3844, + "margin_dpo/margin_mean": 0.023894816637039185, + "margin_dpo/margin_std": 0.4568118453025818, + "step": 16 + }, + { + "KL/chosen_KL_mean": -0.007831573486328125, + "KL/mean": -0.02137354016304016, + "KL/rejected_KL_mean": -0.03491973876953125, + "KL/std": 0.29245567321777344, + "epoch": 0.024963289280469897, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.02709352970123291, + "fcm_dpo/q_t": 0.4993228316307068, + "grad_norm": 35.51238250732422, + "learning_rate": 1.1594202898550725e-07, + "logits/chosen": 0.5117985010147095, + "logits/rejected": 0.7446095943450928, + "logps/chosen": -104.37213897705078, + "logps/ref_chosen": -104.36430358886719, + "logps/ref_rejected": -90.46772766113281, + "logps/rejected": -90.50263977050781, + "loss": 1.3841, + "margin_dpo/margin_mean": 0.027093231678009033, + "margin_dpo/margin_std": 0.4270169138908386, + "step": 17 + }, + { + "KL/chosen_KL_mean": 0.024341583251953125, + "KL/mean": 0.0127788707613945, + "KL/rejected_KL_mean": 0.001224517822265625, + "KL/std": 0.31795647740364075, + "epoch": 0.02643171806167401, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.02311234176158905, + "fcm_dpo/q_t": 0.49942252039909363, + "grad_norm": 41.704132080078125, + "learning_rate": 1.2318840579710146e-07, + "logits/chosen": 0.34284430742263794, + "logits/rejected": 0.6835780143737793, + "logps/chosen": -87.06761169433594, + "logps/ref_chosen": -87.09195709228516, + "logps/ref_rejected": -81.85072326660156, + "logps/rejected": -81.84949493408203, + "loss": 1.3845, + "margin_dpo/margin_mean": 0.023112982511520386, + "margin_dpo/margin_std": 0.4300282597541809, + "step": 18 + }, + { + "KL/chosen_KL_mean": 0.020893096923828125, + "KL/mean": -0.011265799403190613, + "KL/rejected_KL_mean": -0.043422698974609375, + "KL/std": 0.31101712584495544, + "epoch": 0.027900146842878122, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.064310222864151, + "fcm_dpo/q_t": 0.4983947277069092, + "grad_norm": 31.057170867919922, + "learning_rate": 1.3043478260869563e-07, + "logits/chosen": 0.6548206806182861, + "logits/rejected": 0.9354963302612305, + "logps/chosen": -105.85264587402344, + "logps/ref_chosen": -105.87354278564453, + "logps/ref_rejected": -96.93023681640625, + "logps/rejected": -96.9736557006836, + "loss": 1.3805, + "margin_dpo/margin_mean": 0.06431007385253906, + "margin_dpo/margin_std": 0.47644898295402527, + "step": 19 + }, + { + "KL/chosen_KL_mean": 0.016139984130859375, + "KL/mean": 0.009934380650520325, + "KL/rejected_KL_mean": 0.00372314453125, + "KL/std": 0.3077821433544159, + "epoch": 0.02936857562408223, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.012421995401382446, + "fcm_dpo/q_t": 0.49968940019607544, + "grad_norm": 32.689388275146484, + "learning_rate": 1.3768115942028986e-07, + "logits/chosen": 0.1846219301223755, + "logits/rejected": 0.4561445116996765, + "logps/chosen": -90.74198150634766, + "logps/ref_chosen": -90.75811767578125, + "logps/ref_rejected": -85.91232299804688, + "logps/rejected": -85.90859985351562, + "loss": 1.3854, + "margin_dpo/margin_mean": 0.012422174215316772, + "margin_dpo/margin_std": 0.3584854006767273, + "step": 20 + }, + { + "KL/chosen_KL_mean": 0.018489837646484375, + "KL/mean": -0.019078142940998077, + "KL/rejected_KL_mean": -0.056652069091796875, + "KL/std": 0.3066573143005371, + "epoch": 0.030837004405286344, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.07514522969722748, + "fcm_dpo/q_t": 0.4981222450733185, + "grad_norm": 31.387516021728516, + "learning_rate": 1.4492753623188405e-07, + "logits/chosen": 0.45054763555526733, + "logits/rejected": 0.6211205124855042, + "logps/chosen": -80.31497192382812, + "logps/ref_chosen": -80.33345794677734, + "logps/ref_rejected": -83.93373107910156, + "logps/rejected": -83.99038696289062, + "loss": 1.3794, + "margin_dpo/margin_mean": 0.07514509558677673, + "margin_dpo/margin_std": 0.47712084650993347, + "step": 21 + }, + { + "KL/chosen_KL_mean": 0.062107086181640625, + "KL/mean": 0.018313273787498474, + "KL/rejected_KL_mean": -0.025478363037109375, + "KL/std": 0.3286937475204468, + "epoch": 0.032305433186490456, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.08757480978965759, + "fcm_dpo/q_t": 0.49781161546707153, + "grad_norm": 42.74930191040039, + "learning_rate": 1.5217391304347825e-07, + "logits/chosen": 0.7168225049972534, + "logits/rejected": 0.8415087461471558, + "logps/chosen": -95.33319091796875, + "logps/ref_chosen": -95.39530181884766, + "logps/ref_rejected": -103.47351837158203, + "logps/rejected": -103.49899291992188, + "loss": 1.378, + "margin_dpo/margin_mean": 0.08757475018501282, + "margin_dpo/margin_std": 0.4211307764053345, + "step": 22 + }, + { + "KL/chosen_KL_mean": -0.020757675170898438, + "KL/mean": 0.00687447190284729, + "KL/rejected_KL_mean": 0.034511566162109375, + "KL/std": 0.3154371380805969, + "epoch": 0.033773861967694566, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.055268943309783936, + "fcm_dpo/q_t": 0.5013798475265503, + "grad_norm": 31.475515365600586, + "learning_rate": 1.5942028985507245e-07, + "logits/chosen": 0.7378020286560059, + "logits/rejected": 0.917765736579895, + "logps/chosen": -90.65827178955078, + "logps/ref_chosen": -90.63751220703125, + "logps/ref_rejected": -86.59425354003906, + "logps/rejected": -86.55974578857422, + "loss": 1.3923, + "margin_dpo/margin_mean": -0.05526900291442871, + "margin_dpo/margin_std": 0.4065503180027008, + "step": 23 + }, + { + "KL/chosen_KL_mean": 0.015657424926757812, + "KL/mean": -0.00726298987865448, + "KL/rejected_KL_mean": -0.030185699462890625, + "KL/std": 0.304160475730896, + "epoch": 0.03524229074889868, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.04584333300590515, + "fcm_dpo/q_t": 0.4988551437854767, + "grad_norm": 44.36646270751953, + "learning_rate": 1.6666666666666665e-07, + "logits/chosen": 0.4017108678817749, + "logits/rejected": 0.4368298053741455, + "logps/chosen": -69.9016342163086, + "logps/ref_chosen": -69.91728973388672, + "logps/ref_rejected": -106.60990142822266, + "logps/rejected": -106.64009094238281, + "loss": 1.3822, + "margin_dpo/margin_mean": 0.04584340751171112, + "margin_dpo/margin_std": 0.4237392544746399, + "step": 24 + }, + { + "KL/chosen_KL_mean": -0.021209716796875, + "KL/mean": -0.019529297947883606, + "KL/rejected_KL_mean": -0.017848968505859375, + "KL/std": 0.3543895184993744, + "epoch": 0.03671071953010279, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.0033625662326812744, + "fcm_dpo/q_t": 0.5000829696655273, + "grad_norm": 37.1446418762207, + "learning_rate": 1.7391304347826085e-07, + "logits/chosen": 0.3448099195957184, + "logits/rejected": 0.5056936740875244, + "logps/chosen": -80.84669494628906, + "logps/ref_chosen": -80.82548522949219, + "logps/ref_rejected": -95.95710754394531, + "logps/rejected": -95.97496032714844, + "loss": 1.3873, + "margin_dpo/margin_mean": -0.0033624768257141113, + "margin_dpo/margin_std": 0.5277082920074463, + "step": 25 + }, + { + "KL/chosen_KL_mean": -0.018627166748046875, + "KL/mean": -0.02942410111427307, + "KL/rejected_KL_mean": -0.040218353271484375, + "KL/std": 0.3203151822090149, + "epoch": 0.0381791483113069, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.02159792184829712, + "fcm_dpo/q_t": 0.49945974349975586, + "grad_norm": 41.023773193359375, + "learning_rate": 1.8115942028985507e-07, + "logits/chosen": 0.3951873481273651, + "logits/rejected": 0.60643470287323, + "logps/chosen": -88.9197998046875, + "logps/ref_chosen": -88.90116882324219, + "logps/ref_rejected": -109.82819366455078, + "logps/rejected": -109.868408203125, + "loss": 1.3846, + "margin_dpo/margin_mean": 0.021598011255264282, + "margin_dpo/margin_std": 0.4305553734302521, + "step": 26 + }, + { + "KL/chosen_KL_mean": 0.010311126708984375, + "KL/mean": -0.030757389962673187, + "KL/rejected_KL_mean": -0.07183074951171875, + "KL/std": 0.34087151288986206, + "epoch": 0.039647577092511016, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.08214427530765533, + "fcm_dpo/q_t": 0.49794501066207886, + "grad_norm": 44.443389892578125, + "learning_rate": 1.8840579710144927e-07, + "logits/chosen": 0.17309018969535828, + "logits/rejected": 0.36056604981422424, + "logps/chosen": -77.58570098876953, + "logps/ref_chosen": -77.59600830078125, + "logps/ref_rejected": -103.93850708007812, + "logps/rejected": -104.01033782958984, + "loss": 1.3787, + "margin_dpo/margin_mean": 0.08214372396469116, + "margin_dpo/margin_std": 0.4637642502784729, + "step": 27 + }, + { + "KL/chosen_KL_mean": 0.11543655395507812, + "KL/mean": 0.025208979845046997, + "KL/rejected_KL_mean": -0.06502151489257812, + "KL/std": 0.30955493450164795, + "epoch": 0.041116005873715125, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.1804606318473816, + "fcm_dpo/q_t": 0.49549025297164917, + "grad_norm": 36.613136291503906, + "learning_rate": 1.9565217391304347e-07, + "logits/chosen": 0.5012938380241394, + "logits/rejected": 0.7116572856903076, + "logps/chosen": -102.11312866210938, + "logps/ref_chosen": -102.22856903076172, + "logps/ref_rejected": -96.9594955444336, + "logps/rejected": -97.02452087402344, + "loss": 1.3688, + "margin_dpo/margin_mean": 0.18046081066131592, + "margin_dpo/margin_std": 0.42057356238365173, + "step": 28 + }, + { + "KL/chosen_KL_mean": -0.0085296630859375, + "KL/mean": -0.08421530574560165, + "KL/rejected_KL_mean": -0.15990066528320312, + "KL/std": 0.37015920877456665, + "epoch": 0.042584434654919234, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.15137828886508942, + "fcm_dpo/q_t": 0.49621906876564026, + "grad_norm": 42.10970687866211, + "learning_rate": 2.028985507246377e-07, + "logits/chosen": 0.11546057462692261, + "logits/rejected": 0.3588726222515106, + "logps/chosen": -88.65557861328125, + "logps/ref_chosen": -88.64704895019531, + "logps/ref_rejected": -102.96011352539062, + "logps/rejected": -103.12001037597656, + "loss": 1.3719, + "margin_dpo/margin_mean": 0.15137861669063568, + "margin_dpo/margin_std": 0.5257235765457153, + "step": 29 + }, + { + "KL/chosen_KL_mean": 0.03961181640625, + "KL/mean": -0.024579517543315887, + "KL/rejected_KL_mean": -0.088775634765625, + "KL/std": 0.32620030641555786, + "epoch": 0.04405286343612335, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.12838895618915558, + "fcm_dpo/q_t": 0.4967923164367676, + "grad_norm": 39.2256965637207, + "learning_rate": 2.1014492753623187e-07, + "logits/chosen": 0.5374374389648438, + "logits/rejected": 0.6512571573257446, + "logps/chosen": -88.34877014160156, + "logps/ref_chosen": -88.38838195800781, + "logps/ref_rejected": -102.31889343261719, + "logps/rejected": -102.40766906738281, + "loss": 1.374, + "margin_dpo/margin_mean": 0.12838923931121826, + "margin_dpo/margin_std": 0.4373652935028076, + "step": 30 + }, + { + "KL/chosen_KL_mean": -0.039501190185546875, + "KL/mean": -0.05637046694755554, + "KL/rejected_KL_mean": -0.07323837280273438, + "KL/std": 0.3393365740776062, + "epoch": 0.04552129221732746, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.03373754024505615, + "fcm_dpo/q_t": 0.4991576671600342, + "grad_norm": 31.257678985595703, + "learning_rate": 2.1739130434782607e-07, + "logits/chosen": 0.8123838305473328, + "logits/rejected": 1.068558692932129, + "logps/chosen": -101.1651611328125, + "logps/ref_chosen": -101.12565612792969, + "logps/ref_rejected": -79.8584213256836, + "logps/rejected": -79.93165588378906, + "loss": 1.3834, + "margin_dpo/margin_mean": 0.033737361431121826, + "margin_dpo/margin_std": 0.4375351071357727, + "step": 31 + }, + { + "KL/chosen_KL_mean": -0.059062957763671875, + "KL/mean": -0.09265387058258057, + "KL/rejected_KL_mean": -0.12624359130859375, + "KL/std": 0.3508501350879669, + "epoch": 0.04698972099853157, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.06718266010284424, + "fcm_dpo/q_t": 0.4983203411102295, + "grad_norm": 35.197174072265625, + "learning_rate": 2.2463768115942027e-07, + "logits/chosen": 0.4451717138290405, + "logits/rejected": 0.5880470871925354, + "logps/chosen": -96.67608642578125, + "logps/ref_chosen": -96.61702728271484, + "logps/ref_rejected": -96.224365234375, + "logps/rejected": -96.3506088256836, + "loss": 1.3802, + "margin_dpo/margin_mean": 0.06718306243419647, + "margin_dpo/margin_std": 0.4782736897468567, + "step": 32 + }, + { + "KL/chosen_KL_mean": 0.01537322998046875, + "KL/mean": -0.033518873155117035, + "KL/rejected_KL_mean": -0.08240890502929688, + "KL/std": 0.2934386730194092, + "epoch": 0.048458149779735685, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.09778077900409698, + "fcm_dpo/q_t": 0.49755772948265076, + "grad_norm": 33.55974197387695, + "learning_rate": 2.318840579710145e-07, + "logits/chosen": 0.6998616456985474, + "logits/rejected": 0.8830456733703613, + "logps/chosen": -81.50564575195312, + "logps/ref_chosen": -81.52101135253906, + "logps/ref_rejected": -93.80595397949219, + "logps/rejected": -93.88836669921875, + "loss": 1.377, + "margin_dpo/margin_mean": 0.09778153896331787, + "margin_dpo/margin_std": 0.40736132860183716, + "step": 33 + }, + { + "KL/chosen_KL_mean": 0.0535430908203125, + "KL/mean": -0.0476461797952652, + "KL/rejected_KL_mean": -0.14883804321289062, + "KL/std": 0.3794667422771454, + "epoch": 0.049926578560939794, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.20238515734672546, + "fcm_dpo/q_t": 0.4949512183666229, + "grad_norm": 40.8648796081543, + "learning_rate": 2.391304347826087e-07, + "logits/chosen": 0.660446047782898, + "logits/rejected": 0.8321809768676758, + "logps/chosen": -77.14849853515625, + "logps/ref_chosen": -77.20204162597656, + "logps/ref_rejected": -106.71876525878906, + "logps/rejected": -106.86759948730469, + "loss": 1.3669, + "margin_dpo/margin_mean": 0.20238518714904785, + "margin_dpo/margin_std": 0.5540159940719604, + "step": 34 + }, + { + "KL/chosen_KL_mean": 0.008678436279296875, + "KL/mean": -0.11909741163253784, + "KL/rejected_KL_mean": -0.2468719482421875, + "KL/std": 0.4011994004249573, + "epoch": 0.0513950073421439, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.2555462121963501, + "fcm_dpo/q_t": 0.49361878633499146, + "grad_norm": 42.36187744140625, + "learning_rate": 2.463768115942029e-07, + "logits/chosen": 0.3200201094150543, + "logits/rejected": 0.4755074381828308, + "logps/chosen": -77.56167602539062, + "logps/ref_chosen": -77.57035064697266, + "logps/ref_rejected": -112.18855285644531, + "logps/rejected": -112.4354248046875, + "loss": 1.3617, + "margin_dpo/margin_mean": 0.25554654002189636, + "margin_dpo/margin_std": 0.5492804646492004, + "step": 35 + }, + { + "KL/chosen_KL_mean": 0.00325775146484375, + "KL/mean": -0.0820009633898735, + "KL/rejected_KL_mean": -0.16726303100585938, + "KL/std": 0.4162295162677765, + "epoch": 0.05286343612334802, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.17051883041858673, + "fcm_dpo/q_t": 0.4957442283630371, + "grad_norm": 32.467105865478516, + "learning_rate": 2.536231884057971e-07, + "logits/chosen": 0.4792647957801819, + "logits/rejected": 0.6717060804367065, + "logps/chosen": -83.28498840332031, + "logps/ref_chosen": -83.28823852539062, + "logps/ref_rejected": -91.75741577148438, + "logps/rejected": -91.9246826171875, + "loss": 1.3701, + "margin_dpo/margin_mean": 0.1705191731452942, + "margin_dpo/margin_std": 0.5299290418624878, + "step": 36 + }, + { + "KL/chosen_KL_mean": -0.028217315673828125, + "KL/mean": -0.14107736945152283, + "KL/rejected_KL_mean": -0.253936767578125, + "KL/std": 0.43520498275756836, + "epoch": 0.05433186490455213, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.22571538388729095, + "fcm_dpo/q_t": 0.4943671226501465, + "grad_norm": 35.993526458740234, + "learning_rate": 2.6086956521739126e-07, + "logits/chosen": 0.5401657819747925, + "logits/rejected": 0.6905125975608826, + "logps/chosen": -94.79930877685547, + "logps/ref_chosen": -94.77108764648438, + "logps/ref_rejected": -85.71726989746094, + "logps/rejected": -85.97120666503906, + "loss": 1.3648, + "margin_dpo/margin_mean": 0.2257155328989029, + "margin_dpo/margin_std": 0.5866572856903076, + "step": 37 + }, + { + "KL/chosen_KL_mean": 0.08259963989257812, + "KL/mean": -0.07191865891218185, + "KL/rejected_KL_mean": -0.22642898559570312, + "KL/std": 0.45193982124328613, + "epoch": 0.055800293685756244, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.3090275526046753, + "fcm_dpo/q_t": 0.492290735244751, + "grad_norm": 36.67866516113281, + "learning_rate": 2.681159420289855e-07, + "logits/chosen": 0.681549072265625, + "logits/rejected": 0.8211994171142578, + "logps/chosen": -75.89590454101562, + "logps/ref_chosen": -75.97850036621094, + "logps/ref_rejected": -104.10401916503906, + "logps/rejected": -104.3304443359375, + "loss": 1.3567, + "margin_dpo/margin_mean": 0.30902788043022156, + "margin_dpo/margin_std": 0.6416710615158081, + "step": 38 + }, + { + "KL/chosen_KL_mean": -0.011474609375, + "KL/mean": -0.1568700671195984, + "KL/rejected_KL_mean": -0.3022727966308594, + "KL/std": 0.38306891918182373, + "epoch": 0.05726872246696035, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.29079556465148926, + "fcm_dpo/q_t": 0.49273791909217834, + "grad_norm": 36.345123291015625, + "learning_rate": 2.753623188405797e-07, + "logits/chosen": 0.6276674270629883, + "logits/rejected": 0.8255791664123535, + "logps/chosen": -81.19725036621094, + "logps/ref_chosen": -81.18577575683594, + "logps/ref_rejected": -84.1959228515625, + "logps/rejected": -84.4981918334961, + "loss": 1.3582, + "margin_dpo/margin_mean": 0.2907947897911072, + "margin_dpo/margin_std": 0.5327550768852234, + "step": 39 + }, + { + "KL/chosen_KL_mean": 0.026821136474609375, + "KL/mean": -0.07124792039394379, + "KL/rejected_KL_mean": -0.16931533813476562, + "KL/std": 0.33206266164779663, + "epoch": 0.05873715124816446, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.19613954424858093, + "fcm_dpo/q_t": 0.4951009154319763, + "grad_norm": 29.182266235351562, + "learning_rate": 2.8260869565217386e-07, + "logits/chosen": 0.6739002466201782, + "logits/rejected": 0.8751500844955444, + "logps/chosen": -83.30575561523438, + "logps/ref_chosen": -83.33257293701172, + "logps/ref_rejected": -80.25591278076172, + "logps/rejected": -80.42523193359375, + "loss": 1.3673, + "margin_dpo/margin_mean": 0.1961391270160675, + "margin_dpo/margin_std": 0.4317895174026489, + "step": 40 + }, + { + "KL/chosen_KL_mean": -0.0812530517578125, + "KL/mean": -0.2116875797510147, + "KL/rejected_KL_mean": -0.3421211242675781, + "KL/std": 0.4825252592563629, + "epoch": 0.06020558002936858, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.26086869835853577, + "fcm_dpo/q_t": 0.4934980273246765, + "grad_norm": 33.234317779541016, + "learning_rate": 2.898550724637681e-07, + "logits/chosen": 0.6371617317199707, + "logits/rejected": 0.8245461583137512, + "logps/chosen": -93.22991943359375, + "logps/ref_chosen": -93.14866638183594, + "logps/ref_rejected": -102.07920837402344, + "logps/rejected": -102.42132568359375, + "loss": 1.3617, + "margin_dpo/margin_mean": 0.2608690559864044, + "margin_dpo/margin_std": 0.7232835292816162, + "step": 41 + }, + { + "KL/chosen_KL_mean": -0.00684356689453125, + "KL/mean": -0.2057129293680191, + "KL/rejected_KL_mean": -0.4045829772949219, + "KL/std": 0.4727141261100769, + "epoch": 0.06167400881057269, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.39773795008659363, + "fcm_dpo/q_t": 0.4900670647621155, + "grad_norm": 40.909812927246094, + "learning_rate": 2.971014492753623e-07, + "logits/chosen": 0.48165372014045715, + "logits/rejected": 0.6244504451751709, + "logps/chosen": -90.70846557617188, + "logps/ref_chosen": -90.70162200927734, + "logps/ref_rejected": -113.8646469116211, + "logps/rejected": -114.26922607421875, + "loss": 1.3478, + "margin_dpo/margin_mean": 0.39773809909820557, + "margin_dpo/margin_std": 0.5912743806838989, + "step": 42 + }, + { + "KL/chosen_KL_mean": 0.033107757568359375, + "KL/mean": -0.21173666417598724, + "KL/rejected_KL_mean": -0.4565849304199219, + "KL/std": 0.5068866014480591, + "epoch": 0.0631424375917768, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.489694744348526, + "fcm_dpo/q_t": 0.4877869188785553, + "grad_norm": 37.69319534301758, + "learning_rate": 3.043478260869565e-07, + "logits/chosen": 0.5057786703109741, + "logits/rejected": 0.6928372979164124, + "logps/chosen": -89.61092376708984, + "logps/ref_chosen": -89.64402770996094, + "logps/ref_rejected": -103.95185852050781, + "logps/rejected": -104.40843963623047, + "loss": 1.3391, + "margin_dpo/margin_mean": 0.48969531059265137, + "margin_dpo/margin_std": 0.6913731694221497, + "step": 43 + }, + { + "KL/chosen_KL_mean": -0.05744171142578125, + "KL/mean": -0.23107439279556274, + "KL/rejected_KL_mean": -0.4047088623046875, + "KL/std": 0.4452526271343231, + "epoch": 0.06461086637298091, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.3472658693790436, + "fcm_dpo/q_t": 0.49132949113845825, + "grad_norm": 34.49655532836914, + "learning_rate": 3.115942028985507e-07, + "logits/chosen": 0.9959243535995483, + "logits/rejected": 1.1132128238677979, + "logps/chosen": -81.93576049804688, + "logps/ref_chosen": -81.8783187866211, + "logps/ref_rejected": -113.421630859375, + "logps/rejected": -113.82633972167969, + "loss": 1.3527, + "margin_dpo/margin_mean": 0.3472673296928406, + "margin_dpo/margin_std": 0.5616138577461243, + "step": 44 + }, + { + "KL/chosen_KL_mean": -0.06879806518554688, + "KL/mean": -0.19089767336845398, + "KL/rejected_KL_mean": -0.31299591064453125, + "KL/std": 0.4350392818450928, + "epoch": 0.06607929515418502, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.24419839680194855, + "fcm_dpo/q_t": 0.49390164017677307, + "grad_norm": 24.39482307434082, + "learning_rate": 3.188405797101449e-07, + "logits/chosen": 0.722280740737915, + "logits/rejected": 0.835638165473938, + "logps/chosen": -77.41339111328125, + "logps/ref_chosen": -77.34459686279297, + "logps/ref_rejected": -84.24774169921875, + "logps/rejected": -84.56074523925781, + "loss": 1.3628, + "margin_dpo/margin_mean": 0.24419908225536346, + "margin_dpo/margin_std": 0.5641046762466431, + "step": 45 + }, + { + "KL/chosen_KL_mean": -0.01519012451171875, + "KL/mean": -0.22886362671852112, + "KL/rejected_KL_mean": -0.4425392150878906, + "KL/std": 0.5174437165260315, + "epoch": 0.06754772393538913, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.42735230922698975, + "fcm_dpo/q_t": 0.4893287420272827, + "grad_norm": 33.09678649902344, + "learning_rate": 3.260869565217391e-07, + "logits/chosen": 0.5283759832382202, + "logits/rejected": 0.7729748487472534, + "logps/chosen": -90.35601806640625, + "logps/ref_chosen": -90.3408203125, + "logps/ref_rejected": -100.676513671875, + "logps/rejected": -101.11905670166016, + "loss": 1.3448, + "margin_dpo/margin_mean": 0.4273521900177002, + "margin_dpo/margin_std": 0.561310887336731, + "step": 46 + }, + { + "KL/chosen_KL_mean": -0.07775497436523438, + "KL/mean": -0.28309375047683716, + "KL/rejected_KL_mean": -0.488433837890625, + "KL/std": 0.5084018111228943, + "epoch": 0.06901615271659324, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.41067540645599365, + "fcm_dpo/q_t": 0.4897534251213074, + "grad_norm": 33.92341995239258, + "learning_rate": 3.333333333333333e-07, + "logits/chosen": 0.5192316770553589, + "logits/rejected": 0.7062826156616211, + "logps/chosen": -104.48905944824219, + "logps/ref_chosen": -104.41130065917969, + "logps/ref_rejected": -101.25489807128906, + "logps/rejected": -101.74333190917969, + "loss": 1.347, + "margin_dpo/margin_mean": 0.4106749892234802, + "margin_dpo/margin_std": 0.7300633192062378, + "step": 47 + }, + { + "KL/chosen_KL_mean": -0.061695098876953125, + "KL/mean": -0.3085932731628418, + "KL/rejected_KL_mean": -0.5554962158203125, + "KL/std": 0.6243237257003784, + "epoch": 0.07048458149779736, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.49380332231521606, + "fcm_dpo/q_t": 0.48767757415771484, + "grad_norm": 39.03615188598633, + "learning_rate": 3.4057971014492755e-07, + "logits/chosen": 0.19509437680244446, + "logits/rejected": 0.3942684829235077, + "logps/chosen": -91.16197204589844, + "logps/ref_chosen": -91.10027313232422, + "logps/ref_rejected": -95.08058166503906, + "logps/rejected": -95.63607788085938, + "loss": 1.339, + "margin_dpo/margin_mean": 0.49380412697792053, + "margin_dpo/margin_std": 0.7588241100311279, + "step": 48 + }, + { + "KL/chosen_KL_mean": 0.033519744873046875, + "KL/mean": -0.3305027484893799, + "KL/rejected_KL_mean": -0.6945266723632812, + "KL/std": 0.7838465571403503, + "epoch": 0.07195301027900147, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.728039026260376, + "fcm_dpo/q_t": 0.4818696677684784, + "grad_norm": 39.64306640625, + "learning_rate": 3.478260869565217e-07, + "logits/chosen": 0.2486877143383026, + "logits/rejected": 0.7374652624130249, + "logps/chosen": -92.97016143798828, + "logps/ref_chosen": -93.00367736816406, + "logps/ref_rejected": -91.74899291992188, + "logps/rejected": -92.44352722167969, + "loss": 1.3171, + "margin_dpo/margin_mean": 0.7280387878417969, + "margin_dpo/margin_std": 0.9538377523422241, + "step": 49 + }, + { + "KL/chosen_KL_mean": -0.038837432861328125, + "KL/mean": -0.40678519010543823, + "KL/rejected_KL_mean": -0.7747268676757812, + "KL/std": 0.7677117586135864, + "epoch": 0.07342143906020558, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.7358890771865845, + "fcm_dpo/q_t": 0.48165565729141235, + "grad_norm": 33.14576721191406, + "learning_rate": 3.5507246376811595e-07, + "logits/chosen": 0.45622560381889343, + "logits/rejected": 0.6242513656616211, + "logps/chosen": -94.6656494140625, + "logps/ref_chosen": -94.62681579589844, + "logps/ref_rejected": -103.57435607910156, + "logps/rejected": -104.34909057617188, + "loss": 1.3161, + "margin_dpo/margin_mean": 0.7358888387680054, + "margin_dpo/margin_std": 0.9077410697937012, + "step": 50 + }, + { + "KL/chosen_KL_mean": -0.06505584716796875, + "KL/mean": -0.3700931966304779, + "KL/rejected_KL_mean": -0.6751251220703125, + "KL/std": 0.8226094245910645, + "epoch": 0.07488986784140969, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.6100632548332214, + "fcm_dpo/q_t": 0.4848117232322693, + "grad_norm": 30.207778930664062, + "learning_rate": 3.6231884057971015e-07, + "logits/chosen": 0.7389651536941528, + "logits/rejected": 0.8867373466491699, + "logps/chosen": -87.57233428955078, + "logps/ref_chosen": -87.50727844238281, + "logps/ref_rejected": -83.47235107421875, + "logps/rejected": -84.14747619628906, + "loss": 1.3287, + "margin_dpo/margin_mean": 0.6100629568099976, + "margin_dpo/margin_std": 0.9975903630256653, + "step": 51 + }, + { + "KL/chosen_KL_mean": 0.039501190185546875, + "KL/mean": -0.41753852367401123, + "KL/rejected_KL_mean": -0.8745880126953125, + "KL/std": 0.7825153470039368, + "epoch": 0.0763582966226138, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.9140852689743042, + "fcm_dpo/q_t": 0.4772176146507263, + "grad_norm": 31.828645706176758, + "learning_rate": 3.695652173913043e-07, + "logits/chosen": 0.379444420337677, + "logits/rejected": 0.5888497829437256, + "logps/chosen": -90.59075927734375, + "logps/ref_chosen": -90.63026428222656, + "logps/ref_rejected": -87.0390625, + "logps/rejected": -87.91365051269531, + "loss": 1.2989, + "margin_dpo/margin_mean": 0.9140851497650146, + "margin_dpo/margin_std": 0.8735721111297607, + "step": 52 + }, + { + "KL/chosen_KL_mean": 0.00278472900390625, + "KL/mean": -0.54905104637146, + "KL/rejected_KL_mean": -1.1008872985839844, + "KL/std": 1.1362104415893555, + "epoch": 0.07782672540381791, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.1036782264709473, + "fcm_dpo/q_t": 0.4727194905281067, + "grad_norm": 39.239871978759766, + "learning_rate": 3.7681159420289855e-07, + "logits/chosen": 0.23845317959785461, + "logits/rejected": 0.3551448881626129, + "logps/chosen": -81.58029174804688, + "logps/ref_chosen": -81.58307647705078, + "logps/ref_rejected": -95.66152954101562, + "logps/rejected": -96.76242065429688, + "loss": 1.2844, + "margin_dpo/margin_mean": 1.1036782264709473, + "margin_dpo/margin_std": 1.4056880474090576, + "step": 53 + }, + { + "KL/chosen_KL_mean": 0.00627899169921875, + "KL/mean": -0.5997223854064941, + "KL/rejected_KL_mean": -1.2057304382324219, + "KL/std": 1.359240174293518, + "epoch": 0.07929515418502203, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.212003469467163, + "fcm_dpo/q_t": 0.469951868057251, + "grad_norm": 39.345611572265625, + "learning_rate": 3.8405797101449274e-07, + "logits/chosen": 0.3955309987068176, + "logits/rejected": 0.575438380241394, + "logps/chosen": -88.90388488769531, + "logps/ref_chosen": -88.91017150878906, + "logps/ref_rejected": -99.1175537109375, + "logps/rejected": -100.32328796386719, + "loss": 1.2736, + "margin_dpo/margin_mean": 1.2120035886764526, + "margin_dpo/margin_std": 1.3928658962249756, + "step": 54 + }, + { + "KL/chosen_KL_mean": 0.14455795288085938, + "KL/mean": -0.46645084023475647, + "KL/rejected_KL_mean": -1.07745361328125, + "KL/std": 1.2257211208343506, + "epoch": 0.08076358296622614, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.2220126390457153, + "fcm_dpo/q_t": 0.469801127910614, + "grad_norm": 35.018096923828125, + "learning_rate": 3.9130434782608694e-07, + "logits/chosen": 0.09322050958871841, + "logits/rejected": 0.4001709818840027, + "logps/chosen": -92.31136322021484, + "logps/ref_chosen": -92.45591735839844, + "logps/ref_rejected": -92.97093200683594, + "logps/rejected": -94.04838562011719, + "loss": 1.2746, + "margin_dpo/margin_mean": 1.2220122814178467, + "margin_dpo/margin_std": 1.6103571653366089, + "step": 55 + }, + { + "KL/chosen_KL_mean": -0.046146392822265625, + "KL/mean": -0.6011971235275269, + "KL/rejected_KL_mean": -1.1562461853027344, + "KL/std": 1.2121695280075073, + "epoch": 0.08223201174743025, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.1101038455963135, + "fcm_dpo/q_t": 0.4725731313228607, + "grad_norm": 33.058006286621094, + "learning_rate": 3.9855072463768114e-07, + "logits/chosen": 0.2968568503856659, + "logits/rejected": 0.6376580595970154, + "logps/chosen": -87.28280639648438, + "logps/ref_chosen": -87.23665618896484, + "logps/ref_rejected": -100.64553833007812, + "logps/rejected": -101.80178833007812, + "loss": 1.2846, + "margin_dpo/margin_mean": 1.110103726387024, + "margin_dpo/margin_std": 1.578169584274292, + "step": 56 + }, + { + "KL/chosen_KL_mean": -0.12443923950195312, + "KL/mean": -0.6088008880615234, + "KL/rejected_KL_mean": -1.0931587219238281, + "KL/std": 1.192524790763855, + "epoch": 0.08370044052863436, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.968733549118042, + "fcm_dpo/q_t": 0.4759920537471771, + "grad_norm": 34.070369720458984, + "learning_rate": 4.057971014492754e-07, + "logits/chosen": 0.44745880365371704, + "logits/rejected": 0.6691286563873291, + "logps/chosen": -98.27517700195312, + "logps/ref_chosen": -98.15074157714844, + "logps/ref_rejected": -101.05284118652344, + "logps/rejected": -102.14599609375, + "loss": 1.2974, + "margin_dpo/margin_mean": 0.9687336683273315, + "margin_dpo/margin_std": 1.5028915405273438, + "step": 57 + }, + { + "KL/chosen_KL_mean": 0.2667388916015625, + "KL/mean": -0.49684733152389526, + "KL/rejected_KL_mean": -1.2604255676269531, + "KL/std": 1.4403879642486572, + "epoch": 0.08516886930983847, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.5271626710891724, + "fcm_dpo/q_t": 0.46250051259994507, + "grad_norm": 37.94102096557617, + "learning_rate": 4.1304347826086954e-07, + "logits/chosen": 0.08838152885437012, + "logits/rejected": 0.4025894105434418, + "logps/chosen": -99.3142318725586, + "logps/ref_chosen": -99.58097076416016, + "logps/ref_rejected": -91.22227478027344, + "logps/rejected": -92.48269653320312, + "loss": 1.2493, + "margin_dpo/margin_mean": 1.5271626710891724, + "margin_dpo/margin_std": 1.8847169876098633, + "step": 58 + }, + { + "KL/chosen_KL_mean": 0.021205902099609375, + "KL/mean": -0.57396399974823, + "KL/rejected_KL_mean": -1.1691322326660156, + "KL/std": 1.1711525917053223, + "epoch": 0.08663729809104258, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.1903326511383057, + "fcm_dpo/q_t": 0.4704800248146057, + "grad_norm": 34.30070495605469, + "learning_rate": 4.2028985507246374e-07, + "logits/chosen": 0.11888501048088074, + "logits/rejected": 0.30946826934814453, + "logps/chosen": -89.78112030029297, + "logps/ref_chosen": -89.80232238769531, + "logps/ref_rejected": -94.27667236328125, + "logps/rejected": -95.44580841064453, + "loss": 1.2759, + "margin_dpo/margin_mean": 1.190332293510437, + "margin_dpo/margin_std": 1.4320528507232666, + "step": 59 + }, + { + "KL/chosen_KL_mean": 0.04380035400390625, + "KL/mean": -0.558405876159668, + "KL/rejected_KL_mean": -1.1606216430664062, + "KL/std": 1.2293000221252441, + "epoch": 0.0881057268722467, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.2044206857681274, + "fcm_dpo/q_t": 0.4701216220855713, + "grad_norm": 28.800195693969727, + "learning_rate": 4.2753623188405794e-07, + "logits/chosen": 0.34888800978660583, + "logits/rejected": 0.6163915395736694, + "logps/chosen": -95.11190795898438, + "logps/ref_chosen": -95.15571594238281, + "logps/ref_rejected": -91.3724365234375, + "logps/rejected": -92.5330581665039, + "loss": 1.2744, + "margin_dpo/margin_mean": 1.2044206857681274, + "margin_dpo/margin_std": 1.4062446355819702, + "step": 60 + }, + { + "KL/chosen_KL_mean": -0.2785377502441406, + "KL/mean": -0.8395862579345703, + "KL/rejected_KL_mean": -1.400634765625, + "KL/std": 1.4459847211837769, + "epoch": 0.08957415565345081, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.122102975845337, + "fcm_dpo/q_t": 0.47252535820007324, + "grad_norm": 28.436363220214844, + "learning_rate": 4.3478260869565214e-07, + "logits/chosen": 0.20631279051303864, + "logits/rejected": 0.3471667468547821, + "logps/chosen": -86.01084899902344, + "logps/ref_chosen": -85.73230743408203, + "logps/ref_rejected": -97.96575927734375, + "logps/rejected": -99.36639404296875, + "loss": 1.287, + "margin_dpo/margin_mean": 1.1221034526824951, + "margin_dpo/margin_std": 1.9247691631317139, + "step": 61 + }, + { + "KL/chosen_KL_mean": -0.11776351928710938, + "KL/mean": -0.7294133901596069, + "KL/rejected_KL_mean": -1.3410682678222656, + "KL/std": 1.4079499244689941, + "epoch": 0.09104258443465492, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.223311185836792, + "fcm_dpo/q_t": 0.46987247467041016, + "grad_norm": 28.47466468811035, + "learning_rate": 4.420289855072464e-07, + "logits/chosen": 0.10442984104156494, + "logits/rejected": 0.2933581471443176, + "logps/chosen": -81.75314331054688, + "logps/ref_chosen": -81.63538360595703, + "logps/ref_rejected": -84.03831481933594, + "logps/rejected": -85.37937927246094, + "loss": 1.2769, + "margin_dpo/margin_mean": 1.2233108282089233, + "margin_dpo/margin_std": 1.7898602485656738, + "step": 62 + }, + { + "KL/chosen_KL_mean": -0.06484222412109375, + "KL/mean": -0.8377429246902466, + "KL/rejected_KL_mean": -1.6106376647949219, + "KL/std": 1.4270102977752686, + "epoch": 0.09251101321585903, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.5457981824874878, + "fcm_dpo/q_t": 0.4617375433444977, + "grad_norm": 34.867984771728516, + "learning_rate": 4.4927536231884053e-07, + "logits/chosen": 0.21503260731697083, + "logits/rejected": 0.418557345867157, + "logps/chosen": -103.68890380859375, + "logps/ref_chosen": -103.62405395507812, + "logps/ref_rejected": -103.40303039550781, + "logps/rejected": -105.013671875, + "loss": 1.2442, + "margin_dpo/margin_mean": 1.54579758644104, + "margin_dpo/margin_std": 1.6060906648635864, + "step": 63 + }, + { + "KL/chosen_KL_mean": -0.09991836547851562, + "KL/mean": -0.9311131238937378, + "KL/rejected_KL_mean": -1.7623176574707031, + "KL/std": 1.8329956531524658, + "epoch": 0.09397944199706314, + "fcm_dpo/beta": 0.10238748043775558, + "fcm_dpo/delta": 0.11797122657299042, + "fcm_dpo/margin": 1.6624014377593994, + "fcm_dpo/q_t": 0.4589458703994751, + "grad_norm": 35.67680740356445, + "learning_rate": 4.5652173913043473e-07, + "logits/chosen": 0.8073688745498657, + "logits/rejected": 1.0393321514129639, + "logps/chosen": -87.10150146484375, + "logps/ref_chosen": -87.0015869140625, + "logps/ref_rejected": -100.5854721069336, + "logps/rejected": -102.34779357910156, + "loss": 1.2353, + "margin_dpo/margin_mean": 1.6624014377593994, + "margin_dpo/margin_std": 2.069509267807007, + "step": 64 + }, + { + "KL/chosen_KL_mean": -0.029155731201171875, + "KL/mean": -0.9522594809532166, + "KL/rejected_KL_mean": -1.8753700256347656, + "KL/std": 1.6730495691299438, + "epoch": 0.09544787077826726, + "fcm_dpo/beta": 0.10238748043775558, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.8462045192718506, + "fcm_dpo/q_t": 0.45357537269592285, + "grad_norm": 38.958038330078125, + "learning_rate": 4.63768115942029e-07, + "logits/chosen": -0.022555742412805557, + "logits/rejected": 0.12171991169452667, + "logps/chosen": -91.25106811523438, + "logps/ref_chosen": -91.22191619873047, + "logps/ref_rejected": -115.33553314208984, + "logps/rejected": -117.21090698242188, + "loss": 1.217, + "margin_dpo/margin_mean": 1.8462042808532715, + "margin_dpo/margin_std": 2.042468786239624, + "step": 65 + }, + { + "KL/chosen_KL_mean": -0.283233642578125, + "KL/mean": -0.9032148122787476, + "KL/rejected_KL_mean": -1.523193359375, + "KL/std": 1.5640734434127808, + "epoch": 0.09691629955947137, + "fcm_dpo/beta": 0.10238748043775558, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.239959478378296, + "fcm_dpo/q_t": 0.46878179907798767, + "grad_norm": 26.243160247802734, + "learning_rate": 4.7101449275362313e-07, + "logits/chosen": 0.1345851719379425, + "logits/rejected": 0.3374342918395996, + "logps/chosen": -84.06745910644531, + "logps/ref_chosen": -83.78422546386719, + "logps/ref_rejected": -82.7520980834961, + "logps/rejected": -84.2752914428711, + "loss": 1.2733, + "margin_dpo/margin_mean": 1.239959716796875, + "margin_dpo/margin_std": 1.9022430181503296, + "step": 66 + }, + { + "KL/chosen_KL_mean": -0.8461761474609375, + "KL/mean": -1.401423692703247, + "KL/rejected_KL_mean": -1.9566688537597656, + "KL/std": 2.0096323490142822, + "epoch": 0.09838472834067548, + "fcm_dpo/beta": 0.10238748043775558, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.1104950904846191, + "fcm_dpo/q_t": 0.47256746888160706, + "grad_norm": 24.86586570739746, + "learning_rate": 4.782608695652174e-07, + "logits/chosen": 0.28954365849494934, + "logits/rejected": 0.5612607598304749, + "logps/chosen": -88.51913452148438, + "logps/ref_chosen": -87.67295837402344, + "logps/ref_rejected": -79.08674621582031, + "logps/rejected": -81.04341888427734, + "loss": 1.2953, + "margin_dpo/margin_mean": 1.1104950904846191, + "margin_dpo/margin_std": 2.7410237789154053, + "step": 67 + }, + { + "KL/chosen_KL_mean": -0.21075057983398438, + "KL/mean": -1.270378589630127, + "KL/rejected_KL_mean": -2.3300018310546875, + "KL/std": 2.157897472381592, + "epoch": 0.09985315712187959, + "fcm_dpo/beta": 0.10602904856204987, + "fcm_dpo/delta": 0.23036563396453857, + "fcm_dpo/margin": 2.1192517280578613, + "fcm_dpo/q_t": 0.4469185769557953, + "grad_norm": 29.703475952148438, + "learning_rate": 4.855072463768116e-07, + "logits/chosen": -0.051017943769693375, + "logits/rejected": 0.24151284992694855, + "logps/chosen": -97.28959655761719, + "logps/ref_chosen": -97.07884216308594, + "logps/ref_rejected": -84.11872863769531, + "logps/rejected": -86.44873046875, + "loss": 1.1931, + "margin_dpo/margin_mean": 2.1192517280578613, + "margin_dpo/margin_std": 2.628087043762207, + "step": 68 + }, + { + "KL/chosen_KL_mean": -0.26189422607421875, + "KL/mean": -1.4020657539367676, + "KL/rejected_KL_mean": -2.542236328125, + "KL/std": 2.4624533653259277, + "epoch": 0.1013215859030837, + "fcm_dpo/beta": 0.11060892045497894, + "fcm_dpo/delta": 0.20270594954490662, + "fcm_dpo/margin": 2.2803425788879395, + "fcm_dpo/q_t": 0.44102779030799866, + "grad_norm": 32.46053695678711, + "learning_rate": 4.927536231884058e-07, + "logits/chosen": 0.1682801991701126, + "logits/rejected": 0.2794885039329529, + "logps/chosen": -85.98160552978516, + "logps/ref_chosen": -85.71971130371094, + "logps/ref_rejected": -109.48020935058594, + "logps/rejected": -112.02244567871094, + "loss": 1.1768, + "margin_dpo/margin_mean": 2.2803428173065186, + "margin_dpo/margin_std": 3.052189350128174, + "step": 69 + }, + { + "KL/chosen_KL_mean": -0.6263198852539062, + "KL/mean": -1.899319052696228, + "KL/rejected_KL_mean": -3.172313690185547, + "KL/std": 2.73360013961792, + "epoch": 0.1027900146842878, + "fcm_dpo/beta": 0.11450241506099701, + "fcm_dpo/delta": 0.16308385133743286, + "fcm_dpo/margin": 2.545989513397217, + "fcm_dpo/q_t": 0.43262767791748047, + "grad_norm": 31.931215286254883, + "learning_rate": 5e-07, + "logits/chosen": 0.21033185720443726, + "logits/rejected": 0.38455793261528015, + "logps/chosen": -95.63626861572266, + "logps/ref_chosen": -95.00994873046875, + "logps/ref_rejected": -96.21272277832031, + "logps/rejected": -99.38504028320312, + "loss": 1.1523, + "margin_dpo/margin_mean": 2.545989751815796, + "margin_dpo/margin_std": 3.3493716716766357, + "step": 70 + }, + { + "KL/chosen_KL_mean": 0.001708984375, + "KL/mean": -1.8702661991119385, + "KL/rejected_KL_mean": -3.7422447204589844, + "KL/std": 3.5025901794433594, + "epoch": 0.10425844346549193, + "fcm_dpo/beta": 0.11530126631259918, + "fcm_dpo/delta": 0.01898610219359398, + "fcm_dpo/margin": 3.743950128555298, + "fcm_dpo/q_t": 0.400613009929657, + "grad_norm": 37.1617317199707, + "learning_rate": 4.999967061337492e-07, + "logits/chosen": -0.19057126343250275, + "logits/rejected": 0.1714491844177246, + "logps/chosen": -90.97564697265625, + "logps/ref_chosen": -90.97735595703125, + "logps/ref_rejected": -102.59103393554688, + "logps/rejected": -106.3332748413086, + "loss": 1.0462, + "margin_dpo/margin_mean": 3.7439494132995605, + "margin_dpo/margin_std": 3.907170295715332, + "step": 71 + }, + { + "KL/chosen_KL_mean": -0.5383262634277344, + "KL/mean": -1.9858675003051758, + "KL/rejected_KL_mean": -3.43341064453125, + "KL/std": 2.893824577331543, + "epoch": 0.10572687224669604, + "fcm_dpo/beta": 0.11765197664499283, + "fcm_dpo/delta": 0.1133369654417038, + "fcm_dpo/margin": 2.895083427429199, + "fcm_dpo/q_t": 0.4203723669052124, + "grad_norm": 32.836151123046875, + "learning_rate": 4.999868246217933e-07, + "logits/chosen": -0.24647319316864014, + "logits/rejected": 0.07103689014911652, + "logps/chosen": -98.42985534667969, + "logps/ref_chosen": -97.89152526855469, + "logps/ref_rejected": -100.19171142578125, + "logps/rejected": -103.6251220703125, + "loss": 1.1114, + "margin_dpo/margin_mean": 2.895082712173462, + "margin_dpo/margin_std": 3.351975679397583, + "step": 72 + }, + { + "KL/chosen_KL_mean": -0.7371978759765625, + "KL/mean": -2.3258001804351807, + "KL/rejected_KL_mean": -3.9144020080566406, + "KL/std": 3.848926067352295, + "epoch": 0.10719530102790015, + "fcm_dpo/beta": 0.11976398527622223, + "fcm_dpo/delta": 0.07221053540706635, + "fcm_dpo/margin": 3.1772067546844482, + "fcm_dpo/q_t": 0.4168751835823059, + "grad_norm": 30.715152740478516, + "learning_rate": 4.999703557245192e-07, + "logits/chosen": -0.25766175985336304, + "logits/rejected": 0.05361177772283554, + "logps/chosen": -96.50621795654297, + "logps/ref_chosen": -95.7690200805664, + "logps/ref_rejected": -95.93243408203125, + "logps/rejected": -99.84683990478516, + "loss": 1.1183, + "margin_dpo/margin_mean": 3.177206516265869, + "margin_dpo/margin_std": 4.9230732917785645, + "step": 73 + }, + { + "KL/chosen_KL_mean": -0.98834228515625, + "KL/mean": -2.741278648376465, + "KL/rejected_KL_mean": -4.494209289550781, + "KL/std": 3.77854585647583, + "epoch": 0.10866372980910426, + "fcm_dpo/beta": 0.1204468160867691, + "fcm_dpo/delta": 0.02892753668129444, + "fcm_dpo/margin": 3.505868434906006, + "fcm_dpo/q_t": 0.40502387285232544, + "grad_norm": 33.72903823852539, + "learning_rate": 4.999472998758977e-07, + "logits/chosen": -0.35920172929763794, + "logits/rejected": -0.21716484427452087, + "logps/chosen": -79.79673767089844, + "logps/ref_chosen": -78.80839538574219, + "logps/ref_rejected": -101.64677429199219, + "logps/rejected": -106.14097595214844, + "loss": 1.0762, + "margin_dpo/margin_mean": 3.5058681964874268, + "margin_dpo/margin_std": 4.543154716491699, + "step": 74 + }, + { + "KL/chosen_KL_mean": -1.028656005859375, + "KL/mean": -3.3344218730926514, + "KL/rejected_KL_mean": -5.640190124511719, + "KL/std": 4.607555866241455, + "epoch": 0.11013215859030837, + "fcm_dpo/beta": 0.11946292221546173, + "fcm_dpo/delta": -0.10670093446969986, + "fcm_dpo/margin": 4.611537933349609, + "fcm_dpo/q_t": 0.3799246549606323, + "grad_norm": 34.582496643066406, + "learning_rate": 4.999176576834721e-07, + "logits/chosen": -0.2411120980978012, + "logits/rejected": -0.039592910557985306, + "logps/chosen": -79.31050872802734, + "logps/ref_chosen": -78.28185272216797, + "logps/ref_rejected": -115.40311431884766, + "logps/rejected": -121.04330444335938, + "loss": 0.999, + "margin_dpo/margin_mean": 4.611536979675293, + "margin_dpo/margin_std": 5.436832427978516, + "step": 75 + }, + { + "KL/chosen_KL_mean": -1.4529953002929688, + "KL/mean": -3.011784791946411, + "KL/rejected_KL_mean": -4.570579528808594, + "KL/std": 3.8695666790008545, + "epoch": 0.11160058737151249, + "fcm_dpo/beta": 0.11969182640314102, + "fcm_dpo/delta": 0.07985444366931915, + "fcm_dpo/margin": 3.117582321166992, + "fcm_dpo/q_t": 0.41714996099472046, + "grad_norm": 26.333070755004883, + "learning_rate": 4.998814299283415e-07, + "logits/chosen": -0.6458679437637329, + "logits/rejected": -0.27020227909088135, + "logps/chosen": -89.33013916015625, + "logps/ref_chosen": -87.87713623046875, + "logps/ref_rejected": -85.71968078613281, + "logps/rejected": -90.2902603149414, + "loss": 1.1185, + "margin_dpo/margin_mean": 3.117582321166992, + "margin_dpo/margin_std": 4.7150468826293945, + "step": 76 + }, + { + "KL/chosen_KL_mean": -1.6510658264160156, + "KL/mean": -3.3294880390167236, + "KL/rejected_KL_mean": -5.007907867431641, + "KL/std": 4.483570098876953, + "epoch": 0.1130690161527166, + "fcm_dpo/beta": 0.12075696885585785, + "fcm_dpo/delta": 0.046513475477695465, + "fcm_dpo/margin": 3.3568429946899414, + "fcm_dpo/q_t": 0.4093211591243744, + "grad_norm": 31.489459991455078, + "learning_rate": 4.998386175651409e-07, + "logits/chosen": -0.2016545534133911, + "logits/rejected": 0.16604572534561157, + "logps/chosen": -101.35140991210938, + "logps/ref_chosen": -99.70034790039062, + "logps/ref_rejected": -98.20576477050781, + "logps/rejected": -103.21366882324219, + "loss": 1.109, + "margin_dpo/margin_mean": 3.356842517852783, + "margin_dpo/margin_std": 5.0975823402404785, + "step": 77 + }, + { + "KL/chosen_KL_mean": -0.7189369201660156, + "KL/mean": -2.515406608581543, + "KL/rejected_KL_mean": -4.3118743896484375, + "KL/std": 3.712735652923584, + "epoch": 0.1145374449339207, + "fcm_dpo/beta": 0.1215987429022789, + "fcm_dpo/delta": 0.013700693845748901, + "fcm_dpo/margin": 3.592939853668213, + "fcm_dpo/q_t": 0.40087586641311646, + "grad_norm": 28.228641510009766, + "learning_rate": 4.997892217220159e-07, + "logits/chosen": -0.07256608456373215, + "logits/rejected": 0.1554630994796753, + "logps/chosen": -91.0156478881836, + "logps/ref_chosen": -90.29670715332031, + "logps/ref_rejected": -91.13772583007812, + "logps/rejected": -95.44960021972656, + "loss": 1.0586, + "margin_dpo/margin_mean": 3.5929393768310547, + "margin_dpo/margin_std": 4.356261253356934, + "step": 78 + }, + { + "KL/chosen_KL_mean": -0.8799591064453125, + "KL/mean": -3.268341541290283, + "KL/rejected_KL_mean": -5.656734466552734, + "KL/std": 5.356631278991699, + "epoch": 0.11600587371512482, + "fcm_dpo/beta": 0.11956053972244263, + "fcm_dpo/delta": -0.128304123878479, + "fcm_dpo/margin": 4.776772499084473, + "fcm_dpo/q_t": 0.37577077746391296, + "grad_norm": 28.946622848510742, + "learning_rate": 4.997332437005931e-07, + "logits/chosen": -0.3763652741909027, + "logits/rejected": 0.024601154029369354, + "logps/chosen": -87.25828552246094, + "logps/ref_chosen": -86.37832641601562, + "logps/ref_rejected": -94.10777282714844, + "logps/rejected": -99.7645034790039, + "loss": 0.9994, + "margin_dpo/margin_mean": 4.776772499084473, + "margin_dpo/margin_std": 5.929773330688477, + "step": 79 + }, + { + "KL/chosen_KL_mean": -1.5266952514648438, + "KL/mean": -3.4288413524627686, + "KL/rejected_KL_mean": -5.330986022949219, + "KL/std": 4.7091450691223145, + "epoch": 0.11747430249632893, + "fcm_dpo/beta": 0.11858350038528442, + "fcm_dpo/delta": -0.001179170561954379, + "fcm_dpo/margin": 3.8042891025543213, + "fcm_dpo/q_t": 0.4012908935546875, + "grad_norm": 28.268394470214844, + "learning_rate": 4.996706849759452e-07, + "logits/chosen": -0.5914753675460815, + "logits/rejected": -0.2243952751159668, + "logps/chosen": -95.49700927734375, + "logps/ref_chosen": -93.97032165527344, + "logps/ref_rejected": -92.57441711425781, + "logps/rejected": -97.90540313720703, + "loss": 1.0689, + "margin_dpo/margin_mean": 3.804288864135742, + "margin_dpo/margin_std": 5.219730377197266, + "step": 80 + }, + { + "KL/chosen_KL_mean": -1.0590744018554688, + "KL/mean": -3.745607852935791, + "KL/rejected_KL_mean": -6.432132720947266, + "KL/std": 5.830990314483643, + "epoch": 0.11894273127753303, + "fcm_dpo/beta": 0.11468231678009033, + "fcm_dpo/delta": -0.17923633754253387, + "fcm_dpo/margin": 5.373061180114746, + "fcm_dpo/q_t": 0.3717917203903198, + "grad_norm": 29.017410278320312, + "learning_rate": 4.996015471965529e-07, + "logits/chosen": -0.5799692273139954, + "logits/rejected": -0.1557232141494751, + "logps/chosen": -100.88919830322266, + "logps/ref_chosen": -99.83012390136719, + "logps/ref_rejected": -133.67247009277344, + "logps/rejected": -140.10459899902344, + "loss": 0.9883, + "margin_dpo/margin_mean": 5.373061180114746, + "margin_dpo/margin_std": 6.918301582336426, + "step": 81 + }, + { + "KL/chosen_KL_mean": -1.9359970092773438, + "KL/mean": -3.785255193710327, + "KL/rejected_KL_mean": -5.6345062255859375, + "KL/std": 4.574352264404297, + "epoch": 0.12041116005873716, + "fcm_dpo/beta": 0.11558075249195099, + "fcm_dpo/delta": 0.022369179874658585, + "fcm_dpo/margin": 3.6985130310058594, + "fcm_dpo/q_t": 0.4048781394958496, + "grad_norm": 28.42616081237793, + "learning_rate": 4.995258321842611e-07, + "logits/chosen": -0.1761595904827118, + "logits/rejected": 0.04140507057309151, + "logps/chosen": -84.98197937011719, + "logps/ref_chosen": -83.04598236083984, + "logps/ref_rejected": -94.52595520019531, + "logps/rejected": -100.16046142578125, + "loss": 1.0848, + "margin_dpo/margin_mean": 3.698512554168701, + "margin_dpo/margin_std": 5.055111885070801, + "step": 82 + }, + { + "KL/chosen_KL_mean": -2.3091964721679688, + "KL/mean": -4.382638931274414, + "KL/rejected_KL_mean": -6.456077575683594, + "KL/std": 5.10935115814209, + "epoch": 0.12187958883994127, + "fcm_dpo/beta": 0.11439836025238037, + "fcm_dpo/delta": -0.02563435398042202, + "fcm_dpo/margin": 4.146886825561523, + "fcm_dpo/q_t": 0.3988873362541199, + "grad_norm": 30.49053382873535, + "learning_rate": 4.994435419342304e-07, + "logits/chosen": -0.5838781595230103, + "logits/rejected": -0.16465207934379578, + "logps/chosen": -94.48541259765625, + "logps/ref_chosen": -92.17621612548828, + "logps/ref_rejected": -107.74464416503906, + "logps/rejected": -114.20072174072266, + "loss": 1.0859, + "margin_dpo/margin_mean": 4.146885871887207, + "margin_dpo/margin_std": 6.455161094665527, + "step": 83 + }, + { + "KL/chosen_KL_mean": -2.8626060485839844, + "KL/mean": -4.194946765899658, + "KL/rejected_KL_mean": -5.527286529541016, + "KL/std": 4.59609842300415, + "epoch": 0.12334801762114538, + "fcm_dpo/beta": 0.1138247549533844, + "fcm_dpo/delta": -0.021521486341953278, + "fcm_dpo/margin": 2.664675235748291, + "fcm_dpo/q_t": 0.433022141456604, + "grad_norm": 32.54677963256836, + "learning_rate": 4.993546786148857e-07, + "logits/chosen": -0.4822829067707062, + "logits/rejected": -0.09913340210914612, + "logps/chosen": -104.38909912109375, + "logps/ref_chosen": -101.5264892578125, + "logps/ref_rejected": -92.42608642578125, + "logps/rejected": -97.953369140625, + "loss": 1.1864, + "margin_dpo/margin_mean": 2.66467547416687, + "margin_dpo/margin_std": 4.930209159851074, + "step": 84 + }, + { + "KL/chosen_KL_mean": -2.5848312377929688, + "KL/mean": -4.304393768310547, + "KL/rejected_KL_mean": -6.023960113525391, + "KL/std": 5.050421237945557, + "epoch": 0.12481644640234948, + "fcm_dpo/beta": 0.11529316753149033, + "fcm_dpo/delta": 0.05501023307442665, + "fcm_dpo/margin": 3.4391307830810547, + "fcm_dpo/q_t": 0.41383618116378784, + "grad_norm": 31.917598724365234, + "learning_rate": 4.992592445678582e-07, + "logits/chosen": -0.2992115318775177, + "logits/rejected": 0.03511541709303856, + "logps/chosen": -98.71220397949219, + "logps/ref_chosen": -96.12738037109375, + "logps/ref_rejected": -85.05519104003906, + "logps/rejected": -91.07914733886719, + "loss": 1.1438, + "margin_dpo/margin_mean": 3.439131736755371, + "margin_dpo/margin_std": 6.141880035400391, + "step": 85 + }, + { + "KL/chosen_KL_mean": -2.005725860595703, + "KL/mean": -4.218888759613037, + "KL/rejected_KL_mean": -6.432056427001953, + "KL/std": 5.8354387283325195, + "epoch": 0.1262848751835536, + "fcm_dpo/beta": 0.1139456182718277, + "fcm_dpo/delta": -0.05737065523862839, + "fcm_dpo/margin": 4.426335334777832, + "fcm_dpo/q_t": 0.3978298604488373, + "grad_norm": 24.41836929321289, + "learning_rate": 4.991572423079235e-07, + "logits/chosen": -0.5035943984985352, + "logits/rejected": -0.258251428604126, + "logps/chosen": -83.70999145507812, + "logps/ref_chosen": -81.70426177978516, + "logps/ref_rejected": -93.65550231933594, + "logps/rejected": -100.08755493164062, + "loss": 1.0791, + "margin_dpo/margin_mean": 4.426335334777832, + "margin_dpo/margin_std": 7.23484992980957, + "step": 86 + }, + { + "KL/chosen_KL_mean": -2.282470703125, + "KL/mean": -4.64469051361084, + "KL/rejected_KL_mean": -7.006904602050781, + "KL/std": 6.405066013336182, + "epoch": 0.1277533039647577, + "fcm_dpo/beta": 0.112020343542099, + "fcm_dpo/delta": -0.08409902453422546, + "fcm_dpo/margin": 4.724432945251465, + "fcm_dpo/q_t": 0.38618338108062744, + "grad_norm": 29.623733520507812, + "learning_rate": 4.990486745229364e-07, + "logits/chosen": -0.7301775217056274, + "logits/rejected": -0.3559700846672058, + "logps/chosen": -94.96843719482422, + "logps/ref_chosen": -92.68596649169922, + "logps/ref_rejected": -102.91818237304688, + "logps/rejected": -109.92507934570312, + "loss": 1.0774, + "margin_dpo/margin_mean": 4.724432945251465, + "margin_dpo/margin_std": 7.575509071350098, + "step": 87 + }, + { + "KL/chosen_KL_mean": -2.8142852783203125, + "KL/mean": -4.718484878540039, + "KL/rejected_KL_mean": -6.622676849365234, + "KL/std": 6.107346534729004, + "epoch": 0.12922173274596183, + "fcm_dpo/beta": 0.11150763928890228, + "fcm_dpo/delta": 0.0256040021777153, + "fcm_dpo/margin": 3.8083901405334473, + "fcm_dpo/q_t": 0.4089120030403137, + "grad_norm": 27.321096420288086, + "learning_rate": 4.989335440737586e-07, + "logits/chosen": -0.1671774983406067, + "logits/rejected": 0.020983021706342697, + "logps/chosen": -103.5772705078125, + "logps/ref_chosen": -100.76298522949219, + "logps/ref_rejected": -113.15037536621094, + "logps/rejected": -119.7730484008789, + "loss": 1.1211, + "margin_dpo/margin_mean": 3.808389902114868, + "margin_dpo/margin_std": 6.349940299987793, + "step": 88 + }, + { + "KL/chosen_KL_mean": -2.3073387145996094, + "KL/mean": -4.367650985717773, + "KL/rejected_KL_mean": -6.4279632568359375, + "KL/std": 5.265946388244629, + "epoch": 0.13069016152716592, + "fcm_dpo/beta": 0.11189480125904083, + "fcm_dpo/delta": -0.011854719370603561, + "fcm_dpo/margin": 4.120621204376221, + "fcm_dpo/q_t": 0.4049159288406372, + "grad_norm": 25.438533782958984, + "learning_rate": 4.988118539941847e-07, + "logits/chosen": -0.8401930332183838, + "logits/rejected": -0.5166289806365967, + "logps/chosen": -91.99842071533203, + "logps/ref_chosen": -89.69108581542969, + "logps/ref_rejected": -88.56832885742188, + "logps/rejected": -94.99629211425781, + "loss": 1.0967, + "margin_dpo/margin_mean": 4.1206207275390625, + "margin_dpo/margin_std": 6.960460186004639, + "step": 89 + }, + { + "KL/chosen_KL_mean": -2.1090965270996094, + "KL/mean": -4.7426629066467285, + "KL/rejected_KL_mean": -7.376224517822266, + "KL/std": 6.103512763977051, + "epoch": 0.13215859030837004, + "fcm_dpo/beta": 0.11034879833459854, + "fcm_dpo/delta": -0.13909685611724854, + "fcm_dpo/margin": 5.2671403884887695, + "fcm_dpo/q_t": 0.37826719880104065, + "grad_norm": 27.8148136138916, + "learning_rate": 4.986836074908615e-07, + "logits/chosen": -0.35358595848083496, + "logits/rejected": -0.2331031858921051, + "logps/chosen": -83.49165344238281, + "logps/ref_chosen": -81.38255310058594, + "logps/ref_rejected": -117.77714538574219, + "logps/rejected": -125.15336608886719, + "loss": 1.0209, + "margin_dpo/margin_mean": 5.267140865325928, + "margin_dpo/margin_std": 7.285268783569336, + "step": 90 + }, + { + "KL/chosen_KL_mean": -2.2910118103027344, + "KL/mean": -4.56092643737793, + "KL/rejected_KL_mean": -6.830844879150391, + "KL/std": 6.004371166229248, + "epoch": 0.13362701908957417, + "fcm_dpo/beta": 0.10842406004667282, + "fcm_dpo/delta": -0.04439987987279892, + "fcm_dpo/margin": 4.539830207824707, + "fcm_dpo/q_t": 0.39736443758010864, + "grad_norm": 28.05781364440918, + "learning_rate": 4.985488079432037e-07, + "logits/chosen": -0.3208407461643219, + "logits/rejected": 0.09519103169441223, + "logps/chosen": -99.51289367675781, + "logps/ref_chosen": -97.22187805175781, + "logps/ref_rejected": -92.97674560546875, + "logps/rejected": -99.80758666992188, + "loss": 1.0725, + "margin_dpo/margin_mean": 4.539829730987549, + "margin_dpo/margin_std": 6.988130569458008, + "step": 91 + }, + { + "KL/chosen_KL_mean": -2.587505340576172, + "KL/mean": -4.551479816436768, + "KL/rejected_KL_mean": -6.515453338623047, + "KL/std": 5.919102668762207, + "epoch": 0.13509544787077826, + "fcm_dpo/beta": 0.10871409624814987, + "fcm_dpo/delta": 0.023897580802440643, + "fcm_dpo/margin": 3.9279491901397705, + "fcm_dpo/q_t": 0.40844058990478516, + "grad_norm": 25.167949676513672, + "learning_rate": 4.984074589033043e-07, + "logits/chosen": -0.8719907999038696, + "logits/rejected": -0.512382447719574, + "logps/chosen": -87.11778259277344, + "logps/ref_chosen": -84.5302734375, + "logps/ref_rejected": -84.5013198852539, + "logps/rejected": -91.01676940917969, + "loss": 1.1075, + "margin_dpo/margin_mean": 3.9279487133026123, + "margin_dpo/margin_std": 6.353695392608643, + "step": 92 + }, + { + "KL/chosen_KL_mean": -2.7571678161621094, + "KL/mean": -4.692116737365723, + "KL/rejected_KL_mean": -6.627071380615234, + "KL/std": 5.639936447143555, + "epoch": 0.13656387665198239, + "fcm_dpo/beta": 0.1093287318944931, + "fcm_dpo/delta": 0.02795940451323986, + "fcm_dpo/margin": 3.869906425476074, + "fcm_dpo/q_t": 0.4095788300037384, + "grad_norm": 29.23712158203125, + "learning_rate": 4.982595640958425e-07, + "logits/chosen": -0.984565794467926, + "logits/rejected": -0.43060898780822754, + "logps/chosen": -93.00759887695312, + "logps/ref_chosen": -90.25043487548828, + "logps/ref_rejected": -84.0942153930664, + "logps/rejected": -90.72128295898438, + "loss": 1.1339, + "margin_dpo/margin_mean": 3.869905948638916, + "margin_dpo/margin_std": 6.868067741394043, + "step": 93 + }, + { + "KL/chosen_KL_mean": -2.8524551391601562, + "KL/mean": -5.7169413566589355, + "KL/rejected_KL_mean": -8.581424713134766, + "KL/std": 6.534344673156738, + "epoch": 0.13803230543318648, + "fcm_dpo/beta": 0.10555772483348846, + "fcm_dpo/delta": -0.16819366812705994, + "fcm_dpo/margin": 5.72897481918335, + "fcm_dpo/q_t": 0.3694302439689636, + "grad_norm": 27.815052032470703, + "learning_rate": 4.98105127417984e-07, + "logits/chosen": -0.7305547595024109, + "logits/rejected": -0.43267369270324707, + "logps/chosen": -95.30667114257812, + "logps/ref_chosen": -92.4542236328125, + "logps/ref_rejected": -105.2472915649414, + "logps/rejected": -113.82872009277344, + "loss": 1.0087, + "margin_dpo/margin_mean": 5.728974342346191, + "margin_dpo/margin_std": 7.491952896118164, + "step": 94 + }, + { + "KL/chosen_KL_mean": -2.2870559692382812, + "KL/mean": -4.574873447418213, + "KL/rejected_KL_mean": -6.862682342529297, + "KL/std": 5.51740837097168, + "epoch": 0.1395007342143906, + "fcm_dpo/beta": 0.10563249886035919, + "fcm_dpo/delta": -0.0353727787733078, + "fcm_dpo/margin": 4.575626850128174, + "fcm_dpo/q_t": 0.3979508578777313, + "grad_norm": 23.662145614624023, + "learning_rate": 4.979441529392784e-07, + "logits/chosen": -0.7484630942344666, + "logits/rejected": -0.4083735942840576, + "logps/chosen": -81.1607666015625, + "logps/ref_chosen": -78.87370300292969, + "logps/ref_rejected": -83.59121704101562, + "logps/rejected": -90.45389556884766, + "loss": 1.0736, + "margin_dpo/margin_mean": 4.575626373291016, + "margin_dpo/margin_std": 6.794055938720703, + "step": 95 + }, + { + "KL/chosen_KL_mean": -1.9295196533203125, + "KL/mean": -4.763896465301514, + "KL/rejected_KL_mean": -7.5982666015625, + "KL/std": 6.609318733215332, + "epoch": 0.14096916299559473, + "fcm_dpo/beta": 0.10252609103918076, + "fcm_dpo/delta": -0.1397034227848053, + "fcm_dpo/margin": 5.668753623962402, + "fcm_dpo/q_t": 0.37637150287628174, + "grad_norm": 23.820087432861328, + "learning_rate": 4.977766449015534e-07, + "logits/chosen": -0.7242352962493896, + "logits/rejected": -0.2868039608001709, + "logps/chosen": -108.52168273925781, + "logps/ref_chosen": -106.5921630859375, + "logps/ref_rejected": -101.76802825927734, + "logps/rejected": -109.36629486083984, + "loss": 1.0104, + "margin_dpo/margin_mean": 5.668753623962402, + "margin_dpo/margin_std": 7.402196884155273, + "step": 96 + }, + { + "KL/chosen_KL_mean": -2.9581756591796875, + "KL/mean": -5.535565376281738, + "KL/rejected_KL_mean": -8.112953186035156, + "KL/std": 5.711134433746338, + "epoch": 0.14243759177679882, + "fcm_dpo/beta": 0.10165230184793472, + "fcm_dpo/delta": -0.07878708839416504, + "fcm_dpo/margin": 5.154780387878418, + "fcm_dpo/q_t": 0.38353899121284485, + "grad_norm": 26.42591667175293, + "learning_rate": 4.976026077188012e-07, + "logits/chosen": -0.6754403114318848, + "logits/rejected": -0.29090479016304016, + "logps/chosen": -91.6380615234375, + "logps/ref_chosen": -88.67988586425781, + "logps/ref_rejected": -84.81229400634766, + "logps/rejected": -92.92524719238281, + "loss": 1.017, + "margin_dpo/margin_mean": 5.15477991104126, + "margin_dpo/margin_std": 5.9855170249938965, + "step": 97 + }, + { + "KL/chosen_KL_mean": -2.1783065795898438, + "KL/mean": -5.075955390930176, + "KL/rejected_KL_mean": -7.973602294921875, + "KL/std": 5.512714385986328, + "epoch": 0.14390602055800295, + "fcm_dpo/beta": 0.09801796078681946, + "fcm_dpo/delta": -0.12645207345485687, + "fcm_dpo/margin": 5.795301914215088, + "fcm_dpo/q_t": 0.37369024753570557, + "grad_norm": 24.648876190185547, + "learning_rate": 4.974220459770639e-07, + "logits/chosen": -0.7000224590301514, + "logits/rejected": -0.4411921799182892, + "logps/chosen": -94.42080688476562, + "logps/ref_chosen": -92.24249267578125, + "logps/ref_rejected": -101.51948547363281, + "logps/rejected": -109.49308776855469, + "loss": 0.9792, + "margin_dpo/margin_mean": 5.79530143737793, + "margin_dpo/margin_std": 6.1865949630737305, + "step": 98 + }, + { + "KL/chosen_KL_mean": -1.6604843139648438, + "KL/mean": -4.779559135437012, + "KL/rejected_KL_mean": -7.8986358642578125, + "KL/std": 6.573156833648682, + "epoch": 0.14537444933920704, + "fcm_dpo/beta": 0.0955786406993866, + "fcm_dpo/delta": -0.15569189190864563, + "fcm_dpo/margin": 6.238154411315918, + "fcm_dpo/q_t": 0.37738341093063354, + "grad_norm": 23.617198944091797, + "learning_rate": 4.972349644343108e-07, + "logits/chosen": -0.4310641884803772, + "logits/rejected": -0.21981525421142578, + "logps/chosen": -73.84513854980469, + "logps/ref_chosen": -72.18464660644531, + "logps/ref_rejected": -91.88131713867188, + "logps/rejected": -99.77995300292969, + "loss": 1.0005, + "margin_dpo/margin_mean": 6.238154411315918, + "margin_dpo/margin_std": 8.400813102722168, + "step": 99 + }, + { + "KL/chosen_KL_mean": -1.9705314636230469, + "KL/mean": -4.247406005859375, + "KL/rejected_KL_mean": -6.524280548095703, + "KL/std": 4.973667144775391, + "epoch": 0.14684287812041116, + "fcm_dpo/beta": 0.09482486546039581, + "fcm_dpo/delta": 0.018798498436808586, + "fcm_dpo/margin": 4.553745269775391, + "fcm_dpo/q_t": 0.4016741216182709, + "grad_norm": 21.318092346191406, + "learning_rate": 4.970413680203148e-07, + "logits/chosen": -0.7095170021057129, + "logits/rejected": -0.2382434904575348, + "logps/chosen": -91.48435974121094, + "logps/ref_chosen": -89.51382446289062, + "logps/ref_rejected": -81.21713256835938, + "logps/rejected": -87.74140930175781, + "loss": 1.0581, + "margin_dpo/margin_mean": 4.553744792938232, + "margin_dpo/margin_std": 5.306511878967285, + "step": 100 + }, + { + "KL/chosen_KL_mean": -3.2380905151367188, + "KL/mean": -5.267632961273193, + "KL/rejected_KL_mean": -7.297176361083984, + "KL/std": 6.405241012573242, + "epoch": 0.14831130690161526, + "fcm_dpo/beta": 0.09572748839855194, + "fcm_dpo/delta": 0.0637625902891159, + "fcm_dpo/margin": 4.059075832366943, + "fcm_dpo/q_t": 0.41352343559265137, + "grad_norm": 26.655681610107422, + "learning_rate": 4.968412618365215e-07, + "logits/chosen": -0.9653269052505493, + "logits/rejected": -0.714327335357666, + "logps/chosen": -97.64128112792969, + "logps/ref_chosen": -94.4031982421875, + "logps/ref_rejected": -89.91001892089844, + "logps/rejected": -97.20719146728516, + "loss": 1.1382, + "margin_dpo/margin_mean": 4.059075355529785, + "margin_dpo/margin_std": 6.960895538330078, + "step": 101 + }, + { + "KL/chosen_KL_mean": -3.7383384704589844, + "KL/mean": -5.492098808288574, + "KL/rejected_KL_mean": -7.245857238769531, + "KL/std": 6.08369255065918, + "epoch": 0.14977973568281938, + "fcm_dpo/beta": 0.09616687148809433, + "fcm_dpo/delta": -0.03382991999387741, + "fcm_dpo/margin": 3.5075225830078125, + "fcm_dpo/q_t": 0.4243978261947632, + "grad_norm": 31.925813674926758, + "learning_rate": 4.966346511559149e-07, + "logits/chosen": -0.734827995300293, + "logits/rejected": -0.3681231737136841, + "logps/chosen": -95.34776306152344, + "logps/ref_chosen": -91.60942077636719, + "logps/ref_rejected": -75.7689208984375, + "logps/rejected": -83.01477813720703, + "loss": 1.1806, + "margin_dpo/margin_mean": 3.507523536682129, + "margin_dpo/margin_std": 6.865536689758301, + "step": 102 + }, + { + "KL/chosen_KL_mean": -2.8404159545898438, + "KL/mean": -5.731388092041016, + "KL/rejected_KL_mean": -8.622367858886719, + "KL/std": 6.446306228637695, + "epoch": 0.1512481644640235, + "fcm_dpo/beta": 0.0942225307226181, + "fcm_dpo/delta": -0.10055366903543472, + "fcm_dpo/margin": 5.781953811645508, + "fcm_dpo/q_t": 0.37916356325149536, + "grad_norm": 25.041677474975586, + "learning_rate": 4.964215414228785e-07, + "logits/chosen": -0.7607011198997498, + "logits/rejected": -0.3788653612136841, + "logps/chosen": -100.2427978515625, + "logps/ref_chosen": -97.40238952636719, + "logps/ref_rejected": -98.981689453125, + "logps/rejected": -107.60404968261719, + "loss": 1.0197, + "margin_dpo/margin_mean": 5.781954288482666, + "margin_dpo/margin_std": 7.250732421875, + "step": 103 + }, + { + "KL/chosen_KL_mean": -2.1470069885253906, + "KL/mean": -5.268563270568848, + "KL/rejected_KL_mean": -8.390117645263672, + "KL/std": 7.010969161987305, + "epoch": 0.1527165932452276, + "fcm_dpo/beta": 0.09195356070995331, + "fcm_dpo/delta": -0.13192078471183777, + "fcm_dpo/margin": 6.24310827255249, + "fcm_dpo/q_t": 0.3788053095340729, + "grad_norm": 20.705169677734375, + "learning_rate": 4.96201938253052e-07, + "logits/chosen": -0.9428690671920776, + "logits/rejected": -0.6052085757255554, + "logps/chosen": -97.00337219238281, + "logps/ref_chosen": -94.85636138916016, + "logps/ref_rejected": -97.02669525146484, + "logps/rejected": -105.41680908203125, + "loss": 1.006, + "margin_dpo/margin_mean": 6.243109703063965, + "margin_dpo/margin_std": 8.268304824829102, + "step": 104 + }, + { + "KL/chosen_KL_mean": -2.795276641845703, + "KL/mean": -5.931557655334473, + "KL/rejected_KL_mean": -9.06783676147461, + "KL/std": 6.289705276489258, + "epoch": 0.15418502202643172, + "fcm_dpo/beta": 0.09026394784450531, + "fcm_dpo/delta": -0.1230391189455986, + "fcm_dpo/margin": 6.272557735443115, + "fcm_dpo/q_t": 0.37542030215263367, + "grad_norm": 23.900367736816406, + "learning_rate": 4.959758474331832e-07, + "logits/chosen": -0.8815001249313354, + "logits/rejected": -0.41239750385284424, + "logps/chosen": -99.55317687988281, + "logps/ref_chosen": -96.75790405273438, + "logps/ref_rejected": -101.5774917602539, + "logps/rejected": -110.64532470703125, + "loss": 1.0079, + "margin_dpo/margin_mean": 6.272557735443115, + "margin_dpo/margin_std": 7.820766448974609, + "step": 105 + }, + { + "KL/chosen_KL_mean": -2.7666282653808594, + "KL/mean": -5.792070388793945, + "KL/rejected_KL_mean": -8.817512512207031, + "KL/std": 6.627781867980957, + "epoch": 0.15565345080763582, + "fcm_dpo/beta": 0.0884179174900055, + "fcm_dpo/delta": -0.08976646512746811, + "fcm_dpo/margin": 6.05088472366333, + "fcm_dpo/q_t": 0.3848966062068939, + "grad_norm": 21.478395462036133, + "learning_rate": 4.957432749209755e-07, + "logits/chosen": -0.8322691321372986, + "logits/rejected": -0.41586387157440186, + "logps/chosen": -95.20453643798828, + "logps/ref_chosen": -92.43791198730469, + "logps/ref_rejected": -91.00970458984375, + "logps/rejected": -99.82720947265625, + "loss": 1.025, + "margin_dpo/margin_mean": 6.050884246826172, + "margin_dpo/margin_std": 7.906839370727539, + "step": 106 + }, + { + "KL/chosen_KL_mean": -1.926666259765625, + "KL/mean": -5.380865097045898, + "KL/rejected_KL_mean": -8.835067749023438, + "KL/std": 8.281155586242676, + "epoch": 0.15712187958883994, + "fcm_dpo/beta": 0.08573895692825317, + "fcm_dpo/delta": -0.1512652188539505, + "fcm_dpo/margin": 6.90839958190918, + "fcm_dpo/q_t": 0.3809913694858551, + "grad_norm": 21.756074905395508, + "learning_rate": 4.955042268449307e-07, + "logits/chosen": -0.6202389001846313, + "logits/rejected": -0.10301902145147324, + "logps/chosen": -107.05167388916016, + "logps/ref_chosen": -105.12500762939453, + "logps/ref_rejected": -101.1824951171875, + "logps/rejected": -110.01756286621094, + "loss": 1.0173, + "margin_dpo/margin_mean": 6.908400058746338, + "margin_dpo/margin_std": 10.279182434082031, + "step": 107 + }, + { + "KL/chosen_KL_mean": -3.0530052185058594, + "KL/mean": -6.2047247886657715, + "KL/rejected_KL_mean": -9.356449127197266, + "KL/std": 7.492901802062988, + "epoch": 0.15859030837004406, + "fcm_dpo/beta": 0.08362379670143127, + "fcm_dpo/delta": -0.08228491991758347, + "fcm_dpo/margin": 6.303438186645508, + "fcm_dpo/q_t": 0.38573557138442993, + "grad_norm": 21.562427520751953, + "learning_rate": 4.952587095041881e-07, + "logits/chosen": -0.8234080672264099, + "logits/rejected": -0.31772857904434204, + "logps/chosen": -92.26236724853516, + "logps/ref_chosen": -89.20936584472656, + "logps/ref_rejected": -101.83784484863281, + "logps/rejected": -111.19429779052734, + "loss": 1.0301, + "margin_dpo/margin_mean": 6.303438186645508, + "margin_dpo/margin_std": 8.144371032714844, + "step": 108 + }, + { + "KL/chosen_KL_mean": -2.3106842041015625, + "KL/mean": -6.337649345397949, + "KL/rejected_KL_mean": -10.364620208740234, + "KL/std": 7.792704105377197, + "epoch": 0.16005873715124816, + "fcm_dpo/beta": 0.08079753816127777, + "fcm_dpo/delta": -0.21555772423744202, + "fcm_dpo/margin": 8.053934097290039, + "fcm_dpo/q_t": 0.36050570011138916, + "grad_norm": 20.57724380493164, + "learning_rate": 4.95006729368358e-07, + "logits/chosen": -0.5087655186653137, + "logits/rejected": -0.2829950153827667, + "logps/chosen": -102.49777221679688, + "logps/ref_chosen": -100.18708801269531, + "logps/ref_rejected": -103.53739929199219, + "logps/rejected": -113.90201568603516, + "loss": 0.9607, + "margin_dpo/margin_mean": 8.053934097290039, + "margin_dpo/margin_std": 9.591377258300781, + "step": 109 + }, + { + "KL/chosen_KL_mean": -3.0492591857910156, + "KL/mean": -5.415680408477783, + "KL/rejected_KL_mean": -7.782108306884766, + "KL/std": 6.225885391235352, + "epoch": 0.16152716593245228, + "fcm_dpo/beta": 0.08057047426700592, + "fcm_dpo/delta": 0.0711936354637146, + "fcm_dpo/margin": 4.732847690582275, + "fcm_dpo/q_t": 0.4142294228076935, + "grad_norm": 22.01658821105957, + "learning_rate": 4.947482930773511e-07, + "logits/chosen": -0.26160767674446106, + "logits/rejected": 0.1310604214668274, + "logps/chosen": -109.94497680664062, + "logps/ref_chosen": -106.89572143554688, + "logps/ref_rejected": -85.81120300292969, + "logps/rejected": -93.59330749511719, + "loss": 1.1182, + "margin_dpo/margin_mean": 4.732847213745117, + "margin_dpo/margin_std": 7.055660724639893, + "step": 110 + }, + { + "KL/chosen_KL_mean": -2.6614913940429688, + "KL/mean": -7.284836292266846, + "KL/rejected_KL_mean": -11.908180236816406, + "KL/std": 8.842585563659668, + "epoch": 0.16299559471365638, + "fcm_dpo/beta": 0.07738985121250153, + "fcm_dpo/delta": -0.2887975573539734, + "fcm_dpo/margin": 9.246686935424805, + "fcm_dpo/q_t": 0.3454093933105469, + "grad_norm": 19.24679183959961, + "learning_rate": 4.944834074412042e-07, + "logits/chosen": -0.8856979608535767, + "logits/rejected": -0.6032519340515137, + "logps/chosen": -95.99085998535156, + "logps/ref_chosen": -93.3293685913086, + "logps/ref_rejected": -104.177490234375, + "logps/rejected": -116.08566284179688, + "loss": 0.9146, + "margin_dpo/margin_mean": 9.246686935424805, + "margin_dpo/margin_std": 9.710762977600098, + "step": 111 + }, + { + "KL/chosen_KL_mean": -3.026447296142578, + "KL/mean": -5.440064430236816, + "KL/rejected_KL_mean": -7.8536834716796875, + "KL/std": 5.122225761413574, + "epoch": 0.1644640234948605, + "fcm_dpo/beta": 0.07755783945322037, + "fcm_dpo/delta": 0.07860900461673737, + "fcm_dpo/margin": 4.827238082885742, + "fcm_dpo/q_t": 0.41483229398727417, + "grad_norm": 19.261314392089844, + "learning_rate": 4.942120794399002e-07, + "logits/chosen": -1.0721787214279175, + "logits/rejected": -0.5804400444030762, + "logps/chosen": -85.77349090576172, + "logps/ref_chosen": -82.74703979492188, + "logps/ref_rejected": -71.5584487915039, + "logps/rejected": -79.4121322631836, + "loss": 1.1127, + "margin_dpo/margin_mean": 4.8272385597229, + "margin_dpo/margin_std": 6.858962059020996, + "step": 112 + }, + { + "KL/chosen_KL_mean": -3.902820587158203, + "KL/mean": -6.444548606872559, + "KL/rejected_KL_mean": -8.986282348632812, + "KL/std": 6.719594478607178, + "epoch": 0.16593245227606462, + "fcm_dpo/beta": 0.07859328389167786, + "fcm_dpo/delta": 0.05254785716533661, + "fcm_dpo/margin": 5.083460807800293, + "fcm_dpo/q_t": 0.40949755907058716, + "grad_norm": 22.944673538208008, + "learning_rate": 4.939343162231841e-07, + "logits/chosen": -0.8270721435546875, + "logits/rejected": -0.30553609132766724, + "logps/chosen": -101.80231475830078, + "logps/ref_chosen": -97.89949035644531, + "logps/ref_rejected": -84.41960144042969, + "logps/rejected": -93.4058837890625, + "loss": 1.0913, + "margin_dpo/margin_mean": 5.083460807800293, + "margin_dpo/margin_std": 6.480808734893799, + "step": 113 + }, + { + "KL/chosen_KL_mean": -3.5971412658691406, + "KL/mean": -7.166137218475342, + "KL/rejected_KL_mean": -10.735137939453125, + "KL/std": 7.941897869110107, + "epoch": 0.16740088105726872, + "fcm_dpo/beta": 0.07726357132196426, + "fcm_dpo/delta": -0.1080031767487526, + "fcm_dpo/margin": 7.137995719909668, + "fcm_dpo/q_t": 0.3839123249053955, + "grad_norm": 22.6549129486084, + "learning_rate": 4.936501251103751e-07, + "logits/chosen": -0.8440374135971069, + "logits/rejected": -0.2608720660209656, + "logps/chosen": -100.09225463867188, + "logps/ref_chosen": -96.4951171875, + "logps/ref_rejected": -92.01536560058594, + "logps/rejected": -102.75050354003906, + "loss": 1.0423, + "margin_dpo/margin_mean": 7.137994766235352, + "margin_dpo/margin_std": 10.222505569458008, + "step": 114 + }, + { + "KL/chosen_KL_mean": -3.5465164184570312, + "KL/mean": -7.230834007263184, + "KL/rejected_KL_mean": -10.915145874023438, + "KL/std": 8.279781341552734, + "epoch": 0.16886930983847284, + "fcm_dpo/beta": 0.07585737854242325, + "fcm_dpo/delta": -0.11531846225261688, + "fcm_dpo/margin": 7.36862325668335, + "fcm_dpo/q_t": 0.37747281789779663, + "grad_norm": 19.451929092407227, + "learning_rate": 4.933595135901732e-07, + "logits/chosen": -0.8900731801986694, + "logits/rejected": -0.4940794110298157, + "logps/chosen": -106.72663879394531, + "logps/ref_chosen": -103.18013000488281, + "logps/ref_rejected": -105.71595764160156, + "logps/rejected": -116.631103515625, + "loss": 1.0032, + "margin_dpo/margin_mean": 7.368623733520508, + "margin_dpo/margin_std": 8.848615646362305, + "step": 115 + }, + { + "KL/chosen_KL_mean": -3.2444305419921875, + "KL/mean": -6.25869083404541, + "KL/rejected_KL_mean": -9.272945404052734, + "KL/std": 6.523632049560547, + "epoch": 0.17033773861967694, + "fcm_dpo/beta": 0.07550537586212158, + "fcm_dpo/delta": -0.0056202951818704605, + "fcm_dpo/margin": 6.028512954711914, + "fcm_dpo/q_t": 0.3978078067302704, + "grad_norm": 18.775808334350586, + "learning_rate": 4.930624893204624e-07, + "logits/chosen": -0.978033185005188, + "logits/rejected": -0.5550253391265869, + "logps/chosen": -84.40280151367188, + "logps/ref_chosen": -81.15837097167969, + "logps/ref_rejected": -86.30586242675781, + "logps/rejected": -95.57880401611328, + "loss": 1.056, + "margin_dpo/margin_mean": 6.028513431549072, + "margin_dpo/margin_std": 7.502448558807373, + "step": 116 + }, + { + "KL/chosen_KL_mean": -4.012409210205078, + "KL/mean": -6.524020671844482, + "KL/rejected_KL_mean": -9.035633087158203, + "KL/std": 6.570239067077637, + "epoch": 0.17180616740088106, + "fcm_dpo/beta": 0.0756017416715622, + "fcm_dpo/delta": 0.07268474996089935, + "fcm_dpo/margin": 5.023226737976074, + "fcm_dpo/q_t": 0.41434288024902344, + "grad_norm": 26.403039932250977, + "learning_rate": 4.927590601281083e-07, + "logits/chosen": -0.5369125008583069, + "logits/rejected": -0.12467953562736511, + "logps/chosen": -101.09336853027344, + "logps/ref_chosen": -97.0809555053711, + "logps/ref_rejected": -75.1399154663086, + "logps/rejected": -84.17555236816406, + "loss": 1.1271, + "margin_dpo/margin_mean": 5.023226261138916, + "margin_dpo/margin_std": 7.87533712387085, + "step": 117 + }, + { + "KL/chosen_KL_mean": -2.8037071228027344, + "KL/mean": -6.297224998474121, + "KL/rejected_KL_mean": -9.790740966796875, + "KL/std": 7.187756538391113, + "epoch": 0.17327459618208516, + "fcm_dpo/beta": 0.07533556967973709, + "fcm_dpo/delta": -0.08075873553752899, + "fcm_dpo/margin": 6.987035751342773, + "fcm_dpo/q_t": 0.3795852065086365, + "grad_norm": 18.19528579711914, + "learning_rate": 4.924492340087524e-07, + "logits/chosen": -0.9829877614974976, + "logits/rejected": -0.6505037546157837, + "logps/chosen": -89.3399429321289, + "logps/ref_chosen": -86.53623962402344, + "logps/ref_rejected": -82.93563842773438, + "logps/rejected": -92.72637939453125, + "loss": 0.9982, + "margin_dpo/margin_mean": 6.987035751342773, + "margin_dpo/margin_std": 7.422086238861084, + "step": 118 + }, + { + "KL/chosen_KL_mean": -4.230094909667969, + "KL/mean": -7.863368034362793, + "KL/rejected_KL_mean": -11.496639251708984, + "KL/std": 8.438607215881348, + "epoch": 0.17474302496328928, + "fcm_dpo/beta": 0.07391680032014847, + "fcm_dpo/delta": -0.09241504222154617, + "fcm_dpo/margin": 7.266541957855225, + "fcm_dpo/q_t": 0.3817199468612671, + "grad_norm": 20.986663818359375, + "learning_rate": 4.92133019126601e-07, + "logits/chosen": -1.16860830783844, + "logits/rejected": -0.8600578308105469, + "logps/chosen": -108.83917236328125, + "logps/ref_chosen": -104.60908508300781, + "logps/ref_rejected": -109.15780639648438, + "logps/rejected": -120.65444946289062, + "loss": 1.0506, + "margin_dpo/margin_mean": 7.266542434692383, + "margin_dpo/margin_std": 10.496493339538574, + "step": 119 + }, + { + "KL/chosen_KL_mean": -3.249011993408203, + "KL/mean": -8.116844177246094, + "KL/rejected_KL_mean": -12.984676361083984, + "KL/std": 10.410438537597656, + "epoch": 0.1762114537444934, + "fcm_dpo/beta": 0.07091405242681503, + "fcm_dpo/delta": -0.2584834694862366, + "fcm_dpo/margin": 9.735661506652832, + "fcm_dpo/q_t": 0.3579162359237671, + "grad_norm": 23.868812561035156, + "learning_rate": 4.918104238142103e-07, + "logits/chosen": -1.3024057149887085, + "logits/rejected": -0.7799468040466309, + "logps/chosen": -117.93643951416016, + "logps/ref_chosen": -114.68742370605469, + "logps/ref_rejected": -111.18586730957031, + "logps/rejected": -124.17054748535156, + "loss": 0.9846, + "margin_dpo/margin_mean": 9.735660552978516, + "margin_dpo/margin_std": 13.09634780883789, + "step": 120 + }, + { + "KL/chosen_KL_mean": -2.951152801513672, + "KL/mean": -7.305971145629883, + "KL/rejected_KL_mean": -11.660797119140625, + "KL/std": 8.056767463684082, + "epoch": 0.1776798825256975, + "fcm_dpo/beta": 0.06817714869976044, + "fcm_dpo/delta": -0.1538236141204834, + "fcm_dpo/margin": 8.709644317626953, + "fcm_dpo/q_t": 0.3668941259384155, + "grad_norm": 20.163454055786133, + "learning_rate": 4.91481456572267e-07, + "logits/chosen": -1.070889949798584, + "logits/rejected": -0.749244213104248, + "logps/chosen": -95.06781005859375, + "logps/ref_chosen": -92.11666107177734, + "logps/ref_rejected": -114.972900390625, + "logps/rejected": -126.63369750976562, + "loss": 0.9593, + "margin_dpo/margin_mean": 8.70964527130127, + "margin_dpo/margin_std": 8.971741676330566, + "step": 121 + }, + { + "KL/chosen_KL_mean": -3.3295440673828125, + "KL/mean": -8.211607933044434, + "KL/rejected_KL_mean": -13.093673706054688, + "KL/std": 8.696558952331543, + "epoch": 0.17914831130690162, + "fcm_dpo/beta": 0.06610635668039322, + "fcm_dpo/delta": -0.20845989882946014, + "fcm_dpo/margin": 9.764131546020508, + "fcm_dpo/q_t": 0.3557118773460388, + "grad_norm": 20.708974838256836, + "learning_rate": 4.911461260693638e-07, + "logits/chosen": -0.9383471012115479, + "logits/rejected": -0.8294496536254883, + "logps/chosen": -88.20711517333984, + "logps/ref_chosen": -84.87757110595703, + "logps/ref_rejected": -111.0829849243164, + "logps/rejected": -124.1766586303711, + "loss": 0.9401, + "margin_dpo/margin_mean": 9.764131546020508, + "margin_dpo/margin_std": 10.03492546081543, + "step": 122 + }, + { + "KL/chosen_KL_mean": -3.7645301818847656, + "KL/mean": -7.370052337646484, + "KL/rejected_KL_mean": -10.975570678710938, + "KL/std": 8.020940780639648, + "epoch": 0.18061674008810572, + "fcm_dpo/beta": 0.06455057859420776, + "fcm_dpo/delta": -0.017208658158779144, + "fcm_dpo/margin": 7.211041450500488, + "fcm_dpo/q_t": 0.39527085423469543, + "grad_norm": 17.73777198791504, + "learning_rate": 4.908044411417711e-07, + "logits/chosen": -1.1302647590637207, + "logits/rejected": -0.8233848810195923, + "logps/chosen": -102.24327850341797, + "logps/ref_chosen": -98.47874450683594, + "logps/ref_rejected": -93.85417175292969, + "logps/rejected": -104.82974243164062, + "loss": 1.056, + "margin_dpo/margin_mean": 7.211041450500488, + "margin_dpo/margin_std": 9.136102676391602, + "step": 123 + }, + { + "KL/chosen_KL_mean": -3.4926681518554688, + "KL/mean": -7.897808074951172, + "KL/rejected_KL_mean": -12.30295181274414, + "KL/std": 10.150493621826172, + "epoch": 0.18208516886930984, + "fcm_dpo/beta": 0.06381617486476898, + "fcm_dpo/delta": -0.11886176466941833, + "fcm_dpo/margin": 8.810285568237305, + "fcm_dpo/q_t": 0.3854610323905945, + "grad_norm": 18.02562141418457, + "learning_rate": 4.904564107932048e-07, + "logits/chosen": -0.7194161415100098, + "logits/rejected": -0.4542362093925476, + "logps/chosen": -106.25556945800781, + "logps/ref_chosen": -102.76290130615234, + "logps/ref_rejected": -123.005615234375, + "logps/rejected": -135.30856323242188, + "loss": 1.041, + "margin_dpo/margin_mean": 8.810285568237305, + "margin_dpo/margin_std": 13.129676818847656, + "step": 124 + }, + { + "KL/chosen_KL_mean": -4.987255096435547, + "KL/mean": -8.224781036376953, + "KL/rejected_KL_mean": -11.462310791015625, + "KL/std": 8.092391967773438, + "epoch": 0.18355359765051396, + "fcm_dpo/beta": 0.0637354701757431, + "fcm_dpo/delta": 0.03892642632126808, + "fcm_dpo/margin": 6.4750494956970215, + "fcm_dpo/q_t": 0.40852275490760803, + "grad_norm": 18.638343811035156, + "learning_rate": 4.90102044194588e-07, + "logits/chosen": -0.6182900667190552, + "logits/rejected": -0.40189093351364136, + "logps/chosen": -95.41352844238281, + "logps/ref_chosen": -90.42627716064453, + "logps/ref_rejected": -89.24901580810547, + "logps/rejected": -100.7113265991211, + "loss": 1.1068, + "margin_dpo/margin_mean": 6.4750494956970215, + "margin_dpo/margin_std": 9.89763355255127, + "step": 125 + }, + { + "KL/chosen_KL_mean": -3.870624542236328, + "KL/mean": -8.586780548095703, + "KL/rejected_KL_mean": -13.302947998046875, + "KL/std": 8.751096725463867, + "epoch": 0.18502202643171806, + "fcm_dpo/beta": 0.06251658499240875, + "fcm_dpo/delta": -0.14837688207626343, + "fcm_dpo/margin": 9.432323455810547, + "fcm_dpo/q_t": 0.3651265799999237, + "grad_norm": 17.993650436401367, + "learning_rate": 4.897413506838102e-07, + "logits/chosen": -0.7394363284111023, + "logits/rejected": -0.45875149965286255, + "logps/chosen": -98.29350280761719, + "logps/ref_chosen": -94.42288208007812, + "logps/ref_rejected": -103.88156127929688, + "logps/rejected": -117.18450927734375, + "loss": 0.9524, + "margin_dpo/margin_mean": 9.432323455810547, + "margin_dpo/margin_std": 9.009821891784668, + "step": 126 + }, + { + "KL/chosen_KL_mean": -3.8602333068847656, + "KL/mean": -7.447772979736328, + "KL/rejected_KL_mean": -11.03531265258789, + "KL/std": 8.205819129943848, + "epoch": 0.18649045521292218, + "fcm_dpo/beta": 0.06226357817649841, + "fcm_dpo/delta": 0.003141850233078003, + "fcm_dpo/margin": 7.175076484680176, + "fcm_dpo/q_t": 0.4015953242778778, + "grad_norm": 18.24762535095215, + "learning_rate": 4.89374339765481e-07, + "logits/chosen": -0.9668242335319519, + "logits/rejected": -0.6396434903144836, + "logps/chosen": -88.61128234863281, + "logps/ref_chosen": -84.75105285644531, + "logps/ref_rejected": -83.54523468017578, + "logps/rejected": -94.58055114746094, + "loss": 1.0801, + "margin_dpo/margin_mean": 7.175076484680176, + "margin_dpo/margin_std": 10.24130916595459, + "step": 127 + }, + { + "KL/chosen_KL_mean": -3.8696937561035156, + "KL/mean": -7.551077365875244, + "KL/rejected_KL_mean": -11.232461929321289, + "KL/std": 9.975054740905762, + "epoch": 0.18795888399412627, + "fcm_dpo/beta": 0.06226147711277008, + "fcm_dpo/delta": -0.009228337556123734, + "fcm_dpo/margin": 7.362769603729248, + "fcm_dpo/q_t": 0.39746347069740295, + "grad_norm": 20.354082107543945, + "learning_rate": 4.890010211106795e-07, + "logits/chosen": -0.7365471124649048, + "logits/rejected": -0.28447097539901733, + "logps/chosen": -94.48933410644531, + "logps/ref_chosen": -90.61964416503906, + "logps/ref_rejected": -83.10347747802734, + "logps/rejected": -94.3359375, + "loss": 1.0974, + "margin_dpo/margin_mean": 7.362770080566406, + "margin_dpo/margin_std": 12.026878356933594, + "step": 128 + }, + { + "KL/chosen_KL_mean": -4.365577697753906, + "KL/mean": -8.200631141662598, + "KL/rejected_KL_mean": -12.035690307617188, + "KL/std": 8.236621856689453, + "epoch": 0.1894273127753304, + "fcm_dpo/beta": 0.06190914288163185, + "fcm_dpo/delta": -0.026235422119498253, + "fcm_dpo/margin": 7.670107364654541, + "fcm_dpo/q_t": 0.39817970991134644, + "grad_norm": 16.664791107177734, + "learning_rate": 4.88621404556699e-07, + "logits/chosen": -0.7015722990036011, + "logits/rejected": -0.3987889587879181, + "logps/chosen": -103.28428649902344, + "logps/ref_chosen": -98.91870880126953, + "logps/ref_rejected": -103.28419494628906, + "logps/rejected": -115.31988525390625, + "loss": 1.0725, + "margin_dpo/margin_mean": 7.670106887817383, + "margin_dpo/margin_std": 10.961736679077148, + "step": 129 + }, + { + "KL/chosen_KL_mean": -3.4742355346679688, + "KL/mean": -8.451114654541016, + "KL/rejected_KL_mean": -13.427993774414062, + "KL/std": 9.836666107177734, + "epoch": 0.19089574155653452, + "fcm_dpo/beta": 0.06065458431839943, + "fcm_dpo/delta": -0.16370603442192078, + "fcm_dpo/margin": 9.953760147094727, + "fcm_dpo/q_t": 0.3699725866317749, + "grad_norm": 15.890512466430664, + "learning_rate": 4.882355001067891e-07, + "logits/chosen": -1.00321364402771, + "logits/rejected": -0.7537674307823181, + "logps/chosen": -76.65792846679688, + "logps/ref_chosen": -73.18368530273438, + "logps/ref_rejected": -88.02154541015625, + "logps/rejected": -101.44953918457031, + "loss": 0.9811, + "margin_dpo/margin_mean": 9.953760147094727, + "margin_dpo/margin_std": 11.440141677856445, + "step": 130 + }, + { + "KL/chosen_KL_mean": -2.8554420471191406, + "KL/mean": -7.318256378173828, + "KL/rejected_KL_mean": -11.781074523925781, + "KL/std": 9.620319366455078, + "epoch": 0.19236417033773862, + "fcm_dpo/beta": 0.059274956583976746, + "fcm_dpo/delta": -0.08378984034061432, + "fcm_dpo/margin": 8.92563533782959, + "fcm_dpo/q_t": 0.3831636905670166, + "grad_norm": 17.686756134033203, + "learning_rate": 4.878433179298909e-07, + "logits/chosen": -1.0559083223342896, + "logits/rejected": -0.8412282466888428, + "logps/chosen": -76.8118896484375, + "logps/ref_chosen": -73.9564437866211, + "logps/ref_rejected": -94.45582580566406, + "logps/rejected": -106.23690032958984, + "loss": 1.0228, + "margin_dpo/margin_mean": 8.92563533782959, + "margin_dpo/margin_std": 10.949589729309082, + "step": 131 + }, + { + "KL/chosen_KL_mean": -4.320400238037109, + "KL/mean": -8.87709903717041, + "KL/rejected_KL_mean": -13.433792114257812, + "KL/std": 10.098474502563477, + "epoch": 0.19383259911894274, + "fcm_dpo/beta": 0.05771227926015854, + "fcm_dpo/delta": -0.080808125436306, + "fcm_dpo/margin": 9.113384246826172, + "fcm_dpo/q_t": 0.3815780580043793, + "grad_norm": 18.335140228271484, + "learning_rate": 4.874448683603694e-07, + "logits/chosen": -1.1837594509124756, + "logits/rejected": -0.7761355042457581, + "logps/chosen": -96.7598876953125, + "logps/ref_chosen": -92.43948364257812, + "logps/ref_rejected": -94.13252258300781, + "logps/rejected": -107.56631469726562, + "loss": 1.0207, + "margin_dpo/margin_mean": 9.113384246826172, + "margin_dpo/margin_std": 11.179327964782715, + "step": 132 + }, + { + "KL/chosen_KL_mean": -3.47509765625, + "KL/mean": -8.17091178894043, + "KL/rejected_KL_mean": -12.86672592163086, + "KL/std": 10.175192832946777, + "epoch": 0.19530102790014683, + "fcm_dpo/beta": 0.0570124089717865, + "fcm_dpo/delta": -0.09018716216087341, + "fcm_dpo/margin": 9.391630172729492, + "fcm_dpo/q_t": 0.3806670606136322, + "grad_norm": 16.736326217651367, + "learning_rate": 4.870401618977415e-07, + "logits/chosen": -1.52720308303833, + "logits/rejected": -1.1478118896484375, + "logps/chosen": -96.59175872802734, + "logps/ref_chosen": -93.11666107177734, + "logps/ref_rejected": -102.09490966796875, + "logps/rejected": -114.96163940429688, + "loss": 1.0005, + "margin_dpo/margin_mean": 9.391630172729492, + "margin_dpo/margin_std": 10.577903747558594, + "step": 133 + }, + { + "KL/chosen_KL_mean": -3.762378692626953, + "KL/mean": -8.813262939453125, + "KL/rejected_KL_mean": -13.864151000976562, + "KL/std": 9.70240592956543, + "epoch": 0.19676945668135096, + "fcm_dpo/beta": 0.05603744089603424, + "fcm_dpo/delta": -0.1231902688741684, + "fcm_dpo/margin": 10.10176944732666, + "fcm_dpo/q_t": 0.3723108172416687, + "grad_norm": 15.872366905212402, + "learning_rate": 4.866292092063986e-07, + "logits/chosen": -1.6060965061187744, + "logits/rejected": -1.2849180698394775, + "logps/chosen": -87.3125, + "logps/ref_chosen": -83.55012512207031, + "logps/ref_rejected": -93.8815689086914, + "logps/rejected": -107.74571990966797, + "loss": 0.9789, + "margin_dpo/margin_mean": 10.101770401000977, + "margin_dpo/margin_std": 10.374430656433105, + "step": 134 + }, + { + "KL/chosen_KL_mean": -3.5816001892089844, + "KL/mean": -8.457603454589844, + "KL/rejected_KL_mean": -13.333610534667969, + "KL/std": 10.658963203430176, + "epoch": 0.19823788546255505, + "fcm_dpo/beta": 0.054719895124435425, + "fcm_dpo/delta": -0.08825526386499405, + "fcm_dpo/margin": 9.752004623413086, + "fcm_dpo/q_t": 0.3835597038269043, + "grad_norm": 21.14170265197754, + "learning_rate": 4.862120211153265e-07, + "logits/chosen": -1.071568489074707, + "logits/rejected": -0.942071795463562, + "logps/chosen": -81.16423034667969, + "logps/ref_chosen": -77.58262634277344, + "logps/ref_rejected": -121.05648040771484, + "logps/rejected": -134.3900909423828, + "loss": 1.015, + "margin_dpo/margin_mean": 9.752005577087402, + "margin_dpo/margin_std": 11.867852210998535, + "step": 135 + }, + { + "KL/chosen_KL_mean": -3.9735069274902344, + "KL/mean": -8.918501853942871, + "KL/rejected_KL_mean": -13.86349868774414, + "KL/std": 10.705445289611816, + "epoch": 0.19970631424375918, + "fcm_dpo/beta": 0.05359436944127083, + "fcm_dpo/delta": -0.08462314307689667, + "fcm_dpo/margin": 9.889991760253906, + "fcm_dpo/q_t": 0.37985050678253174, + "grad_norm": 15.652995109558105, + "learning_rate": 4.857886086178193e-07, + "logits/chosen": -1.3203887939453125, + "logits/rejected": -0.8870661854743958, + "logps/chosen": -98.146240234375, + "logps/ref_chosen": -94.1727294921875, + "logps/ref_rejected": -103.26428985595703, + "logps/rejected": -117.12779235839844, + "loss": 0.9944, + "margin_dpo/margin_mean": 9.889989852905273, + "margin_dpo/margin_std": 10.464974403381348, + "step": 136 + }, + { + "KL/chosen_KL_mean": -3.146820068359375, + "KL/mean": -10.815319061279297, + "KL/rejected_KL_mean": -18.483814239501953, + "KL/std": 13.656440734863281, + "epoch": 0.2011747430249633, + "fcm_dpo/beta": 0.05032534897327423, + "fcm_dpo/delta": -0.35249799489974976, + "fcm_dpo/margin": 15.336999893188477, + "fcm_dpo/q_t": 0.3331122398376465, + "grad_norm": 18.101778030395508, + "learning_rate": 4.853589828711902e-07, + "logits/chosen": -1.2057225704193115, + "logits/rejected": -0.9818614721298218, + "logps/chosen": -81.26556396484375, + "logps/ref_chosen": -78.11874389648438, + "logps/ref_rejected": -117.92237091064453, + "logps/rejected": -136.40618896484375, + "loss": 0.8664, + "margin_dpo/margin_mean": 15.336999893188477, + "margin_dpo/margin_std": 14.805900573730469, + "step": 137 + }, + { + "KL/chosen_KL_mean": -5.702308654785156, + "KL/mean": -10.494245529174805, + "KL/rejected_KL_mean": -15.286182403564453, + "KL/std": 10.31187629699707, + "epoch": 0.2026431718061674, + "fcm_dpo/beta": 0.04949317127466202, + "fcm_dpo/delta": -0.02566656284034252, + "fcm_dpo/margin": 9.583876609802246, + "fcm_dpo/q_t": 0.3929169178009033, + "grad_norm": 14.387374877929688, + "learning_rate": 4.849231551964771e-07, + "logits/chosen": -0.6397614479064941, + "logits/rejected": -0.3746693730354309, + "logps/chosen": -98.53225708007812, + "logps/ref_chosen": -92.82994079589844, + "logps/ref_rejected": -98.64840698242188, + "logps/rejected": -113.93458557128906, + "loss": 1.0379, + "margin_dpo/margin_mean": 9.583876609802246, + "margin_dpo/margin_std": 11.318662643432617, + "step": 138 + }, + { + "KL/chosen_KL_mean": -3.7344131469726562, + "KL/mean": -9.354459762573242, + "KL/rejected_KL_mean": -14.974510192871094, + "KL/std": 11.291858673095703, + "epoch": 0.20411160058737152, + "fcm_dpo/beta": 0.048662759363651276, + "fcm_dpo/delta": -0.10290683060884476, + "fcm_dpo/margin": 11.240102767944336, + "fcm_dpo/q_t": 0.3744027018547058, + "grad_norm": 14.602463722229004, + "learning_rate": 4.844811370781446e-07, + "logits/chosen": -1.3200509548187256, + "logits/rejected": -1.003678560256958, + "logps/chosen": -82.7374267578125, + "logps/ref_chosen": -79.00301361083984, + "logps/ref_rejected": -85.82733154296875, + "logps/rejected": -100.80184173583984, + "loss": 1.0031, + "margin_dpo/margin_mean": 11.240104675292969, + "margin_dpo/margin_std": 13.213805198669434, + "step": 139 + }, + { + "KL/chosen_KL_mean": -4.216518402099609, + "KL/mean": -9.270885467529297, + "KL/rejected_KL_mean": -14.325252532958984, + "KL/std": 10.672473907470703, + "epoch": 0.2055800293685756, + "fcm_dpo/beta": 0.04824919253587723, + "fcm_dpo/delta": -0.039659008383750916, + "fcm_dpo/margin": 10.108734130859375, + "fcm_dpo/q_t": 0.3884885907173157, + "grad_norm": 14.080016136169434, + "learning_rate": 4.840329401637809e-07, + "logits/chosen": -1.6188607215881348, + "logits/rejected": -1.2256109714508057, + "logps/chosen": -92.5689697265625, + "logps/ref_chosen": -88.35244750976562, + "logps/ref_rejected": -90.57476806640625, + "logps/rejected": -104.9000244140625, + "loss": 1.0231, + "margin_dpo/margin_mean": 10.108734130859375, + "margin_dpo/margin_std": 11.146553993225098, + "step": 140 + }, + { + "KL/chosen_KL_mean": -5.553092956542969, + "KL/mean": -11.420736312866211, + "KL/rejected_KL_mean": -17.288379669189453, + "KL/std": 13.659140586853027, + "epoch": 0.20704845814977973, + "fcm_dpo/beta": 0.047365203499794006, + "fcm_dpo/delta": -0.11200984567403793, + "fcm_dpo/margin": 11.735284805297852, + "fcm_dpo/q_t": 0.3748253881931305, + "grad_norm": 16.944189071655273, + "learning_rate": 4.83578576263792e-07, + "logits/chosen": -1.092730164527893, + "logits/rejected": -0.704372227191925, + "logps/chosen": -112.77325439453125, + "logps/ref_chosen": -107.22016143798828, + "logps/ref_rejected": -104.71223449707031, + "logps/rejected": -122.0006103515625, + "loss": 0.9987, + "margin_dpo/margin_mean": 11.735283851623535, + "margin_dpo/margin_std": 13.693428039550781, + "step": 141 + }, + { + "KL/chosen_KL_mean": -6.127674102783203, + "KL/mean": -11.769968032836914, + "KL/rejected_KL_mean": -17.412261962890625, + "KL/std": 12.742448806762695, + "epoch": 0.20851688693098386, + "fcm_dpo/beta": 0.0464097335934639, + "fcm_dpo/delta": -0.07808461785316467, + "fcm_dpo/margin": 11.284584045410156, + "fcm_dpo/q_t": 0.3839187026023865, + "grad_norm": 14.27704906463623, + "learning_rate": 4.83118057351089e-07, + "logits/chosen": -1.030910849571228, + "logits/rejected": -0.7692275047302246, + "logps/chosen": -94.85662078857422, + "logps/ref_chosen": -88.72894287109375, + "logps/ref_rejected": -99.73038482666016, + "logps/rejected": -117.14264678955078, + "loss": 1.0091, + "margin_dpo/margin_mean": 11.284584045410156, + "margin_dpo/margin_std": 13.189620971679688, + "step": 142 + }, + { + "KL/chosen_KL_mean": -4.1996307373046875, + "KL/mean": -8.925056457519531, + "KL/rejected_KL_mean": -13.650474548339844, + "KL/std": 10.892311096191406, + "epoch": 0.20998531571218795, + "fcm_dpo/beta": 0.04621533304452896, + "fcm_dpo/delta": 0.013468053191900253, + "fcm_dpo/margin": 9.450836181640625, + "fcm_dpo/q_t": 0.4005667269229889, + "grad_norm": 14.099587440490723, + "learning_rate": 4.826513955607734e-07, + "logits/chosen": -1.5993086099624634, + "logits/rejected": -1.1545026302337646, + "logps/chosen": -89.70420837402344, + "logps/ref_chosen": -85.50457763671875, + "logps/ref_rejected": -85.8056640625, + "logps/rejected": -99.45613098144531, + "loss": 1.0679, + "margin_dpo/margin_mean": 9.450836181640625, + "margin_dpo/margin_std": 11.928359985351562, + "step": 143 + }, + { + "KL/chosen_KL_mean": -5.068080902099609, + "KL/mean": -9.908295631408691, + "KL/rejected_KL_mean": -14.748512268066406, + "KL/std": 11.455781936645508, + "epoch": 0.21145374449339208, + "fcm_dpo/beta": 0.04674536734819412, + "fcm_dpo/delta": -0.0035240501165390015, + "fcm_dpo/margin": 9.680429458618164, + "fcm_dpo/q_t": 0.39962923526763916, + "grad_norm": 18.266056060791016, + "learning_rate": 4.821786031898176e-07, + "logits/chosen": -1.4986419677734375, + "logits/rejected": -0.9835749864578247, + "logps/chosen": -109.75517272949219, + "logps/ref_chosen": -104.68708801269531, + "logps/ref_rejected": -87.71908569335938, + "logps/rejected": -102.46759033203125, + "loss": 1.085, + "margin_dpo/margin_mean": 9.680429458618164, + "margin_dpo/margin_std": 14.155384063720703, + "step": 144 + }, + { + "KL/chosen_KL_mean": -4.3048095703125, + "KL/mean": -9.987184524536133, + "KL/rejected_KL_mean": -15.669563293457031, + "KL/std": 11.09414291381836, + "epoch": 0.21292217327459617, + "fcm_dpo/beta": 0.045895203948020935, + "fcm_dpo/delta": -0.07554689049720764, + "fcm_dpo/margin": 11.364750862121582, + "fcm_dpo/q_t": 0.3813377916812897, + "grad_norm": 14.497042655944824, + "learning_rate": 4.816996926967401e-07, + "logits/chosen": -1.2646183967590332, + "logits/rejected": -0.707251787185669, + "logps/chosen": -92.82469940185547, + "logps/ref_chosen": -88.51988983154297, + "logps/ref_rejected": -83.76513671875, + "logps/rejected": -99.43470764160156, + "loss": 1.0046, + "margin_dpo/margin_mean": 11.364751815795898, + "margin_dpo/margin_std": 12.453365325927734, + "step": 145 + }, + { + "KL/chosen_KL_mean": -4.763313293457031, + "KL/mean": -11.054548263549805, + "KL/rejected_KL_mean": -17.345783233642578, + "KL/std": 12.090798377990723, + "epoch": 0.2143906020558003, + "fcm_dpo/beta": 0.044664401561021805, + "fcm_dpo/delta": -0.12001601606607437, + "fcm_dpo/margin": 12.582468032836914, + "fcm_dpo/q_t": 0.3720228672027588, + "grad_norm": 14.317770004272461, + "learning_rate": 4.812146767012779e-07, + "logits/chosen": -1.3493354320526123, + "logits/rejected": -0.7054879665374756, + "logps/chosen": -107.66917419433594, + "logps/ref_chosen": -102.90586853027344, + "logps/ref_rejected": -88.9493637084961, + "logps/rejected": -106.29515075683594, + "loss": 0.9807, + "margin_dpo/margin_mean": 12.582468032836914, + "margin_dpo/margin_std": 13.1751708984375, + "step": 146 + }, + { + "KL/chosen_KL_mean": -4.699851989746094, + "KL/mean": -11.160320281982422, + "KL/rejected_KL_mean": -17.62078857421875, + "KL/std": 11.679289817810059, + "epoch": 0.21585903083700442, + "fcm_dpo/beta": 0.04384419322013855, + "fcm_dpo/delta": -0.1234760582447052, + "fcm_dpo/margin": 12.920938491821289, + "fcm_dpo/q_t": 0.3704032301902771, + "grad_norm": 13.309486389160156, + "learning_rate": 4.807235679840536e-07, + "logits/chosen": -1.292638897895813, + "logits/rejected": -0.7031066417694092, + "logps/chosen": -93.9620361328125, + "logps/ref_chosen": -89.26219177246094, + "logps/ref_rejected": -76.50011444091797, + "logps/rejected": -94.12090301513672, + "loss": 0.9648, + "margin_dpo/margin_mean": 12.920938491821289, + "margin_dpo/margin_std": 12.50632095336914, + "step": 147 + }, + { + "KL/chosen_KL_mean": -3.066070556640625, + "KL/mean": -8.936927795410156, + "KL/rejected_KL_mean": -14.807792663574219, + "KL/std": 11.961725234985352, + "epoch": 0.2173274596182085, + "fcm_dpo/beta": 0.04350690543651581, + "fcm_dpo/delta": -0.06505133211612701, + "fcm_dpo/margin": 11.741720199584961, + "fcm_dpo/q_t": 0.3875482678413391, + "grad_norm": 14.381302833557129, + "learning_rate": 4.802263794862384e-07, + "logits/chosen": -1.5446524620056152, + "logits/rejected": -1.1513751745224, + "logps/chosen": -94.82894897460938, + "logps/ref_chosen": -91.76287841796875, + "logps/ref_rejected": -110.38552856445312, + "logps/rejected": -125.19332122802734, + "loss": 1.0254, + "margin_dpo/margin_mean": 11.741720199584961, + "margin_dpo/margin_std": 13.526103973388672, + "step": 148 + }, + { + "KL/chosen_KL_mean": -6.082599639892578, + "KL/mean": -11.758880615234375, + "KL/rejected_KL_mean": -17.43515396118164, + "KL/std": 12.793182373046875, + "epoch": 0.21879588839941264, + "fcm_dpo/beta": 0.04260088503360748, + "fcm_dpo/delta": -0.03577341139316559, + "fcm_dpo/margin": 11.352545738220215, + "fcm_dpo/q_t": 0.396129846572876, + "grad_norm": 13.894159317016602, + "learning_rate": 4.797231243092118e-07, + "logits/chosen": -1.283447265625, + "logits/rejected": -0.9648770689964294, + "logps/chosen": -99.56661987304688, + "logps/ref_chosen": -93.48401641845703, + "logps/ref_rejected": -105.6436996459961, + "logps/rejected": -123.078857421875, + "loss": 1.0569, + "margin_dpo/margin_mean": 11.352546691894531, + "margin_dpo/margin_std": 15.676870346069336, + "step": 149 + }, + { + "KL/chosen_KL_mean": -4.868541717529297, + "KL/mean": -10.874984741210938, + "KL/rejected_KL_mean": -16.881431579589844, + "KL/std": 13.660394668579102, + "epoch": 0.22026431718061673, + "fcm_dpo/beta": 0.042201243340969086, + "fcm_dpo/delta": -0.060271378606557846, + "fcm_dpo/margin": 12.01289176940918, + "fcm_dpo/q_t": 0.38769829273223877, + "grad_norm": 12.558541297912598, + "learning_rate": 4.792138157142157e-07, + "logits/chosen": -1.2181215286254883, + "logits/rejected": -0.9505946636199951, + "logps/chosen": -80.64723205566406, + "logps/ref_chosen": -75.7786865234375, + "logps/ref_rejected": -90.02676391601562, + "logps/rejected": -106.90819549560547, + "loss": 1.0293, + "margin_dpo/margin_mean": 12.01289176940918, + "margin_dpo/margin_std": 15.103593826293945, + "step": 150 + }, + { + "KL/chosen_KL_mean": -6.614654541015625, + "KL/mean": -12.416755676269531, + "KL/rejected_KL_mean": -18.21884536743164, + "KL/std": 13.398006439208984, + "epoch": 0.22173274596182085, + "fcm_dpo/beta": 0.04193378984928131, + "fcm_dpo/delta": -0.03847986459732056, + "fcm_dpo/margin": 11.604190826416016, + "fcm_dpo/q_t": 0.3916553854942322, + "grad_norm": 15.315407752990723, + "learning_rate": 4.786984671220053e-07, + "logits/chosen": -2.1024904251098633, + "logits/rejected": -1.5320215225219727, + "logps/chosen": -109.348876953125, + "logps/ref_chosen": -102.73422241210938, + "logps/ref_rejected": -106.03781127929688, + "logps/rejected": -124.25665283203125, + "loss": 1.0433, + "margin_dpo/margin_mean": 11.604188919067383, + "margin_dpo/margin_std": 14.949684143066406, + "step": 151 + }, + { + "KL/chosen_KL_mean": -3.824298858642578, + "KL/mean": -10.796260833740234, + "KL/rejected_KL_mean": -17.768226623535156, + "KL/std": 13.248649597167969, + "epoch": 0.22320117474302498, + "fcm_dpo/beta": 0.04098113998770714, + "fcm_dpo/delta": -0.1291038691997528, + "fcm_dpo/margin": 13.943924903869629, + "fcm_dpo/q_t": 0.3707083761692047, + "grad_norm": 13.648688316345215, + "learning_rate": 4.78177092112495e-07, + "logits/chosen": -1.5742216110229492, + "logits/rejected": -1.20149564743042, + "logps/chosen": -101.4150390625, + "logps/ref_chosen": -97.59074401855469, + "logps/ref_rejected": -110.49913024902344, + "logps/rejected": -128.26736450195312, + "loss": 0.9782, + "margin_dpo/margin_mean": 13.943923950195312, + "margin_dpo/margin_std": 15.15260124206543, + "step": 152 + }, + { + "KL/chosen_KL_mean": -6.445198059082031, + "KL/mean": -11.51744270324707, + "KL/rejected_KL_mean": -16.58969497680664, + "KL/std": 12.89784049987793, + "epoch": 0.22466960352422907, + "fcm_dpo/beta": 0.040995821356773376, + "fcm_dpo/delta": 0.03560230880975723, + "fcm_dpo/margin": 10.144502639770508, + "fcm_dpo/q_t": 0.40656381845474243, + "grad_norm": 13.640460968017578, + "learning_rate": 4.776497044244016e-07, + "logits/chosen": -1.5161995887756348, + "logits/rejected": -1.1859508752822876, + "logps/chosen": -93.47079467773438, + "logps/ref_chosen": -87.02560424804688, + "logps/ref_rejected": -91.06854248046875, + "logps/rejected": -107.65823364257812, + "loss": 1.0979, + "margin_dpo/margin_mean": 10.144502639770508, + "margin_dpo/margin_std": 14.666120529174805, + "step": 153 + }, + { + "KL/chosen_KL_mean": -4.714118957519531, + "KL/mean": -12.884841918945312, + "KL/rejected_KL_mean": -21.05556869506836, + "KL/std": 14.184276580810547, + "epoch": 0.2261380323054332, + "fcm_dpo/beta": 0.040085356682538986, + "fcm_dpo/delta": -0.21898622810840607, + "fcm_dpo/margin": 16.341453552246094, + "fcm_dpo/q_t": 0.3526090979576111, + "grad_norm": 13.28144359588623, + "learning_rate": 4.771163179548808e-07, + "logits/chosen": -1.4826412200927734, + "logits/rejected": -1.1600244045257568, + "logps/chosen": -95.81978607177734, + "logps/ref_chosen": -91.10566711425781, + "logps/ref_rejected": -111.37238311767578, + "logps/rejected": -132.42794799804688, + "loss": 0.9173, + "margin_dpo/margin_mean": 16.341453552246094, + "margin_dpo/margin_std": 14.729225158691406, + "step": 154 + }, + { + "KL/chosen_KL_mean": -6.760845184326172, + "KL/mean": -12.999425888061523, + "KL/rejected_KL_mean": -19.237995147705078, + "KL/std": 14.634838104248047, + "epoch": 0.2276064610866373, + "fcm_dpo/beta": 0.03888711705803871, + "fcm_dpo/delta": -0.037780579179525375, + "fcm_dpo/margin": 12.477148056030273, + "fcm_dpo/q_t": 0.38871341943740845, + "grad_norm": 16.313806533813477, + "learning_rate": 4.7657694675916247e-07, + "logits/chosen": -1.1022429466247559, + "logits/rejected": -0.7117606401443481, + "logps/chosen": -101.32182312011719, + "logps/ref_chosen": -94.56097412109375, + "logps/ref_rejected": -84.77359008789062, + "logps/rejected": -104.01158142089844, + "loss": 1.0683, + "margin_dpo/margin_mean": 12.477148056030273, + "margin_dpo/margin_std": 17.88715362548828, + "step": 155 + }, + { + "KL/chosen_KL_mean": -8.382980346679688, + "KL/mean": -13.902236938476562, + "KL/rejected_KL_mean": -19.421493530273438, + "KL/std": 12.998016357421875, + "epoch": 0.2290748898678414, + "fcm_dpo/beta": 0.03896789997816086, + "fcm_dpo/delta": 0.02044248953461647, + "fcm_dpo/margin": 11.0385103225708, + "fcm_dpo/q_t": 0.4047124683856964, + "grad_norm": 14.493485450744629, + "learning_rate": 4.7603160505017893e-07, + "logits/chosen": -1.380091905593872, + "logits/rejected": -0.8696132898330688, + "logps/chosen": -103.03836822509766, + "logps/ref_chosen": -94.65538787841797, + "logps/ref_rejected": -84.41934204101562, + "logps/rejected": -103.84083557128906, + "loss": 1.0885, + "margin_dpo/margin_mean": 11.038509368896484, + "margin_dpo/margin_std": 15.625204086303711, + "step": 156 + }, + { + "KL/chosen_KL_mean": -7.839717864990234, + "KL/mean": -15.43508529663086, + "KL/rejected_KL_mean": -23.03044891357422, + "KL/std": 13.96629810333252, + "epoch": 0.2305433186490455, + "fcm_dpo/beta": 0.038414839655160904, + "fcm_dpo/delta": -0.1416759192943573, + "fcm_dpo/margin": 15.19072437286377, + "fcm_dpo/q_t": 0.36670780181884766, + "grad_norm": 14.359320640563965, + "learning_rate": 4.7548030719819154e-07, + "logits/chosen": -1.9664299488067627, + "logits/rejected": -1.5126819610595703, + "logps/chosen": -104.85211181640625, + "logps/ref_chosen": -97.01239013671875, + "logps/ref_rejected": -111.70133972167969, + "logps/rejected": -134.73179626464844, + "loss": 0.9577, + "margin_dpo/margin_mean": 15.19072437286377, + "margin_dpo/margin_std": 14.695119857788086, + "step": 157 + }, + { + "KL/chosen_KL_mean": -7.228157043457031, + "KL/mean": -15.251038551330566, + "KL/rejected_KL_mean": -23.27391815185547, + "KL/std": 14.975448608398438, + "epoch": 0.23201174743024963, + "fcm_dpo/beta": 0.037267886102199554, + "fcm_dpo/delta": -0.15720441937446594, + "fcm_dpo/margin": 16.0457706451416, + "fcm_dpo/q_t": 0.3649120628833771, + "grad_norm": 13.368614196777344, + "learning_rate": 4.7492306773041136e-07, + "logits/chosen": -1.1402028799057007, + "logits/rejected": -0.8593883514404297, + "logps/chosen": -101.94111633300781, + "logps/ref_chosen": -94.71296691894531, + "logps/ref_rejected": -120.93733978271484, + "logps/rejected": -144.2112579345703, + "loss": 0.9518, + "margin_dpo/margin_mean": 16.04576873779297, + "margin_dpo/margin_std": 15.876977920532227, + "step": 158 + }, + { + "KL/chosen_KL_mean": -6.558082580566406, + "KL/mean": -13.74714469909668, + "KL/rejected_KL_mean": -20.93621063232422, + "KL/std": 14.688760757446289, + "epoch": 0.23348017621145375, + "fcm_dpo/beta": 0.03646499663591385, + "fcm_dpo/delta": -0.07844078540802002, + "fcm_dpo/margin": 14.378135681152344, + "fcm_dpo/q_t": 0.381900817155838, + "grad_norm": 12.028036117553711, + "learning_rate": 4.743599013306165e-07, + "logits/chosen": -1.9074044227600098, + "logits/rejected": -1.418100357055664, + "logps/chosen": -114.77632141113281, + "logps/ref_chosen": -108.2182388305664, + "logps/ref_rejected": -94.83901977539062, + "logps/rejected": -115.77523803710938, + "loss": 1.012, + "margin_dpo/margin_mean": 14.378135681152344, + "margin_dpo/margin_std": 16.724992752075195, + "step": 159 + }, + { + "KL/chosen_KL_mean": -5.5895233154296875, + "KL/mean": -12.679683685302734, + "KL/rejected_KL_mean": -19.769851684570312, + "KL/std": 13.713754653930664, + "epoch": 0.23494860499265785, + "fcm_dpo/beta": 0.03592487424612045, + "fcm_dpo/delta": -0.06281135976314545, + "fcm_dpo/margin": 14.18032455444336, + "fcm_dpo/q_t": 0.3823738992214203, + "grad_norm": 12.476420402526855, + "learning_rate": 4.737908228387656e-07, + "logits/chosen": -1.5746586322784424, + "logits/rejected": -1.0254801511764526, + "logps/chosen": -103.7529296875, + "logps/ref_chosen": -98.16340637207031, + "logps/ref_rejected": -102.28097534179688, + "logps/rejected": -122.05082702636719, + "loss": 1.004, + "margin_dpo/margin_mean": 14.180325508117676, + "margin_dpo/margin_std": 14.713062286376953, + "step": 160 + }, + { + "KL/chosen_KL_mean": -7.667057037353516, + "KL/mean": -14.123154640197754, + "KL/rejected_KL_mean": -20.579246520996094, + "KL/std": 12.620899200439453, + "epoch": 0.23641703377386197, + "fcm_dpo/beta": 0.035670921206474304, + "fcm_dpo/delta": -0.01148887351155281, + "fcm_dpo/margin": 12.912195205688477, + "fcm_dpo/q_t": 0.39526236057281494, + "grad_norm": 12.445168495178223, + "learning_rate": 4.7321584725060594e-07, + "logits/chosen": -1.8524129390716553, + "logits/rejected": -1.3672497272491455, + "logps/chosen": -94.25892639160156, + "logps/ref_chosen": -86.59187316894531, + "logps/ref_rejected": -89.01643371582031, + "logps/rejected": -109.5956802368164, + "loss": 1.0558, + "margin_dpo/margin_mean": 12.912195205688477, + "margin_dpo/margin_std": 16.13299560546875, + "step": 161 + }, + { + "KL/chosen_KL_mean": -6.879505157470703, + "KL/mean": -13.900179862976074, + "KL/rejected_KL_mean": -20.920848846435547, + "KL/std": 14.575180053710938, + "epoch": 0.23788546255506607, + "fcm_dpo/beta": 0.035436250269412994, + "fcm_dpo/delta": -0.050333380699157715, + "fcm_dpo/margin": 14.041352272033691, + "fcm_dpo/q_t": 0.38521867990493774, + "grad_norm": 11.967451095581055, + "learning_rate": 4.7263498971727905e-07, + "logits/chosen": -1.1731939315795898, + "logits/rejected": -0.7678380012512207, + "logps/chosen": -104.28239440917969, + "logps/ref_chosen": -97.40288543701172, + "logps/ref_rejected": -95.86923217773438, + "logps/rejected": -116.79008483886719, + "loss": 1.018, + "margin_dpo/margin_mean": 14.041352272033691, + "margin_dpo/margin_std": 15.449185371398926, + "step": 162 + }, + { + "KL/chosen_KL_mean": -6.612934112548828, + "KL/mean": -15.126335144042969, + "KL/rejected_KL_mean": -23.639739990234375, + "KL/std": 16.71318244934082, + "epoch": 0.2393538913362702, + "fcm_dpo/beta": 0.034580446779727936, + "fcm_dpo/delta": -0.1479780077934265, + "fcm_dpo/margin": 17.02680206298828, + "fcm_dpo/q_t": 0.36851170659065247, + "grad_norm": 10.702917098999023, + "learning_rate": 4.720482655449212e-07, + "logits/chosen": -1.5282377004623413, + "logits/rejected": -1.1244807243347168, + "logps/chosen": -88.788330078125, + "logps/ref_chosen": -82.1753921508789, + "logps/ref_rejected": -82.86473846435547, + "logps/rejected": -106.50447845458984, + "loss": 0.9625, + "margin_dpo/margin_mean": 17.02680206298828, + "margin_dpo/margin_std": 17.83454132080078, + "step": 163 + }, + { + "KL/chosen_KL_mean": -10.24361801147461, + "KL/mean": -15.968530654907227, + "KL/rejected_KL_mean": -21.693431854248047, + "KL/std": 13.200516700744629, + "epoch": 0.24082232011747431, + "fcm_dpo/beta": 0.03443962335586548, + "fcm_dpo/delta": 0.05723126232624054, + "fcm_dpo/margin": 11.449810981750488, + "fcm_dpo/q_t": 0.4088016152381897, + "grad_norm": 12.716316223144531, + "learning_rate": 4.714556901942599e-07, + "logits/chosen": -1.2501944303512573, + "logits/rejected": -0.7770636677742004, + "logps/chosen": -100.71788787841797, + "logps/ref_chosen": -90.47427368164062, + "logps/ref_rejected": -87.02809143066406, + "logps/rejected": -108.72152709960938, + "loss": 1.0932, + "margin_dpo/margin_mean": 11.449811935424805, + "margin_dpo/margin_std": 14.289133071899414, + "step": 164 + }, + { + "KL/chosen_KL_mean": -6.868404388427734, + "KL/mean": -15.295869827270508, + "KL/rejected_KL_mean": -23.72333526611328, + "KL/std": 15.947157859802246, + "epoch": 0.2422907488986784, + "fcm_dpo/beta": 0.03420202434062958, + "fcm_dpo/delta": -0.13400211930274963, + "fcm_dpo/margin": 16.85493278503418, + "fcm_dpo/q_t": 0.3720381259918213, + "grad_norm": 10.454062461853027, + "learning_rate": 4.708572792802069e-07, + "logits/chosen": -1.6756125688552856, + "logits/rejected": -1.2562980651855469, + "logps/chosen": -94.18163299560547, + "logps/ref_chosen": -87.313232421875, + "logps/ref_rejected": -81.902587890625, + "logps/rejected": -105.62591552734375, + "loss": 0.9691, + "margin_dpo/margin_mean": 16.854930877685547, + "margin_dpo/margin_std": 18.01761817932129, + "step": 165 + }, + { + "KL/chosen_KL_mean": -11.347457885742188, + "KL/mean": -19.54320526123047, + "KL/rejected_KL_mean": -27.73895263671875, + "KL/std": 17.023427963256836, + "epoch": 0.24375917767988253, + "fcm_dpo/beta": 0.03336024656891823, + "fcm_dpo/delta": -0.1024494618177414, + "fcm_dpo/margin": 16.391490936279297, + "fcm_dpo/q_t": 0.38189011812210083, + "grad_norm": 11.76332950592041, + "learning_rate": 4.702530485714461e-07, + "logits/chosen": -1.5171177387237549, + "logits/rejected": -1.311524510383606, + "logps/chosen": -92.93352508544922, + "logps/ref_chosen": -81.58606719970703, + "logps/ref_rejected": -103.16517639160156, + "logps/rejected": -130.9041290283203, + "loss": 1.0065, + "margin_dpo/margin_mean": 16.391490936279297, + "margin_dpo/margin_std": 20.421768188476562, + "step": 166 + }, + { + "KL/chosen_KL_mean": -10.083026885986328, + "KL/mean": -18.919815063476562, + "KL/rejected_KL_mean": -27.756610870361328, + "KL/std": 15.828325271606445, + "epoch": 0.24522760646108663, + "fcm_dpo/beta": 0.03236294537782669, + "fcm_dpo/delta": -0.13084493577480316, + "fcm_dpo/margin": 17.673582077026367, + "fcm_dpo/q_t": 0.36864155530929565, + "grad_norm": 11.045381546020508, + "learning_rate": 4.6964301399001877e-07, + "logits/chosen": -1.0328116416931152, + "logits/rejected": -0.8548140525817871, + "logps/chosen": -93.64469909667969, + "logps/ref_chosen": -83.56167602539062, + "logps/ref_rejected": -99.05305480957031, + "logps/rejected": -126.80966186523438, + "loss": 0.9549, + "margin_dpo/margin_mean": 17.673582077026367, + "margin_dpo/margin_std": 15.923975944519043, + "step": 167 + }, + { + "KL/chosen_KL_mean": -9.740535736083984, + "KL/mean": -17.327659606933594, + "KL/rejected_KL_mean": -24.91478729248047, + "KL/std": 15.80506706237793, + "epoch": 0.24669603524229075, + "fcm_dpo/beta": 0.032199449837207794, + "fcm_dpo/delta": -0.04057988524436951, + "fcm_dpo/margin": 15.174245834350586, + "fcm_dpo/q_t": 0.38755232095718384, + "grad_norm": 12.069757461547852, + "learning_rate": 4.690271916109034e-07, + "logits/chosen": -1.6021510362625122, + "logits/rejected": -1.2641432285308838, + "logps/chosen": -89.67176818847656, + "logps/ref_chosen": -79.93123626708984, + "logps/ref_rejected": -80.10216522216797, + "logps/rejected": -105.01695251464844, + "loss": 1.0162, + "margin_dpo/margin_mean": 15.174247741699219, + "margin_dpo/margin_std": 15.809932708740234, + "step": 168 + }, + { + "KL/chosen_KL_mean": -8.248973846435547, + "KL/mean": -16.996950149536133, + "KL/rejected_KL_mean": -25.74492645263672, + "KL/std": 16.920543670654297, + "epoch": 0.24816446402349487, + "fcm_dpo/beta": 0.03156084194779396, + "fcm_dpo/delta": -0.10816548019647598, + "fcm_dpo/margin": 17.495952606201172, + "fcm_dpo/q_t": 0.375136137008667, + "grad_norm": 11.632828712463379, + "learning_rate": 4.6840559766159235e-07, + "logits/chosen": -1.4784125089645386, + "logits/rejected": -1.2528705596923828, + "logps/chosen": -91.15267944335938, + "logps/ref_chosen": -82.90370178222656, + "logps/ref_rejected": -90.12191009521484, + "logps/rejected": -115.86683654785156, + "loss": 0.9796, + "margin_dpo/margin_mean": 17.495952606201172, + "margin_dpo/margin_std": 17.988906860351562, + "step": 169 + }, + { + "KL/chosen_KL_mean": -9.879325866699219, + "KL/mean": -17.5045166015625, + "KL/rejected_KL_mean": -25.12970733642578, + "KL/std": 14.28281021118164, + "epoch": 0.24963289280469897, + "fcm_dpo/beta": 0.031117241829633713, + "fcm_dpo/delta": -0.026353713124990463, + "fcm_dpo/margin": 15.250386238098145, + "fcm_dpo/q_t": 0.3886600434780121, + "grad_norm": 11.964139938354492, + "learning_rate": 4.6777824852166437e-07, + "logits/chosen": -1.4093681573867798, + "logits/rejected": -1.190025806427002, + "logps/chosen": -84.86129760742188, + "logps/ref_chosen": -74.98197174072266, + "logps/ref_rejected": -82.16560363769531, + "logps/rejected": -107.2953109741211, + "loss": 1.0143, + "margin_dpo/margin_mean": 15.250385284423828, + "margin_dpo/margin_std": 14.157035827636719, + "step": 170 + }, + { + "KL/chosen_KL_mean": -12.461994171142578, + "KL/mean": -20.921245574951172, + "KL/rejected_KL_mean": -29.380496978759766, + "KL/std": 19.991989135742188, + "epoch": 0.2511013215859031, + "fcm_dpo/beta": 0.03089229390025139, + "fcm_dpo/delta": -0.07661177217960358, + "fcm_dpo/margin": 16.91851043701172, + "fcm_dpo/q_t": 0.38857120275497437, + "grad_norm": 13.428890228271484, + "learning_rate": 4.6714516072235273e-07, + "logits/chosen": -1.5971887111663818, + "logits/rejected": -1.0483250617980957, + "logps/chosen": -131.44052124023438, + "logps/ref_chosen": -118.97853088378906, + "logps/ref_rejected": -115.76930236816406, + "logps/rejected": -145.14981079101562, + "loss": 1.0393, + "margin_dpo/margin_mean": 16.918508529663086, + "margin_dpo/margin_std": 23.625953674316406, + "step": 171 + }, + { + "KL/chosen_KL_mean": -13.391273498535156, + "KL/mean": -21.045547485351562, + "KL/rejected_KL_mean": -28.699817657470703, + "KL/std": 14.668384552001953, + "epoch": 0.2525697503671072, + "fcm_dpo/beta": 0.030571138486266136, + "fcm_dpo/delta": -0.019111724570393562, + "fcm_dpo/margin": 15.308544158935547, + "fcm_dpo/q_t": 0.3887174129486084, + "grad_norm": 11.000654220581055, + "learning_rate": 4.6650635094610966e-07, + "logits/chosen": -2.1800696849823, + "logits/rejected": -1.4203687906265259, + "logps/chosen": -114.54505920410156, + "logps/ref_chosen": -101.15379333496094, + "logps/ref_rejected": -91.00568389892578, + "logps/rejected": -119.70550537109375, + "loss": 1.0056, + "margin_dpo/margin_mean": 15.308544158935547, + "margin_dpo/margin_std": 12.103874206542969, + "step": 172 + }, + { + "KL/chosen_KL_mean": -12.206268310546875, + "KL/mean": -18.89590072631836, + "KL/rejected_KL_mean": -25.585529327392578, + "KL/std": 14.004446983337402, + "epoch": 0.2540381791483113, + "fcm_dpo/beta": 0.03084501624107361, + "fcm_dpo/delta": 0.03884591534733772, + "fcm_dpo/margin": 13.379262924194336, + "fcm_dpo/q_t": 0.4037604331970215, + "grad_norm": 10.733778953552246, + "learning_rate": 4.6586183602616687e-07, + "logits/chosen": -1.5182697772979736, + "logits/rejected": -1.044596552848816, + "logps/chosen": -111.26295471191406, + "logps/ref_chosen": -99.05668640136719, + "logps/ref_rejected": -83.8507308959961, + "logps/rejected": -109.43626403808594, + "loss": 1.065, + "margin_dpo/margin_mean": 13.379262924194336, + "margin_dpo/margin_std": 14.841697692871094, + "step": 173 + }, + { + "KL/chosen_KL_mean": -11.841604232788086, + "KL/mean": -20.907365798950195, + "KL/rejected_KL_mean": -29.973129272460938, + "KL/std": 18.7991886138916, + "epoch": 0.2555066079295154, + "fcm_dpo/beta": 0.030422156676650047, + "fcm_dpo/delta": -0.10742342472076416, + "fcm_dpo/margin": 18.13152313232422, + "fcm_dpo/q_t": 0.3739241361618042, + "grad_norm": 12.568257331848145, + "learning_rate": 4.652116329460919e-07, + "logits/chosen": -1.813513159751892, + "logits/rejected": -1.7639669179916382, + "logps/chosen": -92.2505111694336, + "logps/ref_chosen": -80.40890502929688, + "logps/ref_rejected": -108.20171356201172, + "logps/rejected": -138.17483520507812, + "loss": 0.98, + "margin_dpo/margin_mean": 18.13152313232422, + "margin_dpo/margin_std": 18.54364776611328, + "step": 174 + }, + { + "KL/chosen_KL_mean": -15.296707153320312, + "KL/mean": -24.05184555053711, + "KL/rejected_KL_mean": -32.806983947753906, + "KL/std": 16.83472442626953, + "epoch": 0.25697503671071953, + "fcm_dpo/beta": 0.029946405440568924, + "fcm_dpo/delta": -0.07842611521482468, + "fcm_dpo/margin": 17.510276794433594, + "fcm_dpo/q_t": 0.3779052495956421, + "grad_norm": 12.207069396972656, + "learning_rate": 4.645557588393406e-07, + "logits/chosen": -1.1669551134109497, + "logits/rejected": -0.8998315334320068, + "logps/chosen": -98.01319885253906, + "logps/ref_chosen": -82.71649169921875, + "logps/ref_rejected": -96.66712951660156, + "logps/rejected": -129.47412109375, + "loss": 0.9836, + "margin_dpo/margin_mean": 17.510276794433594, + "margin_dpo/margin_std": 15.949636459350586, + "step": 175 + }, + { + "KL/chosen_KL_mean": -12.60012435913086, + "KL/mean": -21.525157928466797, + "KL/rejected_KL_mean": -30.4501953125, + "KL/std": 18.355226516723633, + "epoch": 0.25844346549192365, + "fcm_dpo/beta": 0.029383402317762375, + "fcm_dpo/delta": -0.07866019010543823, + "fcm_dpo/margin": 17.85007095336914, + "fcm_dpo/q_t": 0.3824155926704407, + "grad_norm": 11.708728790283203, + "learning_rate": 4.638942309888058e-07, + "logits/chosen": -1.534851312637329, + "logits/rejected": -1.3692426681518555, + "logps/chosen": -92.22943115234375, + "logps/ref_chosen": -79.62930297851562, + "logps/ref_rejected": -102.6885986328125, + "logps/rejected": -133.1387939453125, + "loss": 1.0142, + "margin_dpo/margin_mean": 17.85007095336914, + "margin_dpo/margin_std": 20.96698760986328, + "step": 176 + }, + { + "KL/chosen_KL_mean": -13.994693756103516, + "KL/mean": -24.452960968017578, + "KL/rejected_KL_mean": -34.91122817993164, + "KL/std": 20.982011795043945, + "epoch": 0.2599118942731278, + "fcm_dpo/beta": 0.028721442446112633, + "fcm_dpo/delta": -0.16008484363555908, + "fcm_dpo/margin": 20.916528701782227, + "fcm_dpo/q_t": 0.36762043833732605, + "grad_norm": 10.58277702331543, + "learning_rate": 4.6322706682636137e-07, + "logits/chosen": -1.4493741989135742, + "logits/rejected": -1.0991737842559814, + "logps/chosen": -106.55579376220703, + "logps/ref_chosen": -92.56109619140625, + "logps/ref_rejected": -103.33839416503906, + "logps/rejected": -138.24961853027344, + "loss": 0.9556, + "margin_dpo/margin_mean": 20.91653060913086, + "margin_dpo/margin_std": 21.6851806640625, + "step": 177 + }, + { + "KL/chosen_KL_mean": -12.885887145996094, + "KL/mean": -25.311311721801758, + "KL/rejected_KL_mean": -37.736717224121094, + "KL/std": 20.406723022460938, + "epoch": 0.26138032305433184, + "fcm_dpo/beta": 0.02731304243206978, + "fcm_dpo/delta": -0.2457062304019928, + "fcm_dpo/margin": 24.850833892822266, + "fcm_dpo/q_t": 0.347068727016449, + "grad_norm": 11.803115844726562, + "learning_rate": 4.6255428393240354e-07, + "logits/chosen": -1.683790683746338, + "logits/rejected": -1.457137107849121, + "logps/chosen": -97.15168762207031, + "logps/ref_chosen": -84.26579284667969, + "logps/ref_rejected": -110.79730987548828, + "logps/rejected": -148.53402709960938, + "loss": 0.8943, + "margin_dpo/margin_mean": 24.850830078125, + "margin_dpo/margin_std": 22.01804542541504, + "step": 178 + }, + { + "KL/chosen_KL_mean": -16.510086059570312, + "KL/mean": -24.588722229003906, + "KL/rejected_KL_mean": -32.667362213134766, + "KL/std": 19.8905029296875, + "epoch": 0.26284875183553597, + "fcm_dpo/beta": 0.026959318667650223, + "fcm_dpo/delta": 0.014857767149806023, + "fcm_dpo/margin": 16.15727996826172, + "fcm_dpo/q_t": 0.402656614780426, + "grad_norm": 11.33733081817627, + "learning_rate": 4.6187590003538724e-07, + "logits/chosen": -1.4103319644927979, + "logits/rejected": -0.9745727181434631, + "logps/chosen": -109.95806121826172, + "logps/ref_chosen": -93.4479751586914, + "logps/ref_rejected": -94.25541687011719, + "logps/rejected": -126.92277526855469, + "loss": 1.0857, + "margin_dpo/margin_mean": 16.15727996826172, + "margin_dpo/margin_std": 22.873689651489258, + "step": 179 + }, + { + "KL/chosen_KL_mean": -13.921092987060547, + "KL/mean": -24.237394332885742, + "KL/rejected_KL_mean": -34.55370330810547, + "KL/std": 19.22161102294922, + "epoch": 0.2643171806167401, + "fcm_dpo/beta": 0.02657276764512062, + "fcm_dpo/delta": -0.10452157258987427, + "fcm_dpo/margin": 20.632612228393555, + "fcm_dpo/q_t": 0.3743304908275604, + "grad_norm": 13.59306526184082, + "learning_rate": 4.611919330113591e-07, + "logits/chosen": -1.1502811908721924, + "logits/rejected": -0.901892364025116, + "logps/chosen": -102.75663757324219, + "logps/ref_chosen": -88.83554077148438, + "logps/ref_rejected": -103.90696716308594, + "logps/rejected": -138.46066284179688, + "loss": 0.9707, + "margin_dpo/margin_mean": 20.632612228393555, + "margin_dpo/margin_std": 19.694652557373047, + "step": 180 + }, + { + "KL/chosen_KL_mean": -13.958858489990234, + "KL/mean": -21.948686599731445, + "KL/rejected_KL_mean": -29.938507080078125, + "KL/std": 18.260334014892578, + "epoch": 0.2657856093979442, + "fcm_dpo/beta": 0.026639170944690704, + "fcm_dpo/delta": 0.025360099971294403, + "fcm_dpo/margin": 15.979652404785156, + "fcm_dpo/q_t": 0.40332111716270447, + "grad_norm": 10.452695846557617, + "learning_rate": 4.605024008834863e-07, + "logits/chosen": -1.4390556812286377, + "logits/rejected": -0.9684023261070251, + "logps/chosen": -93.19282531738281, + "logps/ref_chosen": -79.23396301269531, + "logps/ref_rejected": -69.17442321777344, + "logps/rejected": -99.11293029785156, + "loss": 1.0657, + "margin_dpo/margin_mean": 15.979652404785156, + "margin_dpo/margin_std": 19.490962982177734, + "step": 181 + }, + { + "KL/chosen_KL_mean": -17.644939422607422, + "KL/mean": -27.92449951171875, + "KL/rejected_KL_mean": -38.20405578613281, + "KL/std": 21.12765121459961, + "epoch": 0.26725403817914833, + "fcm_dpo/beta": 0.02617749758064747, + "fcm_dpo/delta": -0.09390002489089966, + "fcm_dpo/margin": 20.55911636352539, + "fcm_dpo/q_t": 0.37889379262924194, + "grad_norm": 10.89322280883789, + "learning_rate": 4.598073218215817e-07, + "logits/chosen": -1.104196548461914, + "logits/rejected": -0.8755874633789062, + "logps/chosen": -93.72712707519531, + "logps/ref_chosen": -76.08218383789062, + "logps/ref_rejected": -94.9686050415039, + "logps/rejected": -133.17266845703125, + "loss": 1.0015, + "margin_dpo/margin_mean": 20.559114456176758, + "margin_dpo/margin_std": 23.262847900390625, + "step": 182 + }, + { + "KL/chosen_KL_mean": -18.160797119140625, + "KL/mean": -28.280651092529297, + "KL/rejected_KL_mean": -38.40050506591797, + "KL/std": 19.251916885375977, + "epoch": 0.2687224669603524, + "fcm_dpo/beta": 0.02576177939772606, + "fcm_dpo/delta": -0.0759400725364685, + "fcm_dpo/margin": 20.23970603942871, + "fcm_dpo/q_t": 0.3797719180583954, + "grad_norm": 11.388766288757324, + "learning_rate": 4.5910671414162484e-07, + "logits/chosen": -1.658898115158081, + "logits/rejected": -1.0657981634140015, + "logps/chosen": -112.0687026977539, + "logps/ref_chosen": -93.90790557861328, + "logps/ref_rejected": -83.56061553955078, + "logps/rejected": -121.96112060546875, + "loss": 0.992, + "margin_dpo/margin_mean": 20.23970603942871, + "margin_dpo/margin_std": 19.79551887512207, + "step": 183 + }, + { + "KL/chosen_KL_mean": -16.416858673095703, + "KL/mean": -23.260700225830078, + "KL/rejected_KL_mean": -30.104541778564453, + "KL/std": 19.949066162109375, + "epoch": 0.2701908957415565, + "fcm_dpo/beta": 0.026149462908506393, + "fcm_dpo/delta": 0.09541141986846924, + "fcm_dpo/margin": 13.687676429748535, + "fcm_dpo/q_t": 0.417415976524353, + "grad_norm": 10.762918472290039, + "learning_rate": 4.5840059630527985e-07, + "logits/chosen": -1.542266845703125, + "logits/rejected": -1.2651708126068115, + "logps/chosen": -103.2847671508789, + "logps/ref_chosen": -86.86790466308594, + "logps/ref_rejected": -84.28456115722656, + "logps/rejected": -114.38909912109375, + "loss": 1.1357, + "margin_dpo/margin_mean": 13.687676429748535, + "margin_dpo/margin_std": 21.762523651123047, + "step": 184 + }, + { + "KL/chosen_KL_mean": -17.35260009765625, + "KL/mean": -25.63970184326172, + "KL/rejected_KL_mean": -33.92680358886719, + "KL/std": 19.54541778564453, + "epoch": 0.27165932452276065, + "fcm_dpo/beta": 0.02639647200703621, + "fcm_dpo/delta": 0.012794554233551025, + "fcm_dpo/margin": 16.57420539855957, + "fcm_dpo/q_t": 0.40176862478256226, + "grad_norm": 12.104183197021484, + "learning_rate": 4.5768898691940836e-07, + "logits/chosen": -2.1059458255767822, + "logits/rejected": -1.4902725219726562, + "logps/chosen": -111.3740234375, + "logps/ref_chosen": -94.02142333984375, + "logps/ref_rejected": -81.45668029785156, + "logps/rejected": -115.38348388671875, + "loss": 1.0775, + "margin_dpo/margin_mean": 16.574207305908203, + "margin_dpo/margin_std": 22.642166137695312, + "step": 185 + }, + { + "KL/chosen_KL_mean": -21.38967514038086, + "KL/mean": -30.390981674194336, + "KL/rejected_KL_mean": -39.39229202270508, + "KL/std": 18.92312240600586, + "epoch": 0.27312775330396477, + "fcm_dpo/beta": 0.02614632062613964, + "fcm_dpo/delta": -0.022055618464946747, + "fcm_dpo/margin": 18.00262451171875, + "fcm_dpo/q_t": 0.39177870750427246, + "grad_norm": 12.5892915725708, + "learning_rate": 4.5697190473557947e-07, + "logits/chosen": -2.2961578369140625, + "logits/rejected": -1.4019124507904053, + "logps/chosen": -135.58135986328125, + "logps/ref_chosen": -114.19168090820312, + "logps/ref_rejected": -93.33912658691406, + "logps/rejected": -132.73141479492188, + "loss": 1.0364, + "margin_dpo/margin_mean": 18.00262451171875, + "margin_dpo/margin_std": 20.395862579345703, + "step": 186 + }, + { + "KL/chosen_KL_mean": -18.42379379272461, + "KL/mean": -26.974987030029297, + "KL/rejected_KL_mean": -35.526180267333984, + "KL/std": 21.24410629272461, + "epoch": 0.2745961820851689, + "fcm_dpo/beta": 0.02628886140882969, + "fcm_dpo/delta": 0.00022946670651435852, + "fcm_dpo/margin": 17.10239028930664, + "fcm_dpo/q_t": 0.39897704124450684, + "grad_norm": 11.098430633544922, + "learning_rate": 4.5624936864957555e-07, + "logits/chosen": -1.6193220615386963, + "logits/rejected": -1.3785688877105713, + "logps/chosen": -95.29334259033203, + "logps/ref_chosen": -76.86955261230469, + "logps/ref_rejected": -88.16494750976562, + "logps/rejected": -123.69112396240234, + "loss": 1.071, + "margin_dpo/margin_mean": 17.10239028930664, + "margin_dpo/margin_std": 23.12689208984375, + "step": 187 + }, + { + "KL/chosen_KL_mean": -14.091022491455078, + "KL/mean": -23.916259765625, + "KL/rejected_KL_mean": -33.74150848388672, + "KL/std": 20.944185256958008, + "epoch": 0.27606461086637296, + "fcm_dpo/beta": 0.025859780609607697, + "fcm_dpo/delta": -0.061903007328510284, + "fcm_dpo/margin": 19.650480270385742, + "fcm_dpo/q_t": 0.3814808130264282, + "grad_norm": 11.369409561157227, + "learning_rate": 4.5552139770089454e-07, + "logits/chosen": -1.9536044597625732, + "logits/rejected": -1.7075963020324707, + "logps/chosen": -93.28582763671875, + "logps/ref_chosen": -79.1948013305664, + "logps/ref_rejected": -96.82889556884766, + "logps/rejected": -130.57040405273438, + "loss": 1.0053, + "margin_dpo/margin_mean": 19.65047836303711, + "margin_dpo/margin_std": 19.69561767578125, + "step": 188 + }, + { + "KL/chosen_KL_mean": -18.532405853271484, + "KL/mean": -28.625364303588867, + "KL/rejected_KL_mean": -38.718326568603516, + "KL/std": 21.00287628173828, + "epoch": 0.2775330396475771, + "fcm_dpo/beta": 0.025762559846043587, + "fcm_dpo/delta": -0.07420355081558228, + "fcm_dpo/margin": 20.18592071533203, + "fcm_dpo/q_t": 0.38445261120796204, + "grad_norm": 10.222798347473145, + "learning_rate": 4.5478801107224794e-07, + "logits/chosen": -1.5331592559814453, + "logits/rejected": -0.865401029586792, + "logps/chosen": -102.42520141601562, + "logps/ref_chosen": -83.8927993774414, + "logps/ref_rejected": -79.46589660644531, + "logps/rejected": -118.18421936035156, + "loss": 1.0148, + "margin_dpo/margin_mean": 20.18592071533203, + "margin_dpo/margin_std": 23.509119033813477, + "step": 189 + }, + { + "KL/chosen_KL_mean": -14.602851867675781, + "KL/mean": -26.681779861450195, + "KL/rejected_KL_mean": -38.760711669921875, + "KL/std": 23.119483947753906, + "epoch": 0.2790014684287812, + "fcm_dpo/beta": 0.02488422766327858, + "fcm_dpo/delta": -0.16096192598342896, + "fcm_dpo/margin": 24.157867431640625, + "fcm_dpo/q_t": 0.36203956604003906, + "grad_norm": 11.011933326721191, + "learning_rate": 4.5404922808905543e-07, + "logits/chosen": -1.7000792026519775, + "logits/rejected": -1.091031789779663, + "logps/chosen": -118.04431915283203, + "logps/ref_chosen": -103.44146728515625, + "logps/ref_rejected": -109.21502685546875, + "logps/rejected": -147.97573852539062, + "loss": 0.9398, + "margin_dpo/margin_mean": 24.157869338989258, + "margin_dpo/margin_std": 21.921974182128906, + "step": 190 + }, + { + "KL/chosen_KL_mean": -13.981407165527344, + "KL/mean": -27.475727081298828, + "KL/rejected_KL_mean": -40.97004699707031, + "KL/std": 24.972049713134766, + "epoch": 0.28046989720998533, + "fcm_dpo/beta": 0.0239790640771389, + "fcm_dpo/delta": -0.21051949262619019, + "fcm_dpo/margin": 26.98863983154297, + "fcm_dpo/q_t": 0.35482460260391235, + "grad_norm": 11.797168731689453, + "learning_rate": 4.5330506821893565e-07, + "logits/chosen": -1.5692925453186035, + "logits/rejected": -1.2564661502838135, + "logps/chosen": -113.76692199707031, + "logps/ref_chosen": -99.78551483154297, + "logps/ref_rejected": -116.00028228759766, + "logps/rejected": -156.9703369140625, + "loss": 0.9215, + "margin_dpo/margin_mean": 26.98863983154297, + "margin_dpo/margin_std": 25.576162338256836, + "step": 191 + }, + { + "KL/chosen_KL_mean": -23.067813873291016, + "KL/mean": -32.510894775390625, + "KL/rejected_KL_mean": -41.9539680480957, + "KL/std": 20.102333068847656, + "epoch": 0.28193832599118945, + "fcm_dpo/beta": 0.02366233989596367, + "fcm_dpo/delta": 0.003228548914194107, + "fcm_dpo/margin": 18.886154174804688, + "fcm_dpo/q_t": 0.3986278176307678, + "grad_norm": 12.40970230102539, + "learning_rate": 4.5255555107119336e-07, + "logits/chosen": -1.56754469871521, + "logits/rejected": -1.313439130783081, + "logps/chosen": -123.92813873291016, + "logps/ref_chosen": -100.86032104492188, + "logps/ref_rejected": -109.44500732421875, + "logps/rejected": -151.3989715576172, + "loss": 1.0588, + "margin_dpo/margin_mean": 18.886154174804688, + "margin_dpo/margin_std": 23.490009307861328, + "step": 192 + }, + { + "KL/chosen_KL_mean": -18.929611206054688, + "KL/mean": -26.77655792236328, + "KL/rejected_KL_mean": -34.623504638671875, + "KL/std": 21.295259475708008, + "epoch": 0.2834067547723935, + "fcm_dpo/beta": 0.02403874322772026, + "fcm_dpo/delta": 0.07514029741287231, + "fcm_dpo/margin": 15.693893432617188, + "fcm_dpo/q_t": 0.41458457708358765, + "grad_norm": 11.779868125915527, + "learning_rate": 4.5180069639630236e-07, + "logits/chosen": -2.00793719291687, + "logits/rejected": -1.4219706058502197, + "logps/chosen": -111.86792755126953, + "logps/ref_chosen": -92.93831634521484, + "logps/ref_rejected": -83.10917663574219, + "logps/rejected": -117.73268127441406, + "loss": 1.1358, + "margin_dpo/margin_mean": 15.693893432617188, + "margin_dpo/margin_std": 25.76551055908203, + "step": 193 + }, + { + "KL/chosen_KL_mean": -19.625965118408203, + "KL/mean": -28.691383361816406, + "KL/rejected_KL_mean": -37.75680160522461, + "KL/std": 21.91200828552246, + "epoch": 0.28487518355359764, + "fcm_dpo/beta": 0.024184411391615868, + "fcm_dpo/delta": 0.011726578697562218, + "fcm_dpo/margin": 18.130844116210938, + "fcm_dpo/q_t": 0.40050068497657776, + "grad_norm": 9.958355903625488, + "learning_rate": 4.510405240853854e-07, + "logits/chosen": -2.278360605239868, + "logits/rejected": -1.8964219093322754, + "logps/chosen": -83.41754150390625, + "logps/ref_chosen": -63.79157257080078, + "logps/ref_rejected": -68.72665405273438, + "logps/rejected": -106.48345947265625, + "loss": 1.0675, + "margin_dpo/margin_mean": 18.130842208862305, + "margin_dpo/margin_std": 23.099246978759766, + "step": 194 + }, + { + "KL/chosen_KL_mean": -18.747093200683594, + "KL/mean": -30.734756469726562, + "KL/rejected_KL_mean": -42.7224235534668, + "KL/std": 22.262435913085938, + "epoch": 0.28634361233480177, + "fcm_dpo/beta": 0.02373277023434639, + "fcm_dpo/delta": -0.12598851323127747, + "fcm_dpo/margin": 23.975330352783203, + "fcm_dpo/q_t": 0.3703756332397461, + "grad_norm": 10.308981895446777, + "learning_rate": 4.5027505416968985e-07, + "logits/chosen": -1.4115322828292847, + "logits/rejected": -1.2881722450256348, + "logps/chosen": -102.63255310058594, + "logps/ref_chosen": -83.88545989990234, + "logps/ref_rejected": -104.00125122070312, + "logps/rejected": -146.7236785888672, + "loss": 0.9804, + "margin_dpo/margin_mean": 23.975330352783203, + "margin_dpo/margin_std": 25.573593139648438, + "step": 195 + }, + { + "KL/chosen_KL_mean": -21.64429473876953, + "KL/mean": -33.10361099243164, + "KL/rejected_KL_mean": -44.562931060791016, + "KL/std": 23.96672821044922, + "epoch": 0.2878120411160059, + "fcm_dpo/beta": 0.023208342492580414, + "fcm_dpo/delta": -0.08651156723499298, + "fcm_dpo/margin": 22.918628692626953, + "fcm_dpo/q_t": 0.37599894404411316, + "grad_norm": 9.788215637207031, + "learning_rate": 4.495043068200599e-07, + "logits/chosen": -1.357054591178894, + "logits/rejected": -0.9101194143295288, + "logps/chosen": -111.045654296875, + "logps/ref_chosen": -89.4013671875, + "logps/ref_rejected": -83.34735107421875, + "logps/rejected": -127.9102783203125, + "loss": 0.981, + "margin_dpo/margin_mean": 22.918628692626953, + "margin_dpo/margin_std": 21.413265228271484, + "step": 196 + }, + { + "KL/chosen_KL_mean": -22.132862091064453, + "KL/mean": -29.120304107666016, + "KL/rejected_KL_mean": -36.10774612426758, + "KL/std": 19.36894989013672, + "epoch": 0.28928046989721, + "fcm_dpo/beta": 0.023539701476693153, + "fcm_dpo/delta": 0.1251697987318039, + "fcm_dpo/margin": 13.97488021850586, + "fcm_dpo/q_t": 0.4233154058456421, + "grad_norm": 10.3875150680542, + "learning_rate": 4.4872830234640493e-07, + "logits/chosen": -1.2650811672210693, + "logits/rejected": -0.9297800064086914, + "logps/chosen": -108.97357177734375, + "logps/ref_chosen": -86.84070587158203, + "logps/ref_rejected": -82.79289245605469, + "logps/rejected": -118.90064239501953, + "loss": 1.1384, + "margin_dpo/margin_mean": 13.974882125854492, + "margin_dpo/margin_std": 20.425912857055664, + "step": 197 + }, + { + "KL/chosen_KL_mean": -25.586280822753906, + "KL/mean": -36.8443603515625, + "KL/rejected_KL_mean": -48.102439880371094, + "KL/std": 22.67544937133789, + "epoch": 0.2907488986784141, + "fcm_dpo/beta": 0.023422207683324814, + "fcm_dpo/delta": -0.0816422700881958, + "fcm_dpo/margin": 22.516151428222656, + "fcm_dpo/q_t": 0.3797655999660492, + "grad_norm": 12.176416397094727, + "learning_rate": 4.479470611971645e-07, + "logits/chosen": -1.6032984256744385, + "logits/rejected": -1.4243170022964478, + "logps/chosen": -110.634521484375, + "logps/ref_chosen": -85.04824829101562, + "logps/ref_rejected": -104.10365295410156, + "logps/rejected": -152.20608520507812, + "loss": 0.994, + "margin_dpo/margin_mean": 22.516151428222656, + "margin_dpo/margin_std": 23.426759719848633, + "step": 198 + }, + { + "KL/chosen_KL_mean": -25.030391693115234, + "KL/mean": -34.420745849609375, + "KL/rejected_KL_mean": -43.811100006103516, + "KL/std": 22.346843719482422, + "epoch": 0.2922173274596182, + "fcm_dpo/beta": 0.0233822762966156, + "fcm_dpo/delta": 0.011303754523396492, + "fcm_dpo/margin": 18.780710220336914, + "fcm_dpo/q_t": 0.3995858430862427, + "grad_norm": 10.67126750946045, + "learning_rate": 4.471606039587695e-07, + "logits/chosen": -1.5150001049041748, + "logits/rejected": -1.250755786895752, + "logps/chosen": -106.81709289550781, + "logps/ref_chosen": -81.78669738769531, + "logps/ref_rejected": -90.790771484375, + "logps/rejected": -134.60186767578125, + "loss": 1.0646, + "margin_dpo/margin_mean": 18.78070831298828, + "margin_dpo/margin_std": 23.448162078857422, + "step": 199 + }, + { + "KL/chosen_KL_mean": -24.523277282714844, + "KL/mean": -35.61384582519531, + "KL/rejected_KL_mean": -46.70440673828125, + "KL/std": 23.738174438476562, + "epoch": 0.2936857562408223, + "fcm_dpo/beta": 0.023026688024401665, + "fcm_dpo/delta": -0.06481599807739258, + "fcm_dpo/margin": 22.181123733520508, + "fcm_dpo/q_t": 0.38517606258392334, + "grad_norm": 10.47972583770752, + "learning_rate": 4.4636895135509966e-07, + "logits/chosen": -1.624913215637207, + "logits/rejected": -1.0862584114074707, + "logps/chosen": -108.79136657714844, + "logps/ref_chosen": -84.26809692382812, + "logps/ref_rejected": -86.96786499023438, + "logps/rejected": -133.67227172851562, + "loss": 1.0313, + "margin_dpo/margin_mean": 22.181121826171875, + "margin_dpo/margin_std": 27.287662506103516, + "step": 200 + }, + { + "KL/chosen_KL_mean": -26.327220916748047, + "KL/mean": -37.032928466796875, + "KL/rejected_KL_mean": -47.7386474609375, + "KL/std": 25.674232482910156, + "epoch": 0.29515418502202645, + "fcm_dpo/beta": 0.022968199104070663, + "fcm_dpo/delta": -0.0439612939953804, + "fcm_dpo/margin": 21.411434173583984, + "fcm_dpo/q_t": 0.3894343376159668, + "grad_norm": 10.895137786865234, + "learning_rate": 4.455721242469372e-07, + "logits/chosen": -1.4008371829986572, + "logits/rejected": -1.0123724937438965, + "logps/chosen": -134.09971618652344, + "logps/ref_chosen": -107.77249145507812, + "logps/ref_rejected": -119.79248046875, + "logps/rejected": -167.5311279296875, + "loss": 1.0482, + "margin_dpo/margin_mean": 21.411434173583984, + "margin_dpo/margin_std": 28.15359115600586, + "step": 201 + }, + { + "KL/chosen_KL_mean": -24.698772430419922, + "KL/mean": -34.60642623901367, + "KL/rejected_KL_mean": -44.51408004760742, + "KL/std": 21.26959228515625, + "epoch": 0.2966226138032305, + "fcm_dpo/beta": 0.022876489907503128, + "fcm_dpo/delta": -0.0034904824569821358, + "fcm_dpo/margin": 19.81529998779297, + "fcm_dpo/q_t": 0.39716821908950806, + "grad_norm": 11.087641716003418, + "learning_rate": 4.4477014363141755e-07, + "logits/chosen": -1.5747380256652832, + "logits/rejected": -1.4448705911636353, + "logps/chosen": -100.67122650146484, + "logps/ref_chosen": -75.97245788574219, + "logps/ref_rejected": -94.4599838256836, + "logps/rejected": -138.97406005859375, + "loss": 1.0554, + "margin_dpo/margin_mean": 19.81529998779297, + "margin_dpo/margin_std": 24.043787002563477, + "step": 202 + }, + { + "KL/chosen_KL_mean": -22.10192108154297, + "KL/mean": -33.389915466308594, + "KL/rejected_KL_mean": -44.67791748046875, + "KL/std": 22.240814208984375, + "epoch": 0.29809104258443464, + "fcm_dpo/beta": 0.022657740861177444, + "fcm_dpo/delta": -0.06478673964738846, + "fcm_dpo/margin": 22.57599639892578, + "fcm_dpo/q_t": 0.38190820813179016, + "grad_norm": 9.961379051208496, + "learning_rate": 4.439630306414758e-07, + "logits/chosen": -1.8847124576568604, + "logits/rejected": -1.4328787326812744, + "logps/chosen": -117.06907653808594, + "logps/ref_chosen": -94.96715545654297, + "logps/ref_rejected": -92.8876724243164, + "logps/rejected": -137.56558227539062, + "loss": 1.0021, + "margin_dpo/margin_mean": 22.57599639892578, + "margin_dpo/margin_std": 23.117809295654297, + "step": 203 + }, + { + "KL/chosen_KL_mean": -27.695480346679688, + "KL/mean": -37.42096710205078, + "KL/rejected_KL_mean": -47.146453857421875, + "KL/std": 22.132984161376953, + "epoch": 0.29955947136563876, + "fcm_dpo/beta": 0.02265116758644581, + "fcm_dpo/delta": 0.009622307494282722, + "fcm_dpo/margin": 19.45098304748535, + "fcm_dpo/q_t": 0.4003278613090515, + "grad_norm": 11.085248947143555, + "learning_rate": 4.431508065452897e-07, + "logits/chosen": -1.8878350257873535, + "logits/rejected": -1.2349971532821655, + "logps/chosen": -144.05267333984375, + "logps/ref_chosen": -116.35719299316406, + "logps/ref_rejected": -93.39759063720703, + "logps/rejected": -140.54403686523438, + "loss": 1.0745, + "margin_dpo/margin_mean": 19.45098114013672, + "margin_dpo/margin_std": 25.756441116333008, + "step": 204 + }, + { + "KL/chosen_KL_mean": -25.701007843017578, + "KL/mean": -38.21492004394531, + "KL/rejected_KL_mean": -50.72883605957031, + "KL/std": 24.512920379638672, + "epoch": 0.3010279001468429, + "fcm_dpo/beta": 0.022171659395098686, + "fcm_dpo/delta": -0.11142764240503311, + "fcm_dpo/margin": 25.02783203125, + "fcm_dpo/q_t": 0.36982864141464233, + "grad_norm": 9.582030296325684, + "learning_rate": 4.4233349274571974e-07, + "logits/chosen": -2.1238603591918945, + "logits/rejected": -1.6858125925064087, + "logps/chosen": -114.56035614013672, + "logps/ref_chosen": -88.85934448242188, + "logps/ref_rejected": -91.8544921875, + "logps/rejected": -142.5833282470703, + "loss": 0.9569, + "margin_dpo/margin_mean": 25.027830123901367, + "margin_dpo/margin_std": 21.043460845947266, + "step": 205 + }, + { + "KL/chosen_KL_mean": -26.342140197753906, + "KL/mean": -37.848567962646484, + "KL/rejected_KL_mean": -49.35499954223633, + "KL/std": 23.870223999023438, + "epoch": 0.302496328928047, + "fcm_dpo/beta": 0.02183135598897934, + "fcm_dpo/delta": -0.05566246807575226, + "fcm_dpo/margin": 23.012861251831055, + "fcm_dpo/q_t": 0.3818650245666504, + "grad_norm": 11.548815727233887, + "learning_rate": 4.415111107797445e-07, + "logits/chosen": -1.4087283611297607, + "logits/rejected": -1.2552969455718994, + "logps/chosen": -102.88848876953125, + "logps/ref_chosen": -76.54634857177734, + "logps/ref_rejected": -102.95314025878906, + "logps/rejected": -152.30813598632812, + "loss": 0.9927, + "margin_dpo/margin_mean": 23.012861251831055, + "margin_dpo/margin_std": 20.281452178955078, + "step": 206 + }, + { + "KL/chosen_KL_mean": -27.59354019165039, + "KL/mean": -40.39479064941406, + "KL/rejected_KL_mean": -53.196048736572266, + "KL/std": 29.01898765563965, + "epoch": 0.3039647577092511, + "fcm_dpo/beta": 0.021515972912311554, + "fcm_dpo/delta": -0.106700599193573, + "fcm_dpo/margin": 25.602506637573242, + "fcm_dpo/q_t": 0.3801648020744324, + "grad_norm": 11.600892066955566, + "learning_rate": 4.4068368231789365e-07, + "logits/chosen": -2.1540088653564453, + "logits/rejected": -1.7385635375976562, + "logps/chosen": -113.82518768310547, + "logps/ref_chosen": -86.23164367675781, + "logps/ref_rejected": -90.65512084960938, + "logps/rejected": -143.85116577148438, + "loss": 1.0211, + "margin_dpo/margin_mean": 25.60250473022461, + "margin_dpo/margin_std": 33.49293518066406, + "step": 207 + }, + { + "KL/chosen_KL_mean": -28.55429458618164, + "KL/mean": -41.814109802246094, + "KL/rejected_KL_mean": -55.07392883300781, + "KL/std": 26.690162658691406, + "epoch": 0.3054331864904552, + "fcm_dpo/beta": 0.021119004115462303, + "fcm_dpo/delta": -0.11649561673402786, + "fcm_dpo/margin": 26.519634246826172, + "fcm_dpo/q_t": 0.3698372542858124, + "grad_norm": 10.919686317443848, + "learning_rate": 4.398512291636768e-07, + "logits/chosen": -1.7398350238800049, + "logits/rejected": -1.3810641765594482, + "logps/chosen": -122.71383666992188, + "logps/ref_chosen": -94.1595458984375, + "logps/ref_rejected": -100.96233367919922, + "logps/rejected": -156.0362548828125, + "loss": 0.967, + "margin_dpo/margin_mean": 26.519634246826172, + "margin_dpo/margin_std": 24.634014129638672, + "step": 208 + }, + { + "KL/chosen_KL_mean": -24.12353515625, + "KL/mean": -35.77311706542969, + "KL/rejected_KL_mean": -47.42270278930664, + "KL/std": 23.735620498657227, + "epoch": 0.3069016152716593, + "fcm_dpo/beta": 0.02069961279630661, + "fcm_dpo/delta": -0.03445186838507652, + "fcm_dpo/margin": 23.29916763305664, + "fcm_dpo/q_t": 0.3899701237678528, + "grad_norm": 9.853930473327637, + "learning_rate": 4.3901377325300857e-07, + "logits/chosen": -2.117767810821533, + "logits/rejected": -1.647392749786377, + "logps/chosen": -108.29409790039062, + "logps/ref_chosen": -84.17056274414062, + "logps/ref_rejected": -87.61955261230469, + "logps/rejected": -135.04225158691406, + "loss": 1.034, + "margin_dpo/margin_mean": 23.29916763305664, + "margin_dpo/margin_std": 26.88469696044922, + "step": 209 + }, + { + "KL/chosen_KL_mean": -27.380931854248047, + "KL/mean": -38.50312042236328, + "KL/rejected_KL_mean": -49.62531661987305, + "KL/std": 25.168006896972656, + "epoch": 0.30837004405286345, + "fcm_dpo/beta": 0.020676523447036743, + "fcm_dpo/delta": -0.010528551414608955, + "fcm_dpo/margin": 22.244384765625, + "fcm_dpo/q_t": 0.39471542835235596, + "grad_norm": 9.15931224822998, + "learning_rate": 4.381713366536311e-07, + "logits/chosen": -1.8130569458007812, + "logits/rejected": -1.417151689529419, + "logps/chosen": -108.55210876464844, + "logps/ref_chosen": -81.17117309570312, + "logps/ref_rejected": -84.17478942871094, + "logps/rejected": -133.80010986328125, + "loss": 1.0421, + "margin_dpo/margin_mean": 22.244382858276367, + "margin_dpo/margin_std": 25.39181137084961, + "step": 210 + }, + { + "KL/chosen_KL_mean": -30.4791259765625, + "KL/mean": -41.428985595703125, + "KL/rejected_KL_mean": -52.378849029541016, + "KL/std": 26.59711456298828, + "epoch": 0.30983847283406757, + "fcm_dpo/beta": 0.020739322528243065, + "fcm_dpo/delta": -0.00446801632642746, + "fcm_dpo/margin": 21.899728775024414, + "fcm_dpo/q_t": 0.39676520228385925, + "grad_norm": 11.158641815185547, + "learning_rate": 4.373239415645323e-07, + "logits/chosen": -1.9388091564178467, + "logits/rejected": -1.3017692565917969, + "logps/chosen": -139.19183349609375, + "logps/ref_chosen": -108.71270751953125, + "logps/ref_rejected": -93.55564880371094, + "logps/rejected": -145.93450927734375, + "loss": 1.0539, + "margin_dpo/margin_mean": 21.899728775024414, + "margin_dpo/margin_std": 26.995410919189453, + "step": 211 + }, + { + "KL/chosen_KL_mean": -21.315662384033203, + "KL/mean": -37.14613723754883, + "KL/rejected_KL_mean": -52.97661590576172, + "KL/std": 29.989168167114258, + "epoch": 0.31130690161527164, + "fcm_dpo/beta": 0.02004014141857624, + "fcm_dpo/delta": -0.1976533830165863, + "fcm_dpo/margin": 31.66094970703125, + "fcm_dpo/q_t": 0.3564627468585968, + "grad_norm": 9.618505477905273, + "learning_rate": 4.3647161031536086e-07, + "logits/chosen": -1.7199327945709229, + "logits/rejected": -1.2956655025482178, + "logps/chosen": -119.6776123046875, + "logps/ref_chosen": -98.36195373535156, + "logps/ref_rejected": -109.88999938964844, + "logps/rejected": -162.86660766601562, + "loss": 0.9275, + "margin_dpo/margin_mean": 31.660951614379883, + "margin_dpo/margin_std": 29.891117095947266, + "step": 212 + }, + { + "KL/chosen_KL_mean": -25.102352142333984, + "KL/mean": -37.12311935424805, + "KL/rejected_KL_mean": -49.14387512207031, + "KL/std": 27.50717544555664, + "epoch": 0.31277533039647576, + "fcm_dpo/beta": 0.019762098789215088, + "fcm_dpo/delta": -0.02654850482940674, + "fcm_dpo/margin": 24.041526794433594, + "fcm_dpo/q_t": 0.3912399411201477, + "grad_norm": 12.485479354858398, + "learning_rate": 4.3561436536583774e-07, + "logits/chosen": -1.7787501811981201, + "logits/rejected": -0.9773513078689575, + "logps/chosen": -133.15765380859375, + "logps/ref_chosen": -108.05531311035156, + "logps/ref_rejected": -100.14414978027344, + "logps/rejected": -149.28802490234375, + "loss": 1.0464, + "margin_dpo/margin_mean": 24.041526794433594, + "margin_dpo/margin_std": 29.77020835876465, + "step": 213 + }, + { + "KL/chosen_KL_mean": -26.931499481201172, + "KL/mean": -38.158233642578125, + "KL/rejected_KL_mean": -49.384971618652344, + "KL/std": 29.45815658569336, + "epoch": 0.3142437591776799, + "fcm_dpo/beta": 0.01982264406979084, + "fcm_dpo/delta": 0.005104076117277145, + "fcm_dpo/margin": 22.45347785949707, + "fcm_dpo/q_t": 0.4010830521583557, + "grad_norm": 9.625554084777832, + "learning_rate": 4.3475222930516473e-07, + "logits/chosen": -1.8941937685012817, + "logits/rejected": -1.77357816696167, + "logps/chosen": -104.73623657226562, + "logps/ref_chosen": -77.80473327636719, + "logps/ref_rejected": -84.27578735351562, + "logps/rejected": -133.6607666015625, + "loss": 1.084, + "margin_dpo/margin_mean": 22.45347785949707, + "margin_dpo/margin_std": 32.474544525146484, + "step": 214 + }, + { + "KL/chosen_KL_mean": -23.280712127685547, + "KL/mean": -38.35309600830078, + "KL/rejected_KL_mean": -53.425479888916016, + "KL/std": 29.038955688476562, + "epoch": 0.315712187958884, + "fcm_dpo/beta": 0.01939084380865097, + "fcm_dpo/delta": -0.14286500215530396, + "fcm_dpo/margin": 30.144752502441406, + "fcm_dpo/q_t": 0.3668321669101715, + "grad_norm": 9.348302841186523, + "learning_rate": 4.3388522485142885e-07, + "logits/chosen": -1.4726921319961548, + "logits/rejected": -0.9769058227539062, + "logps/chosen": -108.39460754394531, + "logps/ref_chosen": -85.1138916015625, + "logps/ref_rejected": -96.86151123046875, + "logps/rejected": -150.2869873046875, + "loss": 0.9528, + "margin_dpo/margin_mean": 30.144752502441406, + "margin_dpo/margin_std": 28.670223236083984, + "step": 215 + }, + { + "KL/chosen_KL_mean": -25.23155975341797, + "KL/mean": -38.3830451965332, + "KL/rejected_KL_mean": -51.53453826904297, + "KL/std": 26.145912170410156, + "epoch": 0.31718061674008813, + "fcm_dpo/beta": 0.019167980179190636, + "fcm_dpo/delta": -0.05720216780900955, + "fcm_dpo/margin": 26.30297088623047, + "fcm_dpo/q_t": 0.38458746671676636, + "grad_norm": 8.929350852966309, + "learning_rate": 4.330133748510036e-07, + "logits/chosen": -1.9628796577453613, + "logits/rejected": -1.5714607238769531, + "logps/chosen": -105.82386779785156, + "logps/ref_chosen": -80.59230041503906, + "logps/ref_rejected": -81.41983795166016, + "logps/rejected": -132.95437622070312, + "loss": 1.0071, + "margin_dpo/margin_mean": 26.3029727935791, + "margin_dpo/margin_std": 27.05829620361328, + "step": 216 + }, + { + "KL/chosen_KL_mean": -30.480587005615234, + "KL/mean": -41.486656188964844, + "KL/rejected_KL_mean": -52.492733001708984, + "KL/std": 26.829389572143555, + "epoch": 0.3186490455212922, + "fcm_dpo/beta": 0.019149091094732285, + "fcm_dpo/delta": 0.02959912270307541, + "fcm_dpo/margin": 22.012144088745117, + "fcm_dpo/q_t": 0.4052965044975281, + "grad_norm": 10.928380012512207, + "learning_rate": 4.3213670227794757e-07, + "logits/chosen": -1.852818489074707, + "logits/rejected": -1.5402162075042725, + "logps/chosen": -123.95317077636719, + "logps/ref_chosen": -93.47257995605469, + "logps/ref_rejected": -103.488525390625, + "logps/rejected": -155.98126220703125, + "loss": 1.0859, + "margin_dpo/margin_mean": 22.012142181396484, + "margin_dpo/margin_std": 30.480266571044922, + "step": 217 + }, + { + "KL/chosen_KL_mean": -29.74217987060547, + "KL/mean": -42.884925842285156, + "KL/rejected_KL_mean": -56.027671813964844, + "KL/std": 27.209476470947266, + "epoch": 0.3201174743024963, + "fcm_dpo/beta": 0.018940910696983337, + "fcm_dpo/delta": -0.05069791525602341, + "fcm_dpo/margin": 26.285490036010742, + "fcm_dpo/q_t": 0.3873566687107086, + "grad_norm": 9.492467880249023, + "learning_rate": 4.3125523023339815e-07, + "logits/chosen": -1.284709095954895, + "logits/rejected": -0.893244743347168, + "logps/chosen": -118.80101013183594, + "logps/ref_chosen": -89.05883026123047, + "logps/ref_rejected": -94.30680847167969, + "logps/rejected": -150.33447265625, + "loss": 1.0359, + "margin_dpo/margin_mean": 26.28548812866211, + "margin_dpo/margin_std": 32.44743728637695, + "step": 218 + }, + { + "KL/chosen_KL_mean": -26.7744140625, + "KL/mean": -38.57316970825195, + "KL/rejected_KL_mean": -50.371925354003906, + "KL/std": 31.48541259765625, + "epoch": 0.32158590308370044, + "fcm_dpo/beta": 0.019057394936680794, + "fcm_dpo/delta": -0.00039842724800109863, + "fcm_dpo/margin": 23.59750747680664, + "fcm_dpo/q_t": 0.398790180683136, + "grad_norm": 10.948594093322754, + "learning_rate": 4.303689819449636e-07, + "logits/chosen": -1.679978609085083, + "logits/rejected": -1.2237684726715088, + "logps/chosen": -127.78175354003906, + "logps/ref_chosen": -101.00733947753906, + "logps/ref_rejected": -92.46794128417969, + "logps/rejected": -142.83987426757812, + "loss": 1.0807, + "margin_dpo/margin_mean": 23.597511291503906, + "margin_dpo/margin_std": 33.49624252319336, + "step": 219 + }, + { + "KL/chosen_KL_mean": -27.571399688720703, + "KL/mean": -38.54227828979492, + "KL/rejected_KL_mean": -49.513153076171875, + "KL/std": 26.086421966552734, + "epoch": 0.32305433186490456, + "fcm_dpo/beta": 0.019048605114221573, + "fcm_dpo/delta": 0.03341676667332649, + "fcm_dpo/margin": 21.941770553588867, + "fcm_dpo/q_t": 0.4032576084136963, + "grad_norm": 11.072999000549316, + "learning_rate": 4.2947798076611047e-07, + "logits/chosen": -1.5224353075027466, + "logits/rejected": -1.1750617027282715, + "logps/chosen": -123.10861206054688, + "logps/ref_chosen": -95.53721618652344, + "logps/ref_rejected": -94.30703735351562, + "logps/rejected": -143.8201904296875, + "loss": 1.0687, + "margin_dpo/margin_mean": 21.9417724609375, + "margin_dpo/margin_std": 25.74374008178711, + "step": 220 + }, + { + "KL/chosen_KL_mean": -26.08209228515625, + "KL/mean": -42.82952880859375, + "KL/rejected_KL_mean": -59.576961517333984, + "KL/std": 32.706634521484375, + "epoch": 0.3245227606461087, + "fcm_dpo/beta": 0.018682332709431648, + "fcm_dpo/delta": -0.1871957927942276, + "fcm_dpo/margin": 33.494869232177734, + "fcm_dpo/q_t": 0.357664555311203, + "grad_norm": 10.636537551879883, + "learning_rate": 4.285822501755485e-07, + "logits/chosen": -1.2643651962280273, + "logits/rejected": -1.1244919300079346, + "logps/chosen": -108.92695617675781, + "logps/ref_chosen": -82.84486389160156, + "logps/ref_rejected": -110.81179809570312, + "logps/rejected": -170.38876342773438, + "loss": 0.9268, + "margin_dpo/margin_mean": 33.494869232177734, + "margin_dpo/margin_std": 28.816444396972656, + "step": 221 + }, + { + "KL/chosen_KL_mean": -33.369239807128906, + "KL/mean": -47.110816955566406, + "KL/rejected_KL_mean": -60.85240173339844, + "KL/std": 31.279739379882812, + "epoch": 0.32599118942731276, + "fcm_dpo/beta": 0.01830419711768627, + "fcm_dpo/delta": -0.056051403284072876, + "fcm_dpo/margin": 27.483156204223633, + "fcm_dpo/q_t": 0.38697564601898193, + "grad_norm": 11.075545310974121, + "learning_rate": 4.276818137766118e-07, + "logits/chosen": -1.9451358318328857, + "logits/rejected": -1.5141632556915283, + "logps/chosen": -128.51121520996094, + "logps/ref_chosen": -95.14198303222656, + "logps/ref_rejected": -106.80441284179688, + "logps/rejected": -167.6568145751953, + "loss": 1.0509, + "margin_dpo/margin_mean": 27.483158111572266, + "margin_dpo/margin_std": 36.74195861816406, + "step": 222 + }, + { + "KL/chosen_KL_mean": -32.391475677490234, + "KL/mean": -42.24231719970703, + "KL/rejected_KL_mean": -52.09315872192383, + "KL/std": 27.651885986328125, + "epoch": 0.3274596182085169, + "fcm_dpo/beta": 0.01839200034737587, + "fcm_dpo/delta": 0.0910235345363617, + "fcm_dpo/margin": 19.70168685913086, + "fcm_dpo/q_t": 0.41741105914115906, + "grad_norm": 10.734125137329102, + "learning_rate": 4.2677669529663686e-07, + "logits/chosen": -2.1838834285736084, + "logits/rejected": -1.7929104566574097, + "logps/chosen": -117.96658325195312, + "logps/ref_chosen": -85.57511138916016, + "logps/ref_rejected": -86.45237731933594, + "logps/rejected": -138.5455322265625, + "loss": 1.1292, + "margin_dpo/margin_mean": 19.70168685913086, + "margin_dpo/margin_std": 30.232542037963867, + "step": 223 + }, + { + "KL/chosen_KL_mean": -29.820423126220703, + "KL/mean": -43.66676712036133, + "KL/rejected_KL_mean": -57.51311111450195, + "KL/std": 30.592058181762695, + "epoch": 0.328928046989721, + "fcm_dpo/beta": 0.018324650824069977, + "fcm_dpo/delta": -0.06056816875934601, + "fcm_dpo/margin": 27.692689895629883, + "fcm_dpo/q_t": 0.38475412130355835, + "grad_norm": 10.024768829345703, + "learning_rate": 4.2586691858633747e-07, + "logits/chosen": -1.8784904479980469, + "logits/rejected": -1.4315197467803955, + "logps/chosen": -112.54422760009766, + "logps/ref_chosen": -82.72380065917969, + "logps/ref_rejected": -82.59538269042969, + "logps/rejected": -140.10848999023438, + "loss": 1.0197, + "margin_dpo/margin_mean": 27.692691802978516, + "margin_dpo/margin_std": 31.912351608276367, + "step": 224 + }, + { + "KL/chosen_KL_mean": -33.354976654052734, + "KL/mean": -47.91131591796875, + "KL/rejected_KL_mean": -62.46765899658203, + "KL/std": 32.20822525024414, + "epoch": 0.3303964757709251, + "fcm_dpo/beta": 0.018125828355550766, + "fcm_dpo/delta": -0.08200725167989731, + "fcm_dpo/margin": 29.112688064575195, + "fcm_dpo/q_t": 0.38220465183258057, + "grad_norm": 11.154870986938477, + "learning_rate": 4.249525076191759e-07, + "logits/chosen": -1.878772258758545, + "logits/rejected": -1.3943595886230469, + "logps/chosen": -129.03265380859375, + "logps/ref_chosen": -95.67768859863281, + "logps/ref_rejected": -105.09687805175781, + "logps/rejected": -167.56454467773438, + "loss": 1.0206, + "margin_dpo/margin_mean": 29.112688064575195, + "margin_dpo/margin_std": 35.91593933105469, + "step": 225 + }, + { + "KL/chosen_KL_mean": -27.574657440185547, + "KL/mean": -41.40182876586914, + "KL/rejected_KL_mean": -55.22899627685547, + "KL/std": 31.36874008178711, + "epoch": 0.33186490455212925, + "fcm_dpo/beta": 0.017766930162906647, + "fcm_dpo/delta": -0.04428989440202713, + "fcm_dpo/margin": 27.654335021972656, + "fcm_dpo/q_t": 0.3866173326969147, + "grad_norm": 9.041952133178711, + "learning_rate": 4.2403348649073167e-07, + "logits/chosen": -1.857753038406372, + "logits/rejected": -1.326967477798462, + "logps/chosen": -121.03558349609375, + "logps/ref_chosen": -93.46092224121094, + "logps/ref_rejected": -86.7017593383789, + "logps/rejected": -141.93075561523438, + "loss": 1.0131, + "margin_dpo/margin_mean": 27.654335021972656, + "margin_dpo/margin_std": 27.987306594848633, + "step": 226 + }, + { + "KL/chosen_KL_mean": -37.83222198486328, + "KL/mean": -52.051429748535156, + "KL/rejected_KL_mean": -66.27063751220703, + "KL/std": 30.28497314453125, + "epoch": 0.3333333333333333, + "fcm_dpo/beta": 0.017683152109384537, + "fcm_dpo/delta": -0.055658984929323196, + "fcm_dpo/margin": 28.438419342041016, + "fcm_dpo/q_t": 0.3871355652809143, + "grad_norm": 10.029187202453613, + "learning_rate": 4.2310987941806615e-07, + "logits/chosen": -1.5599040985107422, + "logits/rejected": -1.2161908149719238, + "logps/chosen": -130.646484375, + "logps/ref_chosen": -92.81427001953125, + "logps/ref_rejected": -104.73693084716797, + "logps/rejected": -171.007568359375, + "loss": 1.038, + "margin_dpo/margin_mean": 28.438419342041016, + "margin_dpo/margin_std": 35.90989685058594, + "step": 227 + }, + { + "KL/chosen_KL_mean": -35.73163604736328, + "KL/mean": -46.58448791503906, + "KL/rejected_KL_mean": -57.43733596801758, + "KL/std": 28.182226181030273, + "epoch": 0.33480176211453744, + "fcm_dpo/beta": 0.017788201570510864, + "fcm_dpo/delta": 0.06649549305438995, + "fcm_dpo/margin": 21.705698013305664, + "fcm_dpo/q_t": 0.4111056923866272, + "grad_norm": 10.444371223449707, + "learning_rate": 4.2218171073908463e-07, + "logits/chosen": -1.8236147165298462, + "logits/rejected": -1.3874504566192627, + "logps/chosen": -129.76876831054688, + "logps/ref_chosen": -94.03712463378906, + "logps/ref_rejected": -96.02151489257812, + "logps/rejected": -153.4588623046875, + "loss": 1.1114, + "margin_dpo/margin_mean": 21.705698013305664, + "margin_dpo/margin_std": 31.48227310180664, + "step": 228 + }, + { + "KL/chosen_KL_mean": -31.76717758178711, + "KL/mean": -49.680389404296875, + "KL/rejected_KL_mean": -67.59359741210938, + "KL/std": 34.649620056152344, + "epoch": 0.33627019089574156, + "fcm_dpo/beta": 0.017390307039022446, + "fcm_dpo/delta": -0.18441686034202576, + "fcm_dpo/margin": 35.82642364501953, + "fcm_dpo/q_t": 0.361656129360199, + "grad_norm": 9.87229061126709, + "learning_rate": 4.212490049118951e-07, + "logits/chosen": -1.7755630016326904, + "logits/rejected": -1.1372489929199219, + "logps/chosen": -127.3448486328125, + "logps/ref_chosen": -95.57766723632812, + "logps/ref_rejected": -89.17379760742188, + "logps/rejected": -156.76739501953125, + "loss": 0.9497, + "margin_dpo/margin_mean": 35.82642364501953, + "margin_dpo/margin_std": 37.666080474853516, + "step": 229 + }, + { + "KL/chosen_KL_mean": -33.3121452331543, + "KL/mean": -51.62913513183594, + "KL/rejected_KL_mean": -69.94612121582031, + "KL/std": 34.056793212890625, + "epoch": 0.3377386196769457, + "fcm_dpo/beta": 0.016771184280514717, + "fcm_dpo/delta": -0.17516855895519257, + "fcm_dpo/margin": 36.63397979736328, + "fcm_dpo/q_t": 0.3589358925819397, + "grad_norm": 10.829142570495605, + "learning_rate": 4.203117865141635e-07, + "logits/chosen": -1.6221001148223877, + "logits/rejected": -1.5562043190002441, + "logps/chosen": -97.02577209472656, + "logps/ref_chosen": -63.713626861572266, + "logps/ref_rejected": -91.90870666503906, + "logps/rejected": -161.85482788085938, + "loss": 0.9346, + "margin_dpo/margin_mean": 36.63397979736328, + "margin_dpo/margin_std": 33.538787841796875, + "step": 230 + }, + { + "KL/chosen_KL_mean": -37.03425598144531, + "KL/mean": -52.77891159057617, + "KL/rejected_KL_mean": -68.5235595703125, + "KL/std": 32.92323303222656, + "epoch": 0.3392070484581498, + "fcm_dpo/beta": 0.01636582799255848, + "fcm_dpo/delta": -0.06978515535593033, + "fcm_dpo/margin": 31.48931121826172, + "fcm_dpo/q_t": 0.3800153434276581, + "grad_norm": 8.929129600524902, + "learning_rate": 4.1937008024246625e-07, + "logits/chosen": -1.8607261180877686, + "logits/rejected": -1.3150267601013184, + "logps/chosen": -132.48992919921875, + "logps/ref_chosen": -95.45567321777344, + "logps/ref_rejected": -80.95568084716797, + "logps/rejected": -149.479248046875, + "loss": 0.9881, + "margin_dpo/margin_mean": 31.48931312561035, + "margin_dpo/margin_std": 28.539264678955078, + "step": 231 + }, + { + "KL/chosen_KL_mean": -38.97697448730469, + "KL/mean": -52.253780364990234, + "KL/rejected_KL_mean": -65.53057861328125, + "KL/std": 30.39307403564453, + "epoch": 0.3406754772393539, + "fcm_dpo/beta": 0.016406547278165817, + "fcm_dpo/delta": 0.014987261965870857, + "fcm_dpo/margin": 26.553607940673828, + "fcm_dpo/q_t": 0.39936891198158264, + "grad_norm": 9.567919731140137, + "learning_rate": 4.1842391091163933e-07, + "logits/chosen": -1.363256812095642, + "logits/rejected": -0.8709152936935425, + "logps/chosen": -135.87423706054688, + "logps/ref_chosen": -96.89726257324219, + "logps/ref_rejected": -89.76461791992188, + "logps/rejected": -155.29519653320312, + "loss": 1.0591, + "margin_dpo/margin_mean": 26.553607940673828, + "margin_dpo/margin_std": 31.093616485595703, + "step": 232 + }, + { + "KL/chosen_KL_mean": -40.90872573852539, + "KL/mean": -59.25398254394531, + "KL/rejected_KL_mean": -77.59922790527344, + "KL/std": 38.152381896972656, + "epoch": 0.342143906020558, + "fcm_dpo/beta": 0.016056008636951447, + "fcm_dpo/delta": -0.1481083482503891, + "fcm_dpo/margin": 36.69049835205078, + "fcm_dpo/q_t": 0.36577850580215454, + "grad_norm": 10.34557056427002, + "learning_rate": 4.174733034541245e-07, + "logits/chosen": -1.6242706775665283, + "logits/rejected": -1.4135671854019165, + "logps/chosen": -129.95904541015625, + "logps/ref_chosen": -89.05032348632812, + "logps/ref_rejected": -112.75917053222656, + "logps/rejected": -190.3583984375, + "loss": 0.9602, + "margin_dpo/margin_mean": 36.69050216674805, + "margin_dpo/margin_std": 36.75348663330078, + "step": 233 + }, + { + "KL/chosen_KL_mean": -39.851036071777344, + "KL/mean": -59.33667755126953, + "KL/rejected_KL_mean": -78.82231903076172, + "KL/std": 36.616676330566406, + "epoch": 0.3436123348017621, + "fcm_dpo/beta": 0.01545102708041668, + "fcm_dpo/delta": -0.1645185798406601, + "fcm_dpo/margin": 38.971282958984375, + "fcm_dpo/q_t": 0.3646003007888794, + "grad_norm": 9.245274543762207, + "learning_rate": 4.165182829193126e-07, + "logits/chosen": -1.4410686492919922, + "logits/rejected": -1.3687561750411987, + "logps/chosen": -114.16940307617188, + "logps/ref_chosen": -74.318359375, + "logps/ref_rejected": -106.38758850097656, + "logps/rejected": -185.20989990234375, + "loss": 0.9603, + "margin_dpo/margin_mean": 38.971282958984375, + "margin_dpo/margin_std": 40.08073425292969, + "step": 234 + }, + { + "KL/chosen_KL_mean": -47.7650260925293, + "KL/mean": -62.5955924987793, + "KL/rejected_KL_mean": -77.4261474609375, + "KL/std": 32.802406311035156, + "epoch": 0.34508076358296624, + "fcm_dpo/beta": 0.015446878969669342, + "fcm_dpo/delta": -0.008598173037171364, + "fcm_dpo/margin": 29.66112518310547, + "fcm_dpo/q_t": 0.3935566544532776, + "grad_norm": 9.225977897644043, + "learning_rate": 4.1555887447288255e-07, + "logits/chosen": -1.6084418296813965, + "logits/rejected": -1.0536733865737915, + "logps/chosen": -145.98207092285156, + "logps/ref_chosen": -98.217041015625, + "logps/ref_rejected": -97.24677276611328, + "logps/rejected": -174.67291259765625, + "loss": 1.0335, + "margin_dpo/margin_mean": 29.66112518310547, + "margin_dpo/margin_std": 31.05475425720215, + "step": 235 + }, + { + "KL/chosen_KL_mean": -40.224674224853516, + "KL/mean": -59.228267669677734, + "KL/rejected_KL_mean": -78.23187255859375, + "KL/std": 41.153526306152344, + "epoch": 0.3465491923641703, + "fcm_dpo/beta": 0.015149770304560661, + "fcm_dpo/delta": -0.13328908383846283, + "fcm_dpo/margin": 38.00719451904297, + "fcm_dpo/q_t": 0.37542450428009033, + "grad_norm": 9.858134269714355, + "learning_rate": 4.1459510339613946e-07, + "logits/chosen": -1.3845889568328857, + "logits/rejected": -1.3828411102294922, + "logps/chosen": -119.06240844726562, + "logps/ref_chosen": -78.83773040771484, + "logps/ref_rejected": -109.06343078613281, + "logps/rejected": -187.29530334472656, + "loss": 0.9855, + "margin_dpo/margin_mean": 38.00719451904297, + "margin_dpo/margin_std": 44.75410461425781, + "step": 236 + }, + { + "KL/chosen_KL_mean": -47.65378952026367, + "KL/mean": -61.71543884277344, + "KL/rejected_KL_mean": -75.7770767211914, + "KL/std": 36.32740020751953, + "epoch": 0.34801762114537443, + "fcm_dpo/beta": 0.015071479603648186, + "fcm_dpo/delta": 0.02729911357164383, + "fcm_dpo/margin": 28.123275756835938, + "fcm_dpo/q_t": 0.4083618223667145, + "grad_norm": 9.366373062133789, + "learning_rate": 4.136269950853473e-07, + "logits/chosen": -1.4898028373718262, + "logits/rejected": -1.1301348209381104, + "logps/chosen": -132.86508178710938, + "logps/ref_chosen": -85.21128845214844, + "logps/ref_rejected": -99.90999603271484, + "logps/rejected": -175.68707275390625, + "loss": 1.1194, + "margin_dpo/margin_mean": 28.123275756835938, + "margin_dpo/margin_std": 47.189090728759766, + "step": 237 + }, + { + "KL/chosen_KL_mean": -52.024654388427734, + "KL/mean": -68.32868957519531, + "KL/rejected_KL_mean": -84.63272094726562, + "KL/std": 37.93627166748047, + "epoch": 0.34948604992657856, + "fcm_dpo/beta": 0.014987221919000149, + "fcm_dpo/delta": -0.040753819048404694, + "fcm_dpo/margin": 32.60807800292969, + "fcm_dpo/q_t": 0.38997578620910645, + "grad_norm": 8.857922554016113, + "learning_rate": 4.126545750510605e-07, + "logits/chosen": -1.2862224578857422, + "logits/rejected": -1.1352065801620483, + "logps/chosen": -130.75588989257812, + "logps/ref_chosen": -78.73123168945312, + "logps/ref_rejected": -95.41840362548828, + "logps/rejected": -180.05111694335938, + "loss": 1.0342, + "margin_dpo/margin_mean": 32.60807800292969, + "margin_dpo/margin_std": 39.337013244628906, + "step": 238 + }, + { + "KL/chosen_KL_mean": -47.21555709838867, + "KL/mean": -62.49492645263672, + "KL/rejected_KL_mean": -77.77430725097656, + "KL/std": 33.71417236328125, + "epoch": 0.3509544787077827, + "fcm_dpo/beta": 0.014858734793961048, + "fcm_dpo/delta": -0.00522758811712265, + "fcm_dpo/margin": 30.55875015258789, + "fcm_dpo/q_t": 0.3967739939689636, + "grad_norm": 9.374117851257324, + "learning_rate": 4.116778689174514e-07, + "logits/chosen": -1.7733337879180908, + "logits/rejected": -1.2934105396270752, + "logps/chosen": -139.81649780273438, + "logps/ref_chosen": -92.60093688964844, + "logps/ref_rejected": -100.51769256591797, + "logps/rejected": -178.2919921875, + "loss": 1.0604, + "margin_dpo/margin_mean": 30.55875015258789, + "margin_dpo/margin_std": 37.68760299682617, + "step": 239 + }, + { + "KL/chosen_KL_mean": -49.013118743896484, + "KL/mean": -62.57707977294922, + "KL/rejected_KL_mean": -76.14105224609375, + "KL/std": 33.897212982177734, + "epoch": 0.3524229074889868, + "fcm_dpo/beta": 0.015017041936516762, + "fcm_dpo/delta": 0.04441224783658981, + "fcm_dpo/margin": 27.1279296875, + "fcm_dpo/q_t": 0.4065741300582886, + "grad_norm": 10.550976753234863, + "learning_rate": 4.106969024216348e-07, + "logits/chosen": -2.0598533153533936, + "logits/rejected": -1.7029976844787598, + "logps/chosen": -135.17288208007812, + "logps/ref_chosen": -86.15977478027344, + "logps/ref_rejected": -80.45567321777344, + "logps/rejected": -156.59671020507812, + "loss": 1.0894, + "margin_dpo/margin_mean": 27.127927780151367, + "margin_dpo/margin_std": 36.040008544921875, + "step": 240 + }, + { + "KL/chosen_KL_mean": -44.52872085571289, + "KL/mean": -68.604736328125, + "KL/rejected_KL_mean": -92.68074035644531, + "KL/std": 39.203582763671875, + "epoch": 0.35389133627019087, + "fcm_dpo/beta": 0.014577025547623634, + "fcm_dpo/delta": -0.27007779479026794, + "fcm_dpo/margin": 48.15202331542969, + "fcm_dpo/q_t": 0.34311577677726746, + "grad_norm": 9.134992599487305, + "learning_rate": 4.097117014129903e-07, + "logits/chosen": -1.63786780834198, + "logits/rejected": -0.9519686698913574, + "logps/chosen": -145.5746612548828, + "logps/ref_chosen": -101.04594421386719, + "logps/ref_rejected": -94.04934692382812, + "logps/rejected": -186.73008728027344, + "loss": 0.8934, + "margin_dpo/margin_mean": 48.15202331542969, + "margin_dpo/margin_std": 43.82901382446289, + "step": 241 + }, + { + "KL/chosen_KL_mean": -52.94053268432617, + "KL/mean": -70.04161071777344, + "KL/rejected_KL_mean": -87.1427001953125, + "KL/std": 34.48725891113281, + "epoch": 0.355359765051395, + "fcm_dpo/beta": 0.014258917421102524, + "fcm_dpo/delta": -0.039715249091386795, + "fcm_dpo/margin": 34.20215606689453, + "fcm_dpo/q_t": 0.3874158263206482, + "grad_norm": 9.531928062438965, + "learning_rate": 4.087222918524807e-07, + "logits/chosen": -1.6932920217514038, + "logits/rejected": -1.013551115989685, + "logps/chosen": -148.61318969726562, + "logps/ref_chosen": -95.67266845703125, + "logps/ref_rejected": -90.65454864501953, + "logps/rejected": -177.7972412109375, + "loss": 1.027, + "margin_dpo/margin_mean": 34.20215606689453, + "margin_dpo/margin_std": 38.174041748046875, + "step": 242 + }, + { + "KL/chosen_KL_mean": -51.86396789550781, + "KL/mean": -68.801513671875, + "KL/rejected_KL_mean": -85.73905181884766, + "KL/std": 36.9276123046875, + "epoch": 0.3568281938325991, + "fcm_dpo/beta": 0.014229689724743366, + "fcm_dpo/delta": -0.034833505749702454, + "fcm_dpo/margin": 33.875083923339844, + "fcm_dpo/q_t": 0.39147675037384033, + "grad_norm": 9.327872276306152, + "learning_rate": 4.07728699811968e-07, + "logits/chosen": -1.8189271688461304, + "logits/rejected": -0.9876618385314941, + "logps/chosen": -149.89537048339844, + "logps/ref_chosen": -98.03140258789062, + "logps/ref_rejected": -83.18806457519531, + "logps/rejected": -168.9271240234375, + "loss": 1.0562, + "margin_dpo/margin_mean": 33.875083923339844, + "margin_dpo/margin_std": 43.51741027832031, + "step": 243 + }, + { + "KL/chosen_KL_mean": -54.084632873535156, + "KL/mean": -72.06623840332031, + "KL/rejected_KL_mean": -90.0478515625, + "KL/std": 40.88813018798828, + "epoch": 0.35829662261380324, + "fcm_dpo/beta": 0.013942182995378971, + "fcm_dpo/delta": -0.05431248992681503, + "fcm_dpo/margin": 35.963226318359375, + "fcm_dpo/q_t": 0.3859729468822479, + "grad_norm": 9.260563850402832, + "learning_rate": 4.067309514735267e-07, + "logits/chosen": -2.1343822479248047, + "logits/rejected": -1.8000872135162354, + "logps/chosen": -142.97854614257812, + "logps/ref_chosen": -88.8939208984375, + "logps/ref_rejected": -102.57278442382812, + "logps/rejected": -192.62063598632812, + "loss": 1.0208, + "margin_dpo/margin_mean": 35.963226318359375, + "margin_dpo/margin_std": 41.07027816772461, + "step": 244 + }, + { + "KL/chosen_KL_mean": -45.61378860473633, + "KL/mean": -64.84752655029297, + "KL/rejected_KL_mean": -84.08126068115234, + "KL/std": 38.089500427246094, + "epoch": 0.35976505139500736, + "fcm_dpo/beta": 0.013776318170130253, + "fcm_dpo/delta": -0.08433786034584045, + "fcm_dpo/margin": 38.467464447021484, + "fcm_dpo/q_t": 0.3781154155731201, + "grad_norm": 8.69044303894043, + "learning_rate": 4.057290731287531e-07, + "logits/chosen": -2.269152879714966, + "logits/rejected": -1.4394030570983887, + "logps/chosen": -149.8077850341797, + "logps/ref_chosen": -104.19400024414062, + "logps/ref_rejected": -92.65645599365234, + "logps/rejected": -176.7377166748047, + "loss": 0.9899, + "margin_dpo/margin_mean": 38.467464447021484, + "margin_dpo/margin_std": 38.598819732666016, + "step": 245 + }, + { + "KL/chosen_KL_mean": -59.491981506347656, + "KL/mean": -74.54719543457031, + "KL/rejected_KL_mean": -89.60240173339844, + "KL/std": 36.99351119995117, + "epoch": 0.36123348017621143, + "fcm_dpo/beta": 0.01372530311346054, + "fcm_dpo/delta": 0.03816515952348709, + "fcm_dpo/margin": 30.110424041748047, + "fcm_dpo/q_t": 0.4057997465133667, + "grad_norm": 8.911885261535645, + "learning_rate": 4.047230911780736e-07, + "logits/chosen": -1.8584641218185425, + "logits/rejected": -1.3190970420837402, + "logps/chosen": -162.71102905273438, + "logps/ref_chosen": -103.21904754638672, + "logps/ref_rejected": -90.99221801757812, + "logps/rejected": -180.5946044921875, + "loss": 1.0846, + "margin_dpo/margin_mean": 30.11042594909668, + "margin_dpo/margin_std": 39.586910247802734, + "step": 246 + }, + { + "KL/chosen_KL_mean": -53.546241760253906, + "KL/mean": -77.9178695678711, + "KL/rejected_KL_mean": -102.28948974609375, + "KL/std": 41.55171203613281, + "epoch": 0.36270190895741555, + "fcm_dpo/beta": 0.013310082256793976, + "fcm_dpo/delta": -0.21401762962341309, + "fcm_dpo/margin": 48.74325942993164, + "fcm_dpo/q_t": 0.354427695274353, + "grad_norm": 8.816831588745117, + "learning_rate": 4.0371303213004814e-07, + "logits/chosen": -2.1500349044799805, + "logits/rejected": -1.96571946144104, + "logps/chosen": -140.54061889648438, + "logps/ref_chosen": -86.99437713623047, + "logps/ref_rejected": -111.33802795410156, + "logps/rejected": -213.6275177001953, + "loss": 0.9249, + "margin_dpo/margin_mean": 48.743263244628906, + "margin_dpo/margin_std": 46.65956115722656, + "step": 247 + }, + { + "KL/chosen_KL_mean": -61.57766342163086, + "KL/mean": -77.98873901367188, + "KL/rejected_KL_mean": -94.39981842041016, + "KL/std": 36.767066955566406, + "epoch": 0.3641703377386197, + "fcm_dpo/beta": 0.013294167816638947, + "fcm_dpo/delta": 0.014036701992154121, + "fcm_dpo/margin": 32.822166442871094, + "fcm_dpo/q_t": 0.39929088950157166, + "grad_norm": 9.573286056518555, + "learning_rate": 4.0269892260067197e-07, + "logits/chosen": -1.8852083683013916, + "logits/rejected": -1.853461742401123, + "logps/chosen": -136.36318969726562, + "logps/ref_chosen": -74.7855224609375, + "logps/ref_rejected": -98.27689361572266, + "logps/rejected": -192.6767120361328, + "loss": 1.056, + "margin_dpo/margin_mean": 32.82216262817383, + "margin_dpo/margin_std": 37.72064208984375, + "step": 248 + }, + { + "KL/chosen_KL_mean": -67.73788452148438, + "KL/mean": -82.79411315917969, + "KL/rejected_KL_mean": -97.85035705566406, + "KL/std": 39.32136535644531, + "epoch": 0.3656387665198238, + "fcm_dpo/beta": 0.013312667608261108, + "fcm_dpo/delta": 0.0510699599981308, + "fcm_dpo/margin": 30.112468719482422, + "fcm_dpo/q_t": 0.40923529863357544, + "grad_norm": 10.465920448303223, + "learning_rate": 4.0168078931267426e-07, + "logits/chosen": -2.1551313400268555, + "logits/rejected": -1.6154589653015137, + "logps/chosen": -163.44168090820312, + "logps/ref_chosen": -95.70379638671875, + "logps/ref_rejected": -87.14646911621094, + "logps/rejected": -184.996826171875, + "loss": 1.1095, + "margin_dpo/margin_mean": 30.112470626831055, + "margin_dpo/margin_std": 44.744789123535156, + "step": 249 + }, + { + "KL/chosen_KL_mean": -60.65462875366211, + "KL/mean": -77.22984313964844, + "KL/rejected_KL_mean": -93.80506896972656, + "KL/std": 39.87385177612305, + "epoch": 0.3671071953010279, + "fcm_dpo/beta": 0.013342011719942093, + "fcm_dpo/delta": 0.007517479360103607, + "fcm_dpo/margin": 33.15043258666992, + "fcm_dpo/q_t": 0.39819926023483276, + "grad_norm": 10.026691436767578, + "learning_rate": 4.006586590948141e-07, + "logits/chosen": -1.9156348705291748, + "logits/rejected": -0.9815178513526917, + "logps/chosen": -174.70684814453125, + "logps/ref_chosen": -114.05221557617188, + "logps/ref_rejected": -81.08767700195312, + "logps/rejected": -174.8927459716797, + "loss": 1.0735, + "margin_dpo/margin_mean": 33.15043258666992, + "margin_dpo/margin_std": 42.58338165283203, + "step": 250 + }, + { + "KL/chosen_KL_mean": -60.36662292480469, + "KL/mean": -75.32786560058594, + "KL/rejected_KL_mean": -90.28909301757812, + "KL/std": 37.09200668334961, + "epoch": 0.368575624082232, + "fcm_dpo/beta": 0.013437955640256405, + "fcm_dpo/delta": 0.04947926104068756, + "fcm_dpo/margin": 29.9224796295166, + "fcm_dpo/q_t": 0.40989112854003906, + "grad_norm": 10.422486305236816, + "learning_rate": 3.9963255888117325e-07, + "logits/chosen": -1.9565802812576294, + "logits/rejected": -1.2686119079589844, + "logps/chosen": -158.07791137695312, + "logps/ref_chosen": -97.71128845214844, + "logps/ref_rejected": -83.52742004394531, + "logps/rejected": -173.81651306152344, + "loss": 1.1108, + "margin_dpo/margin_mean": 29.9224796295166, + "margin_dpo/margin_std": 44.29001998901367, + "step": 251 + }, + { + "KL/chosen_KL_mean": -60.423797607421875, + "KL/mean": -75.56443786621094, + "KL/rejected_KL_mean": -90.70506286621094, + "KL/std": 39.300804138183594, + "epoch": 0.3700440528634361, + "fcm_dpo/beta": 0.01364966668188572, + "fcm_dpo/delta": 0.03814868628978729, + "fcm_dpo/margin": 30.281265258789062, + "fcm_dpo/q_t": 0.40516144037246704, + "grad_norm": 10.538346290588379, + "learning_rate": 3.9860251571044666e-07, + "logits/chosen": -2.0909392833709717, + "logits/rejected": -1.5616133213043213, + "logps/chosen": -169.409912109375, + "logps/ref_chosen": -108.9861068725586, + "logps/ref_rejected": -91.56423950195312, + "logps/rejected": -182.26931762695312, + "loss": 1.093, + "margin_dpo/margin_mean": 30.281265258789062, + "margin_dpo/margin_std": 41.773738861083984, + "step": 252 + }, + { + "KL/chosen_KL_mean": -66.49739074707031, + "KL/mean": -80.59434509277344, + "KL/rejected_KL_mean": -94.69131469726562, + "KL/std": 42.872676849365234, + "epoch": 0.37151248164464024, + "fcm_dpo/beta": 0.013871677219867706, + "fcm_dpo/delta": 0.05996757373213768, + "fcm_dpo/margin": 28.193923950195312, + "fcm_dpo/q_t": 0.4091401696205139, + "grad_norm": 10.558040618896484, + "learning_rate": 3.9756855672522986e-07, + "logits/chosen": -1.8401331901550293, + "logits/rejected": -1.6228747367858887, + "logps/chosen": -166.71368408203125, + "logps/ref_chosen": -100.21630096435547, + "logps/ref_rejected": -105.67670440673828, + "logps/rejected": -200.36801147460938, + "loss": 1.099, + "margin_dpo/margin_mean": 28.193923950195312, + "margin_dpo/margin_std": 37.02585220336914, + "step": 253 + }, + { + "KL/chosen_KL_mean": -64.74571228027344, + "KL/mean": -79.40669250488281, + "KL/rejected_KL_mean": -94.06766510009766, + "KL/std": 40.28760528564453, + "epoch": 0.37298091042584436, + "fcm_dpo/beta": 0.013849547132849693, + "fcm_dpo/delta": 0.04552397131919861, + "fcm_dpo/margin": 29.32195472717285, + "fcm_dpo/q_t": 0.40663406252861023, + "grad_norm": 10.165385246276855, + "learning_rate": 3.965307091713037e-07, + "logits/chosen": -1.99363374710083, + "logits/rejected": -1.4603936672210693, + "logps/chosen": -163.48089599609375, + "logps/ref_chosen": -98.73518371582031, + "logps/ref_rejected": -93.73825073242188, + "logps/rejected": -187.805908203125, + "loss": 1.1056, + "margin_dpo/margin_mean": 29.32195281982422, + "margin_dpo/margin_std": 42.65867614746094, + "step": 254 + }, + { + "KL/chosen_KL_mean": -65.8162841796875, + "KL/mean": -87.46654510498047, + "KL/rejected_KL_mean": -109.11680603027344, + "KL/std": 45.29289245605469, + "epoch": 0.3744493392070485, + "fcm_dpo/beta": 0.013654084876179695, + "fcm_dpo/delta": -0.1499733328819275, + "fcm_dpo/margin": 43.30052947998047, + "fcm_dpo/q_t": 0.3662024140357971, + "grad_norm": 9.934861183166504, + "learning_rate": 3.954890003969163e-07, + "logits/chosen": -1.779320478439331, + "logits/rejected": -1.4321630001068115, + "logps/chosen": -156.1988525390625, + "logps/ref_chosen": -90.382568359375, + "logps/ref_rejected": -97.07624816894531, + "logps/rejected": -206.19305419921875, + "loss": 0.9687, + "margin_dpo/margin_mean": 43.30052947998047, + "margin_dpo/margin_std": 45.886634826660156, + "step": 255 + }, + { + "KL/chosen_KL_mean": -63.502994537353516, + "KL/mean": -83.19683837890625, + "KL/rejected_KL_mean": -102.89068603515625, + "KL/std": 46.93716049194336, + "epoch": 0.37591776798825255, + "fcm_dpo/beta": 0.013406183570623398, + "fcm_dpo/delta": -0.08231563866138458, + "fcm_dpo/margin": 39.3876953125, + "fcm_dpo/q_t": 0.38315078616142273, + "grad_norm": 10.551952362060547, + "learning_rate": 3.944434578520628e-07, + "logits/chosen": -2.149305820465088, + "logits/rejected": -1.8200596570968628, + "logps/chosen": -152.25579833984375, + "logps/ref_chosen": -88.7528076171875, + "logps/ref_rejected": -98.49383544921875, + "logps/rejected": -201.384521484375, + "loss": 1.046, + "margin_dpo/margin_mean": 39.3876953125, + "margin_dpo/margin_std": 54.235328674316406, + "step": 256 + }, + { + "KL/chosen_KL_mean": -65.27300262451172, + "KL/mean": -85.2200927734375, + "KL/rejected_KL_mean": -105.16717529296875, + "KL/std": 48.23341751098633, + "epoch": 0.37738619676945667, + "fcm_dpo/beta": 0.01303618960082531, + "fcm_dpo/delta": -0.07658988237380981, + "fcm_dpo/margin": 39.894187927246094, + "fcm_dpo/q_t": 0.3826878070831299, + "grad_norm": 9.485233306884766, + "learning_rate": 3.933941090877615e-07, + "logits/chosen": -1.8701181411743164, + "logits/rejected": -1.4704567193984985, + "logps/chosen": -148.07652282714844, + "logps/ref_chosen": -82.80352020263672, + "logps/ref_rejected": -85.8677978515625, + "logps/rejected": -191.03497314453125, + "loss": 1.0246, + "margin_dpo/margin_mean": 39.894187927246094, + "margin_dpo/margin_std": 46.648895263671875, + "step": 257 + }, + { + "KL/chosen_KL_mean": -68.06910705566406, + "KL/mean": -86.16755676269531, + "KL/rejected_KL_mean": -104.2660140991211, + "KL/std": 45.84845733642578, + "epoch": 0.3788546255506608, + "fcm_dpo/beta": 0.013144619762897491, + "fcm_dpo/delta": -0.02780015394091606, + "fcm_dpo/margin": 36.196903228759766, + "fcm_dpo/q_t": 0.39086252450942993, + "grad_norm": 10.46627426147461, + "learning_rate": 3.923409817553284e-07, + "logits/chosen": -2.121572494506836, + "logits/rejected": -1.7644829750061035, + "logps/chosen": -158.25685119628906, + "logps/ref_chosen": -90.187744140625, + "logps/ref_rejected": -103.47068786621094, + "logps/rejected": -207.7366943359375, + "loss": 1.0464, + "margin_dpo/margin_mean": 36.196903228759766, + "margin_dpo/margin_std": 43.453529357910156, + "step": 258 + }, + { + "KL/chosen_KL_mean": -74.21199798583984, + "KL/mean": -89.64044952392578, + "KL/rejected_KL_mean": -105.06889343261719, + "KL/std": 45.95478820800781, + "epoch": 0.3803230543318649, + "fcm_dpo/beta": 0.013054771348834038, + "fcm_dpo/delta": 0.04869488999247551, + "fcm_dpo/margin": 30.85689926147461, + "fcm_dpo/q_t": 0.408587783575058, + "grad_norm": 10.180980682373047, + "learning_rate": 3.9128410360564793e-07, + "logits/chosen": -1.2585045099258423, + "logits/rejected": -1.0040603876113892, + "logps/chosen": -164.9845428466797, + "logps/ref_chosen": -90.77254486083984, + "logps/ref_rejected": -94.58816528320312, + "logps/rejected": -199.6570587158203, + "loss": 1.1307, + "margin_dpo/margin_mean": 30.856897354125977, + "margin_dpo/margin_std": 51.554019927978516, + "step": 259 + }, + { + "KL/chosen_KL_mean": -75.8703384399414, + "KL/mean": -89.35120391845703, + "KL/rejected_KL_mean": -102.83206176757812, + "KL/std": 46.482208251953125, + "epoch": 0.38179148311306904, + "fcm_dpo/beta": 0.01333899050951004, + "fcm_dpo/delta": 0.0938006117939949, + "fcm_dpo/margin": 26.961729049682617, + "fcm_dpo/q_t": 0.4186316728591919, + "grad_norm": 12.10897159576416, + "learning_rate": 3.9022350248844246e-07, + "logits/chosen": -1.7895164489746094, + "logits/rejected": -1.812833309173584, + "logps/chosen": -151.46304321289062, + "logps/ref_chosen": -75.59269714355469, + "logps/ref_rejected": -100.84554290771484, + "logps/rejected": -203.6776123046875, + "loss": 1.1666, + "margin_dpo/margin_mean": 26.961729049682617, + "margin_dpo/margin_std": 50.884849548339844, + "step": 260 + }, + { + "KL/chosen_KL_mean": -59.24699783325195, + "KL/mean": -82.04939270019531, + "KL/rejected_KL_mean": -104.8517837524414, + "KL/std": 46.922218322753906, + "epoch": 0.3832599118942731, + "fcm_dpo/beta": 0.013034466654062271, + "fcm_dpo/delta": -0.15515339374542236, + "fcm_dpo/margin": 45.60478973388672, + "fcm_dpo/q_t": 0.3651568591594696, + "grad_norm": 9.479791641235352, + "learning_rate": 3.891592063515376e-07, + "logits/chosen": -2.04919171333313, + "logits/rejected": -1.6856176853179932, + "logps/chosen": -153.5019073486328, + "logps/ref_chosen": -94.25491333007812, + "logps/ref_rejected": -93.65699005126953, + "logps/rejected": -198.50877380371094, + "loss": 0.9571, + "margin_dpo/margin_mean": 45.60478973388672, + "margin_dpo/margin_std": 45.41069412231445, + "step": 261 + }, + { + "KL/chosen_KL_mean": -63.915008544921875, + "KL/mean": -85.09645080566406, + "KL/rejected_KL_mean": -106.27789306640625, + "KL/std": 49.078636169433594, + "epoch": 0.38472834067547723, + "fcm_dpo/beta": 0.012775479815900326, + "fcm_dpo/delta": -0.09700441360473633, + "fcm_dpo/margin": 42.362884521484375, + "fcm_dpo/q_t": 0.3776552975177765, + "grad_norm": 9.474481582641602, + "learning_rate": 3.880912432401264e-07, + "logits/chosen": -1.8166594505310059, + "logits/rejected": -1.2828445434570312, + "logps/chosen": -149.18231201171875, + "logps/ref_chosen": -85.26730346679688, + "logps/ref_rejected": -90.82609558105469, + "logps/rejected": -197.10397338867188, + "loss": 1.0037, + "margin_dpo/margin_mean": 42.362884521484375, + "margin_dpo/margin_std": 48.5179443359375, + "step": 262 + }, + { + "KL/chosen_KL_mean": -51.72381591796875, + "KL/mean": -80.45867919921875, + "KL/rejected_KL_mean": -109.19354248046875, + "KL/std": 53.76057815551758, + "epoch": 0.38619676945668135, + "fcm_dpo/beta": 0.012269420549273491, + "fcm_dpo/delta": -0.27408623695373535, + "fcm_dpo/margin": 57.4697265625, + "fcm_dpo/q_t": 0.34254956245422363, + "grad_norm": 9.069549560546875, + "learning_rate": 3.870196412960302e-07, + "logits/chosen": -2.10990047454834, + "logits/rejected": -1.5606751441955566, + "logps/chosen": -147.48171997070312, + "logps/ref_chosen": -95.75790405273438, + "logps/ref_rejected": -101.83377075195312, + "logps/rejected": -211.02731323242188, + "loss": 0.9073, + "margin_dpo/margin_mean": 57.4697265625, + "margin_dpo/margin_std": 56.68013000488281, + "step": 263 + }, + { + "KL/chosen_KL_mean": -60.24684143066406, + "KL/mean": -84.13995361328125, + "KL/rejected_KL_mean": -108.03305053710938, + "KL/std": 50.6898307800293, + "epoch": 0.3876651982378855, + "fcm_dpo/beta": 0.011833982542157173, + "fcm_dpo/delta": -0.12274721264839172, + "fcm_dpo/margin": 47.786216735839844, + "fcm_dpo/q_t": 0.3716976046562195, + "grad_norm": 8.860624313354492, + "learning_rate": 3.8594442875695665e-07, + "logits/chosen": -2.292080879211426, + "logits/rejected": -2.016915798187256, + "logps/chosen": -150.8695068359375, + "logps/ref_chosen": -90.6226577758789, + "logps/ref_rejected": -100.32554626464844, + "logps/rejected": -208.3585968017578, + "loss": 0.9837, + "margin_dpo/margin_mean": 47.786216735839844, + "margin_dpo/margin_std": 51.46210479736328, + "step": 264 + }, + { + "KL/chosen_KL_mean": -62.92628479003906, + "KL/mean": -84.69291687011719, + "KL/rejected_KL_mean": -106.45956420898438, + "KL/std": 51.68177032470703, + "epoch": 0.3891336270190896, + "fcm_dpo/beta": 0.011622949503362179, + "fcm_dpo/delta": -0.059719592332839966, + "fcm_dpo/margin": 43.53327941894531, + "fcm_dpo/q_t": 0.3872433602809906, + "grad_norm": 9.03610897064209, + "learning_rate": 3.848656339557562e-07, + "logits/chosen": -2.082602024078369, + "logits/rejected": -1.6820037364959717, + "logps/chosen": -155.29861450195312, + "logps/ref_chosen": -92.37232971191406, + "logps/ref_rejected": -94.62757873535156, + "logps/rejected": -201.08714294433594, + "loss": 1.0401, + "margin_dpo/margin_mean": 43.53327941894531, + "margin_dpo/margin_std": 55.71157455444336, + "step": 265 + }, + { + "KL/chosen_KL_mean": -73.42080688476562, + "KL/mean": -93.02494049072266, + "KL/rejected_KL_mean": -112.62907409667969, + "KL/std": 49.6181526184082, + "epoch": 0.39060205580029367, + "fcm_dpo/beta": 0.01160573773086071, + "fcm_dpo/delta": -0.005409969016909599, + "fcm_dpo/margin": 39.20826721191406, + "fcm_dpo/q_t": 0.397316575050354, + "grad_norm": 10.500243186950684, + "learning_rate": 3.8378328531967507e-07, + "logits/chosen": -2.4886555671691895, + "logits/rejected": -1.707979679107666, + "logps/chosen": -175.62083435058594, + "logps/ref_chosen": -102.20002746582031, + "logps/ref_rejected": -74.36642456054688, + "logps/rejected": -186.99549865722656, + "loss": 1.0681, + "margin_dpo/margin_mean": 39.20826721191406, + "margin_dpo/margin_std": 51.86351776123047, + "step": 266 + }, + { + "KL/chosen_KL_mean": -63.99808120727539, + "KL/mean": -86.74386596679688, + "KL/rejected_KL_mean": -109.4896469116211, + "KL/std": 50.218692779541016, + "epoch": 0.3920704845814978, + "fcm_dpo/beta": 0.01145089603960514, + "fcm_dpo/delta": -0.07552241533994675, + "fcm_dpo/margin": 45.4915771484375, + "fcm_dpo/q_t": 0.3806152939796448, + "grad_norm": 9.601058959960938, + "learning_rate": 3.8269741136960646e-07, + "logits/chosen": -1.6615180969238281, + "logits/rejected": -1.0291712284088135, + "logps/chosen": -168.28408813476562, + "logps/ref_chosen": -104.28599548339844, + "logps/ref_rejected": -95.98719024658203, + "logps/rejected": -205.47683715820312, + "loss": 1.0, + "margin_dpo/margin_mean": 45.4915771484375, + "margin_dpo/margin_std": 47.074493408203125, + "step": 267 + }, + { + "KL/chosen_KL_mean": -63.305763244628906, + "KL/mean": -89.70006561279297, + "KL/rejected_KL_mean": -116.09436798095703, + "KL/std": 50.55253601074219, + "epoch": 0.3935389133627019, + "fcm_dpo/beta": 0.0111553855240345, + "fcm_dpo/delta": -0.14865517616271973, + "fcm_dpo/margin": 52.788612365722656, + "fcm_dpo/q_t": 0.36453887820243835, + "grad_norm": 9.604703903198242, + "learning_rate": 3.8160804071933894e-07, + "logits/chosen": -1.68306565284729, + "logits/rejected": -1.524566888809204, + "logps/chosen": -150.00198364257812, + "logps/ref_chosen": -86.69622039794922, + "logps/ref_rejected": -109.19184112548828, + "logps/rejected": -225.2862091064453, + "loss": 0.9527, + "margin_dpo/margin_mean": 52.788612365722656, + "margin_dpo/margin_std": 49.61883544921875, + "step": 268 + }, + { + "KL/chosen_KL_mean": -67.03485107421875, + "KL/mean": -92.83975219726562, + "KL/rejected_KL_mean": -118.64463806152344, + "KL/std": 54.211143493652344, + "epoch": 0.39500734214390604, + "fcm_dpo/beta": 0.010949023067951202, + "fcm_dpo/delta": -0.12179264426231384, + "fcm_dpo/margin": 51.60980987548828, + "fcm_dpo/q_t": 0.37075239419937134, + "grad_norm": 8.980087280273438, + "learning_rate": 3.8051520207480204e-07, + "logits/chosen": -1.77012300491333, + "logits/rejected": -1.3072271347045898, + "logps/chosen": -172.00665283203125, + "logps/ref_chosen": -104.97181701660156, + "logps/ref_rejected": -112.47647094726562, + "logps/rejected": -231.12110900878906, + "loss": 0.9722, + "margin_dpo/margin_mean": 51.60980987548828, + "margin_dpo/margin_std": 51.5111083984375, + "step": 269 + }, + { + "KL/chosen_KL_mean": -74.47264099121094, + "KL/mean": -96.40638732910156, + "KL/rejected_KL_mean": -118.34014892578125, + "KL/std": 53.79513168334961, + "epoch": 0.3964757709251101, + "fcm_dpo/beta": 0.010844661854207516, + "fcm_dpo/delta": -0.027146557345986366, + "fcm_dpo/margin": 43.86751174926758, + "fcm_dpo/q_t": 0.39237266778945923, + "grad_norm": 9.225295066833496, + "learning_rate": 3.794189242333106e-07, + "logits/chosen": -2.355654001235962, + "logits/rejected": -1.9361772537231445, + "logps/chosen": -175.54647827148438, + "logps/ref_chosen": -101.07383728027344, + "logps/ref_rejected": -117.75289154052734, + "logps/rejected": -236.09304809570312, + "loss": 1.0678, + "margin_dpo/margin_mean": 43.86750793457031, + "margin_dpo/margin_std": 60.640625, + "step": 270 + }, + { + "KL/chosen_KL_mean": -60.621280670166016, + "KL/mean": -88.11228942871094, + "KL/rejected_KL_mean": -115.6032943725586, + "KL/std": 50.711997985839844, + "epoch": 0.39794419970631423, + "fcm_dpo/beta": 0.01055808737874031, + "fcm_dpo/delta": -0.1387995183467865, + "fcm_dpo/margin": 54.981998443603516, + "fcm_dpo/q_t": 0.3661203384399414, + "grad_norm": 9.050552368164062, + "learning_rate": 3.7831923608280514e-07, + "logits/chosen": -1.4702354669570923, + "logits/rejected": -1.0143587589263916, + "logps/chosen": -157.34588623046875, + "logps/ref_chosen": -96.72459411621094, + "logps/ref_rejected": -98.5244140625, + "logps/rejected": -214.12771606445312, + "loss": 0.9747, + "margin_dpo/margin_mean": 54.981998443603516, + "margin_dpo/margin_std": 57.74799728393555, + "step": 271 + }, + { + "KL/chosen_KL_mean": -52.731178283691406, + "KL/mean": -78.52455139160156, + "KL/rejected_KL_mean": -104.31791687011719, + "KL/std": 55.571563720703125, + "epoch": 0.39941262848751835, + "fcm_dpo/beta": 0.01035686582326889, + "fcm_dpo/delta": -0.08906619250774384, + "fcm_dpo/margin": 51.58673858642578, + "fcm_dpo/q_t": 0.37962785363197327, + "grad_norm": 9.368598937988281, + "learning_rate": 3.772161666010912e-07, + "logits/chosen": -1.3417601585388184, + "logits/rejected": -1.2235678434371948, + "logps/chosen": -133.70840454101562, + "logps/ref_chosen": -80.97721862792969, + "logps/ref_rejected": -108.55535888671875, + "logps/rejected": -212.87327575683594, + "loss": 1.0108, + "margin_dpo/margin_mean": 51.58673858642578, + "margin_dpo/margin_std": 60.24427032470703, + "step": 272 + }, + { + "KL/chosen_KL_mean": -72.3418960571289, + "KL/mean": -98.98159790039062, + "KL/rejected_KL_mean": -125.62128448486328, + "KL/std": 57.446327209472656, + "epoch": 0.4008810572687225, + "fcm_dpo/beta": 0.010085253976285458, + "fcm_dpo/delta": -0.09449248015880585, + "fcm_dpo/margin": 53.27939224243164, + "fcm_dpo/q_t": 0.378003865480423, + "grad_norm": 8.671329498291016, + "learning_rate": 3.761097448550755e-07, + "logits/chosen": -1.827613353729248, + "logits/rejected": -1.4322266578674316, + "logps/chosen": -164.56649780273438, + "logps/ref_chosen": -92.22460174560547, + "logps/ref_rejected": -97.3630599975586, + "logps/rejected": -222.98434448242188, + "loss": 1.0218, + "margin_dpo/margin_mean": 53.279388427734375, + "margin_dpo/margin_std": 64.60359954833984, + "step": 273 + }, + { + "KL/chosen_KL_mean": -74.13670349121094, + "KL/mean": -94.80648040771484, + "KL/rejected_KL_mean": -115.47625732421875, + "KL/std": 55.86241912841797, + "epoch": 0.4023494860499266, + "fcm_dpo/beta": 0.010142168030142784, + "fcm_dpo/delta": 0.032018598169088364, + "fcm_dpo/margin": 41.33954620361328, + "fcm_dpo/q_t": 0.40365350246429443, + "grad_norm": 13.00503921508789, + "learning_rate": 3.75e-07, + "logits/chosen": -2.043581008911133, + "logits/rejected": -1.6510529518127441, + "logps/chosen": -156.39759826660156, + "logps/ref_chosen": -82.26089477539062, + "logps/ref_rejected": -83.87699890136719, + "logps/rejected": -199.35324096679688, + "loss": 1.0741, + "margin_dpo/margin_mean": 41.33954620361328, + "margin_dpo/margin_std": 51.26081085205078, + "step": 274 + }, + { + "KL/chosen_KL_mean": -63.02811813354492, + "KL/mean": -86.29637145996094, + "KL/rejected_KL_mean": -109.56463623046875, + "KL/std": 55.224388122558594, + "epoch": 0.40381791483113066, + "fcm_dpo/beta": 0.01020372286438942, + "fcm_dpo/delta": -0.026614777743816376, + "fcm_dpo/margin": 46.536502838134766, + "fcm_dpo/q_t": 0.3922935128211975, + "grad_norm": 9.820210456848145, + "learning_rate": 3.738869612786737e-07, + "logits/chosen": -2.1946425437927246, + "logits/rejected": -1.8749346733093262, + "logps/chosen": -142.715087890625, + "logps/ref_chosen": -79.68696594238281, + "logps/ref_rejected": -98.7509765625, + "logps/rejected": -208.31561279296875, + "loss": 1.0727, + "margin_dpo/margin_mean": 46.536502838134766, + "margin_dpo/margin_std": 64.95244598388672, + "step": 275 + }, + { + "KL/chosen_KL_mean": -75.37953186035156, + "KL/mean": -100.19548034667969, + "KL/rejected_KL_mean": -125.01143646240234, + "KL/std": 55.71049499511719, + "epoch": 0.4052863436123348, + "fcm_dpo/beta": 0.00998476892709732, + "fcm_dpo/delta": -0.04935610294342041, + "fcm_dpo/margin": 49.63191604614258, + "fcm_dpo/q_t": 0.3882775902748108, + "grad_norm": 10.756178855895996, + "learning_rate": 3.7277065802070204e-07, + "logits/chosen": -2.2146334648132324, + "logits/rejected": -1.6574091911315918, + "logps/chosen": -161.9192352294922, + "logps/ref_chosen": -86.53970336914062, + "logps/ref_rejected": -77.85394287109375, + "logps/rejected": -202.86538696289062, + "loss": 1.0418, + "margin_dpo/margin_mean": 49.631919860839844, + "margin_dpo/margin_std": 61.964561462402344, + "step": 276 + }, + { + "KL/chosen_KL_mean": -81.672607421875, + "KL/mean": -100.11717224121094, + "KL/rejected_KL_mean": -118.5617446899414, + "KL/std": 53.90569305419922, + "epoch": 0.4067547723935389, + "fcm_dpo/beta": 0.010115118697285652, + "fcm_dpo/delta": 0.07986337691545486, + "fcm_dpo/margin": 36.889129638671875, + "fcm_dpo/q_t": 0.4145438075065613, + "grad_norm": 10.247102737426758, + "learning_rate": 3.71651119641714e-07, + "logits/chosen": -1.983532190322876, + "logits/rejected": -1.5623936653137207, + "logps/chosen": -165.91671752929688, + "logps/ref_chosen": -84.24411010742188, + "logps/ref_rejected": -98.83421325683594, + "logps/rejected": -217.39596557617188, + "loss": 1.1201, + "margin_dpo/margin_mean": 36.889129638671875, + "margin_dpo/margin_std": 54.23590087890625, + "step": 277 + }, + { + "KL/chosen_KL_mean": -78.61597442626953, + "KL/mean": -101.62483215332031, + "KL/rejected_KL_mean": -124.63369750976562, + "KL/std": 57.92980194091797, + "epoch": 0.40822320117474303, + "fcm_dpo/beta": 0.01014047022908926, + "fcm_dpo/delta": -0.017631515860557556, + "fcm_dpo/margin": 46.01770782470703, + "fcm_dpo/q_t": 0.39241647720336914, + "grad_norm": 11.945178985595703, + "learning_rate": 3.705283756425872e-07, + "logits/chosen": -2.090693950653076, + "logits/rejected": -1.8765650987625122, + "logps/chosen": -161.0478515625, + "logps/ref_chosen": -82.431884765625, + "logps/ref_rejected": -97.85691833496094, + "logps/rejected": -222.4906005859375, + "loss": 1.0552, + "margin_dpo/margin_mean": 46.01770782470703, + "margin_dpo/margin_std": 57.620609283447266, + "step": 278 + }, + { + "KL/chosen_KL_mean": -81.6851577758789, + "KL/mean": -110.146484375, + "KL/rejected_KL_mean": -138.60781860351562, + "KL/std": 61.24153137207031, + "epoch": 0.40969162995594716, + "fcm_dpo/beta": 0.009937990456819534, + "fcm_dpo/delta": -0.12331337481737137, + "fcm_dpo/margin": 56.922672271728516, + "fcm_dpo/q_t": 0.37146955728530884, + "grad_norm": 10.838581085205078, + "learning_rate": 3.6940245560867e-07, + "logits/chosen": -1.9057362079620361, + "logits/rejected": -1.6948736906051636, + "logps/chosen": -166.8531494140625, + "logps/ref_chosen": -85.16799926757812, + "logps/ref_rejected": -94.12664794921875, + "logps/rejected": -232.73446655273438, + "loss": 0.9809, + "margin_dpo/margin_mean": 56.922672271728516, + "margin_dpo/margin_std": 60.66328430175781, + "step": 279 + }, + { + "KL/chosen_KL_mean": -84.80870056152344, + "KL/mean": -113.30152893066406, + "KL/rejected_KL_mean": -141.79437255859375, + "KL/std": 56.13744354248047, + "epoch": 0.4111600587371512, + "fcm_dpo/beta": 0.009764859452843666, + "fcm_dpo/delta": -0.1125873476266861, + "fcm_dpo/margin": 56.985660552978516, + "fcm_dpo/q_t": 0.3725709617137909, + "grad_norm": 9.573436737060547, + "learning_rate": 3.6827338920900253e-07, + "logits/chosen": -1.7208222150802612, + "logits/rejected": -1.5179696083068848, + "logps/chosen": -170.6651153564453, + "logps/ref_chosen": -85.85641479492188, + "logps/ref_rejected": -104.11859130859375, + "logps/rejected": -245.9129638671875, + "loss": 0.9809, + "margin_dpo/margin_mean": 56.985660552978516, + "margin_dpo/margin_std": 58.380191802978516, + "step": 280 + }, + { + "KL/chosen_KL_mean": -90.166748046875, + "KL/mean": -118.92213439941406, + "KL/rejected_KL_mean": -147.67752075195312, + "KL/std": 65.98188781738281, + "epoch": 0.41262848751835535, + "fcm_dpo/beta": 0.009521182626485825, + "fcm_dpo/delta": -0.1035277247428894, + "fcm_dpo/margin": 57.510780334472656, + "fcm_dpo/q_t": 0.3771226406097412, + "grad_norm": 9.767707824707031, + "learning_rate": 3.6714120619553435e-07, + "logits/chosen": -1.9440927505493164, + "logits/rejected": -1.1883043050765991, + "logps/chosen": -176.7175750732422, + "logps/ref_chosen": -86.55082702636719, + "logps/ref_rejected": -88.62866973876953, + "logps/rejected": -236.3061981201172, + "loss": 1.0028, + "margin_dpo/margin_mean": 57.510780334472656, + "margin_dpo/margin_std": 66.96490478515625, + "step": 281 + }, + { + "KL/chosen_KL_mean": -88.52243041992188, + "KL/mean": -123.98078918457031, + "KL/rejected_KL_mean": -159.43914794921875, + "KL/std": 63.78892517089844, + "epoch": 0.41409691629955947, + "fcm_dpo/beta": 0.00918116606771946, + "fcm_dpo/delta": -0.2153972089290619, + "fcm_dpo/margin": 70.91671752929688, + "fcm_dpo/q_t": 0.3512480854988098, + "grad_norm": 8.765600204467773, + "learning_rate": 3.660059364023408e-07, + "logits/chosen": -2.027914047241211, + "logits/rejected": -1.6634950637817383, + "logps/chosen": -193.62753295898438, + "logps/ref_chosen": -105.10511016845703, + "logps/ref_rejected": -102.85336303710938, + "logps/rejected": -262.29254150390625, + "loss": 0.9117, + "margin_dpo/margin_mean": 70.91671752929688, + "margin_dpo/margin_std": 63.18724822998047, + "step": 282 + }, + { + "KL/chosen_KL_mean": -102.74705505371094, + "KL/mean": -128.06011962890625, + "KL/rejected_KL_mean": -153.37319946289062, + "KL/std": 63.34912872314453, + "epoch": 0.4155653450807636, + "fcm_dpo/beta": 0.009073339402675629, + "fcm_dpo/delta": -0.009805109351873398, + "fcm_dpo/margin": 50.626121520996094, + "fcm_dpo/q_t": 0.39625316858291626, + "grad_norm": 10.84497356414795, + "learning_rate": 3.6486760974483685e-07, + "logits/chosen": -2.187713623046875, + "logits/rejected": -1.84690260887146, + "logps/chosen": -197.79965209960938, + "logps/ref_chosen": -95.05259704589844, + "logps/ref_rejected": -103.54454803466797, + "logps/rejected": -256.917724609375, + "loss": 1.0705, + "margin_dpo/margin_mean": 50.62611770629883, + "margin_dpo/margin_std": 69.36991882324219, + "step": 283 + }, + { + "KL/chosen_KL_mean": -93.44561767578125, + "KL/mean": -125.770263671875, + "KL/rejected_KL_mean": -158.0948944091797, + "KL/std": 62.93775939941406, + "epoch": 0.4170337738619677, + "fcm_dpo/beta": 0.008855831809341908, + "fcm_dpo/delta": -0.13103410601615906, + "fcm_dpo/margin": 64.64927673339844, + "fcm_dpo/q_t": 0.36667758226394653, + "grad_norm": 10.206171035766602, + "learning_rate": 3.6372625621898863e-07, + "logits/chosen": -1.7394795417785645, + "logits/rejected": -1.4053385257720947, + "logps/chosen": -181.11209106445312, + "logps/ref_chosen": -87.66647338867188, + "logps/ref_rejected": -98.75103759765625, + "logps/rejected": -256.845947265625, + "loss": 0.9497, + "margin_dpo/margin_mean": 64.64927673339844, + "margin_dpo/margin_std": 55.73645782470703, + "step": 284 + }, + { + "KL/chosen_KL_mean": -104.77790069580078, + "KL/mean": -127.95679473876953, + "KL/rejected_KL_mean": -151.13571166992188, + "KL/std": 56.91664123535156, + "epoch": 0.4185022026431718, + "fcm_dpo/beta": 0.00891127996146679, + "fcm_dpo/delta": 0.03824569284915924, + "fcm_dpo/margin": 46.357810974121094, + "fcm_dpo/q_t": 0.4031790494918823, + "grad_norm": 10.669732093811035, + "learning_rate": 3.625819059005228e-07, + "logits/chosen": -2.5109004974365234, + "logits/rejected": -2.2383971214294434, + "logps/chosen": -199.2109375, + "logps/ref_chosen": -94.43303680419922, + "logps/ref_rejected": -104.07194519042969, + "logps/rejected": -255.2076416015625, + "loss": 1.0699, + "margin_dpo/margin_mean": 46.35780334472656, + "margin_dpo/margin_std": 53.50532531738281, + "step": 285 + }, + { + "KL/chosen_KL_mean": -108.8774185180664, + "KL/mean": -139.5762939453125, + "KL/rejected_KL_mean": -170.275146484375, + "KL/std": 61.411163330078125, + "epoch": 0.4199706314243759, + "fcm_dpo/beta": 0.008768124505877495, + "fcm_dpo/delta": -0.09358012676239014, + "fcm_dpo/margin": 61.39774703979492, + "fcm_dpo/q_t": 0.37601304054260254, + "grad_norm": 8.792093276977539, + "learning_rate": 3.614345889441346e-07, + "logits/chosen": -1.7636845111846924, + "logits/rejected": -1.285810947418213, + "logps/chosen": -212.59780883789062, + "logps/ref_chosen": -103.72039794921875, + "logps/ref_rejected": -96.25775909423828, + "logps/rejected": -266.53289794921875, + "loss": 0.9919, + "margin_dpo/margin_mean": 61.39775085449219, + "margin_dpo/margin_std": 63.375953674316406, + "step": 286 + }, + { + "KL/chosen_KL_mean": -125.4956283569336, + "KL/mean": -143.25682067871094, + "KL/rejected_KL_mean": -161.01800537109375, + "KL/std": 56.276981353759766, + "epoch": 0.42143906020558003, + "fcm_dpo/beta": 0.00896537210792303, + "fcm_dpo/delta": 0.13455955684185028, + "fcm_dpo/margin": 35.52237319946289, + "fcm_dpo/q_t": 0.4286572337150574, + "grad_norm": 8.975525856018066, + "learning_rate": 3.6028433558269275e-07, + "logits/chosen": -2.0645358562469482, + "logits/rejected": -1.4521374702453613, + "logps/chosen": -219.38551330566406, + "logps/ref_chosen": -93.88988494873047, + "logps/ref_rejected": -83.33365631103516, + "logps/rejected": -244.35165405273438, + "loss": 1.1753, + "margin_dpo/margin_mean": 35.52237319946289, + "margin_dpo/margin_std": 64.4245376586914, + "step": 287 + }, + { + "KL/chosen_KL_mean": -119.27923583984375, + "KL/mean": -151.07376098632812, + "KL/rejected_KL_mean": -182.86830139160156, + "KL/std": 63.64386749267578, + "epoch": 0.42290748898678415, + "fcm_dpo/beta": 0.00880710780620575, + "fcm_dpo/delta": -0.11687169224023819, + "fcm_dpo/margin": 63.58906173706055, + "fcm_dpo/q_t": 0.3747601807117462, + "grad_norm": 8.365121841430664, + "learning_rate": 3.5913117612644327e-07, + "logits/chosen": -1.878031611442566, + "logits/rejected": -1.4685653448104858, + "logps/chosen": -207.43524169921875, + "logps/ref_chosen": -88.15602111816406, + "logps/ref_rejected": -93.28195190429688, + "logps/rejected": -276.1502685546875, + "loss": 0.989, + "margin_dpo/margin_mean": 63.58906555175781, + "margin_dpo/margin_std": 70.78921508789062, + "step": 288 + }, + { + "KL/chosen_KL_mean": -116.05216217041016, + "KL/mean": -148.35260009765625, + "KL/rejected_KL_mean": -180.6530303955078, + "KL/std": 66.90855407714844, + "epoch": 0.4243759177679883, + "fcm_dpo/beta": 0.008643142879009247, + "fcm_dpo/delta": -0.11460825055837631, + "fcm_dpo/margin": 64.60086059570312, + "fcm_dpo/q_t": 0.3773513436317444, + "grad_norm": 7.985620021820068, + "learning_rate": 3.5797514096221024e-07, + "logits/chosen": -2.0833332538604736, + "logits/rejected": -1.8857743740081787, + "logps/chosen": -191.44509887695312, + "logps/ref_chosen": -75.39292907714844, + "logps/ref_rejected": -93.15428161621094, + "logps/rejected": -273.80731201171875, + "loss": 1.0012, + "margin_dpo/margin_mean": 64.60086059570312, + "margin_dpo/margin_std": 76.6702880859375, + "step": 289 + }, + { + "KL/chosen_KL_mean": -125.43167877197266, + "KL/mean": -159.2182159423828, + "KL/rejected_KL_mean": -193.0047607421875, + "KL/std": 70.69041442871094, + "epoch": 0.42584434654919234, + "fcm_dpo/beta": 0.008467867970466614, + "fcm_dpo/delta": -0.12969830632209778, + "fcm_dpo/margin": 67.57308959960938, + "fcm_dpo/q_t": 0.3721858859062195, + "grad_norm": 8.800580024719238, + "learning_rate": 3.568162605525952e-07, + "logits/chosen": -1.1682285070419312, + "logits/rejected": -1.2103686332702637, + "logps/chosen": -213.47366333007812, + "logps/ref_chosen": -88.04197692871094, + "logps/ref_rejected": -123.212158203125, + "logps/rejected": -316.2169189453125, + "loss": 0.9826, + "margin_dpo/margin_mean": 67.57308959960938, + "margin_dpo/margin_std": 73.43978881835938, + "step": 290 + }, + { + "KL/chosen_KL_mean": -117.7747802734375, + "KL/mean": -146.55003356933594, + "KL/rejected_KL_mean": -175.32530212402344, + "KL/std": 67.0921630859375, + "epoch": 0.42731277533039647, + "fcm_dpo/beta": 0.008327526971697807, + "fcm_dpo/delta": -0.030725864693522453, + "fcm_dpo/margin": 57.550506591796875, + "fcm_dpo/q_t": 0.3891611695289612, + "grad_norm": 9.72525405883789, + "learning_rate": 3.5565456543517485e-07, + "logits/chosen": -2.4693450927734375, + "logits/rejected": -2.0153958797454834, + "logps/chosen": -211.87002563476562, + "logps/ref_chosen": -94.09524536132812, + "logps/ref_rejected": -96.05006408691406, + "logps/rejected": -271.3753662109375, + "loss": 1.0355, + "margin_dpo/margin_mean": 57.550506591796875, + "margin_dpo/margin_std": 66.60820770263672, + "step": 291 + }, + { + "KL/chosen_KL_mean": -121.39501190185547, + "KL/mean": -151.31764221191406, + "KL/rejected_KL_mean": -181.24029541015625, + "KL/std": 64.85142517089844, + "epoch": 0.4287812041116006, + "fcm_dpo/beta": 0.008243793621659279, + "fcm_dpo/delta": -0.04561452940106392, + "fcm_dpo/margin": 59.84527587890625, + "fcm_dpo/q_t": 0.3857780992984772, + "grad_norm": 9.154579162597656, + "learning_rate": 3.5449008622169583e-07, + "logits/chosen": -2.019760847091675, + "logits/rejected": -1.5835098028182983, + "logps/chosen": -209.64541625976562, + "logps/ref_chosen": -88.25041198730469, + "logps/ref_rejected": -96.41764068603516, + "logps/rejected": -277.6579284667969, + "loss": 1.0089, + "margin_dpo/margin_mean": 59.84527587890625, + "margin_dpo/margin_std": 60.23894500732422, + "step": 292 + }, + { + "KL/chosen_KL_mean": -120.842529296875, + "KL/mean": -144.92396545410156, + "KL/rejected_KL_mean": -169.00540161132812, + "KL/std": 69.42327117919922, + "epoch": 0.4302496328928047, + "fcm_dpo/beta": 0.008326580747961998, + "fcm_dpo/delta": 0.05035467445850372, + "fcm_dpo/margin": 48.162879943847656, + "fcm_dpo/q_t": 0.41038620471954346, + "grad_norm": 8.575159072875977, + "learning_rate": 3.5332285359726846e-07, + "logits/chosen": -2.351748466491699, + "logits/rejected": -1.7826613187789917, + "logps/chosen": -208.2190704345703, + "logps/ref_chosen": -87.37654113769531, + "logps/ref_rejected": -85.75579833984375, + "logps/rejected": -254.76121520996094, + "loss": 1.1087, + "margin_dpo/margin_mean": 48.162879943847656, + "margin_dpo/margin_std": 71.87301635742188, + "step": 293 + }, + { + "KL/chosen_KL_mean": -124.97944641113281, + "KL/mean": -144.67184448242188, + "KL/rejected_KL_mean": -164.36422729492188, + "KL/std": 65.12646484375, + "epoch": 0.43171806167400884, + "fcm_dpo/beta": 0.00848595891147852, + "fcm_dpo/delta": 0.11919843405485153, + "fcm_dpo/margin": 39.3847770690918, + "fcm_dpo/q_t": 0.4253361225128174, + "grad_norm": 9.369819641113281, + "learning_rate": 3.5215289831955786e-07, + "logits/chosen": -1.8082489967346191, + "logits/rejected": -1.6395163536071777, + "logps/chosen": -198.48739624023438, + "logps/ref_chosen": -73.50794982910156, + "logps/ref_rejected": -88.08877563476562, + "logps/rejected": -252.4530029296875, + "loss": 1.1789, + "margin_dpo/margin_mean": 39.38477325439453, + "margin_dpo/margin_std": 75.7198486328125, + "step": 294 + }, + { + "KL/chosen_KL_mean": -118.93072509765625, + "KL/mean": -149.1732177734375, + "KL/rejected_KL_mean": -179.41571044921875, + "KL/std": 66.96073150634766, + "epoch": 0.4331864904552129, + "fcm_dpo/beta": 0.008392706513404846, + "fcm_dpo/delta": -0.061477065086364746, + "fcm_dpo/margin": 60.485008239746094, + "fcm_dpo/q_t": 0.3851470351219177, + "grad_norm": 8.884387969970703, + "learning_rate": 3.509802512179737e-07, + "logits/chosen": -1.5656354427337646, + "logits/rejected": -1.409606695175171, + "logps/chosen": -196.69619750976562, + "logps/ref_chosen": -77.76548767089844, + "logps/ref_rejected": -94.24726867675781, + "logps/rejected": -273.6629638671875, + "loss": 1.0254, + "margin_dpo/margin_mean": 60.485008239746094, + "margin_dpo/margin_std": 71.83735656738281, + "step": 295 + }, + { + "KL/chosen_KL_mean": -127.09491729736328, + "KL/mean": -159.75466918945312, + "KL/rejected_KL_mean": -192.41441345214844, + "KL/std": 67.45562744140625, + "epoch": 0.434654919236417, + "fcm_dpo/beta": 0.008258884772658348, + "fcm_dpo/delta": -0.0951654464006424, + "fcm_dpo/margin": 65.31949615478516, + "fcm_dpo/q_t": 0.3758048713207245, + "grad_norm": 11.877053260803223, + "learning_rate": 3.498049431928577e-07, + "logits/chosen": -2.6076111793518066, + "logits/rejected": -2.1342110633850098, + "logps/chosen": -224.95132446289062, + "logps/ref_chosen": -97.85641479492188, + "logps/ref_rejected": -100.81631469726562, + "logps/rejected": -293.230712890625, + "loss": 0.9905, + "margin_dpo/margin_mean": 65.31949615478516, + "margin_dpo/margin_std": 67.6308364868164, + "step": 296 + }, + { + "KL/chosen_KL_mean": -129.87213134765625, + "KL/mean": -156.85235595703125, + "KL/rejected_KL_mean": -183.8325653076172, + "KL/std": 66.62882232666016, + "epoch": 0.43612334801762115, + "fcm_dpo/beta": 0.008269837126135826, + "fcm_dpo/delta": 0.003833906725049019, + "fcm_dpo/margin": 53.96044158935547, + "fcm_dpo/q_t": 0.3997609615325928, + "grad_norm": 12.508593559265137, + "learning_rate": 3.486270052146694e-07, + "logits/chosen": -1.3853625059127808, + "logits/rejected": -1.080672025680542, + "logps/chosen": -218.43797302246094, + "logps/ref_chosen": -88.56583404541016, + "logps/ref_rejected": -101.55656433105469, + "logps/rejected": -285.3891296386719, + "loss": 1.068, + "margin_dpo/margin_mean": 53.96044158935547, + "margin_dpo/margin_std": 71.1764907836914, + "step": 297 + }, + { + "KL/chosen_KL_mean": -109.5050048828125, + "KL/mean": -144.00445556640625, + "KL/rejected_KL_mean": -178.50390625, + "KL/std": 73.2869873046875, + "epoch": 0.43759177679882527, + "fcm_dpo/beta": 0.0081322081387043, + "fcm_dpo/delta": -0.11755289137363434, + "fcm_dpo/margin": 68.9989013671875, + "fcm_dpo/q_t": 0.3743922710418701, + "grad_norm": 12.788277626037598, + "learning_rate": 3.474464683231698e-07, + "logits/chosen": -2.0523953437805176, + "logits/rejected": -2.0216622352600098, + "logps/chosen": -204.38543701171875, + "logps/ref_chosen": -94.88043975830078, + "logps/ref_rejected": -122.31101989746094, + "logps/rejected": -300.81494140625, + "loss": 0.9822, + "margin_dpo/margin_mean": 68.9989013671875, + "margin_dpo/margin_std": 74.03498840332031, + "step": 298 + }, + { + "KL/chosen_KL_mean": -112.6458740234375, + "KL/mean": -147.16741943359375, + "KL/rejected_KL_mean": -181.68899536132812, + "KL/std": 70.91508483886719, + "epoch": 0.4390602055800294, + "fcm_dpo/beta": 0.007931895554065704, + "fcm_dpo/delta": -0.10341779887676239, + "fcm_dpo/margin": 69.0431137084961, + "fcm_dpo/q_t": 0.3754294514656067, + "grad_norm": 10.664654731750488, + "learning_rate": 3.462633636266041e-07, + "logits/chosen": -0.9453160762786865, + "logits/rejected": -0.7165822982788086, + "logps/chosen": -193.05422973632812, + "logps/ref_chosen": -80.40835571289062, + "logps/ref_rejected": -89.53717041015625, + "logps/rejected": -271.22613525390625, + "loss": 0.9897, + "margin_dpo/margin_mean": 69.0431137084961, + "margin_dpo/margin_std": 73.78032684326172, + "step": 299 + }, + { + "KL/chosen_KL_mean": -118.73129272460938, + "KL/mean": -147.56182861328125, + "KL/rejected_KL_mean": -176.392333984375, + "KL/std": 74.04791259765625, + "epoch": 0.44052863436123346, + "fcm_dpo/beta": 0.00785842351615429, + "fcm_dpo/delta": -0.0036856867372989655, + "fcm_dpo/margin": 57.66106414794922, + "fcm_dpo/q_t": 0.39694181084632874, + "grad_norm": 9.227557182312012, + "learning_rate": 3.4507772230088147e-07, + "logits/chosen": -1.436816930770874, + "logits/rejected": -1.2209242582321167, + "logps/chosen": -206.89019775390625, + "logps/ref_chosen": -88.15890502929688, + "logps/ref_rejected": -100.93919372558594, + "logps/rejected": -277.33154296875, + "loss": 1.0727, + "margin_dpo/margin_mean": 57.66106414794922, + "margin_dpo/margin_std": 77.9786148071289, + "step": 300 + }, + { + "KL/chosen_KL_mean": -109.10418701171875, + "KL/mean": -145.15988159179688, + "KL/rejected_KL_mean": -181.21554565429688, + "KL/std": 72.45738983154297, + "epoch": 0.4419970631424376, + "fcm_dpo/beta": 0.007712747436016798, + "fcm_dpo/delta": -0.1138525903224945, + "fcm_dpo/margin": 72.11134338378906, + "fcm_dpo/q_t": 0.3746373653411865, + "grad_norm": 10.497952461242676, + "learning_rate": 3.4388957558875316e-07, + "logits/chosen": -1.5942405462265015, + "logits/rejected": -1.1501681804656982, + "logps/chosen": -194.60910034179688, + "logps/ref_chosen": -85.50491333007812, + "logps/ref_rejected": -98.91683959960938, + "logps/rejected": -280.13238525390625, + "loss": 0.9951, + "margin_dpo/margin_mean": 72.11134338378906, + "margin_dpo/margin_std": 80.96015167236328, + "step": 301 + }, + { + "KL/chosen_KL_mean": -105.61663818359375, + "KL/mean": -136.363037109375, + "KL/rejected_KL_mean": -167.1094512939453, + "KL/std": 75.00934600830078, + "epoch": 0.4434654919236417, + "fcm_dpo/beta": 0.00767013244330883, + "fcm_dpo/delta": -0.0228885430842638, + "fcm_dpo/margin": 61.49281692504883, + "fcm_dpo/q_t": 0.39133208990097046, + "grad_norm": 9.270161628723145, + "learning_rate": 3.426989547989902e-07, + "logits/chosen": -1.4711213111877441, + "logits/rejected": -1.2963201999664307, + "logps/chosen": -185.06704711914062, + "logps/ref_chosen": -79.45040893554688, + "logps/ref_rejected": -103.0591049194336, + "logps/rejected": -270.1685485839844, + "loss": 1.0426, + "margin_dpo/margin_mean": 61.49281311035156, + "margin_dpo/margin_std": 73.03895568847656, + "step": 302 + }, + { + "KL/chosen_KL_mean": -98.44601440429688, + "KL/mean": -132.39842224121094, + "KL/rejected_KL_mean": -166.350830078125, + "KL/std": 69.38361358642578, + "epoch": 0.44493392070484583, + "fcm_dpo/beta": 0.007617846131324768, + "fcm_dpo/delta": -0.07091275602579117, + "fcm_dpo/margin": 67.90481567382812, + "fcm_dpo/q_t": 0.38092440366744995, + "grad_norm": 8.717459678649902, + "learning_rate": 3.4150589130555773e-07, + "logits/chosen": -1.8716776371002197, + "logits/rejected": -1.4163432121276855, + "logps/chosen": -194.0443115234375, + "logps/ref_chosen": -95.59829711914062, + "logps/ref_rejected": -92.0645751953125, + "logps/rejected": -258.4154052734375, + "loss": 1.0075, + "margin_dpo/margin_mean": 67.90481567382812, + "margin_dpo/margin_std": 72.90766906738281, + "step": 303 + }, + { + "KL/chosen_KL_mean": -98.38722229003906, + "KL/mean": -133.69200134277344, + "KL/rejected_KL_mean": -168.99676513671875, + "KL/std": 72.43708801269531, + "epoch": 0.44640234948604995, + "fcm_dpo/beta": 0.0074823698960244656, + "fcm_dpo/delta": -0.08265557885169983, + "fcm_dpo/margin": 70.60956573486328, + "fcm_dpo/q_t": 0.3783053755760193, + "grad_norm": 9.571141242980957, + "learning_rate": 3.403104165467883e-07, + "logits/chosen": -2.0542800426483154, + "logits/rejected": -1.6292614936828613, + "logps/chosen": -197.84036254882812, + "logps/ref_chosen": -99.4531478881836, + "logps/ref_rejected": -98.08226013183594, + "logps/rejected": -267.07904052734375, + "loss": 0.9976, + "margin_dpo/margin_mean": 70.60955810546875, + "margin_dpo/margin_std": 74.43731689453125, + "step": 304 + }, + { + "KL/chosen_KL_mean": -106.96412658691406, + "KL/mean": -140.706787109375, + "KL/rejected_KL_mean": -174.44943237304688, + "KL/std": 75.149169921875, + "epoch": 0.447870778267254, + "fcm_dpo/beta": 0.007345802150666714, + "fcm_dpo/delta": -0.04923749715089798, + "fcm_dpo/margin": 67.48530578613281, + "fcm_dpo/q_t": 0.3862135410308838, + "grad_norm": 8.295806884765625, + "learning_rate": 3.391125620245535e-07, + "logits/chosen": -2.002380132675171, + "logits/rejected": -1.506052851676941, + "logps/chosen": -210.22752380371094, + "logps/ref_chosen": -103.26339721679688, + "logps/ref_rejected": -97.82897186279297, + "logps/rejected": -272.2784118652344, + "loss": 1.0347, + "margin_dpo/margin_mean": 67.48530578613281, + "margin_dpo/margin_std": 79.51161193847656, + "step": 305 + }, + { + "KL/chosen_KL_mean": -95.57650756835938, + "KL/mean": -135.11154174804688, + "KL/rejected_KL_mean": -174.64654541015625, + "KL/std": 71.50088500976562, + "epoch": 0.44933920704845814, + "fcm_dpo/beta": 0.007213468197733164, + "fcm_dpo/delta": -0.12819325923919678, + "fcm_dpo/margin": 79.07003784179688, + "fcm_dpo/q_t": 0.36620384454727173, + "grad_norm": 10.3646240234375, + "learning_rate": 3.3791235930343417e-07, + "logits/chosen": -2.2244176864624023, + "logits/rejected": -1.6680846214294434, + "logps/chosen": -195.88034057617188, + "logps/ref_chosen": -100.3038330078125, + "logps/ref_rejected": -92.23086547851562, + "logps/rejected": -266.87744140625, + "loss": 0.9479, + "margin_dpo/margin_mean": 79.07003784179688, + "margin_dpo/margin_std": 65.43547058105469, + "step": 306 + }, + { + "KL/chosen_KL_mean": -101.2436294555664, + "KL/mean": -130.86199951171875, + "KL/rejected_KL_mean": -160.48036193847656, + "KL/std": 72.14974975585938, + "epoch": 0.45080763582966227, + "fcm_dpo/beta": 0.007188752293586731, + "fcm_dpo/delta": 0.02507771924138069, + "fcm_dpo/margin": 59.23672866821289, + "fcm_dpo/q_t": 0.4036938548088074, + "grad_norm": 9.431406021118164, + "learning_rate": 3.367098400098881e-07, + "logits/chosen": -1.878330945968628, + "logits/rejected": -1.5287479162216187, + "logps/chosen": -197.89340209960938, + "logps/ref_chosen": -96.64976501464844, + "logps/ref_rejected": -93.34860229492188, + "logps/rejected": -253.82896423339844, + "loss": 1.0925, + "margin_dpo/margin_mean": 59.23672866821289, + "margin_dpo/margin_std": 84.82234954833984, + "step": 307 + }, + { + "KL/chosen_KL_mean": -105.28068542480469, + "KL/mean": -134.8682403564453, + "KL/rejected_KL_mean": -164.455810546875, + "KL/std": 73.43102264404297, + "epoch": 0.4522760646108664, + "fcm_dpo/beta": 0.007201895117759705, + "fcm_dpo/delta": 0.02417123317718506, + "fcm_dpo/margin": 59.17512893676758, + "fcm_dpo/q_t": 0.40439844131469727, + "grad_norm": 10.738750457763672, + "learning_rate": 3.355050358314172e-07, + "logits/chosen": -1.7507621049880981, + "logits/rejected": -1.4741660356521606, + "logps/chosen": -191.00535583496094, + "logps/ref_chosen": -85.72467041015625, + "logps/ref_rejected": -86.42317962646484, + "logps/rejected": -250.87899780273438, + "loss": 1.0971, + "margin_dpo/margin_mean": 59.17512893676758, + "margin_dpo/margin_std": 85.20042419433594, + "step": 308 + }, + { + "KL/chosen_KL_mean": -118.82035827636719, + "KL/mean": -145.4488067626953, + "KL/rejected_KL_mean": -172.07725524902344, + "KL/std": 77.83595275878906, + "epoch": 0.45374449339207046, + "fcm_dpo/beta": 0.0073315356858074665, + "fcm_dpo/delta": 0.06189519166946411, + "fcm_dpo/margin": 53.25689697265625, + "fcm_dpo/q_t": 0.4107746481895447, + "grad_norm": 13.122225761413574, + "learning_rate": 3.3429797851573183e-07, + "logits/chosen": -1.624879240989685, + "logits/rejected": -1.1875228881835938, + "logps/chosen": -201.97181701660156, + "logps/ref_chosen": -83.15145874023438, + "logps/ref_rejected": -86.58602142333984, + "logps/rejected": -258.66326904296875, + "loss": 1.1114, + "margin_dpo/margin_mean": 53.25689697265625, + "margin_dpo/margin_std": 76.73574829101562, + "step": 309 + }, + { + "KL/chosen_KL_mean": -106.3636474609375, + "KL/mean": -136.62596130371094, + "KL/rejected_KL_mean": -166.8883056640625, + "KL/std": 71.94114685058594, + "epoch": 0.4552129221732746, + "fcm_dpo/beta": 0.00734830042347312, + "fcm_dpo/delta": 0.005481800530105829, + "fcm_dpo/margin": 60.52464294433594, + "fcm_dpo/q_t": 0.3978798985481262, + "grad_norm": 9.456814765930176, + "learning_rate": 3.3308869986991487e-07, + "logits/chosen": -2.310269832611084, + "logits/rejected": -1.901537537574768, + "logps/chosen": -196.1016387939453, + "logps/ref_chosen": -89.73799133300781, + "logps/ref_rejected": -87.07083129882812, + "logps/rejected": -253.95912170410156, + "loss": 1.0607, + "margin_dpo/margin_mean": 60.52464294433594, + "margin_dpo/margin_std": 74.69645690917969, + "step": 310 + }, + { + "KL/chosen_KL_mean": -126.24038696289062, + "KL/mean": -164.61024475097656, + "KL/rejected_KL_mean": -202.98008728027344, + "KL/std": 73.11846160888672, + "epoch": 0.4566813509544787, + "fcm_dpo/beta": 0.007237396668642759, + "fcm_dpo/delta": -0.11156634986400604, + "fcm_dpo/margin": 76.73970794677734, + "fcm_dpo/q_t": 0.3726646304130554, + "grad_norm": 9.374836921691895, + "learning_rate": 3.3187723175958346e-07, + "logits/chosen": -1.5658994913101196, + "logits/rejected": -0.8260637521743774, + "logps/chosen": -214.76681518554688, + "logps/ref_chosen": -88.52644348144531, + "logps/ref_rejected": -82.26608276367188, + "logps/rejected": -285.24615478515625, + "loss": 0.9696, + "margin_dpo/margin_mean": 76.73970031738281, + "margin_dpo/margin_std": 73.58587646484375, + "step": 311 + }, + { + "KL/chosen_KL_mean": -122.65965270996094, + "KL/mean": -154.30389404296875, + "KL/rejected_KL_mean": -185.9481658935547, + "KL/std": 80.49473571777344, + "epoch": 0.4581497797356828, + "fcm_dpo/beta": 0.007210931275039911, + "fcm_dpo/delta": -0.0068823304027318954, + "fcm_dpo/margin": 63.28850555419922, + "fcm_dpo/q_t": 0.3952620029449463, + "grad_norm": 9.447283744812012, + "learning_rate": 3.306636061080487e-07, + "logits/chosen": -1.6048760414123535, + "logits/rejected": -1.0793572664260864, + "logps/chosen": -203.0603485107422, + "logps/ref_chosen": -80.40069580078125, + "logps/ref_rejected": -82.4471435546875, + "logps/rejected": -268.39532470703125, + "loss": 1.0544, + "margin_dpo/margin_mean": 63.28850555419922, + "margin_dpo/margin_std": 77.36481475830078, + "step": 312 + }, + { + "KL/chosen_KL_mean": -115.8592529296875, + "KL/mean": -147.1273193359375, + "KL/rejected_KL_mean": -178.39537048339844, + "KL/std": 78.93463134765625, + "epoch": 0.45961820851688695, + "fcm_dpo/beta": 0.0072022429667413235, + "fcm_dpo/delta": -0.0005557965487241745, + "fcm_dpo/margin": 62.53610610961914, + "fcm_dpo/q_t": 0.3974963128566742, + "grad_norm": 10.454935073852539, + "learning_rate": 3.2944785489547537e-07, + "logits/chosen": -2.4720208644866943, + "logits/rejected": -2.070781946182251, + "logps/chosen": -200.00115966796875, + "logps/ref_chosen": -84.14190673828125, + "logps/ref_rejected": -93.42684173583984, + "logps/rejected": -271.82220458984375, + "loss": 1.0722, + "margin_dpo/margin_mean": 62.53610610961914, + "margin_dpo/margin_std": 84.5926513671875, + "step": 313 + }, + { + "KL/chosen_KL_mean": -118.06554412841797, + "KL/mean": -148.58663940429688, + "KL/rejected_KL_mean": -179.10772705078125, + "KL/std": 78.61907958984375, + "epoch": 0.461086637298091, + "fcm_dpo/beta": 0.007175165228545666, + "fcm_dpo/delta": 0.012416481971740723, + "fcm_dpo/margin": 61.042179107666016, + "fcm_dpo/q_t": 0.4005988538265228, + "grad_norm": 10.791749000549316, + "learning_rate": 3.2823001015803857e-07, + "logits/chosen": -1.8073995113372803, + "logits/rejected": -1.5847784280776978, + "logps/chosen": -206.87527465820312, + "logps/ref_chosen": -88.80972290039062, + "logps/ref_rejected": -104.15892028808594, + "logps/rejected": -283.26666259765625, + "loss": 1.069, + "margin_dpo/margin_mean": 61.042179107666016, + "margin_dpo/margin_std": 78.028564453125, + "step": 314 + }, + { + "KL/chosen_KL_mean": -115.33090209960938, + "KL/mean": -146.99432373046875, + "KL/rejected_KL_mean": -178.65774536132812, + "KL/std": 83.00553131103516, + "epoch": 0.46255506607929514, + "fcm_dpo/beta": 0.007191159762442112, + "fcm_dpo/delta": -0.005660181399434805, + "fcm_dpo/margin": 63.32685089111328, + "fcm_dpo/q_t": 0.3985491394996643, + "grad_norm": 9.554756164550781, + "learning_rate": 3.270101039870797e-07, + "logits/chosen": -1.351799726486206, + "logits/rejected": -1.1717203855514526, + "logps/chosen": -196.12460327148438, + "logps/ref_chosen": -80.793701171875, + "logps/ref_rejected": -92.13711547851562, + "logps/rejected": -270.79486083984375, + "loss": 1.0895, + "margin_dpo/margin_mean": 63.32685089111328, + "margin_dpo/margin_std": 95.18093872070312, + "step": 315 + }, + { + "KL/chosen_KL_mean": -118.03253173828125, + "KL/mean": -151.88902282714844, + "KL/rejected_KL_mean": -185.74551391601562, + "KL/std": 81.04499816894531, + "epoch": 0.46402349486049926, + "fcm_dpo/beta": 0.0071615735068917274, + "fcm_dpo/delta": -0.03669600933790207, + "fcm_dpo/margin": 67.71298217773438, + "fcm_dpo/q_t": 0.39057457447052, + "grad_norm": 9.565749168395996, + "learning_rate": 3.2578816852826086e-07, + "logits/chosen": -1.9321346282958984, + "logits/rejected": -1.9324355125427246, + "logps/chosen": -202.216796875, + "logps/ref_chosen": -84.18424987792969, + "logps/ref_rejected": -105.6908187866211, + "logps/rejected": -291.43634033203125, + "loss": 1.0399, + "margin_dpo/margin_mean": 67.71298217773438, + "margin_dpo/margin_std": 83.6407241821289, + "step": 316 + }, + { + "KL/chosen_KL_mean": -123.20557403564453, + "KL/mean": -165.17007446289062, + "KL/rejected_KL_mean": -207.13458251953125, + "KL/std": 90.36332702636719, + "epoch": 0.4654919236417034, + "fcm_dpo/beta": 0.007012120448052883, + "fcm_dpo/delta": -0.14692042768001556, + "fcm_dpo/margin": 83.92901611328125, + "fcm_dpo/q_t": 0.36895453929901123, + "grad_norm": 10.822399139404297, + "learning_rate": 3.2456423598071783e-07, + "logits/chosen": -1.8852722644805908, + "logits/rejected": -1.4041595458984375, + "logps/chosen": -211.07907104492188, + "logps/ref_chosen": -87.87348937988281, + "logps/ref_rejected": -108.26519775390625, + "logps/rejected": -315.3997802734375, + "loss": 0.9856, + "margin_dpo/margin_mean": 83.92901611328125, + "margin_dpo/margin_std": 95.92436218261719, + "step": 317 + }, + { + "KL/chosen_KL_mean": -119.85832214355469, + "KL/mean": -157.14173889160156, + "KL/rejected_KL_mean": -194.42515563964844, + "KL/std": 75.50823974609375, + "epoch": 0.4669603524229075, + "fcm_dpo/beta": 0.006859579123556614, + "fcm_dpo/delta": -0.06490808725357056, + "fcm_dpo/margin": 74.56683349609375, + "fcm_dpo/q_t": 0.3828851282596588, + "grad_norm": 11.389307022094727, + "learning_rate": 3.233383385962115e-07, + "logits/chosen": -2.099835157394409, + "logits/rejected": -1.378746747970581, + "logps/chosen": -218.06385803222656, + "logps/ref_chosen": -98.20553588867188, + "logps/ref_rejected": -88.13629150390625, + "logps/rejected": -282.56146240234375, + "loss": 1.0126, + "margin_dpo/margin_mean": 74.56683349609375, + "margin_dpo/margin_std": 82.71302795410156, + "step": 318 + }, + { + "KL/chosen_KL_mean": -104.32672119140625, + "KL/mean": -144.81991577148438, + "KL/rejected_KL_mean": -185.31312561035156, + "KL/std": 78.60574340820312, + "epoch": 0.4684287812041116, + "fcm_dpo/beta": 0.00672875763848424, + "fcm_dpo/delta": -0.100704126060009, + "fcm_dpo/margin": 80.98640441894531, + "fcm_dpo/q_t": 0.37574416399002075, + "grad_norm": 10.522967338562012, + "learning_rate": 3.2211050867827805e-07, + "logits/chosen": -1.8933600187301636, + "logits/rejected": -1.820589303970337, + "logps/chosen": -186.83010864257812, + "logps/ref_chosen": -82.50337219238281, + "logps/ref_rejected": -118.65068054199219, + "logps/rejected": -303.96380615234375, + "loss": 0.9941, + "margin_dpo/margin_mean": 80.98640441894531, + "margin_dpo/margin_std": 88.18949890136719, + "step": 319 + }, + { + "KL/chosen_KL_mean": -107.90133666992188, + "KL/mean": -141.27035522460938, + "KL/rejected_KL_mean": -174.63937377929688, + "KL/std": 80.5972900390625, + "epoch": 0.4698972099853157, + "fcm_dpo/beta": 0.006698761135339737, + "fcm_dpo/delta": 0.0030147507786750793, + "fcm_dpo/margin": 66.73802185058594, + "fcm_dpo/q_t": 0.3972882628440857, + "grad_norm": 8.973281860351562, + "learning_rate": 3.208807785813777e-07, + "logits/chosen": -2.0102860927581787, + "logits/rejected": -1.8216140270233154, + "logps/chosen": -195.03857421875, + "logps/ref_chosen": -87.13722229003906, + "logps/ref_rejected": -103.22412109375, + "logps/rejected": -277.86346435546875, + "loss": 1.0507, + "margin_dpo/margin_mean": 66.73802185058594, + "margin_dpo/margin_std": 77.56290435791016, + "step": 320 + }, + { + "KL/chosen_KL_mean": -119.27159881591797, + "KL/mean": -158.0006103515625, + "KL/rejected_KL_mean": -196.72964477539062, + "KL/std": 82.33745574951172, + "epoch": 0.4713656387665198, + "fcm_dpo/beta": 0.00661929976195097, + "fcm_dpo/delta": -0.06669492274522781, + "fcm_dpo/margin": 77.45805358886719, + "fcm_dpo/q_t": 0.3806576132774353, + "grad_norm": 8.919323921203613, + "learning_rate": 3.1964918071004217e-07, + "logits/chosen": -1.880910873413086, + "logits/rejected": -1.4864349365234375, + "logps/chosen": -212.3762664794922, + "logps/ref_chosen": -93.10466766357422, + "logps/ref_rejected": -97.4681396484375, + "logps/rejected": -294.1977844238281, + "loss": 1.0063, + "margin_dpo/margin_mean": 77.45805358886719, + "margin_dpo/margin_std": 81.26376342773438, + "step": 321 + }, + { + "KL/chosen_KL_mean": -111.59713745117188, + "KL/mean": -153.31509399414062, + "KL/rejected_KL_mean": -195.0330352783203, + "KL/std": 82.57926940917969, + "epoch": 0.47283406754772395, + "fcm_dpo/beta": 0.006480282172560692, + "fcm_dpo/delta": -0.09766355156898499, + "fcm_dpo/margin": 83.43589782714844, + "fcm_dpo/q_t": 0.37637168169021606, + "grad_norm": 13.370867729187012, + "learning_rate": 3.184157475180207e-07, + "logits/chosen": -1.5116252899169922, + "logits/rejected": -1.1432104110717773, + "logps/chosen": -203.79005432128906, + "logps/ref_chosen": -92.19291687011719, + "logps/ref_rejected": -100.89321899414062, + "logps/rejected": -295.92626953125, + "loss": 0.9981, + "margin_dpo/margin_mean": 83.43589782714844, + "margin_dpo/margin_std": 89.31085205078125, + "step": 322 + }, + { + "KL/chosen_KL_mean": -128.08004760742188, + "KL/mean": -163.47006225585938, + "KL/rejected_KL_mean": -198.8600311279297, + "KL/std": 82.79885864257812, + "epoch": 0.47430249632892807, + "fcm_dpo/beta": 0.00647247489541769, + "fcm_dpo/delta": -0.008625701069831848, + "fcm_dpo/margin": 70.77998352050781, + "fcm_dpo/q_t": 0.3960116505622864, + "grad_norm": 10.688695907592773, + "learning_rate": 3.171805115074251e-07, + "logits/chosen": -2.0449767112731934, + "logits/rejected": -1.6692168712615967, + "logps/chosen": -208.35215759277344, + "logps/ref_chosen": -80.27210235595703, + "logps/ref_rejected": -80.3033447265625, + "logps/rejected": -279.16339111328125, + "loss": 1.0508, + "margin_dpo/margin_mean": 70.77998352050781, + "margin_dpo/margin_std": 85.38031005859375, + "step": 323 + }, + { + "KL/chosen_KL_mean": -127.90582275390625, + "KL/mean": -164.725341796875, + "KL/rejected_KL_mean": -201.54489135742188, + "KL/std": 82.27083587646484, + "epoch": 0.47577092511013214, + "fcm_dpo/beta": 0.006465715356171131, + "fcm_dpo/delta": -0.02743496000766754, + "fcm_dpo/margin": 73.63906860351562, + "fcm_dpo/q_t": 0.38912340998649597, + "grad_norm": 9.375765800476074, + "learning_rate": 3.1594350522787295e-07, + "logits/chosen": -1.6269385814666748, + "logits/rejected": -0.9706502556800842, + "logps/chosen": -223.6351776123047, + "logps/ref_chosen": -95.72935485839844, + "logps/ref_rejected": -94.16924285888672, + "logps/rejected": -295.714111328125, + "loss": 1.0244, + "margin_dpo/margin_mean": 73.63906860351562, + "margin_dpo/margin_std": 77.25653076171875, + "step": 324 + }, + { + "KL/chosen_KL_mean": -117.1956787109375, + "KL/mean": -156.62721252441406, + "KL/rejected_KL_mean": -196.05874633789062, + "KL/std": 78.86705017089844, + "epoch": 0.47723935389133626, + "fcm_dpo/beta": 0.00639908853918314, + "fcm_dpo/delta": -0.05752098932862282, + "fcm_dpo/margin": 78.86307525634766, + "fcm_dpo/q_t": 0.3828526735305786, + "grad_norm": 8.189767837524414, + "learning_rate": 3.147047612756302e-07, + "logits/chosen": -1.8238208293914795, + "logits/rejected": -1.0989253520965576, + "logps/chosen": -215.9025421142578, + "logps/ref_chosen": -98.70686340332031, + "logps/ref_rejected": -76.40809631347656, + "logps/rejected": -272.46685791015625, + "loss": 1.0077, + "margin_dpo/margin_mean": 78.86307525634766, + "margin_dpo/margin_std": 81.07893371582031, + "step": 325 + }, + { + "KL/chosen_KL_mean": -130.5388641357422, + "KL/mean": -166.12429809570312, + "KL/rejected_KL_mean": -201.709716796875, + "KL/std": 78.67150115966797, + "epoch": 0.4787077826725404, + "fcm_dpo/beta": 0.00636826828122139, + "fcm_dpo/delta": -0.0033944009337574244, + "fcm_dpo/margin": 71.17085266113281, + "fcm_dpo/q_t": 0.39494964480400085, + "grad_norm": 8.22679615020752, + "learning_rate": 3.134643122927519e-07, + "logits/chosen": -2.344250440597534, + "logits/rejected": -1.6645500659942627, + "logps/chosen": -235.20016479492188, + "logps/ref_chosen": -104.66130828857422, + "logps/ref_rejected": -86.82537841796875, + "logps/rejected": -288.53509521484375, + "loss": 1.0511, + "margin_dpo/margin_mean": 71.17085266113281, + "margin_dpo/margin_std": 84.024658203125, + "step": 326 + }, + { + "KL/chosen_KL_mean": -126.83247375488281, + "KL/mean": -161.19143676757812, + "KL/rejected_KL_mean": -195.55038452148438, + "KL/std": 72.33634948730469, + "epoch": 0.4801762114537445, + "fcm_dpo/beta": 0.006365356035530567, + "fcm_dpo/delta": 0.013049425557255745, + "fcm_dpo/margin": 68.71792602539062, + "fcm_dpo/q_t": 0.4000307321548462, + "grad_norm": 9.14709758758545, + "learning_rate": 3.1222219096622264e-07, + "logits/chosen": -1.6792106628417969, + "logits/rejected": -1.0810998678207397, + "logps/chosen": -234.272705078125, + "logps/ref_chosen": -107.44023895263672, + "logps/ref_rejected": -117.05323028564453, + "logps/rejected": -312.6036376953125, + "loss": 1.0584, + "margin_dpo/margin_mean": 68.71792602539062, + "margin_dpo/margin_std": 81.67416381835938, + "step": 327 + }, + { + "KL/chosen_KL_mean": -123.31852722167969, + "KL/mean": -168.656005859375, + "KL/rejected_KL_mean": -213.99349975585938, + "KL/std": 86.12576293945312, + "epoch": 0.48164464023494863, + "fcm_dpo/beta": 0.006271493621170521, + "fcm_dpo/delta": -0.12576939165592194, + "fcm_dpo/margin": 90.67498779296875, + "fcm_dpo/q_t": 0.36831027269363403, + "grad_norm": 10.82209300994873, + "learning_rate": 3.1097843002709427e-07, + "logits/chosen": -1.7117938995361328, + "logits/rejected": -1.4164050817489624, + "logps/chosen": -209.92922973632812, + "logps/ref_chosen": -86.6106948852539, + "logps/ref_rejected": -97.40188598632812, + "logps/rejected": -311.3953857421875, + "loss": 0.9572, + "margin_dpo/margin_mean": 90.67498779296875, + "margin_dpo/margin_std": 82.11698913574219, + "step": 328 + }, + { + "KL/chosen_KL_mean": -128.87521362304688, + "KL/mean": -164.40403747558594, + "KL/rejected_KL_mean": -199.93287658691406, + "KL/std": 81.97160339355469, + "epoch": 0.4831130690161527, + "fcm_dpo/beta": 0.006211411207914352, + "fcm_dpo/delta": 0.008740127086639404, + "fcm_dpo/margin": 71.05769348144531, + "fcm_dpo/q_t": 0.3989870548248291, + "grad_norm": 10.588635444641113, + "learning_rate": 3.0973306224962437e-07, + "logits/chosen": -1.552767276763916, + "logits/rejected": -1.054271936416626, + "logps/chosen": -229.48828125, + "logps/ref_chosen": -100.61308288574219, + "logps/ref_rejected": -104.95370483398438, + "logps/rejected": -304.8865966796875, + "loss": 1.0613, + "margin_dpo/margin_mean": 71.05769348144531, + "margin_dpo/margin_std": 86.44296264648438, + "step": 329 + }, + { + "KL/chosen_KL_mean": -110.63896942138672, + "KL/mean": -148.5820770263672, + "KL/rejected_KL_mean": -186.5251922607422, + "KL/std": 80.47824096679688, + "epoch": 0.4845814977973568, + "fcm_dpo/beta": 0.006208469159901142, + "fcm_dpo/delta": -0.022268781438469887, + "fcm_dpo/margin": 75.88623046875, + "fcm_dpo/q_t": 0.38848236203193665, + "grad_norm": 8.516870498657227, + "learning_rate": 3.084861204504122e-07, + "logits/chosen": -1.7653732299804688, + "logits/rejected": -1.6858103275299072, + "logps/chosen": -187.597412109375, + "logps/ref_chosen": -76.95843505859375, + "logps/ref_rejected": -92.81494140625, + "logps/rejected": -279.34014892578125, + "loss": 1.0166, + "margin_dpo/margin_mean": 75.88623046875, + "margin_dpo/margin_std": 70.37911987304688, + "step": 330 + }, + { + "KL/chosen_KL_mean": -119.34526824951172, + "KL/mean": -161.75233459472656, + "KL/rejected_KL_mean": -204.15940856933594, + "KL/std": 78.9277572631836, + "epoch": 0.48604992657856094, + "fcm_dpo/beta": 0.0061643123626708984, + "fcm_dpo/delta": -0.07684408873319626, + "fcm_dpo/margin": 84.81415557861328, + "fcm_dpo/q_t": 0.3781358599662781, + "grad_norm": 10.151162147521973, + "learning_rate": 3.072376374875335e-07, + "logits/chosen": -1.8666870594024658, + "logits/rejected": -1.562316656112671, + "logps/chosen": -202.40469360351562, + "logps/ref_chosen": -83.05941772460938, + "logps/ref_rejected": -94.13113403320312, + "logps/rejected": -298.29052734375, + "loss": 0.984, + "margin_dpo/margin_mean": 84.81414794921875, + "margin_dpo/margin_std": 76.9617919921875, + "step": 331 + }, + { + "KL/chosen_KL_mean": -125.17555236816406, + "KL/mean": -154.91082763671875, + "KL/rejected_KL_mean": -184.6461181640625, + "KL/std": 87.22966003417969, + "epoch": 0.48751835535976507, + "fcm_dpo/beta": 0.006201116368174553, + "fcm_dpo/delta": 0.08419719338417053, + "fcm_dpo/margin": 59.470550537109375, + "fcm_dpo/q_t": 0.41479602456092834, + "grad_norm": 8.157636642456055, + "learning_rate": 3.059876462596758e-07, + "logits/chosen": -2.1781392097473145, + "logits/rejected": -1.7544598579406738, + "logps/chosen": -207.1064453125, + "logps/ref_chosen": -81.93089294433594, + "logps/ref_rejected": -82.84590148925781, + "logps/rejected": -267.49200439453125, + "loss": 1.1158, + "margin_dpo/margin_mean": 59.470558166503906, + "margin_dpo/margin_std": 84.43423461914062, + "step": 332 + }, + { + "KL/chosen_KL_mean": -145.17108154296875, + "KL/mean": -186.31817626953125, + "KL/rejected_KL_mean": -227.46524047851562, + "KL/std": 90.61051177978516, + "epoch": 0.4889867841409692, + "fcm_dpo/beta": 0.006128801964223385, + "fcm_dpo/delta": -0.058387283235788345, + "fcm_dpo/margin": 82.2941665649414, + "fcm_dpo/q_t": 0.3855236768722534, + "grad_norm": 8.680399894714355, + "learning_rate": 3.0473617970527015e-07, + "logits/chosen": -1.5334293842315674, + "logits/rejected": -1.1944446563720703, + "logps/chosen": -233.70367431640625, + "logps/ref_chosen": -88.53257751464844, + "logps/ref_rejected": -102.46601867675781, + "logps/rejected": -329.9312744140625, + "loss": 1.0292, + "margin_dpo/margin_mean": 82.29415893554688, + "margin_dpo/margin_std": 97.24163055419922, + "step": 333 + }, + { + "KL/chosen_KL_mean": -131.6866912841797, + "KL/mean": -168.419189453125, + "KL/rejected_KL_mean": -205.1516876220703, + "KL/std": 85.39486694335938, + "epoch": 0.49045521292217326, + "fcm_dpo/beta": 0.00611284002661705, + "fcm_dpo/delta": 0.0002731010317802429, + "fcm_dpo/margin": 73.46499633789062, + "fcm_dpo/q_t": 0.39615800976753235, + "grad_norm": 9.491837501525879, + "learning_rate": 3.034832708016243e-07, + "logits/chosen": -1.3197132349014282, + "logits/rejected": -1.0197309255599976, + "logps/chosen": -229.44447326660156, + "logps/ref_chosen": -97.75778198242188, + "logps/ref_rejected": -100.0058364868164, + "logps/rejected": -305.15753173828125, + "loss": 1.0696, + "margin_dpo/margin_mean": 73.46499633789062, + "margin_dpo/margin_std": 95.07492065429688, + "step": 334 + }, + { + "KL/chosen_KL_mean": -131.00123596191406, + "KL/mean": -163.1631317138672, + "KL/rejected_KL_mean": -195.32501220703125, + "KL/std": 85.37344360351562, + "epoch": 0.4919236417033774, + "fcm_dpo/beta": 0.0061988988891243935, + "fcm_dpo/delta": 0.053380027413368225, + "fcm_dpo/margin": 64.32377624511719, + "fcm_dpo/q_t": 0.40897828340530396, + "grad_norm": 11.817728042602539, + "learning_rate": 3.022289525640531e-07, + "logits/chosen": -2.154109477996826, + "logits/rejected": -1.7217153310775757, + "logps/chosen": -227.3772735595703, + "logps/ref_chosen": -96.37603759765625, + "logps/ref_rejected": -94.92401123046875, + "logps/rejected": -290.2490234375, + "loss": 1.1049, + "margin_dpo/margin_mean": 64.32377624511719, + "margin_dpo/margin_std": 93.01960754394531, + "step": 335 + }, + { + "KL/chosen_KL_mean": -133.98861694335938, + "KL/mean": -174.62728881835938, + "KL/rejected_KL_mean": -215.26593017578125, + "KL/std": 87.84451293945312, + "epoch": 0.4933920704845815, + "fcm_dpo/beta": 0.006177923642098904, + "fcm_dpo/delta": -0.054861292243003845, + "fcm_dpo/margin": 81.27732849121094, + "fcm_dpo/q_t": 0.38504862785339355, + "grad_norm": 11.14604377746582, + "learning_rate": 3.009732580450086e-07, + "logits/chosen": -1.5421233177185059, + "logits/rejected": -1.3872023820877075, + "logps/chosen": -222.5181427001953, + "logps/ref_chosen": -88.52952575683594, + "logps/ref_rejected": -109.5557861328125, + "logps/rejected": -324.82171630859375, + "loss": 1.019, + "margin_dpo/margin_mean": 81.27732849121094, + "margin_dpo/margin_std": 92.28482055664062, + "step": 336 + }, + { + "KL/chosen_KL_mean": -132.160888671875, + "KL/mean": -170.9864501953125, + "KL/rejected_KL_mean": -209.81201171875, + "KL/std": 88.68861389160156, + "epoch": 0.4948604992657856, + "fcm_dpo/beta": 0.006131826899945736, + "fcm_dpo/delta": -0.02746148779988289, + "fcm_dpo/margin": 77.65113067626953, + "fcm_dpo/q_t": 0.39256367087364197, + "grad_norm": 9.663256645202637, + "learning_rate": 2.9971622033320914e-07, + "logits/chosen": -2.5155861377716064, + "logits/rejected": -2.2384302616119385, + "logps/chosen": -229.4732666015625, + "logps/ref_chosen": -97.3123779296875, + "logps/ref_rejected": -110.13134765625, + "logps/rejected": -319.943359375, + "loss": 1.0572, + "margin_dpo/margin_mean": 77.65113067626953, + "margin_dpo/margin_std": 102.71278381347656, + "step": 337 + }, + { + "KL/chosen_KL_mean": -136.95333862304688, + "KL/mean": -172.71170043945312, + "KL/rejected_KL_mean": -208.47006225585938, + "KL/std": 89.64544677734375, + "epoch": 0.49632892804698975, + "fcm_dpo/beta": 0.006117684300988913, + "fcm_dpo/delta": 0.01304161548614502, + "fcm_dpo/margin": 71.51670837402344, + "fcm_dpo/q_t": 0.39828041195869446, + "grad_norm": 9.636395454406738, + "learning_rate": 2.984578725527675e-07, + "logits/chosen": -1.8111103773117065, + "logits/rejected": -1.6560990810394287, + "logps/chosen": -220.58346557617188, + "logps/ref_chosen": -83.63011169433594, + "logps/ref_rejected": -93.69235229492188, + "logps/rejected": -302.16241455078125, + "loss": 1.0626, + "margin_dpo/margin_mean": 71.51671600341797, + "margin_dpo/margin_std": 86.73267364501953, + "step": 338 + }, + { + "KL/chosen_KL_mean": -145.28753662109375, + "KL/mean": -183.42254638671875, + "KL/rejected_KL_mean": -221.55752563476562, + "KL/std": 88.99887084960938, + "epoch": 0.4977973568281938, + "fcm_dpo/beta": 0.006152212619781494, + "fcm_dpo/delta": -0.02100226655602455, + "fcm_dpo/margin": 76.27001953125, + "fcm_dpo/q_t": 0.3927009403705597, + "grad_norm": 8.57188606262207, + "learning_rate": 2.9719824786231796e-07, + "logits/chosen": -2.97969913482666, + "logits/rejected": -2.559129238128662, + "logps/chosen": -245.1393585205078, + "logps/ref_chosen": -99.85182189941406, + "logps/ref_rejected": -109.0743637084961, + "logps/rejected": -330.63189697265625, + "loss": 1.0591, + "margin_dpo/margin_mean": 76.27002716064453, + "margin_dpo/margin_std": 97.40506744384766, + "step": 339 + }, + { + "KL/chosen_KL_mean": -155.9953155517578, + "KL/mean": -189.56283569335938, + "KL/rejected_KL_mean": -223.13035583496094, + "KL/std": 81.51522827148438, + "epoch": 0.49926578560939794, + "fcm_dpo/beta": 0.00614144466817379, + "fcm_dpo/delta": 0.03929678723216057, + "fcm_dpo/margin": 67.13504028320312, + "fcm_dpo/q_t": 0.40457192063331604, + "grad_norm": 10.448555946350098, + "learning_rate": 2.959373794541426e-07, + "logits/chosen": -1.5166605710983276, + "logits/rejected": -1.0412172079086304, + "logps/chosen": -240.96926879882812, + "logps/ref_chosen": -84.97396850585938, + "logps/ref_rejected": -78.80033874511719, + "logps/rejected": -301.9306945800781, + "loss": 1.0843, + "margin_dpo/margin_mean": 67.13504028320312, + "margin_dpo/margin_std": 88.80570983886719, + "step": 340 + }, + { + "KL/chosen_KL_mean": -163.71954345703125, + "KL/mean": -196.01004028320312, + "KL/rejected_KL_mean": -228.30056762695312, + "KL/std": 85.64817810058594, + "epoch": 0.5007342143906021, + "fcm_dpo/beta": 0.0061846161261200905, + "fcm_dpo/delta": 0.05270107463002205, + "fcm_dpo/margin": 64.58100128173828, + "fcm_dpo/q_t": 0.40958231687545776, + "grad_norm": 9.756683349609375, + "learning_rate": 2.946753005532965e-07, + "logits/chosen": -1.4001996517181396, + "logits/rejected": -1.1443207263946533, + "logps/chosen": -248.98353576660156, + "logps/ref_chosen": -85.26399230957031, + "logps/ref_rejected": -106.08595275878906, + "logps/rejected": -334.38653564453125, + "loss": 1.1137, + "margin_dpo/margin_mean": 64.58100128173828, + "margin_dpo/margin_std": 97.7337646484375, + "step": 341 + }, + { + "KL/chosen_KL_mean": -130.64772033691406, + "KL/mean": -177.71002197265625, + "KL/rejected_KL_mean": -224.77235412597656, + "KL/std": 86.38710021972656, + "epoch": 0.5022026431718062, + "fcm_dpo/beta": 0.006058323662728071, + "fcm_dpo/delta": -0.12925466895103455, + "fcm_dpo/margin": 94.12464904785156, + "fcm_dpo/q_t": 0.36713290214538574, + "grad_norm": 8.498157501220703, + "learning_rate": 2.934120444167326e-07, + "logits/chosen": -1.437894582748413, + "logits/rejected": -0.6820884943008423, + "logps/chosen": -231.6863250732422, + "logps/ref_chosen": -101.03860473632812, + "logps/ref_rejected": -98.89851379394531, + "logps/rejected": -323.6708679199219, + "loss": 0.9732, + "margin_dpo/margin_mean": 94.12464904785156, + "margin_dpo/margin_std": 91.66548156738281, + "step": 342 + }, + { + "KL/chosen_KL_mean": -142.7459259033203, + "KL/mean": -181.76763916015625, + "KL/rejected_KL_mean": -220.78933715820312, + "KL/std": 89.77273559570312, + "epoch": 0.5036710719530103, + "fcm_dpo/beta": 0.0060761794447898865, + "fcm_dpo/delta": -0.026198003441095352, + "fcm_dpo/margin": 78.04341125488281, + "fcm_dpo/q_t": 0.39156705141067505, + "grad_norm": 11.090639114379883, + "learning_rate": 2.9214764433242476e-07, + "logits/chosen": -2.0876166820526123, + "logits/rejected": -1.9741833209991455, + "logps/chosen": -226.4932861328125, + "logps/ref_chosen": -83.74736022949219, + "logps/ref_rejected": -116.57908630371094, + "logps/rejected": -337.368408203125, + "loss": 1.0463, + "margin_dpo/margin_mean": 78.04341125488281, + "margin_dpo/margin_std": 92.98764038085938, + "step": 343 + }, + { + "KL/chosen_KL_mean": -142.91702270507812, + "KL/mean": -183.55194091796875, + "KL/rejected_KL_mean": -224.18685913085938, + "KL/std": 95.46722412109375, + "epoch": 0.5051395007342144, + "fcm_dpo/beta": 0.006021049339324236, + "fcm_dpo/delta": -0.04170961305499077, + "fcm_dpo/margin": 81.26984405517578, + "fcm_dpo/q_t": 0.38922470808029175, + "grad_norm": 10.0555419921875, + "learning_rate": 2.9088213361849126e-07, + "logits/chosen": -1.6818866729736328, + "logits/rejected": -1.4340788125991821, + "logps/chosen": -221.17098999023438, + "logps/ref_chosen": -78.25396728515625, + "logps/ref_rejected": -95.42870330810547, + "logps/rejected": -319.6155700683594, + "loss": 1.0427, + "margin_dpo/margin_mean": 81.26985168457031, + "margin_dpo/margin_std": 100.70010375976562, + "step": 344 + }, + { + "KL/chosen_KL_mean": -147.00729370117188, + "KL/mean": -190.7525634765625, + "KL/rejected_KL_mean": -234.49781799316406, + "KL/std": 95.71751403808594, + "epoch": 0.5066079295154186, + "fcm_dpo/beta": 0.0059035178273916245, + "fcm_dpo/delta": -0.07031524181365967, + "fcm_dpo/margin": 87.49053192138672, + "fcm_dpo/q_t": 0.38228651881217957, + "grad_norm": 9.630701065063477, + "learning_rate": 2.896155456223163e-07, + "logits/chosen": -1.591372013092041, + "logits/rejected": -1.2972593307495117, + "logps/chosen": -245.0867919921875, + "logps/ref_chosen": -98.0794906616211, + "logps/ref_rejected": -104.03477478027344, + "logps/rejected": -338.5325927734375, + "loss": 1.012, + "margin_dpo/margin_mean": 87.49052429199219, + "margin_dpo/margin_std": 98.24116516113281, + "step": 345 + }, + { + "KL/chosen_KL_mean": -163.81016540527344, + "KL/mean": -197.99606323242188, + "KL/rejected_KL_mean": -232.18194580078125, + "KL/std": 88.83554077148438, + "epoch": 0.5080763582966226, + "fcm_dpo/beta": 0.005932542495429516, + "fcm_dpo/delta": 0.046264972537755966, + "fcm_dpo/margin": 68.37179565429688, + "fcm_dpo/q_t": 0.40812280774116516, + "grad_norm": 8.503084182739258, + "learning_rate": 2.883479137196714e-07, + "logits/chosen": -1.808495044708252, + "logits/rejected": -1.494280219078064, + "logps/chosen": -244.49365234375, + "logps/ref_chosen": -80.68348693847656, + "logps/ref_rejected": -85.89260864257812, + "logps/rejected": -318.0745544433594, + "loss": 1.104, + "margin_dpo/margin_mean": 68.37179565429688, + "margin_dpo/margin_std": 100.7035140991211, + "step": 346 + }, + { + "KL/chosen_KL_mean": -153.70419311523438, + "KL/mean": -186.50839233398438, + "KL/rejected_KL_mean": -219.3125762939453, + "KL/std": 87.70765686035156, + "epoch": 0.5095447870778267, + "fcm_dpo/beta": 0.006018254440277815, + "fcm_dpo/delta": 0.05700414627790451, + "fcm_dpo/margin": 65.60836791992188, + "fcm_dpo/q_t": 0.4102572798728943, + "grad_norm": 8.964280128479004, + "learning_rate": 2.8707927131383614e-07, + "logits/chosen": -1.884059190750122, + "logits/rejected": -1.5605113506317139, + "logps/chosen": -247.38357543945312, + "logps/ref_chosen": -93.67938232421875, + "logps/ref_rejected": -97.54251098632812, + "logps/rejected": -316.8551025390625, + "loss": 1.1063, + "margin_dpo/margin_mean": 65.60836791992188, + "margin_dpo/margin_std": 94.55398559570312, + "step": 347 + }, + { + "KL/chosen_KL_mean": -133.36244201660156, + "KL/mean": -170.93353271484375, + "KL/rejected_KL_mean": -208.50462341308594, + "KL/std": 83.08096313476562, + "epoch": 0.5110132158590308, + "fcm_dpo/beta": 0.005962018854916096, + "fcm_dpo/delta": 0.0004133060574531555, + "fcm_dpo/margin": 75.14217376708984, + "fcm_dpo/q_t": 0.3959684669971466, + "grad_norm": 8.280686378479004, + "learning_rate": 2.858096518347179e-07, + "logits/chosen": -1.851686954498291, + "logits/rejected": -1.5706337690353394, + "logps/chosen": -223.61468505859375, + "logps/ref_chosen": -90.25224304199219, + "logps/ref_rejected": -95.77027893066406, + "logps/rejected": -304.27490234375, + "loss": 1.0563, + "margin_dpo/margin_mean": 75.14216613769531, + "margin_dpo/margin_std": 84.9925537109375, + "step": 348 + }, + { + "KL/chosen_KL_mean": -143.33042907714844, + "KL/mean": -178.51885986328125, + "KL/rejected_KL_mean": -213.707275390625, + "KL/std": 91.03848266601562, + "epoch": 0.5124816446402349, + "fcm_dpo/beta": 0.006063205190002918, + "fcm_dpo/delta": 0.02386452443897724, + "fcm_dpo/margin": 70.37683868408203, + "fcm_dpo/q_t": 0.4028947353363037, + "grad_norm": 10.386116027832031, + "learning_rate": 2.845390887379706e-07, + "logits/chosen": -2.064490556716919, + "logits/rejected": -1.8402228355407715, + "logps/chosen": -235.38851928710938, + "logps/ref_chosen": -92.05809020996094, + "logps/ref_rejected": -103.96527099609375, + "logps/rejected": -317.67254638671875, + "loss": 1.0795, + "margin_dpo/margin_mean": 70.37683868408203, + "margin_dpo/margin_std": 92.30020904541016, + "step": 349 + }, + { + "KL/chosen_KL_mean": -152.83030700683594, + "KL/mean": -181.812744140625, + "KL/rejected_KL_mean": -210.79519653320312, + "KL/std": 83.527099609375, + "epoch": 0.5139500734214391, + "fcm_dpo/beta": 0.006126874126493931, + "fcm_dpo/delta": 0.09844039380550385, + "fcm_dpo/margin": 57.96487808227539, + "fcm_dpo/q_t": 0.41880983114242554, + "grad_norm": 11.032787322998047, + "learning_rate": 2.8326761550411346e-07, + "logits/chosen": -2.0019736289978027, + "logits/rejected": -1.7813717126846313, + "logps/chosen": -239.88450622558594, + "logps/ref_chosen": -87.05419921875, + "logps/ref_rejected": -95.72193145751953, + "logps/rejected": -306.51715087890625, + "loss": 1.1341, + "margin_dpo/margin_mean": 57.964874267578125, + "margin_dpo/margin_std": 89.42626190185547, + "step": 350 + }, + { + "KL/chosen_KL_mean": -151.509033203125, + "KL/mean": -193.4451141357422, + "KL/rejected_KL_mean": -235.38119506835938, + "KL/std": 93.31094360351562, + "epoch": 0.5154185022026432, + "fcm_dpo/beta": 0.006119257770478725, + "fcm_dpo/delta": -0.06661495566368103, + "fcm_dpo/margin": 83.87215423583984, + "fcm_dpo/q_t": 0.38258159160614014, + "grad_norm": 9.132224082946777, + "learning_rate": 2.819952656376487e-07, + "logits/chosen": -1.707326054573059, + "logits/rejected": -1.4008855819702148, + "logps/chosen": -240.67807006835938, + "logps/ref_chosen": -89.16903686523438, + "logps/ref_rejected": -108.29801940917969, + "logps/rejected": -343.67919921875, + "loss": 1.0223, + "margin_dpo/margin_mean": 83.87214660644531, + "margin_dpo/margin_std": 98.51606750488281, + "step": 351 + }, + { + "KL/chosen_KL_mean": -138.6739501953125, + "KL/mean": -178.33370971679688, + "KL/rejected_KL_mean": -217.99343872070312, + "KL/std": 92.34928894042969, + "epoch": 0.5168869309838473, + "fcm_dpo/beta": 0.006055803969502449, + "fcm_dpo/delta": -0.03186991065740585, + "fcm_dpo/margin": 79.31950378417969, + "fcm_dpo/q_t": 0.39239203929901123, + "grad_norm": 9.162763595581055, + "learning_rate": 2.8072207266617854e-07, + "logits/chosen": -1.8129549026489258, + "logits/rejected": -1.2578930854797363, + "logps/chosen": -235.48544311523438, + "logps/ref_chosen": -96.81147766113281, + "logps/ref_rejected": -84.40483093261719, + "logps/rejected": -302.39825439453125, + "loss": 1.0462, + "margin_dpo/margin_mean": 79.31949615478516, + "margin_dpo/margin_std": 98.95259857177734, + "step": 352 + }, + { + "KL/chosen_KL_mean": -159.00277709960938, + "KL/mean": -184.40956115722656, + "KL/rejected_KL_mean": -209.81634521484375, + "KL/std": 82.14775848388672, + "epoch": 0.5183553597650514, + "fcm_dpo/beta": 0.006155360955744982, + "fcm_dpo/delta": 0.14189966022968292, + "fcm_dpo/margin": 50.813568115234375, + "fcm_dpo/q_t": 0.4280507564544678, + "grad_norm": 10.709700584411621, + "learning_rate": 2.794480701395219e-07, + "logits/chosen": -2.196394681930542, + "logits/rejected": -1.8158748149871826, + "logps/chosen": -244.36526489257812, + "logps/ref_chosen": -85.36248779296875, + "logps/ref_rejected": -87.36732482910156, + "logps/rejected": -297.18365478515625, + "loss": 1.1751, + "margin_dpo/margin_mean": 50.813568115234375, + "margin_dpo/margin_std": 91.15338134765625, + "step": 353 + }, + { + "KL/chosen_KL_mean": -139.20684814453125, + "KL/mean": -178.58595275878906, + "KL/rejected_KL_mean": -217.9650421142578, + "KL/std": 83.68165588378906, + "epoch": 0.5198237885462555, + "fcm_dpo/beta": 0.006197898183017969, + "fcm_dpo/delta": -0.04012412577867508, + "fcm_dpo/margin": 78.75819396972656, + "fcm_dpo/q_t": 0.3867066502571106, + "grad_norm": 9.191285133361816, + "learning_rate": 2.781732916288303e-07, + "logits/chosen": -1.6556808948516846, + "logits/rejected": -1.168824553489685, + "logps/chosen": -235.00802612304688, + "logps/ref_chosen": -95.8011703491211, + "logps/ref_rejected": -94.86614227294922, + "logps/rejected": -312.8311767578125, + "loss": 1.0088, + "margin_dpo/margin_mean": 78.75819396972656, + "margin_dpo/margin_std": 76.52041625976562, + "step": 354 + }, + { + "KL/chosen_KL_mean": -137.57781982421875, + "KL/mean": -174.31130981445312, + "KL/rejected_KL_mean": -211.0447998046875, + "KL/std": 85.07630920410156, + "epoch": 0.5212922173274597, + "fcm_dpo/beta": 0.0061437515541911125, + "fcm_dpo/delta": -0.001744445413351059, + "fcm_dpo/margin": 73.46697235107422, + "fcm_dpo/q_t": 0.39597606658935547, + "grad_norm": 10.174688339233398, + "learning_rate": 2.7689777072570284e-07, + "logits/chosen": -2.5813350677490234, + "logits/rejected": -2.1107242107391357, + "logps/chosen": -223.7633819580078, + "logps/ref_chosen": -86.185546875, + "logps/ref_rejected": -88.4615707397461, + "logps/rejected": -299.50634765625, + "loss": 1.0601, + "margin_dpo/margin_mean": 73.46696472167969, + "margin_dpo/margin_std": 89.4022216796875, + "step": 355 + }, + { + "KL/chosen_KL_mean": -161.46664428710938, + "KL/mean": -189.98309326171875, + "KL/rejected_KL_mean": -218.49954223632812, + "KL/std": 83.07925415039062, + "epoch": 0.5227606461086637, + "fcm_dpo/beta": 0.006275987718254328, + "fcm_dpo/delta": 0.09493474662303925, + "fcm_dpo/margin": 57.03289031982422, + "fcm_dpo/q_t": 0.41823703050613403, + "grad_norm": 10.077198028564453, + "learning_rate": 2.7562154104130176e-07, + "logits/chosen": -1.6768903732299805, + "logits/rejected": -1.1983524560928345, + "logps/chosen": -252.28475952148438, + "logps/ref_chosen": -90.818115234375, + "logps/ref_rejected": -82.1273193359375, + "logps/rejected": -300.6268615722656, + "loss": 1.1432, + "margin_dpo/margin_mean": 57.032894134521484, + "margin_dpo/margin_std": 94.3851547241211, + "step": 356 + }, + { + "KL/chosen_KL_mean": -134.25582885742188, + "KL/mean": -180.50140380859375, + "KL/rejected_KL_mean": -226.74700927734375, + "KL/std": 91.9460678100586, + "epoch": 0.5242290748898678, + "fcm_dpo/beta": 0.006120544858276844, + "fcm_dpo/delta": -0.12536004185676575, + "fcm_dpo/margin": 92.49116516113281, + "fcm_dpo/q_t": 0.36878687143325806, + "grad_norm": 8.692706108093262, + "learning_rate": 2.7434463620546594e-07, + "logits/chosen": -2.0856592655181885, + "logits/rejected": -1.7231680154800415, + "logps/chosen": -221.76058959960938, + "logps/ref_chosen": -87.5047607421875, + "logps/ref_rejected": -92.24937438964844, + "logps/rejected": -318.9963684082031, + "loss": 0.9641, + "margin_dpo/margin_mean": 92.49116516113281, + "margin_dpo/margin_std": 83.96719360351562, + "step": 357 + }, + { + "KL/chosen_KL_mean": -146.8455047607422, + "KL/mean": -180.4937744140625, + "KL/rejected_KL_mean": -214.1420135498047, + "KL/std": 85.15274047851562, + "epoch": 0.5256975036710719, + "fcm_dpo/beta": 0.0061625586822628975, + "fcm_dpo/delta": 0.036812763661146164, + "fcm_dpo/margin": 67.29647827148438, + "fcm_dpo/q_t": 0.4069470167160034, + "grad_norm": 9.665899276733398, + "learning_rate": 2.730670898658255e-07, + "logits/chosen": -1.705322504043579, + "logits/rejected": -1.237384557723999, + "logps/chosen": -241.90097045898438, + "logps/ref_chosen": -95.05546569824219, + "logps/ref_rejected": -96.47611999511719, + "logps/rejected": -310.6181335449219, + "loss": 1.0868, + "margin_dpo/margin_mean": 67.29647827148438, + "margin_dpo/margin_std": 91.71583557128906, + "step": 358 + }, + { + "KL/chosen_KL_mean": -163.5113525390625, + "KL/mean": -194.93548583984375, + "KL/rejected_KL_mean": -226.359619140625, + "KL/std": 80.23503112792969, + "epoch": 0.527165932452276, + "fcm_dpo/beta": 0.006196748465299606, + "fcm_dpo/delta": 0.06266428530216217, + "fcm_dpo/margin": 62.8482666015625, + "fcm_dpo/q_t": 0.4122501611709595, + "grad_norm": 9.812956809997559, + "learning_rate": 2.717889356869146e-07, + "logits/chosen": -1.5021560192108154, + "logits/rejected": -0.928576648235321, + "logps/chosen": -250.8846893310547, + "logps/ref_chosen": -87.37332916259766, + "logps/ref_rejected": -88.77188110351562, + "logps/rejected": -315.1315002441406, + "loss": 1.1143, + "margin_dpo/margin_mean": 62.8482666015625, + "margin_dpo/margin_std": 92.45555114746094, + "step": 359 + }, + { + "KL/chosen_KL_mean": -150.11993408203125, + "KL/mean": -190.4580078125, + "KL/rejected_KL_mean": -230.7960968017578, + "KL/std": 81.05520629882812, + "epoch": 0.5286343612334802, + "fcm_dpo/beta": 0.006197527050971985, + "fcm_dpo/delta": -0.05270250886678696, + "fcm_dpo/margin": 80.67617797851562, + "fcm_dpo/q_t": 0.3835641145706177, + "grad_norm": 12.480203628540039, + "learning_rate": 2.7051020734928443e-07, + "logits/chosen": -1.594376564025879, + "logits/rejected": -1.2482199668884277, + "logps/chosen": -225.3433837890625, + "logps/ref_chosen": -75.22344970703125, + "logps/ref_rejected": -76.09400177001953, + "logps/rejected": -306.89007568359375, + "loss": 1.0112, + "margin_dpo/margin_mean": 80.67617797851562, + "margin_dpo/margin_std": 84.2534408569336, + "step": 360 + }, + { + "KL/chosen_KL_mean": -153.8967742919922, + "KL/mean": -188.97772216796875, + "KL/rejected_KL_mean": -224.05865478515625, + "KL/std": 91.28170776367188, + "epoch": 0.5301027900146843, + "fcm_dpo/beta": 0.006206847261637449, + "fcm_dpo/delta": 0.015146945603191853, + "fcm_dpo/margin": 70.16188049316406, + "fcm_dpo/q_t": 0.4020695984363556, + "grad_norm": 11.036417007446289, + "learning_rate": 2.6923093854861593e-07, + "logits/chosen": -1.7453486919403076, + "logits/rejected": -1.5000078678131104, + "logps/chosen": -235.85011291503906, + "logps/ref_chosen": -81.95333099365234, + "logps/ref_rejected": -96.67919921875, + "logps/rejected": -320.73785400390625, + "loss": 1.0814, + "margin_dpo/margin_mean": 70.16187286376953, + "margin_dpo/margin_std": 97.12460327148438, + "step": 361 + }, + { + "KL/chosen_KL_mean": -158.99661254882812, + "KL/mean": -195.830078125, + "KL/rejected_KL_mean": -232.66351318359375, + "KL/std": 85.33515930175781, + "epoch": 0.5315712187958884, + "fcm_dpo/beta": 0.006188073195517063, + "fcm_dpo/delta": -0.006228707730770111, + "fcm_dpo/margin": 73.66690063476562, + "fcm_dpo/q_t": 0.39482009410858154, + "grad_norm": 10.10085678100586, + "learning_rate": 2.679511629948319e-07, + "logits/chosen": -1.9562934637069702, + "logits/rejected": -1.7549805641174316, + "logps/chosen": -248.33633422851562, + "logps/ref_chosen": -89.33971405029297, + "logps/ref_rejected": -110.50196838378906, + "logps/rejected": -343.16546630859375, + "loss": 1.055, + "margin_dpo/margin_mean": 73.66690063476562, + "margin_dpo/margin_std": 89.30767822265625, + "step": 362 + }, + { + "KL/chosen_KL_mean": -149.33816528320312, + "KL/mean": -196.967529296875, + "KL/rejected_KL_mean": -244.59689331054688, + "KL/std": 90.84678649902344, + "epoch": 0.5330396475770925, + "fcm_dpo/beta": 0.006099109537899494, + "fcm_dpo/delta": -0.13886189460754395, + "fcm_dpo/margin": 95.25874328613281, + "fcm_dpo/q_t": 0.36868971586227417, + "grad_norm": 11.294530868530273, + "learning_rate": 2.6667091441120816e-07, + "logits/chosen": -1.7525750398635864, + "logits/rejected": -1.3133634328842163, + "logps/chosen": -230.1131591796875, + "logps/ref_chosen": -80.77499389648438, + "logps/ref_rejected": -79.43267822265625, + "logps/rejected": -324.0295715332031, + "loss": 0.9702, + "margin_dpo/margin_mean": 95.25874328613281, + "margin_dpo/margin_std": 98.54278564453125, + "step": 363 + }, + { + "KL/chosen_KL_mean": -147.554931640625, + "KL/mean": -186.21170043945312, + "KL/rejected_KL_mean": -224.86843872070312, + "KL/std": 84.30770874023438, + "epoch": 0.5345080763582967, + "fcm_dpo/beta": 0.006051627919077873, + "fcm_dpo/delta": -0.01953640580177307, + "fcm_dpo/margin": 77.31351470947266, + "fcm_dpo/q_t": 0.39090150594711304, + "grad_norm": 12.098045349121094, + "learning_rate": 2.6539022653348575e-07, + "logits/chosen": -2.1194944381713867, + "logits/rejected": -1.9742317199707031, + "logps/chosen": -224.08453369140625, + "logps/ref_chosen": -76.52960205078125, + "logps/ref_rejected": -97.56173706054688, + "logps/rejected": -322.43017578125, + "loss": 1.0315, + "margin_dpo/margin_mean": 77.31351470947266, + "margin_dpo/margin_std": 79.7694091796875, + "step": 364 + }, + { + "KL/chosen_KL_mean": -143.73880004882812, + "KL/mean": -185.5823974609375, + "KL/rejected_KL_mean": -227.4259796142578, + "KL/std": 90.41179656982422, + "epoch": 0.5359765051395007, + "fcm_dpo/beta": 0.005980190820991993, + "fcm_dpo/delta": -0.053211845457553864, + "fcm_dpo/margin": 83.68717956542969, + "fcm_dpo/q_t": 0.38757264614105225, + "grad_norm": 9.37916088104248, + "learning_rate": 2.641091331089811e-07, + "logits/chosen": -1.7654857635498047, + "logits/rejected": -1.6367874145507812, + "logps/chosen": -224.28369140625, + "logps/ref_chosen": -80.54489135742188, + "logps/ref_rejected": -99.32316589355469, + "logps/rejected": -326.7491455078125, + "loss": 1.0336, + "margin_dpo/margin_mean": 83.68717956542969, + "margin_dpo/margin_std": 102.36875915527344, + "step": 365 + }, + { + "KL/chosen_KL_mean": -145.4033203125, + "KL/mean": -180.5133056640625, + "KL/rejected_KL_mean": -215.62332153320312, + "KL/std": 94.08183288574219, + "epoch": 0.5374449339207048, + "fcm_dpo/beta": 0.00597140658646822, + "fcm_dpo/delta": 0.03202168643474579, + "fcm_dpo/margin": 70.22000122070312, + "fcm_dpo/q_t": 0.4060773253440857, + "grad_norm": 9.141742706298828, + "learning_rate": 2.6282766789569736e-07, + "logits/chosen": -1.7277851104736328, + "logits/rejected": -1.7594456672668457, + "logps/chosen": -217.64138793945312, + "logps/ref_chosen": -72.23806762695312, + "logps/ref_rejected": -91.67182922363281, + "logps/rejected": -307.295166015625, + "loss": 1.0767, + "margin_dpo/margin_mean": 70.22000885009766, + "margin_dpo/margin_std": 91.1526870727539, + "step": 366 + }, + { + "KL/chosen_KL_mean": -132.71734619140625, + "KL/mean": -174.53428649902344, + "KL/rejected_KL_mean": -216.35122680664062, + "KL/std": 83.21417999267578, + "epoch": 0.5389133627019089, + "fcm_dpo/beta": 0.005939566530287266, + "fcm_dpo/delta": -0.04917112737894058, + "fcm_dpo/margin": 83.63388061523438, + "fcm_dpo/q_t": 0.38521942496299744, + "grad_norm": 8.794506072998047, + "learning_rate": 2.615458646614349e-07, + "logits/chosen": -1.76117742061615, + "logits/rejected": -1.4162061214447021, + "logps/chosen": -218.13934326171875, + "logps/ref_chosen": -85.42201232910156, + "logps/ref_rejected": -83.51779174804688, + "logps/rejected": -299.8690185546875, + "loss": 1.0102, + "margin_dpo/margin_mean": 83.63388061523438, + "margin_dpo/margin_std": 86.39166259765625, + "step": 367 + }, + { + "KL/chosen_KL_mean": -124.18267059326172, + "KL/mean": -170.06015014648438, + "KL/rejected_KL_mean": -215.93765258789062, + "KL/std": 91.82571411132812, + "epoch": 0.540381791483113, + "fcm_dpo/beta": 0.005866661202162504, + "fcm_dpo/delta": -0.09332215785980225, + "fcm_dpo/margin": 91.75496673583984, + "fcm_dpo/q_t": 0.3754565119743347, + "grad_norm": 9.531155586242676, + "learning_rate": 2.6026375718290083e-07, + "logits/chosen": -1.468907356262207, + "logits/rejected": -1.4333953857421875, + "logps/chosen": -198.0238037109375, + "logps/ref_chosen": -73.84112548828125, + "logps/ref_rejected": -104.69528198242188, + "logps/rejected": -320.6329345703125, + "loss": 0.992, + "margin_dpo/margin_mean": 91.75496673583984, + "margin_dpo/margin_std": 93.23069763183594, + "step": 368 + }, + { + "KL/chosen_KL_mean": -137.4363250732422, + "KL/mean": -181.03756713867188, + "KL/rejected_KL_mean": -224.63880920410156, + "KL/std": 84.01296997070312, + "epoch": 0.5418502202643172, + "fcm_dpo/beta": 0.005774751305580139, + "fcm_dpo/delta": -0.05638273060321808, + "fcm_dpo/margin": 87.20248413085938, + "fcm_dpo/q_t": 0.38302382826805115, + "grad_norm": 9.568777084350586, + "learning_rate": 2.589813792448196e-07, + "logits/chosen": -1.5823924541473389, + "logits/rejected": -1.2088656425476074, + "logps/chosen": -235.89828491210938, + "logps/ref_chosen": -98.46195983886719, + "logps/ref_rejected": -99.15672302246094, + "logps/rejected": -323.7955322265625, + "loss": 0.9992, + "margin_dpo/margin_mean": 87.20248413085938, + "margin_dpo/margin_std": 83.74822998046875, + "step": 369 + }, + { + "KL/chosen_KL_mean": -152.99267578125, + "KL/mean": -182.20016479492188, + "KL/rejected_KL_mean": -211.40765380859375, + "KL/std": 81.64815521240234, + "epoch": 0.5433186490455213, + "fcm_dpo/beta": 0.005848081316798925, + "fcm_dpo/delta": 0.11211700737476349, + "fcm_dpo/margin": 58.41497802734375, + "fcm_dpo/q_t": 0.4209824204444885, + "grad_norm": 9.319167137145996, + "learning_rate": 2.5769876463904263e-07, + "logits/chosen": -1.6919502019882202, + "logits/rejected": -1.4850552082061768, + "logps/chosen": -249.76840209960938, + "logps/ref_chosen": -96.77574157714844, + "logps/ref_rejected": -102.74411010742188, + "logps/rejected": -314.1517639160156, + "loss": 1.1455, + "margin_dpo/margin_mean": 58.41497802734375, + "margin_dpo/margin_std": 93.90602111816406, + "step": 370 + }, + { + "KL/chosen_KL_mean": -145.09376525878906, + "KL/mean": -177.24945068359375, + "KL/rejected_KL_mean": -209.40518188476562, + "KL/std": 82.22689819335938, + "epoch": 0.5447870778267254, + "fcm_dpo/beta": 0.005954510532319546, + "fcm_dpo/delta": 0.0693742036819458, + "fcm_dpo/margin": 64.31141662597656, + "fcm_dpo/q_t": 0.4113882780075073, + "grad_norm": 11.732169151306152, + "learning_rate": 2.5641594716365744e-07, + "logits/chosen": -1.8820791244506836, + "logits/rejected": -1.5166668891906738, + "logps/chosen": -248.08396911621094, + "logps/ref_chosen": -102.99020385742188, + "logps/ref_rejected": -105.35062408447266, + "logps/rejected": -314.75579833984375, + "loss": 1.1068, + "margin_dpo/margin_mean": 64.31141662597656, + "margin_dpo/margin_std": 88.78504943847656, + "step": 371 + }, + { + "KL/chosen_KL_mean": -124.0773696899414, + "KL/mean": -172.28167724609375, + "KL/rejected_KL_mean": -220.4859619140625, + "KL/std": 95.52281951904297, + "epoch": 0.5462555066079295, + "fcm_dpo/beta": 0.00585830956697464, + "fcm_dpo/delta": -0.12156160175800323, + "fcm_dpo/margin": 96.40857696533203, + "fcm_dpo/q_t": 0.3712029457092285, + "grad_norm": 8.286251068115234, + "learning_rate": 2.551329606220976e-07, + "logits/chosen": -2.0169198513031006, + "logits/rejected": -1.3986923694610596, + "logps/chosen": -215.10360717773438, + "logps/ref_chosen": -91.02623748779297, + "logps/ref_rejected": -85.6102294921875, + "logps/rejected": -306.09619140625, + "loss": 0.9733, + "margin_dpo/margin_mean": 96.40858459472656, + "margin_dpo/margin_std": 97.09080505371094, + "step": 372 + }, + { + "KL/chosen_KL_mean": -141.81610107421875, + "KL/mean": -177.65072631835938, + "KL/rejected_KL_mean": -213.48536682128906, + "KL/std": 90.250732421875, + "epoch": 0.5477239353891337, + "fcm_dpo/beta": 0.005838732700794935, + "fcm_dpo/delta": 0.032932497560977936, + "fcm_dpo/margin": 71.66928100585938, + "fcm_dpo/q_t": 0.4041418731212616, + "grad_norm": 10.926004409790039, + "learning_rate": 2.538498388222517e-07, + "logits/chosen": -2.1831014156341553, + "logits/rejected": -1.6671464443206787, + "logps/chosen": -241.88919067382812, + "logps/ref_chosen": -100.07307434082031, + "logps/ref_rejected": -91.11634826660156, + "logps/rejected": -304.6017150878906, + "loss": 1.0851, + "margin_dpo/margin_mean": 71.66926574707031, + "margin_dpo/margin_std": 96.1834945678711, + "step": 373 + }, + { + "KL/chosen_KL_mean": -124.3795166015625, + "KL/mean": -163.28689575195312, + "KL/rejected_KL_mean": -202.1942596435547, + "KL/std": 85.82447814941406, + "epoch": 0.5491923641703378, + "fcm_dpo/beta": 0.005836909636855125, + "fcm_dpo/delta": -0.004515616223216057, + "fcm_dpo/margin": 77.81475830078125, + "fcm_dpo/q_t": 0.3948688507080078, + "grad_norm": 8.858715057373047, + "learning_rate": 2.525666155755725e-07, + "logits/chosen": -2.284390449523926, + "logits/rejected": -1.8986577987670898, + "logps/chosen": -222.39186096191406, + "logps/ref_chosen": -98.01234436035156, + "logps/ref_rejected": -100.75224304199219, + "logps/rejected": -302.9465026855469, + "loss": 1.0365, + "margin_dpo/margin_mean": 77.81475830078125, + "margin_dpo/margin_std": 82.1981201171875, + "step": 374 + }, + { + "KL/chosen_KL_mean": -126.76860809326172, + "KL/mean": -171.35986328125, + "KL/rejected_KL_mean": -215.95111083984375, + "KL/std": 83.42938232421875, + "epoch": 0.5506607929515418, + "fcm_dpo/beta": 0.005751289427280426, + "fcm_dpo/delta": -0.06771738827228546, + "fcm_dpo/margin": 89.18248748779297, + "fcm_dpo/q_t": 0.3805190324783325, + "grad_norm": 9.776904106140137, + "learning_rate": 2.512833246961859e-07, + "logits/chosen": -1.632298231124878, + "logits/rejected": -1.4000362157821655, + "logps/chosen": -211.16488647460938, + "logps/ref_chosen": -84.39627838134766, + "logps/ref_rejected": -95.7188491821289, + "logps/rejected": -311.6699523925781, + "loss": 1.0013, + "margin_dpo/margin_mean": 89.18248748779297, + "margin_dpo/margin_std": 88.08191680908203, + "step": 375 + }, + { + "KL/chosen_KL_mean": -121.34154510498047, + "KL/mean": -170.11871337890625, + "KL/rejected_KL_mean": -218.89590454101562, + "KL/std": 94.6286392211914, + "epoch": 0.5521292217327459, + "fcm_dpo/beta": 0.005672710482031107, + "fcm_dpo/delta": -0.10955370217561722, + "fcm_dpo/margin": 97.55435180664062, + "fcm_dpo/q_t": 0.37314870953559875, + "grad_norm": 10.597599029541016, + "learning_rate": 2.5e-07, + "logits/chosen": -1.7344553470611572, + "logits/rejected": -1.422084093093872, + "logps/chosen": -217.07925415039062, + "logps/ref_chosen": -95.73770141601562, + "logps/ref_rejected": -111.97331237792969, + "logps/rejected": -330.86920166015625, + "loss": 0.9814, + "margin_dpo/margin_mean": 97.55435180664062, + "margin_dpo/margin_std": 100.21503448486328, + "step": 376 + }, + { + "KL/chosen_KL_mean": -129.27520751953125, + "KL/mean": -180.42337036132812, + "KL/rejected_KL_mean": -231.571533203125, + "KL/std": 96.63461303710938, + "epoch": 0.55359765051395, + "fcm_dpo/beta": 0.005520098842680454, + "fcm_dpo/delta": -0.12243510782718658, + "fcm_dpo/margin": 102.29632568359375, + "fcm_dpo/q_t": 0.36939406394958496, + "grad_norm": 8.624650001525879, + "learning_rate": 2.487166753038141e-07, + "logits/chosen": -1.474902629852295, + "logits/rejected": -1.3547477722167969, + "logps/chosen": -208.20216369628906, + "logps/ref_chosen": -78.92694854736328, + "logps/ref_rejected": -105.89892578125, + "logps/rejected": -337.470458984375, + "loss": 0.9649, + "margin_dpo/margin_mean": 102.29632568359375, + "margin_dpo/margin_std": 96.22500610351562, + "step": 377 + }, + { + "KL/chosen_KL_mean": -126.4132080078125, + "KL/mean": -167.10244750976562, + "KL/rejected_KL_mean": -207.79168701171875, + "KL/std": 91.60065460205078, + "epoch": 0.5550660792951542, + "fcm_dpo/beta": 0.005534189287573099, + "fcm_dpo/delta": -0.0008363649249076843, + "fcm_dpo/margin": 81.37847900390625, + "fcm_dpo/q_t": 0.39472198486328125, + "grad_norm": 10.012763977050781, + "learning_rate": 2.4743338442442754e-07, + "logits/chosen": -2.034902334213257, + "logits/rejected": -1.7558884620666504, + "logps/chosen": -201.78561401367188, + "logps/ref_chosen": -75.37240600585938, + "logps/ref_rejected": -93.70409393310547, + "logps/rejected": -301.49578857421875, + "loss": 1.0517, + "margin_dpo/margin_mean": 81.37847900390625, + "margin_dpo/margin_std": 93.79391479492188, + "step": 378 + }, + { + "KL/chosen_KL_mean": -147.85743713378906, + "KL/mean": -183.8848876953125, + "KL/rejected_KL_mean": -219.91232299804688, + "KL/std": 94.57432556152344, + "epoch": 0.5565345080763583, + "fcm_dpo/beta": 0.005548754706978798, + "fcm_dpo/delta": 0.05229198560118675, + "fcm_dpo/margin": 72.05487823486328, + "fcm_dpo/q_t": 0.41010457277297974, + "grad_norm": 9.807597160339355, + "learning_rate": 2.461501611777483e-07, + "logits/chosen": -1.740838646888733, + "logits/rejected": -1.659019112586975, + "logps/chosen": -236.71311950683594, + "logps/ref_chosen": -88.85568237304688, + "logps/ref_rejected": -121.38114166259766, + "logps/rejected": -341.29345703125, + "loss": 1.11, + "margin_dpo/margin_mean": 72.05488586425781, + "margin_dpo/margin_std": 108.07455444335938, + "step": 379 + }, + { + "KL/chosen_KL_mean": -162.68089294433594, + "KL/mean": -206.72476196289062, + "KL/rejected_KL_mean": -250.76861572265625, + "KL/std": 97.05766296386719, + "epoch": 0.5580029368575624, + "fcm_dpo/beta": 0.0055307662114501, + "fcm_dpo/delta": -0.03912016749382019, + "fcm_dpo/margin": 88.08770751953125, + "fcm_dpo/q_t": 0.3904613256454468, + "grad_norm": 13.091938018798828, + "learning_rate": 2.4486703937790243e-07, + "logits/chosen": -1.376621961593628, + "logits/rejected": -1.3481290340423584, + "logps/chosen": -242.51409912109375, + "logps/ref_chosen": -79.83321380615234, + "logps/ref_rejected": -110.7447509765625, + "logps/rejected": -361.51336669921875, + "loss": 1.0387, + "margin_dpo/margin_mean": 88.08770751953125, + "margin_dpo/margin_std": 109.23361206054688, + "step": 380 + }, + { + "KL/chosen_KL_mean": -142.77163696289062, + "KL/mean": -181.06393432617188, + "KL/rejected_KL_mean": -219.35623168945312, + "KL/std": 92.96479797363281, + "epoch": 0.5594713656387665, + "fcm_dpo/beta": 0.005546100903302431, + "fcm_dpo/delta": 0.026361431926488876, + "fcm_dpo/margin": 76.58457946777344, + "fcm_dpo/q_t": 0.40147024393081665, + "grad_norm": 10.48298454284668, + "learning_rate": 2.435840528363426e-07, + "logits/chosen": -1.6954889297485352, + "logits/rejected": -1.1373002529144287, + "logps/chosen": -229.33847045898438, + "logps/ref_chosen": -86.56683349609375, + "logps/ref_rejected": -84.416748046875, + "logps/rejected": -303.7729797363281, + "loss": 1.0714, + "margin_dpo/margin_mean": 76.5845947265625, + "margin_dpo/margin_std": 94.87086486816406, + "step": 381 + }, + { + "KL/chosen_KL_mean": -140.79415893554688, + "KL/mean": -184.33340454101562, + "KL/rejected_KL_mean": -227.87265014648438, + "KL/std": 91.3126220703125, + "epoch": 0.5609397944199707, + "fcm_dpo/beta": 0.005491263698786497, + "fcm_dpo/delta": -0.030609525740146637, + "fcm_dpo/margin": 87.07847595214844, + "fcm_dpo/q_t": 0.3894699811935425, + "grad_norm": 9.28026008605957, + "learning_rate": 2.4230123536095745e-07, + "logits/chosen": -1.9678525924682617, + "logits/rejected": -1.7035901546478271, + "logps/chosen": -245.25596618652344, + "logps/ref_chosen": -104.46180725097656, + "logps/ref_rejected": -114.60223388671875, + "logps/rejected": -342.4748840332031, + "loss": 1.0282, + "margin_dpo/margin_mean": 87.07847595214844, + "margin_dpo/margin_std": 92.77288818359375, + "step": 382 + }, + { + "KL/chosen_KL_mean": -152.9261932373047, + "KL/mean": -189.8676300048828, + "KL/rejected_KL_mean": -226.80906677246094, + "KL/std": 94.07464599609375, + "epoch": 0.5624082232011748, + "fcm_dpo/beta": 0.005541680380702019, + "fcm_dpo/delta": 0.04230116680264473, + "fcm_dpo/margin": 73.88288879394531, + "fcm_dpo/q_t": 0.4054419994354248, + "grad_norm": 10.031593322753906, + "learning_rate": 2.4101862075518037e-07, + "logits/chosen": -1.6223798990249634, + "logits/rejected": -1.500431776046753, + "logps/chosen": -235.04226684570312, + "logps/ref_chosen": -82.11607360839844, + "logps/ref_rejected": -98.77716064453125, + "logps/rejected": -325.58624267578125, + "loss": 1.0805, + "margin_dpo/margin_mean": 73.88288879394531, + "margin_dpo/margin_std": 91.41796875, + "step": 383 + }, + { + "KL/chosen_KL_mean": -163.90792846679688, + "KL/mean": -200.69241333007812, + "KL/rejected_KL_mean": -237.4769287109375, + "KL/std": 97.68135070800781, + "epoch": 0.5638766519823789, + "fcm_dpo/beta": 0.005634545348584652, + "fcm_dpo/delta": 0.035694241523742676, + "fcm_dpo/margin": 73.56901550292969, + "fcm_dpo/q_t": 0.4049611985683441, + "grad_norm": 11.313835144042969, + "learning_rate": 2.397362428170992e-07, + "logits/chosen": -1.7594760656356812, + "logits/rejected": -1.5330588817596436, + "logps/chosen": -244.4931640625, + "logps/ref_chosen": -80.58525085449219, + "logps/ref_rejected": -91.44789123535156, + "logps/rejected": -328.9248046875, + "loss": 1.1073, + "margin_dpo/margin_mean": 73.56900787353516, + "margin_dpo/margin_std": 107.26543426513672, + "step": 384 + }, + { + "KL/chosen_KL_mean": -136.58233642578125, + "KL/mean": -183.89329528808594, + "KL/rejected_KL_mean": -231.2042236328125, + "KL/std": 88.43669891357422, + "epoch": 0.5653450807635829, + "fcm_dpo/beta": 0.005509526468813419, + "fcm_dpo/delta": -0.07635048031806946, + "fcm_dpo/margin": 94.62188720703125, + "fcm_dpo/q_t": 0.38109469413757324, + "grad_norm": 10.951947212219238, + "learning_rate": 2.3845413533856514e-07, + "logits/chosen": -2.3123326301574707, + "logits/rejected": -1.7141015529632568, + "logps/chosen": -229.34251403808594, + "logps/ref_chosen": -92.76016235351562, + "logps/ref_rejected": -84.45997619628906, + "logps/rejected": -315.6641845703125, + "loss": 1.0071, + "margin_dpo/margin_mean": 94.62188720703125, + "margin_dpo/margin_std": 102.41546630859375, + "step": 385 + }, + { + "KL/chosen_KL_mean": -145.4132080078125, + "KL/mean": -190.88641357421875, + "KL/rejected_KL_mean": -236.35960388183594, + "KL/std": 90.42179870605469, + "epoch": 0.566813509544787, + "fcm_dpo/beta": 0.0054764775559306145, + "fcm_dpo/delta": -0.050601229071617126, + "fcm_dpo/margin": 90.94638061523438, + "fcm_dpo/q_t": 0.38333213329315186, + "grad_norm": 8.21135425567627, + "learning_rate": 2.3717233210430254e-07, + "logits/chosen": -1.616806983947754, + "logits/rejected": -1.330208420753479, + "logps/chosen": -230.7721710205078, + "logps/ref_chosen": -85.35896301269531, + "logps/ref_rejected": -96.99065399169922, + "logps/rejected": -333.3502502441406, + "loss": 1.0171, + "margin_dpo/margin_mean": 90.94638061523438, + "margin_dpo/margin_std": 97.93557739257812, + "step": 386 + }, + { + "KL/chosen_KL_mean": -168.45127868652344, + "KL/mean": -209.4113006591797, + "KL/rejected_KL_mean": -250.371337890625, + "KL/std": 87.11465454101562, + "epoch": 0.5682819383259912, + "fcm_dpo/beta": 0.0054756514728069305, + "fcm_dpo/delta": 0.0014210660010576248, + "fcm_dpo/margin": 81.92005920410156, + "fcm_dpo/q_t": 0.39575254917144775, + "grad_norm": 11.177667617797852, + "learning_rate": 2.3589086689101889e-07, + "logits/chosen": -2.4550271034240723, + "logits/rejected": -1.772486925125122, + "logps/chosen": -271.139404296875, + "logps/ref_chosen": -102.68814086914062, + "logps/ref_rejected": -99.42858123779297, + "logps/rejected": -349.7999267578125, + "loss": 1.0443, + "margin_dpo/margin_mean": 81.92005920410156, + "margin_dpo/margin_std": 89.9079818725586, + "step": 387 + }, + { + "KL/chosen_KL_mean": -170.07723999023438, + "KL/mean": -215.07505798339844, + "KL/rejected_KL_mean": -260.0728759765625, + "KL/std": 97.01229858398438, + "epoch": 0.5697503671071953, + "fcm_dpo/beta": 0.005427849944680929, + "fcm_dpo/delta": -0.04050945118069649, + "fcm_dpo/margin": 89.99563598632812, + "fcm_dpo/q_t": 0.3870543837547302, + "grad_norm": 9.495423316955566, + "learning_rate": 2.3460977346651428e-07, + "logits/chosen": -1.6890193223953247, + "logits/rejected": -1.5671308040618896, + "logps/chosen": -253.6624755859375, + "logps/ref_chosen": -83.58523559570312, + "logps/ref_rejected": -110.10037231445312, + "logps/rejected": -370.1732482910156, + "loss": 1.0242, + "margin_dpo/margin_mean": 89.99563598632812, + "margin_dpo/margin_std": 99.14422607421875, + "step": 388 + }, + { + "KL/chosen_KL_mean": -173.98683166503906, + "KL/mean": -213.3004608154297, + "KL/rejected_KL_mean": -252.61410522460938, + "KL/std": 99.34040832519531, + "epoch": 0.5712187958883994, + "fcm_dpo/beta": 0.005437172017991543, + "fcm_dpo/delta": 0.023501064628362656, + "fcm_dpo/margin": 78.62728881835938, + "fcm_dpo/q_t": 0.40241333842277527, + "grad_norm": 11.630664825439453, + "learning_rate": 2.3332908558879177e-07, + "logits/chosen": -2.2833778858184814, + "logits/rejected": -1.8254132270812988, + "logps/chosen": -263.80908203125, + "logps/ref_chosen": -89.822265625, + "logps/ref_rejected": -84.85376739501953, + "logps/rejected": -337.4678649902344, + "loss": 1.0825, + "margin_dpo/margin_mean": 78.62728881835938, + "margin_dpo/margin_std": 107.08319091796875, + "step": 389 + }, + { + "KL/chosen_KL_mean": -175.72840881347656, + "KL/mean": -212.7960205078125, + "KL/rejected_KL_mean": -249.86367797851562, + "KL/std": 96.76557159423828, + "epoch": 0.5726872246696035, + "fcm_dpo/beta": 0.005455967970192432, + "fcm_dpo/delta": 0.04710370674729347, + "fcm_dpo/margin": 74.13525390625, + "fcm_dpo/q_t": 0.41013091802597046, + "grad_norm": 9.654455184936523, + "learning_rate": 2.320488370051681e-07, + "logits/chosen": -1.3380204439163208, + "logits/rejected": -1.121154546737671, + "logps/chosen": -253.26084899902344, + "logps/ref_chosen": -77.53244018554688, + "logps/ref_rejected": -92.58392333984375, + "logps/rejected": -342.44757080078125, + "loss": 1.1081, + "margin_dpo/margin_mean": 74.13525390625, + "margin_dpo/margin_std": 110.93473815917969, + "step": 390 + }, + { + "KL/chosen_KL_mean": -158.9708709716797, + "KL/mean": -199.36349487304688, + "KL/rejected_KL_mean": -239.75613403320312, + "KL/std": 92.69353485107422, + "epoch": 0.5741556534508077, + "fcm_dpo/beta": 0.005487093701958656, + "fcm_dpo/delta": 0.006862584501504898, + "fcm_dpo/margin": 80.78526306152344, + "fcm_dpo/q_t": 0.39808547496795654, + "grad_norm": 9.405406951904297, + "learning_rate": 2.3076906145138405e-07, + "logits/chosen": -2.024125576019287, + "logits/rejected": -1.5786292552947998, + "logps/chosen": -247.20379638671875, + "logps/ref_chosen": -88.2329330444336, + "logps/ref_rejected": -93.00756072998047, + "logps/rejected": -332.7637023925781, + "loss": 1.0608, + "margin_dpo/margin_mean": 80.78526306152344, + "margin_dpo/margin_std": 98.21650695800781, + "step": 391 + }, + { + "KL/chosen_KL_mean": -161.13150024414062, + "KL/mean": -209.9306640625, + "KL/rejected_KL_mean": -258.7298583984375, + "KL/std": 103.97460174560547, + "epoch": 0.5756240822320118, + "fcm_dpo/beta": 0.005453658290207386, + "fcm_dpo/delta": -0.08683225512504578, + "fcm_dpo/margin": 97.59834289550781, + "fcm_dpo/q_t": 0.37842029333114624, + "grad_norm": 10.277584075927734, + "learning_rate": 2.294897926507156e-07, + "logits/chosen": -1.775519609451294, + "logits/rejected": -1.6575126647949219, + "logps/chosen": -248.87591552734375, + "logps/ref_chosen": -87.74442291259766, + "logps/ref_rejected": -108.57071685791016, + "logps/rejected": -367.300537109375, + "loss": 1.0096, + "margin_dpo/margin_mean": 97.59835052490234, + "margin_dpo/margin_std": 111.62520599365234, + "step": 392 + }, + { + "KL/chosen_KL_mean": -143.23472595214844, + "KL/mean": -187.03500366210938, + "KL/rejected_KL_mean": -230.83529663085938, + "KL/std": 98.00140380859375, + "epoch": 0.5770925110132159, + "fcm_dpo/beta": 0.005389312282204628, + "fcm_dpo/delta": -0.02322268858551979, + "fcm_dpo/margin": 87.60055541992188, + "fcm_dpo/q_t": 0.39091381430625916, + "grad_norm": 8.891741752624512, + "learning_rate": 2.2821106431308543e-07, + "logits/chosen": -1.800893783569336, + "logits/rejected": -1.484067678451538, + "logps/chosen": -219.7985382080078, + "logps/ref_chosen": -76.56381225585938, + "logps/ref_rejected": -79.72618103027344, + "logps/rejected": -310.56146240234375, + "loss": 1.0313, + "margin_dpo/margin_mean": 87.60055541992188, + "margin_dpo/margin_std": 96.00604248046875, + "step": 393 + }, + { + "KL/chosen_KL_mean": -151.06854248046875, + "KL/mean": -199.05972290039062, + "KL/rejected_KL_mean": -247.05091857910156, + "KL/std": 93.27932739257812, + "epoch": 0.57856093979442, + "fcm_dpo/beta": 0.005334894172847271, + "fcm_dpo/delta": -0.06539418548345566, + "fcm_dpo/margin": 95.98237609863281, + "fcm_dpo/q_t": 0.380257248878479, + "grad_norm": 9.333141326904297, + "learning_rate": 2.2693291013417452e-07, + "logits/chosen": -1.4963953495025635, + "logits/rejected": -1.2618308067321777, + "logps/chosen": -235.7672119140625, + "logps/ref_chosen": -84.69866180419922, + "logps/ref_rejected": -97.457763671875, + "logps/rejected": -344.5086669921875, + "loss": 0.9934, + "margin_dpo/margin_mean": 95.98236846923828, + "margin_dpo/margin_std": 90.62801361083984, + "step": 394 + }, + { + "KL/chosen_KL_mean": -156.96026611328125, + "KL/mean": -193.9337158203125, + "KL/rejected_KL_mean": -230.90713500976562, + "KL/std": 98.38123321533203, + "epoch": 0.580029368575624, + "fcm_dpo/beta": 0.005336286500096321, + "fcm_dpo/delta": 0.057480037212371826, + "fcm_dpo/margin": 73.94686889648438, + "fcm_dpo/q_t": 0.4133360981941223, + "grad_norm": 9.802746772766113, + "learning_rate": 2.2565536379453404e-07, + "logits/chosen": -2.467777729034424, + "logits/rejected": -2.0613045692443848, + "logps/chosen": -245.84791564941406, + "logps/ref_chosen": -88.88763427734375, + "logps/ref_rejected": -91.0455322265625, + "logps/rejected": -321.9526672363281, + "loss": 1.148, + "margin_dpo/margin_mean": 73.94686889648438, + "margin_dpo/margin_std": 132.66213989257812, + "step": 395 + }, + { + "KL/chosen_KL_mean": -166.8684844970703, + "KL/mean": -199.56668090820312, + "KL/rejected_KL_mean": -232.26490783691406, + "KL/std": 86.83419036865234, + "epoch": 0.5814977973568282, + "fcm_dpo/beta": 0.00545476283878088, + "fcm_dpo/delta": 0.09674517810344696, + "fcm_dpo/margin": 65.39639282226562, + "fcm_dpo/q_t": 0.4183182418346405, + "grad_norm": 12.155906677246094, + "learning_rate": 2.2437845895869825e-07, + "logits/chosen": -1.90079665184021, + "logits/rejected": -1.408474326133728, + "logps/chosen": -269.3398132324219, + "logps/ref_chosen": -102.47132873535156, + "logps/ref_rejected": -93.23478698730469, + "logps/rejected": -325.49969482421875, + "loss": 1.1341, + "margin_dpo/margin_mean": 65.39639282226562, + "margin_dpo/margin_std": 101.74411010742188, + "step": 396 + }, + { + "KL/chosen_KL_mean": -149.63766479492188, + "KL/mean": -208.14910888671875, + "KL/rejected_KL_mean": -266.6605529785156, + "KL/std": 100.67845153808594, + "epoch": 0.5829662261380323, + "fcm_dpo/beta": 0.00532365869730711, + "fcm_dpo/delta": -0.18493586778640747, + "fcm_dpo/margin": 117.02285766601562, + "fcm_dpo/q_t": 0.3586190342903137, + "grad_norm": 9.541089057922363, + "learning_rate": 2.2310222927429716e-07, + "logits/chosen": -2.185824394226074, + "logits/rejected": -1.5791468620300293, + "logps/chosen": -250.68209838867188, + "logps/ref_chosen": -101.04441833496094, + "logps/ref_rejected": -108.8511734008789, + "logps/rejected": -375.51171875, + "loss": 0.9507, + "margin_dpo/margin_mean": 117.02285766601562, + "margin_dpo/margin_std": 120.26101684570312, + "step": 397 + }, + { + "KL/chosen_KL_mean": -158.05035400390625, + "KL/mean": -200.77044677734375, + "KL/rejected_KL_mean": -243.4905548095703, + "KL/std": 98.89729309082031, + "epoch": 0.5844346549192364, + "fcm_dpo/beta": 0.00527543667703867, + "fcm_dpo/delta": -0.0008107442408800125, + "fcm_dpo/margin": 85.44017028808594, + "fcm_dpo/q_t": 0.39793872833251953, + "grad_norm": 8.96392822265625, + "learning_rate": 2.2182670837116972e-07, + "logits/chosen": -2.0579802989959717, + "logits/rejected": -1.7727158069610596, + "logps/chosen": -253.09957885742188, + "logps/ref_chosen": -95.04922485351562, + "logps/ref_rejected": -114.00563049316406, + "logps/rejected": -357.4961853027344, + "loss": 1.0733, + "margin_dpo/margin_mean": 85.44017791748047, + "margin_dpo/margin_std": 117.18036651611328, + "step": 398 + }, + { + "KL/chosen_KL_mean": -138.51805114746094, + "KL/mean": -192.81028747558594, + "KL/rejected_KL_mean": -247.10250854492188, + "KL/std": 89.64627838134766, + "epoch": 0.5859030837004405, + "fcm_dpo/beta": 0.005199803970754147, + "fcm_dpo/delta": -0.12131005525588989, + "fcm_dpo/margin": 108.58445739746094, + "fcm_dpo/q_t": 0.36873090267181396, + "grad_norm": 9.179891586303711, + "learning_rate": 2.2055192986047804e-07, + "logits/chosen": -2.100039005279541, + "logits/rejected": -1.2997509241104126, + "logps/chosen": -235.15660095214844, + "logps/ref_chosen": -96.63854217529297, + "logps/ref_rejected": -83.53645324707031, + "logps/rejected": -330.63897705078125, + "loss": 0.9565, + "margin_dpo/margin_mean": 108.5844497680664, + "margin_dpo/margin_std": 95.44795989990234, + "step": 399 + }, + { + "KL/chosen_KL_mean": -144.21548461914062, + "KL/mean": -199.53765869140625, + "KL/rejected_KL_mean": -254.85983276367188, + "KL/std": 99.18544006347656, + "epoch": 0.5873715124816447, + "fcm_dpo/beta": 0.005064649507403374, + "fcm_dpo/delta": -0.11699317395687103, + "fcm_dpo/margin": 110.64436340332031, + "fcm_dpo/q_t": 0.37149012088775635, + "grad_norm": 13.007590293884277, + "learning_rate": 2.192779273338215e-07, + "logits/chosen": -2.2159924507141113, + "logits/rejected": -1.7322194576263428, + "logps/chosen": -249.5186309814453, + "logps/ref_chosen": -105.30314636230469, + "logps/ref_rejected": -108.6555404663086, + "logps/rejected": -363.515380859375, + "loss": 0.98, + "margin_dpo/margin_mean": 110.64436340332031, + "margin_dpo/margin_std": 113.15621948242188, + "step": 400 + }, + { + "KL/chosen_KL_mean": -159.62994384765625, + "KL/mean": -194.17129516601562, + "KL/rejected_KL_mean": -228.71261596679688, + "KL/std": 99.43193817138672, + "epoch": 0.5888399412628488, + "fcm_dpo/beta": 0.005099102854728699, + "fcm_dpo/delta": 0.10143238306045532, + "fcm_dpo/margin": 69.08269500732422, + "fcm_dpo/q_t": 0.42251038551330566, + "grad_norm": 11.749171257019043, + "learning_rate": 2.1800473436235136e-07, + "logits/chosen": -1.5369623899459839, + "logits/rejected": -1.2931877374649048, + "logps/chosen": -243.9191131591797, + "logps/ref_chosen": -84.28916931152344, + "logps/ref_rejected": -90.943115234375, + "logps/rejected": -319.6557312011719, + "loss": 1.1523, + "margin_dpo/margin_mean": 69.08269500732422, + "margin_dpo/margin_std": 121.23318481445312, + "step": 401 + }, + { + "KL/chosen_KL_mean": -143.09613037109375, + "KL/mean": -207.28289794921875, + "KL/rejected_KL_mean": -271.46966552734375, + "KL/std": 106.4303207397461, + "epoch": 0.5903083700440529, + "fcm_dpo/beta": 0.004995099734514952, + "fcm_dpo/delta": -0.20406782627105713, + "fcm_dpo/margin": 128.37356567382812, + "fcm_dpo/q_t": 0.3507021963596344, + "grad_norm": 9.149144172668457, + "learning_rate": 2.1673238449588665e-07, + "logits/chosen": -1.9318034648895264, + "logits/rejected": -1.465431809425354, + "logps/chosen": -226.6892547607422, + "logps/ref_chosen": -83.59312438964844, + "logps/ref_rejected": -87.81027221679688, + "logps/rejected": -359.27996826171875, + "loss": 0.9069, + "margin_dpo/margin_mean": 128.37356567382812, + "margin_dpo/margin_std": 103.30931854248047, + "step": 402 + }, + { + "KL/chosen_KL_mean": -145.625244140625, + "KL/mean": -192.91964721679688, + "KL/rejected_KL_mean": -240.21401977539062, + "KL/std": 102.69251251220703, + "epoch": 0.591776798825257, + "fcm_dpo/beta": 0.004911697469651699, + "fcm_dpo/delta": -0.015448980033397675, + "fcm_dpo/margin": 94.58877563476562, + "fcm_dpo/q_t": 0.39277374744415283, + "grad_norm": 9.459317207336426, + "learning_rate": 2.154609112620295e-07, + "logits/chosen": -2.316864013671875, + "logits/rejected": -1.924619197845459, + "logps/chosen": -219.37832641601562, + "logps/ref_chosen": -73.75308227539062, + "logps/ref_rejected": -83.92012786865234, + "logps/rejected": -324.1341552734375, + "loss": 1.0434, + "margin_dpo/margin_mean": 94.58877563476562, + "margin_dpo/margin_std": 109.79576110839844, + "step": 403 + }, + { + "KL/chosen_KL_mean": -174.01333618164062, + "KL/mean": -214.6016387939453, + "KL/rejected_KL_mean": -255.18992614746094, + "KL/std": 98.91621398925781, + "epoch": 0.593245227606461, + "fcm_dpo/beta": 0.0049449363723397255, + "fcm_dpo/delta": 0.05062542110681534, + "fcm_dpo/margin": 81.17658233642578, + "fcm_dpo/q_t": 0.40871748328208923, + "grad_norm": 13.269368171691895, + "learning_rate": 2.1419034816528218e-07, + "logits/chosen": -2.0389392375946045, + "logits/rejected": -1.7034201622009277, + "logps/chosen": -253.68951416015625, + "logps/ref_chosen": -79.67617797851562, + "logps/ref_rejected": -84.280517578125, + "logps/rejected": -339.470458984375, + "loss": 1.1085, + "margin_dpo/margin_mean": 81.17658996582031, + "margin_dpo/margin_std": 121.17733764648438, + "step": 404 + }, + { + "KL/chosen_KL_mean": -176.22738647460938, + "KL/mean": -217.8411865234375, + "KL/rejected_KL_mean": -259.45501708984375, + "KL/std": 93.14665222167969, + "epoch": 0.5947136563876652, + "fcm_dpo/beta": 0.00502049271017313, + "fcm_dpo/delta": 0.032795634120702744, + "fcm_dpo/margin": 83.22763061523438, + "fcm_dpo/q_t": 0.4037052392959595, + "grad_norm": 8.550498008728027, + "learning_rate": 2.129207286861638e-07, + "logits/chosen": -1.6770999431610107, + "logits/rejected": -1.209843397140503, + "logps/chosen": -272.6893310546875, + "logps/ref_chosen": -96.46195220947266, + "logps/ref_rejected": -92.87071228027344, + "logps/rejected": -352.32574462890625, + "loss": 1.0703, + "margin_dpo/margin_mean": 83.22763061523438, + "margin_dpo/margin_std": 98.129150390625, + "step": 405 + }, + { + "KL/chosen_KL_mean": -166.81361389160156, + "KL/mean": -204.5878143310547, + "KL/rejected_KL_mean": -242.36203002929688, + "KL/std": 99.61241149902344, + "epoch": 0.5961820851688693, + "fcm_dpo/beta": 0.005056130699813366, + "fcm_dpo/delta": 0.07070192694664001, + "fcm_dpo/margin": 75.54840850830078, + "fcm_dpo/q_t": 0.41290178894996643, + "grad_norm": 16.26015281677246, + "learning_rate": 2.1165208628032861e-07, + "logits/chosen": -1.897040843963623, + "logits/rejected": -1.7272096872329712, + "logps/chosen": -244.94757080078125, + "logps/ref_chosen": -78.13396453857422, + "logps/ref_rejected": -98.28359985351562, + "logps/rejected": -340.6456298828125, + "loss": 1.1218, + "margin_dpo/margin_mean": 75.54840850830078, + "margin_dpo/margin_std": 116.21882629394531, + "step": 406 + }, + { + "KL/chosen_KL_mean": -174.8743896484375, + "KL/mean": -208.07913208007812, + "KL/rejected_KL_mean": -241.28387451171875, + "KL/std": 97.82916259765625, + "epoch": 0.5976505139500734, + "fcm_dpo/beta": 0.005158255808055401, + "fcm_dpo/delta": 0.11125224828720093, + "fcm_dpo/margin": 66.40948486328125, + "fcm_dpo/q_t": 0.4212542176246643, + "grad_norm": 10.842120170593262, + "learning_rate": 2.1038445437768375e-07, + "logits/chosen": -1.99392569065094, + "logits/rejected": -1.4557108879089355, + "logps/chosen": -258.88720703125, + "logps/ref_chosen": -84.01283264160156, + "logps/ref_rejected": -82.78103637695312, + "logps/rejected": -324.0649108886719, + "loss": 1.1406, + "margin_dpo/margin_mean": 66.40948486328125, + "margin_dpo/margin_std": 104.44827270507812, + "step": 407 + }, + { + "KL/chosen_KL_mean": -166.17868041992188, + "KL/mean": -199.85830688476562, + "KL/rejected_KL_mean": -233.53793334960938, + "KL/std": 92.88591003417969, + "epoch": 0.5991189427312775, + "fcm_dpo/beta": 0.0052575608715415, + "fcm_dpo/delta": 0.099473737180233, + "fcm_dpo/margin": 67.3592529296875, + "fcm_dpo/q_t": 0.4203840494155884, + "grad_norm": 10.597918510437012, + "learning_rate": 2.0911786638150872e-07, + "logits/chosen": -2.1638379096984863, + "logits/rejected": -1.534118890762329, + "logps/chosen": -270.64044189453125, + "logps/ref_chosen": -104.46174621582031, + "logps/ref_rejected": -96.37217712402344, + "logps/rejected": -329.91009521484375, + "loss": 1.149, + "margin_dpo/margin_mean": 67.3592529296875, + "margin_dpo/margin_std": 115.18268585205078, + "step": 408 + }, + { + "KL/chosen_KL_mean": -177.18463134765625, + "KL/mean": -211.97927856445312, + "KL/rejected_KL_mean": -246.77394104003906, + "KL/std": 96.11148071289062, + "epoch": 0.6005873715124816, + "fcm_dpo/beta": 0.0053542363457381725, + "fcm_dpo/delta": 0.080415278673172, + "fcm_dpo/margin": 69.58931732177734, + "fcm_dpo/q_t": 0.4129691421985626, + "grad_norm": 9.510979652404785, + "learning_rate": 2.0785235566757517e-07, + "logits/chosen": -1.4348442554473877, + "logits/rejected": -0.9889636635780334, + "logps/chosen": -274.8529357910156, + "logps/ref_chosen": -97.66830444335938, + "logps/ref_rejected": -90.04583740234375, + "logps/rejected": -336.81976318359375, + "loss": 1.1267, + "margin_dpo/margin_mean": 69.58930969238281, + "margin_dpo/margin_std": 105.79730224609375, + "step": 409 + }, + { + "KL/chosen_KL_mean": -172.36636352539062, + "KL/mean": -214.38494873046875, + "KL/rejected_KL_mean": -256.40350341796875, + "KL/std": 101.18763732910156, + "epoch": 0.6020558002936858, + "fcm_dpo/beta": 0.005375073757022619, + "fcm_dpo/delta": -0.0017874417826533318, + "fcm_dpo/margin": 84.03714752197266, + "fcm_dpo/q_t": 0.3948698043823242, + "grad_norm": 9.325094223022461, + "learning_rate": 2.065879555832674e-07, + "logits/chosen": -1.847787857055664, + "logits/rejected": -1.7182414531707764, + "logps/chosen": -248.83560180664062, + "logps/ref_chosen": -76.46923828125, + "logps/ref_rejected": -88.64064025878906, + "logps/rejected": -345.0441589355469, + "loss": 1.0554, + "margin_dpo/margin_mean": 84.03715515136719, + "margin_dpo/margin_std": 102.20069885253906, + "step": 410 + }, + { + "KL/chosen_KL_mean": -183.931640625, + "KL/mean": -223.20455932617188, + "KL/rejected_KL_mean": -262.4775390625, + "KL/std": 93.54214477539062, + "epoch": 0.6035242290748899, + "fcm_dpo/beta": 0.005380285903811455, + "fcm_dpo/delta": 0.02847486548125744, + "fcm_dpo/margin": 78.54588317871094, + "fcm_dpo/q_t": 0.40412479639053345, + "grad_norm": 9.493480682373047, + "learning_rate": 2.0532469944670343e-07, + "logits/chosen": -2.276109218597412, + "logits/rejected": -1.8670759201049805, + "logps/chosen": -271.0979309082031, + "logps/ref_chosen": -87.16630554199219, + "logps/ref_rejected": -87.09603118896484, + "logps/rejected": -349.57354736328125, + "loss": 1.077, + "margin_dpo/margin_mean": 78.54588317871094, + "margin_dpo/margin_std": 100.51069641113281, + "step": 411 + }, + { + "KL/chosen_KL_mean": -174.45590209960938, + "KL/mean": -213.2197723388672, + "KL/rejected_KL_mean": -251.983642578125, + "KL/std": 94.68341064453125, + "epoch": 0.604992657856094, + "fcm_dpo/beta": 0.005417516455054283, + "fcm_dpo/delta": 0.03125259652733803, + "fcm_dpo/margin": 77.52774047851562, + "fcm_dpo/q_t": 0.40670841932296753, + "grad_norm": 11.001174926757812, + "learning_rate": 2.0406262054585738e-07, + "logits/chosen": -2.3869190216064453, + "logits/rejected": -2.1678786277770996, + "logps/chosen": -253.40322875976562, + "logps/ref_chosen": -78.94734191894531, + "logps/ref_rejected": -106.10554504394531, + "logps/rejected": -358.08917236328125, + "loss": 1.1111, + "margin_dpo/margin_mean": 77.52774047851562, + "margin_dpo/margin_std": 122.05264282226562, + "step": 412 + }, + { + "KL/chosen_KL_mean": -187.71133422851562, + "KL/mean": -230.51333618164062, + "KL/rejected_KL_mean": -273.3153076171875, + "KL/std": 99.9842529296875, + "epoch": 0.6064610866372981, + "fcm_dpo/beta": 0.005409596022218466, + "fcm_dpo/delta": -0.014023840427398682, + "fcm_dpo/margin": 85.6039810180664, + "fcm_dpo/q_t": 0.3939126431941986, + "grad_norm": 10.571904182434082, + "learning_rate": 2.0280175213768205e-07, + "logits/chosen": -1.0728927850723267, + "logits/rejected": -0.8086060285568237, + "logps/chosen": -283.40606689453125, + "logps/ref_chosen": -95.69470977783203, + "logps/ref_rejected": -107.96085357666016, + "logps/rejected": -381.27618408203125, + "loss": 1.069, + "margin_dpo/margin_mean": 85.60398864746094, + "margin_dpo/margin_std": 117.49549102783203, + "step": 413 + }, + { + "KL/chosen_KL_mean": -172.75274658203125, + "KL/mean": -212.79336547851562, + "KL/rejected_KL_mean": -252.833984375, + "KL/std": 96.98312377929688, + "epoch": 0.6079295154185022, + "fcm_dpo/beta": 0.005443980917334557, + "fcm_dpo/delta": 0.014622347429394722, + "fcm_dpo/margin": 80.08126831054688, + "fcm_dpo/q_t": 0.40238139033317566, + "grad_norm": 14.67659854888916, + "learning_rate": 2.0154212744723247e-07, + "logits/chosen": -1.318266749382019, + "logits/rejected": -0.9448731541633606, + "logps/chosen": -261.0294189453125, + "logps/ref_chosen": -88.27667236328125, + "logps/ref_rejected": -92.87004089355469, + "logps/rejected": -345.70404052734375, + "loss": 1.1017, + "margin_dpo/margin_mean": 80.08126831054688, + "margin_dpo/margin_std": 123.9937973022461, + "step": 414 + }, + { + "KL/chosen_KL_mean": -187.19625854492188, + "KL/mean": -220.89715576171875, + "KL/rejected_KL_mean": -254.5980682373047, + "KL/std": 87.36563873291016, + "epoch": 0.6093979441997063, + "fcm_dpo/beta": 0.005514100193977356, + "fcm_dpo/delta": 0.08122564852237701, + "fcm_dpo/margin": 67.40179443359375, + "fcm_dpo/q_t": 0.41569462418556213, + "grad_norm": 12.043018341064453, + "learning_rate": 2.002837796667909e-07, + "logits/chosen": -1.5819894075393677, + "logits/rejected": -1.2009024620056152, + "logps/chosen": -296.1121826171875, + "logps/ref_chosen": -108.91590118408203, + "logps/ref_rejected": -107.47135925292969, + "logps/rejected": -362.0694274902344, + "loss": 1.1328, + "margin_dpo/margin_mean": 67.40178680419922, + "margin_dpo/margin_std": 107.94573974609375, + "step": 415 + }, + { + "KL/chosen_KL_mean": -185.39361572265625, + "KL/mean": -232.14376831054688, + "KL/rejected_KL_mean": -278.8938903808594, + "KL/std": 102.91915893554688, + "epoch": 0.6108663729809104, + "fcm_dpo/beta": 0.005518518388271332, + "fcm_dpo/delta": -0.07041682302951813, + "fcm_dpo/margin": 93.50028228759766, + "fcm_dpo/q_t": 0.38325217366218567, + "grad_norm": 9.76031494140625, + "learning_rate": 1.990267419549914e-07, + "logits/chosen": -1.8252341747283936, + "logits/rejected": -1.321199655532837, + "logps/chosen": -278.79248046875, + "logps/ref_chosen": -93.39888000488281, + "logps/ref_rejected": -97.6729736328125, + "logps/rejected": -376.56689453125, + "loss": 1.025, + "margin_dpo/margin_mean": 93.50028228759766, + "margin_dpo/margin_std": 108.26390075683594, + "step": 416 + }, + { + "KL/chosen_KL_mean": -172.47686767578125, + "KL/mean": -216.36038208007812, + "KL/rejected_KL_mean": -260.24383544921875, + "KL/std": 95.44965362548828, + "epoch": 0.6123348017621145, + "fcm_dpo/beta": 0.005436611361801624, + "fcm_dpo/delta": -0.028513753786683083, + "fcm_dpo/margin": 87.76699829101562, + "fcm_dpo/q_t": 0.38996344804763794, + "grad_norm": 9.729513168334961, + "learning_rate": 1.9777104743594686e-07, + "logits/chosen": -2.2194457054138184, + "logits/rejected": -1.305099606513977, + "logps/chosen": -256.01220703125, + "logps/ref_chosen": -83.53533172607422, + "logps/ref_rejected": -74.44184875488281, + "logps/rejected": -334.6856994628906, + "loss": 1.0323, + "margin_dpo/margin_mean": 87.76699829101562, + "margin_dpo/margin_std": 98.33658599853516, + "step": 417 + }, + { + "KL/chosen_KL_mean": -165.25379943847656, + "KL/mean": -219.8912353515625, + "KL/rejected_KL_mean": -274.5286560058594, + "KL/std": 102.3167724609375, + "epoch": 0.6138032305433186, + "fcm_dpo/beta": 0.005361407995223999, + "fcm_dpo/delta": -0.14499600231647491, + "fcm_dpo/margin": 109.27485656738281, + "fcm_dpo/q_t": 0.3664282560348511, + "grad_norm": 11.458939552307129, + "learning_rate": 1.965167291983757e-07, + "logits/chosen": -2.1761512756347656, + "logits/rejected": -1.5184229612350464, + "logps/chosen": -273.475341796875, + "logps/ref_chosen": -108.22152709960938, + "logps/ref_rejected": -111.8646469116211, + "logps/rejected": -386.393310546875, + "loss": 0.9592, + "margin_dpo/margin_mean": 109.27485656738281, + "margin_dpo/margin_std": 99.99575805664062, + "step": 418 + }, + { + "KL/chosen_KL_mean": -171.79110717773438, + "KL/mean": -219.5113525390625, + "KL/rejected_KL_mean": -267.2315979003906, + "KL/std": 91.43870544433594, + "epoch": 0.6152716593245228, + "fcm_dpo/beta": 0.005250250920653343, + "fcm_dpo/delta": -0.054010555148124695, + "fcm_dpo/margin": 95.44049072265625, + "fcm_dpo/q_t": 0.38451141119003296, + "grad_norm": 11.30584716796875, + "learning_rate": 1.9526382029472988e-07, + "logits/chosen": -1.195124626159668, + "logits/rejected": -0.7267280220985413, + "logps/chosen": -268.9743957519531, + "logps/ref_chosen": -97.18328094482422, + "logps/ref_rejected": -98.18531799316406, + "logps/rejected": -365.41693115234375, + "loss": 1.0139, + "margin_dpo/margin_mean": 95.44049072265625, + "margin_dpo/margin_std": 100.7812271118164, + "step": 419 + }, + { + "KL/chosen_KL_mean": -176.08578491210938, + "KL/mean": -218.35128784179688, + "KL/rejected_KL_mean": -260.61676025390625, + "KL/std": 96.43272399902344, + "epoch": 0.6167400881057269, + "fcm_dpo/beta": 0.0052343616262078285, + "fcm_dpo/delta": 0.007600046694278717, + "fcm_dpo/margin": 84.53096771240234, + "fcm_dpo/q_t": 0.3991864323616028, + "grad_norm": 15.280583381652832, + "learning_rate": 1.9401235374032425e-07, + "logits/chosen": -2.2325878143310547, + "logits/rejected": -1.1336956024169922, + "logps/chosen": -290.394287109375, + "logps/ref_chosen": -114.30848693847656, + "logps/ref_rejected": -75.68355560302734, + "logps/rejected": -336.3003234863281, + "loss": 1.0709, + "margin_dpo/margin_mean": 84.53097534179688, + "margin_dpo/margin_std": 110.53395080566406, + "step": 420 + }, + { + "KL/chosen_KL_mean": -168.23854064941406, + "KL/mean": -215.48219299316406, + "KL/rejected_KL_mean": -262.725830078125, + "KL/std": 98.87498474121094, + "epoch": 0.618208516886931, + "fcm_dpo/beta": 0.0051781190559268, + "fcm_dpo/delta": -0.041393768042325974, + "fcm_dpo/margin": 94.48728942871094, + "fcm_dpo/q_t": 0.387340784072876, + "grad_norm": 9.249161720275879, + "learning_rate": 1.9276236251246653e-07, + "logits/chosen": -1.8020110130310059, + "logits/rejected": -1.4270470142364502, + "logps/chosen": -254.11839294433594, + "logps/ref_chosen": -85.87985229492188, + "logps/ref_rejected": -96.33648681640625, + "logps/rejected": -359.06231689453125, + "loss": 1.0307, + "margin_dpo/margin_mean": 94.48728942871094, + "margin_dpo/margin_std": 108.87814331054688, + "step": 421 + }, + { + "KL/chosen_KL_mean": -165.93563842773438, + "KL/mean": -210.9785919189453, + "KL/rejected_KL_mean": -256.02154541015625, + "KL/std": 91.84327697753906, + "epoch": 0.6196769456681351, + "fcm_dpo/beta": 0.005169377662241459, + "fcm_dpo/delta": -0.01647116057574749, + "fcm_dpo/margin": 90.0859375, + "fcm_dpo/q_t": 0.39310258626937866, + "grad_norm": 12.468792915344238, + "learning_rate": 1.9151387954958792e-07, + "logits/chosen": -2.383885383605957, + "logits/rejected": -2.015259265899658, + "logps/chosen": -266.4162292480469, + "logps/ref_chosen": -100.48060607910156, + "logps/ref_rejected": -94.40821838378906, + "logps/rejected": -350.4297790527344, + "loss": 1.0407, + "margin_dpo/margin_mean": 90.0859375, + "margin_dpo/margin_std": 104.03911590576172, + "step": 422 + }, + { + "KL/chosen_KL_mean": -153.79103088378906, + "KL/mean": -199.28440856933594, + "KL/rejected_KL_mean": -244.77780151367188, + "KL/std": 109.79222869873047, + "epoch": 0.6211453744493393, + "fcm_dpo/beta": 0.005162882152944803, + "fcm_dpo/delta": -0.020984144881367683, + "fcm_dpo/margin": 90.98677062988281, + "fcm_dpo/q_t": 0.3953096270561218, + "grad_norm": 11.123176574707031, + "learning_rate": 1.902669377503756e-07, + "logits/chosen": -1.9816217422485352, + "logits/rejected": -1.8644654750823975, + "logps/chosen": -232.240966796875, + "logps/ref_chosen": -78.44993591308594, + "logps/ref_rejected": -92.04652404785156, + "logps/rejected": -336.8243408203125, + "loss": 1.0673, + "margin_dpo/margin_mean": 90.98677062988281, + "margin_dpo/margin_std": 124.88569641113281, + "step": 423 + }, + { + "KL/chosen_KL_mean": -173.92189025878906, + "KL/mean": -218.35202026367188, + "KL/rejected_KL_mean": -262.7821350097656, + "KL/std": 97.39276123046875, + "epoch": 0.6226138032305433, + "fcm_dpo/beta": 0.005129658617079258, + "fcm_dpo/delta": -0.006105954758822918, + "fcm_dpo/margin": 88.8602523803711, + "fcm_dpo/q_t": 0.39420658349990845, + "grad_norm": 9.193302154541016, + "learning_rate": 1.890215699729057e-07, + "logits/chosen": -1.7918660640716553, + "logits/rejected": -1.199621319770813, + "logps/chosen": -261.5642395019531, + "logps/ref_chosen": -87.64236450195312, + "logps/ref_rejected": -72.36566162109375, + "logps/rejected": -335.1477966308594, + "loss": 1.0516, + "margin_dpo/margin_mean": 88.86026000976562, + "margin_dpo/margin_std": 106.21658325195312, + "step": 424 + }, + { + "KL/chosen_KL_mean": -179.52130126953125, + "KL/mean": -210.2912139892578, + "KL/rejected_KL_mean": -241.06114196777344, + "KL/std": 100.02275085449219, + "epoch": 0.6240822320117474, + "fcm_dpo/beta": 0.00522254966199398, + "fcm_dpo/delta": 0.133060485124588, + "fcm_dpo/margin": 61.53984451293945, + "fcm_dpo/q_t": 0.42998671531677246, + "grad_norm": 11.00718879699707, + "learning_rate": 1.8777780903377732e-07, + "logits/chosen": -1.6064822673797607, + "logits/rejected": -1.5371153354644775, + "logps/chosen": -258.04107666015625, + "logps/ref_chosen": -78.51979064941406, + "logps/ref_rejected": -102.74864196777344, + "logps/rejected": -343.8097839355469, + "loss": 1.1894, + "margin_dpo/margin_mean": 61.53984832763672, + "margin_dpo/margin_std": 124.87500762939453, + "step": 425 + }, + { + "KL/chosen_KL_mean": -153.57254028320312, + "KL/mean": -198.11199951171875, + "KL/rejected_KL_mean": -242.65147399902344, + "KL/std": 92.34453582763672, + "epoch": 0.6255506607929515, + "fcm_dpo/beta": 0.005266373511403799, + "fcm_dpo/delta": -0.020152367651462555, + "fcm_dpo/margin": 89.07894897460938, + "fcm_dpo/q_t": 0.39211148023605347, + "grad_norm": 12.84345817565918, + "learning_rate": 1.8653568770724803e-07, + "logits/chosen": -1.8006620407104492, + "logits/rejected": -1.075664758682251, + "logps/chosen": -262.078369140625, + "logps/ref_chosen": -108.50582885742188, + "logps/ref_rejected": -88.300048828125, + "logps/rejected": -330.9515380859375, + "loss": 1.0458, + "margin_dpo/margin_mean": 89.07894897460938, + "margin_dpo/margin_std": 106.60686492919922, + "step": 426 + }, + { + "KL/chosen_KL_mean": -126.95327758789062, + "KL/mean": -168.5754852294922, + "KL/rejected_KL_mean": -210.19769287109375, + "KL/std": 88.51153564453125, + "epoch": 0.6270190895741556, + "fcm_dpo/beta": 0.005263722501695156, + "fcm_dpo/delta": 0.012314190156757832, + "fcm_dpo/margin": 83.24441528320312, + "fcm_dpo/q_t": 0.3982256352901459, + "grad_norm": 8.986059188842773, + "learning_rate": 1.8529523872436977e-07, + "logits/chosen": -2.4963278770446777, + "logits/rejected": -1.872001051902771, + "logps/chosen": -226.07374572753906, + "logps/ref_chosen": -99.12046813964844, + "logps/ref_rejected": -85.724609375, + "logps/rejected": -295.92230224609375, + "loss": 1.0543, + "margin_dpo/margin_mean": 83.24441528320312, + "margin_dpo/margin_std": 95.05308532714844, + "step": 427 + }, + { + "KL/chosen_KL_mean": -179.4477996826172, + "KL/mean": -214.71096801757812, + "KL/rejected_KL_mean": -249.97412109375, + "KL/std": 100.2245101928711, + "epoch": 0.6284875183553598, + "fcm_dpo/beta": 0.005296891555190086, + "fcm_dpo/delta": 0.07929681241512299, + "fcm_dpo/margin": 70.52631378173828, + "fcm_dpo/q_t": 0.4152657389640808, + "grad_norm": 9.958909034729004, + "learning_rate": 1.8405649477212697e-07, + "logits/chosen": -1.7457547187805176, + "logits/rejected": -1.4865880012512207, + "logps/chosen": -285.41705322265625, + "logps/ref_chosen": -105.96925354003906, + "logps/ref_rejected": -109.10218048095703, + "logps/rejected": -359.0762939453125, + "loss": 1.1305, + "margin_dpo/margin_mean": 70.52631378173828, + "margin_dpo/margin_std": 110.92942810058594, + "step": 428 + }, + { + "KL/chosen_KL_mean": -164.07260131835938, + "KL/mean": -209.92584228515625, + "KL/rejected_KL_mean": -255.77906799316406, + "KL/std": 103.61181640625, + "epoch": 0.6299559471365639, + "fcm_dpo/beta": 0.005351818632334471, + "fcm_dpo/delta": -0.04383639991283417, + "fcm_dpo/margin": 91.70643615722656, + "fcm_dpo/q_t": 0.3899858593940735, + "grad_norm": 9.219000816345215, + "learning_rate": 1.828194884925749e-07, + "logits/chosen": -1.9351677894592285, + "logits/rejected": -1.217651605606079, + "logps/chosen": -277.6174621582031, + "logps/ref_chosen": -113.54487609863281, + "logps/ref_rejected": -98.24201965332031, + "logps/rejected": -354.02105712890625, + "loss": 1.0496, + "margin_dpo/margin_mean": 91.70643615722656, + "margin_dpo/margin_std": 116.55728149414062, + "step": 429 + }, + { + "KL/chosen_KL_mean": -157.97642517089844, + "KL/mean": -192.23570251464844, + "KL/rejected_KL_mean": -226.4949951171875, + "KL/std": 96.0548324584961, + "epoch": 0.631424375917768, + "fcm_dpo/beta": 0.0053818728774785995, + "fcm_dpo/delta": 0.0838533490896225, + "fcm_dpo/margin": 68.51856994628906, + "fcm_dpo/q_t": 0.41684645414352417, + "grad_norm": 9.969901084899902, + "learning_rate": 1.8158425248197928e-07, + "logits/chosen": -1.4322922229766846, + "logits/rejected": -1.3960630893707275, + "logps/chosen": -249.29579162597656, + "logps/ref_chosen": -91.31936645507812, + "logps/ref_rejected": -110.1096420288086, + "logps/rejected": -336.6046447753906, + "loss": 1.1364, + "margin_dpo/margin_mean": 68.51856994628906, + "margin_dpo/margin_std": 111.74356079101562, + "step": 430 + }, + { + "KL/chosen_KL_mean": -158.5358123779297, + "KL/mean": -199.85360717773438, + "KL/rejected_KL_mean": -241.1714324951172, + "KL/std": 98.350341796875, + "epoch": 0.6328928046989721, + "fcm_dpo/beta": 0.005383031442761421, + "fcm_dpo/delta": 0.00536933820694685, + "fcm_dpo/margin": 82.63562774658203, + "fcm_dpo/q_t": 0.3984500765800476, + "grad_norm": 11.174623489379883, + "learning_rate": 1.8035081928995788e-07, + "logits/chosen": -1.9584853649139404, + "logits/rejected": -1.574393630027771, + "logps/chosen": -251.71702575683594, + "logps/ref_chosen": -93.18122100830078, + "logps/ref_rejected": -98.13226318359375, + "logps/rejected": -339.3037109375, + "loss": 1.0751, + "margin_dpo/margin_mean": 82.63563537597656, + "margin_dpo/margin_std": 111.85409545898438, + "step": 431 + }, + { + "KL/chosen_KL_mean": -138.10623168945312, + "KL/mean": -188.77899169921875, + "KL/rejected_KL_mean": -239.4517364501953, + "KL/std": 103.22248840332031, + "epoch": 0.6343612334801763, + "fcm_dpo/beta": 0.005334785208106041, + "fcm_dpo/delta": -0.09574855864048004, + "fcm_dpo/margin": 101.34549713134766, + "fcm_dpo/q_t": 0.3753151297569275, + "grad_norm": 9.233789443969727, + "learning_rate": 1.791192214186223e-07, + "logits/chosen": -1.7117465734481812, + "logits/rejected": -1.1737985610961914, + "logps/chosen": -242.541015625, + "logps/ref_chosen": -104.43478393554688, + "logps/ref_rejected": -105.08955383300781, + "logps/rejected": -344.5412902832031, + "loss": 0.9866, + "margin_dpo/margin_mean": 101.34548950195312, + "margin_dpo/margin_std": 99.7818603515625, + "step": 432 + }, + { + "KL/chosen_KL_mean": -165.06680297851562, + "KL/mean": -204.61166381835938, + "KL/rejected_KL_mean": -244.15652465820312, + "KL/std": 96.80538940429688, + "epoch": 0.6358296622613803, + "fcm_dpo/beta": 0.005340633448213339, + "fcm_dpo/delta": 0.028125185519456863, + "fcm_dpo/margin": 79.08970642089844, + "fcm_dpo/q_t": 0.40405088663101196, + "grad_norm": 9.684664726257324, + "learning_rate": 1.7788949132172193e-07, + "logits/chosen": -1.4942045211791992, + "logits/rejected": -1.1620519161224365, + "logps/chosen": -254.91001892089844, + "logps/ref_chosen": -89.84322357177734, + "logps/ref_rejected": -101.73345947265625, + "logps/rejected": -345.8899841308594, + "loss": 1.0911, + "margin_dpo/margin_mean": 79.08970642089844, + "margin_dpo/margin_std": 110.08870697021484, + "step": 433 + }, + { + "KL/chosen_KL_mean": -144.42999267578125, + "KL/mean": -189.54330444335938, + "KL/rejected_KL_mean": -234.6566162109375, + "KL/std": 107.36045837402344, + "epoch": 0.6372980910425844, + "fcm_dpo/beta": 0.005302520468831062, + "fcm_dpo/delta": -0.029848497360944748, + "fcm_dpo/margin": 90.22663879394531, + "fcm_dpo/q_t": 0.3948814868927002, + "grad_norm": 12.283258438110352, + "learning_rate": 1.7666166140378853e-07, + "logits/chosen": -2.553473949432373, + "logits/rejected": -1.9006510972976685, + "logps/chosen": -242.12252807617188, + "logps/ref_chosen": -97.6925277709961, + "logps/ref_rejected": -84.09129333496094, + "logps/rejected": -318.7479248046875, + "loss": 1.0594, + "margin_dpo/margin_mean": 90.22663879394531, + "margin_dpo/margin_std": 124.39888763427734, + "step": 434 + }, + { + "KL/chosen_KL_mean": -142.01278686523438, + "KL/mean": -187.8046875, + "KL/rejected_KL_mean": -233.59661865234375, + "KL/std": 99.80178833007812, + "epoch": 0.6387665198237885, + "fcm_dpo/beta": 0.005268946290016174, + "fcm_dpo/delta": -0.03419119864702225, + "fcm_dpo/margin": 91.58380126953125, + "fcm_dpo/q_t": 0.39059340953826904, + "grad_norm": 10.709205627441406, + "learning_rate": 1.7543576401928218e-07, + "logits/chosen": -2.3253371715545654, + "logits/rejected": -1.870253086090088, + "logps/chosen": -228.18472290039062, + "logps/ref_chosen": -86.17192077636719, + "logps/ref_rejected": -93.751708984375, + "logps/rejected": -327.34832763671875, + "loss": 1.0374, + "margin_dpo/margin_mean": 91.58380126953125, + "margin_dpo/margin_std": 109.91569519042969, + "step": 435 + }, + { + "KL/chosen_KL_mean": -118.00169372558594, + "KL/mean": -166.27487182617188, + "KL/rejected_KL_mean": -214.5480194091797, + "KL/std": 85.56976318359375, + "epoch": 0.6402349486049926, + "fcm_dpo/beta": 0.005205495283007622, + "fcm_dpo/delta": -0.05541032925248146, + "fcm_dpo/margin": 96.54632568359375, + "fcm_dpo/q_t": 0.38232114911079407, + "grad_norm": 10.138195037841797, + "learning_rate": 1.742118314717391e-07, + "logits/chosen": -1.767169713973999, + "logits/rejected": -0.9013456106185913, + "logps/chosen": -223.78878784179688, + "logps/ref_chosen": -105.78710174560547, + "logps/ref_rejected": -88.62471008300781, + "logps/rejected": -303.1727294921875, + "loss": 0.996, + "margin_dpo/margin_mean": 96.54632568359375, + "margin_dpo/margin_std": 89.16836547851562, + "step": 436 + }, + { + "KL/chosen_KL_mean": -138.875, + "KL/mean": -177.42764282226562, + "KL/rejected_KL_mean": -215.9803009033203, + "KL/std": 90.8270263671875, + "epoch": 0.6417033773861968, + "fcm_dpo/beta": 0.005238938145339489, + "fcm_dpo/delta": 0.04788995161652565, + "fcm_dpo/margin": 77.10530090332031, + "fcm_dpo/q_t": 0.4047277569770813, + "grad_norm": 10.809871673583984, + "learning_rate": 1.7298989601292036e-07, + "logits/chosen": -2.010089874267578, + "logits/rejected": -1.3571252822875977, + "logps/chosen": -234.93704223632812, + "logps/ref_chosen": -96.06204223632812, + "logps/ref_rejected": -89.01220703125, + "logps/rejected": -304.99249267578125, + "loss": 1.0759, + "margin_dpo/margin_mean": 77.10530090332031, + "margin_dpo/margin_std": 90.0735092163086, + "step": 437 + }, + { + "KL/chosen_KL_mean": -141.5880126953125, + "KL/mean": -181.06857299804688, + "KL/rejected_KL_mean": -220.54913330078125, + "KL/std": 91.84729766845703, + "epoch": 0.6431718061674009, + "fcm_dpo/beta": 0.00525021692737937, + "fcm_dpo/delta": 0.036726392805576324, + "fcm_dpo/margin": 78.96110534667969, + "fcm_dpo/q_t": 0.40397512912750244, + "grad_norm": 10.72618579864502, + "learning_rate": 1.7176998984196144e-07, + "logits/chosen": -2.027184009552002, + "logits/rejected": -1.1954039335250854, + "logps/chosen": -243.4434051513672, + "logps/ref_chosen": -101.85537719726562, + "logps/ref_rejected": -89.44766235351562, + "logps/rejected": -309.9967956542969, + "loss": 1.0791, + "margin_dpo/margin_mean": 78.96110534667969, + "margin_dpo/margin_std": 98.01394653320312, + "step": 438 + }, + { + "KL/chosen_KL_mean": -152.8901824951172, + "KL/mean": -192.21527099609375, + "KL/rejected_KL_mean": -231.54037475585938, + "KL/std": 99.62709045410156, + "epoch": 0.644640234948605, + "fcm_dpo/beta": 0.005275568459182978, + "fcm_dpo/delta": 0.03587264195084572, + "fcm_dpo/margin": 78.65019226074219, + "fcm_dpo/q_t": 0.40480339527130127, + "grad_norm": 10.864400863647461, + "learning_rate": 1.7055214510452458e-07, + "logits/chosen": -1.7100586891174316, + "logits/rejected": -1.4738038778305054, + "logps/chosen": -234.64581298828125, + "logps/ref_chosen": -81.75563049316406, + "logps/ref_rejected": -90.58636474609375, + "logps/rejected": -322.1267395019531, + "loss": 1.0817, + "margin_dpo/margin_mean": 78.65019226074219, + "margin_dpo/margin_std": 98.31413269042969, + "step": 439 + }, + { + "KL/chosen_KL_mean": -137.06240844726562, + "KL/mean": -185.70657348632812, + "KL/rejected_KL_mean": -234.35072326660156, + "KL/std": 100.66478729248047, + "epoch": 0.6461086637298091, + "fcm_dpo/beta": 0.005241363774985075, + "fcm_dpo/delta": -0.06415960937738419, + "fcm_dpo/margin": 97.28831481933594, + "fcm_dpo/q_t": 0.38413941860198975, + "grad_norm": 8.634608268737793, + "learning_rate": 1.6933639389195134e-07, + "logits/chosen": -2.493295907974243, + "logits/rejected": -2.0014278888702393, + "logps/chosen": -242.7034912109375, + "logps/ref_chosen": -105.64108276367188, + "logps/ref_rejected": -103.4010009765625, + "logps/rejected": -337.7517395019531, + "loss": 1.0193, + "margin_dpo/margin_mean": 97.28831481933594, + "margin_dpo/margin_std": 110.69349670410156, + "step": 440 + }, + { + "KL/chosen_KL_mean": -174.46405029296875, + "KL/mean": -217.0907745361328, + "KL/rejected_KL_mean": -259.71746826171875, + "KL/std": 95.74724578857422, + "epoch": 0.6475770925110133, + "fcm_dpo/beta": 0.00524694100022316, + "fcm_dpo/delta": 0.0027515720576047897, + "fcm_dpo/margin": 85.25341796875, + "fcm_dpo/q_t": 0.398668110370636, + "grad_norm": 9.658367156982422, + "learning_rate": 1.681227682404166e-07, + "logits/chosen": -1.7426241636276245, + "logits/rejected": -1.3799102306365967, + "logps/chosen": -265.99359130859375, + "logps/ref_chosen": -91.529541015625, + "logps/ref_rejected": -103.619384765625, + "logps/rejected": -363.33685302734375, + "loss": 1.0535, + "margin_dpo/margin_mean": 85.25341796875, + "margin_dpo/margin_std": 102.77945709228516, + "step": 441 + }, + { + "KL/chosen_KL_mean": -171.5980224609375, + "KL/mean": -224.1585235595703, + "KL/rejected_KL_mean": -276.71905517578125, + "KL/std": 101.61640930175781, + "epoch": 0.6490455212922174, + "fcm_dpo/beta": 0.005180784035474062, + "fcm_dpo/delta": -0.10002824664115906, + "fcm_dpo/margin": 105.12101745605469, + "fcm_dpo/q_t": 0.37393641471862793, + "grad_norm": 11.204852104187012, + "learning_rate": 1.669113001300851e-07, + "logits/chosen": -1.6903903484344482, + "logits/rejected": -1.2218103408813477, + "logps/chosen": -256.3755798339844, + "logps/ref_chosen": -84.77755737304688, + "logps/ref_rejected": -83.82415771484375, + "logps/rejected": -360.543212890625, + "loss": 0.9777, + "margin_dpo/margin_mean": 105.12100982666016, + "margin_dpo/margin_std": 101.45738220214844, + "step": 442 + }, + { + "KL/chosen_KL_mean": -165.6534423828125, + "KL/mean": -199.2949981689453, + "KL/rejected_KL_mean": -232.93653869628906, + "KL/std": 98.78840637207031, + "epoch": 0.6505139500734214, + "fcm_dpo/beta": 0.0052094897255301476, + "fcm_dpo/delta": 0.10316122323274612, + "fcm_dpo/margin": 67.28306579589844, + "fcm_dpo/q_t": 0.41906964778900146, + "grad_norm": 10.47797679901123, + "learning_rate": 1.6570202148426815e-07, + "logits/chosen": -1.838462233543396, + "logits/rejected": -1.2816200256347656, + "logps/chosen": -268.302734375, + "logps/ref_chosen": -102.64927673339844, + "logps/ref_rejected": -93.03807067871094, + "logps/rejected": -325.974609375, + "loss": 1.128, + "margin_dpo/margin_mean": 67.2830810546875, + "margin_dpo/margin_std": 95.50385284423828, + "step": 443 + }, + { + "KL/chosen_KL_mean": -172.17324829101562, + "KL/mean": -210.02139282226562, + "KL/rejected_KL_mean": -247.86953735351562, + "KL/std": 96.23963165283203, + "epoch": 0.6519823788546255, + "fcm_dpo/beta": 0.005304283928126097, + "fcm_dpo/delta": 0.05045264586806297, + "fcm_dpo/margin": 75.6962890625, + "fcm_dpo/q_t": 0.40898263454437256, + "grad_norm": 11.841695785522461, + "learning_rate": 1.6449496416858282e-07, + "logits/chosen": -1.5317130088806152, + "logits/rejected": -1.3604544401168823, + "logps/chosen": -260.09295654296875, + "logps/ref_chosen": -87.91971588134766, + "logps/ref_rejected": -103.32345581054688, + "logps/rejected": -351.1929931640625, + "loss": 1.1063, + "margin_dpo/margin_mean": 75.6962890625, + "margin_dpo/margin_std": 111.51077270507812, + "step": 444 + }, + { + "KL/chosen_KL_mean": -151.49868774414062, + "KL/mean": -196.0882110595703, + "KL/rejected_KL_mean": -240.67774963378906, + "KL/std": 93.3748550415039, + "epoch": 0.6534508076358296, + "fcm_dpo/beta": 0.005277983844280243, + "fcm_dpo/delta": -0.022016234695911407, + "fcm_dpo/margin": 89.1790771484375, + "fcm_dpo/q_t": 0.3909933865070343, + "grad_norm": 9.416054725646973, + "learning_rate": 1.6329015999011182e-07, + "logits/chosen": -1.9294803142547607, + "logits/rejected": -1.5129873752593994, + "logps/chosen": -252.89956665039062, + "logps/ref_chosen": -101.40086364746094, + "logps/ref_rejected": -99.03790283203125, + "logps/rejected": -339.71563720703125, + "loss": 1.0332, + "margin_dpo/margin_mean": 89.1790771484375, + "margin_dpo/margin_std": 98.0916976928711, + "step": 445 + }, + { + "KL/chosen_KL_mean": -145.10025024414062, + "KL/mean": -185.15869140625, + "KL/rejected_KL_mean": -225.21714782714844, + "KL/std": 99.13099670410156, + "epoch": 0.6549192364170338, + "fcm_dpo/beta": 0.005317972041666508, + "fcm_dpo/delta": 0.024914931505918503, + "fcm_dpo/margin": 80.1169204711914, + "fcm_dpo/q_t": 0.4015638828277588, + "grad_norm": 13.373784065246582, + "learning_rate": 1.6208764069656578e-07, + "logits/chosen": -1.8549327850341797, + "logits/rejected": -1.7110099792480469, + "logps/chosen": -232.5225830078125, + "logps/ref_chosen": -87.4223403930664, + "logps/ref_rejected": -106.70075988769531, + "logps/rejected": -331.91790771484375, + "loss": 1.0651, + "margin_dpo/margin_mean": 80.11691284179688, + "margin_dpo/margin_std": 94.8946762084961, + "step": 446 + }, + { + "KL/chosen_KL_mean": -142.89254760742188, + "KL/mean": -195.46844482421875, + "KL/rejected_KL_mean": -248.04434204101562, + "KL/std": 102.39170837402344, + "epoch": 0.6563876651982379, + "fcm_dpo/beta": 0.005230684764683247, + "fcm_dpo/delta": -0.10589684545993805, + "fcm_dpo/margin": 105.15178680419922, + "fcm_dpo/q_t": 0.37490004301071167, + "grad_norm": 9.607406616210938, + "learning_rate": 1.608874379754465e-07, + "logits/chosen": -2.2026915550231934, + "logits/rejected": -2.157698631286621, + "logps/chosen": -226.5078125, + "logps/ref_chosen": -83.6152572631836, + "logps/ref_rejected": -104.91239929199219, + "logps/rejected": -352.95672607421875, + "loss": 0.9964, + "margin_dpo/margin_mean": 105.15177917480469, + "margin_dpo/margin_std": 116.68319702148438, + "step": 447 + }, + { + "KL/chosen_KL_mean": -154.09796142578125, + "KL/mean": -204.73121643066406, + "KL/rejected_KL_mean": -255.36444091796875, + "KL/std": 107.79026794433594, + "epoch": 0.657856093979442, + "fcm_dpo/beta": 0.005170288495719433, + "fcm_dpo/delta": -0.07774815708398819, + "fcm_dpo/margin": 101.2665023803711, + "fcm_dpo/q_t": 0.3824172616004944, + "grad_norm": 10.670835494995117, + "learning_rate": 1.5968958345321177e-07, + "logits/chosen": -1.6668646335601807, + "logits/rejected": -1.3940508365631104, + "logps/chosen": -246.6737518310547, + "logps/ref_chosen": -92.5757827758789, + "logps/ref_rejected": -107.68977355957031, + "logps/rejected": -363.0542297363281, + "loss": 1.0312, + "margin_dpo/margin_mean": 101.2665023803711, + "margin_dpo/margin_std": 126.43362426757812, + "step": 448 + }, + { + "KL/chosen_KL_mean": -152.4690399169922, + "KL/mean": -204.88592529296875, + "KL/rejected_KL_mean": -257.30279541015625, + "KL/std": 100.44208526611328, + "epoch": 0.6593245227606461, + "fcm_dpo/beta": 0.005061184987425804, + "fcm_dpo/delta": -0.0850830227136612, + "fcm_dpo/margin": 104.83377075195312, + "fcm_dpo/q_t": 0.3781484067440033, + "grad_norm": 9.359891891479492, + "learning_rate": 1.584941086944423e-07, + "logits/chosen": -1.4257352352142334, + "logits/rejected": -0.9795221090316772, + "logps/chosen": -254.86798095703125, + "logps/ref_chosen": -102.39893341064453, + "logps/ref_rejected": -95.14886474609375, + "logps/rejected": -352.45166015625, + "loss": 1.0022, + "margin_dpo/margin_mean": 104.83377075195312, + "margin_dpo/margin_std": 114.44087219238281, + "step": 449 + }, + { + "KL/chosen_KL_mean": -135.41156005859375, + "KL/mean": -184.7716522216797, + "KL/rejected_KL_mean": -234.13174438476562, + "KL/std": 101.62181854248047, + "epoch": 0.6607929515418502, + "fcm_dpo/beta": 0.0049993242137134075, + "fcm_dpo/delta": -0.04585316777229309, + "fcm_dpo/margin": 98.72018432617188, + "fcm_dpo/q_t": 0.3859945833683014, + "grad_norm": 9.926301956176758, + "learning_rate": 1.573010452010098e-07, + "logits/chosen": -2.382091522216797, + "logits/rejected": -2.1891212463378906, + "logps/chosen": -222.4044189453125, + "logps/ref_chosen": -86.99285888671875, + "logps/ref_rejected": -108.53203582763672, + "logps/rejected": -342.6637878417969, + "loss": 1.0259, + "margin_dpo/margin_mean": 98.72018432617188, + "margin_dpo/margin_std": 112.16717529296875, + "step": 450 + }, + { + "KL/chosen_KL_mean": -154.05520629882812, + "KL/mean": -193.57681274414062, + "KL/rejected_KL_mean": -233.09840393066406, + "KL/std": 101.50096893310547, + "epoch": 0.6622613803230544, + "fcm_dpo/beta": 0.005040162242949009, + "fcm_dpo/delta": 0.053614094853401184, + "fcm_dpo/margin": 79.04322814941406, + "fcm_dpo/q_t": 0.4105450510978699, + "grad_norm": 14.635862350463867, + "learning_rate": 1.5611042441124687e-07, + "logits/chosen": -2.411348819732666, + "logits/rejected": -1.9070264101028442, + "logps/chosen": -240.86648559570312, + "logps/ref_chosen": -86.811279296875, + "logps/ref_rejected": -79.8555908203125, + "logps/rejected": -312.9539794921875, + "loss": 1.1218, + "margin_dpo/margin_mean": 79.04322814941406, + "margin_dpo/margin_std": 126.20407104492188, + "step": 451 + }, + { + "KL/chosen_KL_mean": -134.56747436523438, + "KL/mean": -179.08526611328125, + "KL/rejected_KL_mean": -223.6030731201172, + "KL/std": 97.11660766601562, + "epoch": 0.6637298091042585, + "fcm_dpo/beta": 0.0050099738873541355, + "fcm_dpo/delta": 0.0029504671692848206, + "fcm_dpo/margin": 89.03560638427734, + "fcm_dpo/q_t": 0.39591628313064575, + "grad_norm": 8.789139747619629, + "learning_rate": 1.549222776991186e-07, + "logits/chosen": -1.7120274305343628, + "logits/rejected": -1.6523525714874268, + "logps/chosen": -213.94711303710938, + "logps/ref_chosen": -79.379638671875, + "logps/ref_rejected": -103.71539306640625, + "logps/rejected": -327.3184814453125, + "loss": 1.0422, + "margin_dpo/margin_mean": 89.03560638427734, + "margin_dpo/margin_std": 88.60342407226562, + "step": 452 + }, + { + "KL/chosen_KL_mean": -162.24525451660156, + "KL/mean": -215.61207580566406, + "KL/rejected_KL_mean": -268.9789123535156, + "KL/std": 98.82974243164062, + "epoch": 0.6651982378854625, + "fcm_dpo/beta": 0.004989871755242348, + "fcm_dpo/delta": -0.08715921640396118, + "fcm_dpo/margin": 106.73365783691406, + "fcm_dpo/q_t": 0.3777143657207489, + "grad_norm": 12.883195877075195, + "learning_rate": 1.5373663637339584e-07, + "logits/chosen": -1.925933599472046, + "logits/rejected": -1.4531406164169312, + "logps/chosen": -249.94044494628906, + "logps/ref_chosen": -87.6951904296875, + "logps/ref_rejected": -90.0582275390625, + "logps/rejected": -359.0371398925781, + "loss": 0.986, + "margin_dpo/margin_mean": 106.73365783691406, + "margin_dpo/margin_std": 104.16875457763672, + "step": 453 + }, + { + "KL/chosen_KL_mean": -170.08926391601562, + "KL/mean": -213.6710205078125, + "KL/rejected_KL_mean": -257.2528076171875, + "KL/std": 101.20454406738281, + "epoch": 0.6666666666666666, + "fcm_dpo/beta": 0.00499748345464468, + "fcm_dpo/delta": 0.014467813074588776, + "fcm_dpo/margin": 87.16352844238281, + "fcm_dpo/q_t": 0.4020707905292511, + "grad_norm": 10.240715026855469, + "learning_rate": 1.5255353167683017e-07, + "logits/chosen": -1.9337717294692993, + "logits/rejected": -1.4788172245025635, + "logps/chosen": -259.655517578125, + "logps/ref_chosen": -89.56623840332031, + "logps/ref_rejected": -92.92105102539062, + "logps/rejected": -350.173828125, + "loss": 1.0995, + "margin_dpo/margin_mean": 87.16352081298828, + "margin_dpo/margin_std": 131.09112548828125, + "step": 454 + }, + { + "KL/chosen_KL_mean": -160.16110229492188, + "KL/mean": -201.14639282226562, + "KL/rejected_KL_mean": -242.13168334960938, + "KL/std": 99.42855834960938, + "epoch": 0.6681350954478708, + "fcm_dpo/beta": 0.005022745113819838, + "fcm_dpo/delta": 0.039547160267829895, + "fcm_dpo/margin": 81.97059631347656, + "fcm_dpo/q_t": 0.4057035446166992, + "grad_norm": 9.903414726257324, + "learning_rate": 1.5137299478533064e-07, + "logits/chosen": -1.8397959470748901, + "logits/rejected": -1.7978837490081787, + "logps/chosen": -237.79107666015625, + "logps/ref_chosen": -77.62998962402344, + "logps/ref_rejected": -118.97795104980469, + "logps/rejected": -361.1096496582031, + "loss": 1.0944, + "margin_dpo/margin_mean": 81.97059631347656, + "margin_dpo/margin_std": 113.62576293945312, + "step": 455 + }, + { + "KL/chosen_KL_mean": -161.74935913085938, + "KL/mean": -202.62106323242188, + "KL/rejected_KL_mean": -243.4927520751953, + "KL/std": 101.21768188476562, + "epoch": 0.6696035242290749, + "fcm_dpo/beta": 0.005054287146776915, + "fcm_dpo/delta": 0.03827638179063797, + "fcm_dpo/margin": 81.74337768554688, + "fcm_dpo/q_t": 0.40470045804977417, + "grad_norm": 11.922443389892578, + "learning_rate": 1.5019505680714232e-07, + "logits/chosen": -1.9774236679077148, + "logits/rejected": -1.8444124460220337, + "logps/chosen": -251.36624145507812, + "logps/ref_chosen": -89.61686706542969, + "logps/ref_rejected": -109.5597152709961, + "logps/rejected": -353.0524597167969, + "loss": 1.0785, + "margin_dpo/margin_mean": 81.74337768554688, + "margin_dpo/margin_std": 102.49090576171875, + "step": 456 + }, + { + "KL/chosen_KL_mean": -132.38502502441406, + "KL/mean": -186.63055419921875, + "KL/rejected_KL_mean": -240.87606811523438, + "KL/std": 101.0234375, + "epoch": 0.671071953010279, + "fcm_dpo/beta": 0.004944447427988052, + "fcm_dpo/delta": -0.09341508150100708, + "fcm_dpo/margin": 108.49103546142578, + "fcm_dpo/q_t": 0.37410277128219604, + "grad_norm": 10.40605354309082, + "learning_rate": 1.4901974878202627e-07, + "logits/chosen": -2.2007999420166016, + "logits/rejected": -1.8904091119766235, + "logps/chosen": -219.70672607421875, + "logps/ref_chosen": -87.32168579101562, + "logps/ref_rejected": -90.76660919189453, + "logps/rejected": -331.6426696777344, + "loss": 0.9718, + "margin_dpo/margin_mean": 108.49102783203125, + "margin_dpo/margin_std": 90.35786437988281, + "step": 457 + }, + { + "KL/chosen_KL_mean": -153.32867431640625, + "KL/mean": -193.28082275390625, + "KL/rejected_KL_mean": -233.2329864501953, + "KL/std": 100.76258850097656, + "epoch": 0.6725403817914831, + "fcm_dpo/beta": 0.005011053290218115, + "fcm_dpo/delta": 0.05141254514455795, + "fcm_dpo/margin": 79.9043197631836, + "fcm_dpo/q_t": 0.41007083654403687, + "grad_norm": 10.193153381347656, + "learning_rate": 1.4784710168044212e-07, + "logits/chosen": -1.6329656839370728, + "logits/rejected": -1.3935143947601318, + "logps/chosen": -246.84912109375, + "logps/ref_chosen": -93.52044677734375, + "logps/ref_rejected": -103.36898803710938, + "logps/rejected": -336.60198974609375, + "loss": 1.1041, + "margin_dpo/margin_mean": 79.90431213378906, + "margin_dpo/margin_std": 116.63407897949219, + "step": 458 + }, + { + "KL/chosen_KL_mean": -146.66146850585938, + "KL/mean": -196.23553466796875, + "KL/rejected_KL_mean": -245.80963134765625, + "KL/std": 105.66712951660156, + "epoch": 0.6740088105726872, + "fcm_dpo/beta": 0.004953456111252308, + "fcm_dpo/delta": -0.04402646794915199, + "fcm_dpo/margin": 99.14816284179688, + "fcm_dpo/q_t": 0.3881978690624237, + "grad_norm": 7.7661848068237305, + "learning_rate": 1.466771464027316e-07, + "logits/chosen": -1.7135825157165527, + "logits/rejected": -1.4628551006317139, + "logps/chosen": -222.34967041015625, + "logps/ref_chosen": -75.68820190429688, + "logps/ref_rejected": -92.17048645019531, + "logps/rejected": -337.9801025390625, + "loss": 1.0283, + "margin_dpo/margin_mean": 99.14816284179688, + "margin_dpo/margin_std": 112.94024658203125, + "step": 459 + }, + { + "KL/chosen_KL_mean": -149.8790283203125, + "KL/mean": -195.6499786376953, + "KL/rejected_KL_mean": -241.42092895507812, + "KL/std": 102.00483703613281, + "epoch": 0.6754772393538914, + "fcm_dpo/beta": 0.004945947788655758, + "fcm_dpo/delta": -0.003225427120923996, + "fcm_dpo/margin": 91.54191589355469, + "fcm_dpo/q_t": 0.39668217301368713, + "grad_norm": 10.641520500183105, + "learning_rate": 1.4550991377830423e-07, + "logits/chosen": -2.1042251586914062, + "logits/rejected": -2.125800848007202, + "logps/chosen": -230.99691772460938, + "logps/ref_chosen": -81.11788940429688, + "logps/ref_rejected": -110.31238555908203, + "logps/rejected": -351.73333740234375, + "loss": 1.0638, + "margin_dpo/margin_mean": 91.54191589355469, + "margin_dpo/margin_std": 117.19721221923828, + "step": 460 + }, + { + "KL/chosen_KL_mean": -184.72579956054688, + "KL/mean": -221.70619201660156, + "KL/rejected_KL_mean": -258.68658447265625, + "KL/std": 89.47636413574219, + "epoch": 0.6769456681350955, + "fcm_dpo/beta": 0.005012227222323418, + "fcm_dpo/delta": 0.08241581916809082, + "fcm_dpo/margin": 73.9607925415039, + "fcm_dpo/q_t": 0.4152829051017761, + "grad_norm": 9.35671615600586, + "learning_rate": 1.4434543456482518e-07, + "logits/chosen": -1.468023657798767, + "logits/rejected": -1.2711260318756104, + "logps/chosen": -266.309326171875, + "logps/ref_chosen": -81.58352661132812, + "logps/ref_rejected": -93.87710571289062, + "logps/rejected": -352.5636901855469, + "loss": 1.1178, + "margin_dpo/margin_mean": 73.9607925415039, + "margin_dpo/margin_std": 105.83638000488281, + "step": 461 + }, + { + "KL/chosen_KL_mean": -140.54736328125, + "KL/mean": -188.308837890625, + "KL/rejected_KL_mean": -236.0703125, + "KL/std": 96.39823913574219, + "epoch": 0.6784140969162996, + "fcm_dpo/beta": 0.005031496752053499, + "fcm_dpo/delta": -0.03217264637351036, + "fcm_dpo/margin": 95.52291870117188, + "fcm_dpo/q_t": 0.38922619819641113, + "grad_norm": 9.564581871032715, + "learning_rate": 1.4318373944740484e-07, + "logits/chosen": -2.441433906555176, + "logits/rejected": -1.9524383544921875, + "logps/chosen": -234.74591064453125, + "logps/ref_chosen": -94.19854736328125, + "logps/ref_rejected": -85.63162994384766, + "logps/rejected": -321.7019348144531, + "loss": 1.0259, + "margin_dpo/margin_mean": 95.52291870117188, + "margin_dpo/margin_std": 103.81068420410156, + "step": 462 + }, + { + "KL/chosen_KL_mean": -136.15145874023438, + "KL/mean": -192.84652709960938, + "KL/rejected_KL_mean": -249.5416259765625, + "KL/std": 103.62898254394531, + "epoch": 0.6798825256975036, + "fcm_dpo/beta": 0.004961706697940826, + "fcm_dpo/delta": -0.11949680745601654, + "fcm_dpo/margin": 113.39016723632812, + "fcm_dpo/q_t": 0.3697696626186371, + "grad_norm": 14.86490535736084, + "learning_rate": 1.4202485903778976e-07, + "logits/chosen": -1.8930439949035645, + "logits/rejected": -1.4674456119537354, + "logps/chosen": -222.07620239257812, + "logps/ref_chosen": -85.92474365234375, + "logps/ref_rejected": -96.90184020996094, + "logps/rejected": -346.4434814453125, + "loss": 0.9788, + "margin_dpo/margin_mean": 113.39016723632812, + "margin_dpo/margin_std": 110.37509155273438, + "step": 463 + }, + { + "KL/chosen_KL_mean": -167.87515258789062, + "KL/mean": -213.98040771484375, + "KL/rejected_KL_mean": -260.0856628417969, + "KL/std": 102.98896789550781, + "epoch": 0.6813509544787077, + "fcm_dpo/beta": 0.004885343834757805, + "fcm_dpo/delta": -0.0006006844341754913, + "fcm_dpo/margin": 92.21051025390625, + "fcm_dpo/q_t": 0.3967575430870056, + "grad_norm": 12.482551574707031, + "learning_rate": 1.4086882387355658e-07, + "logits/chosen": -1.7639579772949219, + "logits/rejected": -1.8439664840698242, + "logps/chosen": -247.56434631347656, + "logps/ref_chosen": -79.68919372558594, + "logps/ref_rejected": -107.29232025146484, + "logps/rejected": -367.37799072265625, + "loss": 1.0616, + "margin_dpo/margin_mean": 92.21051025390625, + "margin_dpo/margin_std": 117.0827865600586, + "step": 464 + }, + { + "KL/chosen_KL_mean": -159.87017822265625, + "KL/mean": -213.8851318359375, + "KL/rejected_KL_mean": -267.900146484375, + "KL/std": 111.44023132324219, + "epoch": 0.6828193832599119, + "fcm_dpo/beta": 0.004832090809941292, + "fcm_dpo/delta": -0.07617706060409546, + "fcm_dpo/margin": 108.02995300292969, + "fcm_dpo/q_t": 0.37889420986175537, + "grad_norm": 9.909037590026855, + "learning_rate": 1.3971566441730714e-07, + "logits/chosen": -1.5861811637878418, + "logits/rejected": -1.2803863286972046, + "logps/chosen": -251.73040771484375, + "logps/ref_chosen": -91.8602294921875, + "logps/ref_rejected": -118.71001434326172, + "logps/rejected": -386.6101379394531, + "loss": 0.9912, + "margin_dpo/margin_mean": 108.02996063232422, + "margin_dpo/margin_std": 105.53227996826172, + "step": 465 + }, + { + "KL/chosen_KL_mean": -142.86985778808594, + "KL/mean": -198.246826171875, + "KL/rejected_KL_mean": -253.62379455566406, + "KL/std": 107.02445983886719, + "epoch": 0.684287812041116, + "fcm_dpo/beta": 0.004758065566420555, + "fcm_dpo/delta": -0.0813850685954094, + "fcm_dpo/margin": 110.75393676757812, + "fcm_dpo/q_t": 0.37828460335731506, + "grad_norm": 7.444459438323975, + "learning_rate": 1.3856541105586545e-07, + "logits/chosen": -1.5047101974487305, + "logits/rejected": -1.1812858581542969, + "logps/chosen": -227.57125854492188, + "logps/ref_chosen": -84.70140838623047, + "logps/ref_rejected": -96.05084228515625, + "logps/rejected": -349.67462158203125, + "loss": 0.9928, + "margin_dpo/margin_mean": 110.75393676757812, + "margin_dpo/margin_std": 112.03373718261719, + "step": 466 + }, + { + "KL/chosen_KL_mean": -181.15975952148438, + "KL/mean": -227.86563110351562, + "KL/rejected_KL_mean": -274.57147216796875, + "KL/std": 111.49720764160156, + "epoch": 0.6857562408223201, + "fcm_dpo/beta": 0.004727067891508341, + "fcm_dpo/delta": 0.008390307426452637, + "fcm_dpo/margin": 93.41171264648438, + "fcm_dpo/q_t": 0.39889955520629883, + "grad_norm": 13.292122840881348, + "learning_rate": 1.3741809409947729e-07, + "logits/chosen": -2.382991313934326, + "logits/rejected": -1.8103981018066406, + "logps/chosen": -290.45806884765625, + "logps/ref_chosen": -109.29832458496094, + "logps/ref_rejected": -108.8436508178711, + "logps/rejected": -383.4151306152344, + "loss": 1.0777, + "margin_dpo/margin_mean": 93.41171264648438, + "margin_dpo/margin_std": 125.93314361572266, + "step": 467 + }, + { + "KL/chosen_KL_mean": -141.70941162109375, + "KL/mean": -183.89451599121094, + "KL/rejected_KL_mean": -226.07962036132812, + "KL/std": 108.10533905029297, + "epoch": 0.6872246696035242, + "fcm_dpo/beta": 0.004781381692737341, + "fcm_dpo/delta": 0.04858104884624481, + "fcm_dpo/margin": 84.37019348144531, + "fcm_dpo/q_t": 0.4094661772251129, + "grad_norm": 10.17148494720459, + "learning_rate": 1.362737437810114e-07, + "logits/chosen": -1.821883201599121, + "logits/rejected": -1.6187019348144531, + "logps/chosen": -240.03106689453125, + "logps/ref_chosen": -98.32164764404297, + "logps/ref_rejected": -106.68048095703125, + "logps/rejected": -332.76007080078125, + "loss": 1.1036, + "margin_dpo/margin_mean": 84.37019348144531, + "margin_dpo/margin_std": 123.54734802246094, + "step": 468 + }, + { + "KL/chosen_KL_mean": -174.17282104492188, + "KL/mean": -226.3063201904297, + "KL/rejected_KL_mean": -278.4398193359375, + "KL/std": 108.27603149414062, + "epoch": 0.6886930983847284, + "fcm_dpo/beta": 0.004737929441034794, + "fcm_dpo/delta": -0.047216739505529404, + "fcm_dpo/margin": 104.26698303222656, + "fcm_dpo/q_t": 0.3866550028324127, + "grad_norm": 11.116669654846191, + "learning_rate": 1.351323902551631e-07, + "logits/chosen": -2.1750669479370117, + "logits/rejected": -1.7511928081512451, + "logps/chosen": -270.9370422363281, + "logps/ref_chosen": -96.76420593261719, + "logps/ref_rejected": -109.59500885009766, + "logps/rejected": -388.0347900390625, + "loss": 1.0267, + "margin_dpo/margin_mean": 104.26698303222656, + "margin_dpo/margin_std": 117.83578491210938, + "step": 469 + }, + { + "KL/chosen_KL_mean": -160.32565307617188, + "KL/mean": -208.27127075195312, + "KL/rejected_KL_mean": -256.2169189453125, + "KL/std": 101.9165267944336, + "epoch": 0.6901615271659325, + "fcm_dpo/beta": 0.004747654311358929, + "fcm_dpo/delta": -0.005513817071914673, + "fcm_dpo/margin": 95.89126586914062, + "fcm_dpo/q_t": 0.39419037103652954, + "grad_norm": 9.690295219421387, + "learning_rate": 1.339940635976592e-07, + "logits/chosen": -1.669811487197876, + "logits/rejected": -1.2485895156860352, + "logps/chosen": -243.82229614257812, + "logps/ref_chosen": -83.49665832519531, + "logps/ref_rejected": -88.48578643798828, + "logps/rejected": -344.70269775390625, + "loss": 1.0372, + "margin_dpo/margin_mean": 95.89126586914062, + "margin_dpo/margin_std": 102.33804321289062, + "step": 470 + }, + { + "KL/chosen_KL_mean": -185.2115020751953, + "KL/mean": -230.9696044921875, + "KL/rejected_KL_mean": -276.72772216796875, + "KL/std": 100.63410949707031, + "epoch": 0.6916299559471366, + "fcm_dpo/beta": 0.00475767906755209, + "fcm_dpo/delta": 0.01526210643351078, + "fcm_dpo/margin": 91.51620483398438, + "fcm_dpo/q_t": 0.39869433641433716, + "grad_norm": 9.37916088104248, + "learning_rate": 1.3285879380446563e-07, + "logits/chosen": -1.6311432123184204, + "logits/rejected": -1.2209393978118896, + "logps/chosen": -273.685791015625, + "logps/ref_chosen": -88.47430419921875, + "logps/ref_rejected": -90.48171997070312, + "logps/rejected": -367.20941162109375, + "loss": 1.0516, + "margin_dpo/margin_mean": 91.51620483398438, + "margin_dpo/margin_std": 101.12043762207031, + "step": 471 + }, + { + "KL/chosen_KL_mean": -171.4448699951172, + "KL/mean": -226.10107421875, + "KL/rejected_KL_mean": -280.75726318359375, + "KL/std": 100.78599548339844, + "epoch": 0.6930983847283406, + "fcm_dpo/beta": 0.0047001466155052185, + "fcm_dpo/delta": -0.06764907389879227, + "fcm_dpo/margin": 109.31238555908203, + "fcm_dpo/q_t": 0.3814803957939148, + "grad_norm": 8.265682220458984, + "learning_rate": 1.317266107909975e-07, + "logits/chosen": -2.17086124420166, + "logits/rejected": -1.472998023033142, + "logps/chosen": -296.6785888671875, + "logps/ref_chosen": -125.23370361328125, + "logps/ref_rejected": -121.05349731445312, + "logps/rejected": -401.810791015625, + "loss": 0.9951, + "margin_dpo/margin_mean": 109.3123779296875, + "margin_dpo/margin_std": 105.84391784667969, + "step": 472 + }, + { + "KL/chosen_KL_mean": -194.54928588867188, + "KL/mean": -228.7971954345703, + "KL/rejected_KL_mean": -263.04510498046875, + "KL/std": 102.35105895996094, + "epoch": 0.6945668135095447, + "fcm_dpo/beta": 0.0047929780557751656, + "fcm_dpo/delta": 0.1256594955921173, + "fcm_dpo/margin": 68.49583435058594, + "fcm_dpo/q_t": 0.4262773096561432, + "grad_norm": 10.695194244384766, + "learning_rate": 1.3059754439133002e-07, + "logits/chosen": -1.9057073593139648, + "logits/rejected": -1.4209513664245605, + "logps/chosen": -290.16064453125, + "logps/ref_chosen": -95.61137390136719, + "logps/ref_rejected": -88.15115356445312, + "logps/rejected": -351.1962585449219, + "loss": 1.1592, + "margin_dpo/margin_mean": 68.49583435058594, + "margin_dpo/margin_std": 118.30596923828125, + "step": 473 + }, + { + "KL/chosen_KL_mean": -182.2709503173828, + "KL/mean": -217.48263549804688, + "KL/rejected_KL_mean": -252.69432067871094, + "KL/std": 96.60692596435547, + "epoch": 0.6960352422907489, + "fcm_dpo/beta": 0.00481058144941926, + "fcm_dpo/delta": -0.015532290562987328, + "fcm_dpo/margin": 70.4233627319336, + "fcm_dpo/q_t": 0.42211002111434937, + "grad_norm": 13.555959701538086, + "learning_rate": 1.2947162435741277e-07, + "logits/chosen": -1.5289095640182495, + "logits/rejected": -1.3379920721054077, + "logps/chosen": -263.7507019042969, + "logps/ref_chosen": -81.47975158691406, + "logps/ref_rejected": -96.46562957763672, + "logps/rejected": -349.1599426269531, + "loss": 1.1575, + "margin_dpo/margin_mean": 70.4233627319336, + "margin_dpo/margin_std": 116.6693115234375, + "step": 474 + }, + { + "KL/chosen_KL_mean": -137.7569580078125, + "KL/mean": -191.47225952148438, + "KL/rejected_KL_mean": -245.18753051757812, + "KL/std": 103.98016357421875, + "epoch": 0.697503671071953, + "fcm_dpo/beta": 0.0047598229721188545, + "fcm_dpo/delta": -0.06462739408016205, + "fcm_dpo/margin": 107.43055725097656, + "fcm_dpo/q_t": 0.38313812017440796, + "grad_norm": 8.57970142364502, + "learning_rate": 1.2834888035828596e-07, + "logits/chosen": -2.1804757118225098, + "logits/rejected": -2.1264116764068604, + "logps/chosen": -211.95294189453125, + "logps/ref_chosen": -74.19598388671875, + "logps/ref_rejected": -94.69242095947266, + "logps/rejected": -339.87994384765625, + "loss": 1.0099, + "margin_dpo/margin_mean": 107.43057250976562, + "margin_dpo/margin_std": 116.82521057128906, + "step": 475 + }, + { + "KL/chosen_KL_mean": -161.96005249023438, + "KL/mean": -209.29946899414062, + "KL/rejected_KL_mean": -256.6388854980469, + "KL/std": 105.60200500488281, + "epoch": 0.6989720998531571, + "fcm_dpo/beta": 0.004727785475552082, + "fcm_dpo/delta": 0.0022844448685646057, + "fcm_dpo/margin": 94.67884826660156, + "fcm_dpo/q_t": 0.3981456160545349, + "grad_norm": 11.078962326049805, + "learning_rate": 1.2722934197929802e-07, + "logits/chosen": -2.0900261402130127, + "logits/rejected": -1.7165247201919556, + "logps/chosen": -233.93115234375, + "logps/ref_chosen": -71.97109985351562, + "logps/ref_rejected": -80.26224517822266, + "logps/rejected": -336.901123046875, + "loss": 1.057, + "margin_dpo/margin_mean": 94.67884063720703, + "margin_dpo/margin_std": 114.95457458496094, + "step": 476 + }, + { + "KL/chosen_KL_mean": -161.87371826171875, + "KL/mean": -209.43356323242188, + "KL/rejected_KL_mean": -256.993408203125, + "KL/std": 93.7550048828125, + "epoch": 0.7004405286343612, + "fcm_dpo/beta": 0.004747455008327961, + "fcm_dpo/delta": -0.001672297716140747, + "fcm_dpo/margin": 95.11968231201172, + "fcm_dpo/q_t": 0.3954678177833557, + "grad_norm": 9.578988075256348, + "learning_rate": 1.2611303872132631e-07, + "logits/chosen": -2.1565287113189697, + "logits/rejected": -1.2394235134124756, + "logps/chosen": -266.8792724609375, + "logps/ref_chosen": -105.00555419921875, + "logps/ref_rejected": -81.87843322753906, + "logps/rejected": -338.871826171875, + "loss": 1.0576, + "margin_dpo/margin_mean": 95.11968231201172, + "margin_dpo/margin_std": 117.13556671142578, + "step": 477 + }, + { + "KL/chosen_KL_mean": -156.97903442382812, + "KL/mean": -206.59579467773438, + "KL/rejected_KL_mean": -256.21258544921875, + "KL/std": 101.80718994140625, + "epoch": 0.7019089574155654, + "fcm_dpo/beta": 0.004728690255433321, + "fcm_dpo/delta": -0.020189039409160614, + "fcm_dpo/margin": 99.23353576660156, + "fcm_dpo/q_t": 0.39149191975593567, + "grad_norm": 8.809531211853027, + "learning_rate": 1.2500000000000005e-07, + "logits/chosen": -1.5445786714553833, + "logits/rejected": -1.3673609495162964, + "logps/chosen": -233.76724243164062, + "logps/ref_chosen": -76.7882080078125, + "logps/ref_rejected": -90.43994140625, + "logps/rejected": -346.65252685546875, + "loss": 1.0369, + "margin_dpo/margin_mean": 99.23353576660156, + "margin_dpo/margin_std": 112.28427124023438, + "step": 478 + }, + { + "KL/chosen_KL_mean": -161.14065551757812, + "KL/mean": -209.66729736328125, + "KL/rejected_KL_mean": -258.1939697265625, + "KL/std": 101.9322509765625, + "epoch": 0.7033773861967695, + "fcm_dpo/beta": 0.004733966663479805, + "fcm_dpo/delta": -0.010165683925151825, + "fcm_dpo/margin": 97.0533218383789, + "fcm_dpo/q_t": 0.39276695251464844, + "grad_norm": 12.275886535644531, + "learning_rate": 1.2389025514492456e-07, + "logits/chosen": -1.386313796043396, + "logits/rejected": -1.243363857269287, + "logps/chosen": -242.50296020507812, + "logps/ref_chosen": -81.3623046875, + "logps/ref_rejected": -101.09114074707031, + "logps/rejected": -359.28509521484375, + "loss": 1.0381, + "margin_dpo/margin_mean": 97.0533218383789, + "margin_dpo/margin_std": 104.44366455078125, + "step": 479 + }, + { + "KL/chosen_KL_mean": -185.82891845703125, + "KL/mean": -224.86044311523438, + "KL/rejected_KL_mean": -263.8919677734375, + "KL/std": 97.73199462890625, + "epoch": 0.7048458149779736, + "fcm_dpo/beta": 0.0047708675265312195, + "fcm_dpo/delta": 0.08056198805570602, + "fcm_dpo/margin": 78.06306457519531, + "fcm_dpo/q_t": 0.41418278217315674, + "grad_norm": 10.773375511169434, + "learning_rate": 1.227838333989088e-07, + "logits/chosen": -1.8190544843673706, + "logits/rejected": -1.1443976163864136, + "logps/chosen": -282.6028137207031, + "logps/ref_chosen": -96.7739028930664, + "logps/ref_rejected": -86.40473937988281, + "logps/rejected": -350.29669189453125, + "loss": 1.1116, + "margin_dpo/margin_mean": 78.06306457519531, + "margin_dpo/margin_std": 108.42257690429688, + "step": 480 + }, + { + "KL/chosen_KL_mean": -181.53097534179688, + "KL/mean": -227.4097900390625, + "KL/rejected_KL_mean": -273.28863525390625, + "KL/std": 101.75514221191406, + "epoch": 0.7063142437591777, + "fcm_dpo/beta": 0.004794671665877104, + "fcm_dpo/delta": 0.01051416527479887, + "fcm_dpo/margin": 91.75765991210938, + "fcm_dpo/q_t": 0.3983638286590576, + "grad_norm": 12.513395309448242, + "learning_rate": 1.2168076391719489e-07, + "logits/chosen": -2.058073043823242, + "logits/rejected": -1.628892183303833, + "logps/chosen": -273.2011413574219, + "logps/ref_chosen": -91.670166015625, + "logps/ref_rejected": -98.69490051269531, + "logps/rejected": -371.9835205078125, + "loss": 1.0633, + "margin_dpo/margin_mean": 91.7576675415039, + "margin_dpo/margin_std": 113.5112075805664, + "step": 481 + }, + { + "KL/chosen_KL_mean": -175.49473571777344, + "KL/mean": -220.43032836914062, + "KL/rejected_KL_mean": -265.36590576171875, + "KL/std": 99.30690002441406, + "epoch": 0.7077826725403817, + "fcm_dpo/beta": 0.004801459610462189, + "fcm_dpo/delta": 0.019236396998167038, + "fcm_dpo/margin": 89.87120056152344, + "fcm_dpo/q_t": 0.3995790481567383, + "grad_norm": 13.622885704040527, + "learning_rate": 1.2058107576668938e-07, + "logits/chosen": -1.4896271228790283, + "logits/rejected": -1.1253504753112793, + "logps/chosen": -274.01483154296875, + "logps/ref_chosen": -98.52011108398438, + "logps/ref_rejected": -94.82945251464844, + "logps/rejected": -360.19537353515625, + "loss": 1.0672, + "margin_dpo/margin_mean": 89.87120056152344, + "margin_dpo/margin_std": 110.11418914794922, + "step": 482 + }, + { + "KL/chosen_KL_mean": -180.67239379882812, + "KL/mean": -228.32342529296875, + "KL/rejected_KL_mean": -275.9744567871094, + "KL/std": 106.0830078125, + "epoch": 0.7092511013215859, + "fcm_dpo/beta": 0.004840238951146603, + "fcm_dpo/delta": -0.01257888600230217, + "fcm_dpo/margin": 95.30207824707031, + "fcm_dpo/q_t": 0.39278697967529297, + "grad_norm": 8.121712684631348, + "learning_rate": 1.194847979251979e-07, + "logits/chosen": -2.209134101867676, + "logits/rejected": -1.5848324298858643, + "logps/chosen": -287.7909851074219, + "logps/ref_chosen": -107.11860656738281, + "logps/ref_rejected": -101.11499786376953, + "logps/rejected": -377.0894775390625, + "loss": 1.0416, + "margin_dpo/margin_mean": 95.30207824707031, + "margin_dpo/margin_std": 105.41844177246094, + "step": 483 + }, + { + "KL/chosen_KL_mean": -166.78041076660156, + "KL/mean": -219.15415954589844, + "KL/rejected_KL_mean": -271.52789306640625, + "KL/std": 107.94287109375, + "epoch": 0.71071953010279, + "fcm_dpo/beta": 0.004783853888511658, + "fcm_dpo/delta": -0.053837914019823074, + "fcm_dpo/margin": 104.74748229980469, + "fcm_dpo/q_t": 0.38438552618026733, + "grad_norm": 8.95742130279541, + "learning_rate": 1.1839195928066101e-07, + "logits/chosen": -2.4437952041625977, + "logits/rejected": -1.9667483568191528, + "logps/chosen": -253.7603302001953, + "logps/ref_chosen": -86.97991943359375, + "logps/ref_rejected": -90.72367095947266, + "logps/rejected": -362.2515869140625, + "loss": 1.0214, + "margin_dpo/margin_mean": 104.74748229980469, + "margin_dpo/margin_std": 117.56710815429688, + "step": 484 + }, + { + "KL/chosen_KL_mean": -157.0498504638672, + "KL/mean": -214.79684448242188, + "KL/rejected_KL_mean": -272.5438537597656, + "KL/std": 111.50321197509766, + "epoch": 0.7121879588839941, + "fcm_dpo/beta": 0.004689244087785482, + "fcm_dpo/delta": -0.09678236395120621, + "fcm_dpo/margin": 115.4940185546875, + "fcm_dpo/q_t": 0.37508007884025574, + "grad_norm": 9.589292526245117, + "learning_rate": 1.1730258863039347e-07, + "logits/chosen": -1.5288968086242676, + "logits/rejected": -1.0940437316894531, + "logps/chosen": -251.10861206054688, + "logps/ref_chosen": -94.05874633789062, + "logps/ref_rejected": -108.56298828125, + "logps/rejected": -381.1068420410156, + "loss": 0.9801, + "margin_dpo/margin_mean": 115.49401092529297, + "margin_dpo/margin_std": 112.31433868408203, + "step": 485 + }, + { + "KL/chosen_KL_mean": -180.47369384765625, + "KL/mean": -230.94326782226562, + "KL/rejected_KL_mean": -281.412841796875, + "KL/std": 107.85102081298828, + "epoch": 0.7136563876651982, + "fcm_dpo/beta": 0.004661254584789276, + "fcm_dpo/delta": -0.02160482108592987, + "fcm_dpo/margin": 100.93917846679688, + "fcm_dpo/q_t": 0.39127978682518005, + "grad_norm": 10.805781364440918, + "learning_rate": 1.1621671468032493e-07, + "logits/chosen": -2.053313970565796, + "logits/rejected": -1.3463963270187378, + "logps/chosen": -274.2195739746094, + "logps/ref_chosen": -93.74588012695312, + "logps/ref_rejected": -98.07064819335938, + "logps/rejected": -379.4834899902344, + "loss": 1.0399, + "margin_dpo/margin_mean": 100.93917083740234, + "margin_dpo/margin_std": 115.87477111816406, + "step": 486 + }, + { + "KL/chosen_KL_mean": -174.2039794921875, + "KL/mean": -220.03860473632812, + "KL/rejected_KL_mean": -265.87322998046875, + "KL/std": 103.42660522460938, + "epoch": 0.7151248164464024, + "fcm_dpo/beta": 0.0046575660817325115, + "fcm_dpo/delta": 0.02407930977642536, + "fcm_dpo/margin": 91.66924285888672, + "fcm_dpo/q_t": 0.4011977016925812, + "grad_norm": 10.19849681854248, + "learning_rate": 1.1513436604424378e-07, + "logits/chosen": -2.1839797496795654, + "logits/rejected": -1.8756451606750488, + "logps/chosen": -262.237548828125, + "logps/ref_chosen": -88.0335693359375, + "logps/ref_rejected": -98.47209930419922, + "logps/rejected": -364.3453369140625, + "loss": 1.0651, + "margin_dpo/margin_mean": 91.66925048828125, + "margin_dpo/margin_std": 109.32331085205078, + "step": 487 + }, + { + "KL/chosen_KL_mean": -158.35450744628906, + "KL/mean": -208.90673828125, + "KL/rejected_KL_mean": -259.4589538574219, + "KL/std": 109.08303833007812, + "epoch": 0.7165932452276065, + "fcm_dpo/beta": 0.004656763281673193, + "fcm_dpo/delta": -0.02185317687690258, + "fcm_dpo/margin": 101.10443878173828, + "fcm_dpo/q_t": 0.38953208923339844, + "grad_norm": 9.925406455993652, + "learning_rate": 1.1405557124304335e-07, + "logits/chosen": -1.607795238494873, + "logits/rejected": -1.3447988033294678, + "logps/chosen": -243.1441650390625, + "logps/ref_chosen": -84.78964233398438, + "logps/ref_rejected": -90.2734603881836, + "logps/rejected": -349.732421875, + "loss": 1.02, + "margin_dpo/margin_mean": 101.10444641113281, + "margin_dpo/margin_std": 96.80280303955078, + "step": 488 + }, + { + "KL/chosen_KL_mean": -177.58462524414062, + "KL/mean": -223.75230407714844, + "KL/rejected_KL_mean": -269.9199523925781, + "KL/std": 99.37748718261719, + "epoch": 0.7180616740088106, + "fcm_dpo/beta": 0.004661159589886665, + "fcm_dpo/delta": 0.020484985783696175, + "fcm_dpo/margin": 92.3353271484375, + "fcm_dpo/q_t": 0.4008216857910156, + "grad_norm": 10.168931007385254, + "learning_rate": 1.1298035870396985e-07, + "logits/chosen": -1.5888497829437256, + "logits/rejected": -1.0927530527114868, + "logps/chosen": -268.0539245605469, + "logps/ref_chosen": -90.46929931640625, + "logps/ref_rejected": -86.39761352539062, + "logps/rejected": -356.31756591796875, + "loss": 1.0639, + "margin_dpo/margin_mean": 92.33534240722656, + "margin_dpo/margin_std": 110.99066162109375, + "step": 489 + }, + { + "KL/chosen_KL_mean": -179.28955078125, + "KL/mean": -222.39990234375, + "KL/rejected_KL_mean": -265.51025390625, + "KL/std": 107.74891662597656, + "epoch": 0.7195301027900147, + "fcm_dpo/beta": 0.004729720763862133, + "fcm_dpo/delta": 0.04253225028514862, + "fcm_dpo/margin": 86.22069549560547, + "fcm_dpo/q_t": 0.40913423895835876, + "grad_norm": 9.120133399963379, + "learning_rate": 1.1190875675987355e-07, + "logits/chosen": -2.0534164905548096, + "logits/rejected": -2.1311709880828857, + "logps/chosen": -264.60968017578125, + "logps/ref_chosen": -85.32012939453125, + "logps/ref_rejected": -115.99385070800781, + "logps/rejected": -381.50408935546875, + "loss": 1.1287, + "margin_dpo/margin_mean": 86.22069549560547, + "margin_dpo/margin_std": 141.7015838623047, + "step": 490 + }, + { + "KL/chosen_KL_mean": -182.6004180908203, + "KL/mean": -222.75865173339844, + "KL/rejected_KL_mean": -262.9168701171875, + "KL/std": 109.66326141357422, + "epoch": 0.7209985315712188, + "fcm_dpo/beta": 0.004752345383167267, + "fcm_dpo/delta": 0.07104957848787308, + "fcm_dpo/margin": 80.31645202636719, + "fcm_dpo/q_t": 0.4117836356163025, + "grad_norm": 10.851274490356445, + "learning_rate": 1.1084079364846241e-07, + "logits/chosen": -1.7933106422424316, + "logits/rejected": -1.29533052444458, + "logps/chosen": -268.74395751953125, + "logps/ref_chosen": -86.14352416992188, + "logps/ref_rejected": -80.67945861816406, + "logps/rejected": -343.5963439941406, + "loss": 1.1068, + "margin_dpo/margin_mean": 80.31645202636719, + "margin_dpo/margin_std": 111.28205871582031, + "step": 491 + }, + { + "KL/chosen_KL_mean": -169.64657592773438, + "KL/mean": -201.59054565429688, + "KL/rejected_KL_mean": -233.53451538085938, + "KL/std": 107.24102783203125, + "epoch": 0.7224669603524229, + "fcm_dpo/beta": 0.004870216827839613, + "fcm_dpo/delta": 0.14343884587287903, + "fcm_dpo/margin": 63.887962341308594, + "fcm_dpo/q_t": 0.4300411343574524, + "grad_norm": 14.212003707885742, + "learning_rate": 1.097764975115576e-07, + "logits/chosen": -1.9620296955108643, + "logits/rejected": -1.5676814317703247, + "logps/chosen": -250.754150390625, + "logps/ref_chosen": -81.10757446289062, + "logps/ref_rejected": -80.75199890136719, + "logps/rejected": -314.2864990234375, + "loss": 1.1993, + "margin_dpo/margin_mean": 63.887962341308594, + "margin_dpo/margin_std": 133.98292541503906, + "step": 492 + }, + { + "KL/chosen_KL_mean": -167.80459594726562, + "KL/mean": -210.98052978515625, + "KL/rejected_KL_mean": -254.15646362304688, + "KL/std": 97.94412231445312, + "epoch": 0.723935389133627, + "fcm_dpo/beta": 0.004906080663204193, + "fcm_dpo/delta": 0.02713683247566223, + "fcm_dpo/margin": 86.35186767578125, + "fcm_dpo/q_t": 0.40144893527030945, + "grad_norm": 15.690646171569824, + "learning_rate": 1.0871589639435203e-07, + "logits/chosen": -2.3523173332214355, + "logits/rejected": -1.6103328466415405, + "logps/chosen": -280.0119323730469, + "logps/ref_chosen": -112.20733642578125, + "logps/ref_rejected": -93.60719299316406, + "logps/rejected": -347.763671875, + "loss": 1.0698, + "margin_dpo/margin_mean": 86.35186767578125, + "margin_dpo/margin_std": 102.87307739257812, + "step": 493 + }, + { + "KL/chosen_KL_mean": -163.60784912109375, + "KL/mean": -219.94192504882812, + "KL/rejected_KL_mean": -276.2760009765625, + "KL/std": 108.10519409179688, + "epoch": 0.7254038179148311, + "fcm_dpo/beta": 0.004834240768104792, + "fcm_dpo/delta": -0.10148841887712479, + "fcm_dpo/margin": 112.66815185546875, + "fcm_dpo/q_t": 0.3745891749858856, + "grad_norm": 10.725040435791016, + "learning_rate": 1.0765901824467166e-07, + "logits/chosen": -1.388137936592102, + "logits/rejected": -1.3093554973602295, + "logps/chosen": -236.72274780273438, + "logps/ref_chosen": -73.11489868164062, + "logps/ref_rejected": -92.16300201416016, + "logps/rejected": -368.4389953613281, + "loss": 0.9891, + "margin_dpo/margin_mean": 112.66815185546875, + "margin_dpo/margin_std": 116.00218200683594, + "step": 494 + }, + { + "KL/chosen_KL_mean": -167.45440673828125, + "KL/mean": -216.466552734375, + "KL/rejected_KL_mean": -265.47869873046875, + "KL/std": 103.0387954711914, + "epoch": 0.7268722466960352, + "fcm_dpo/beta": 0.00481772143393755, + "fcm_dpo/delta": -0.02338775247335434, + "fcm_dpo/margin": 98.02427673339844, + "fcm_dpo/q_t": 0.3920443654060364, + "grad_norm": 11.115226745605469, + "learning_rate": 1.0660589091223854e-07, + "logits/chosen": -2.2392048835754395, + "logits/rejected": -1.775106430053711, + "logps/chosen": -266.9747314453125, + "logps/ref_chosen": -99.52031707763672, + "logps/ref_rejected": -97.93089294433594, + "logps/rejected": -363.4095764160156, + "loss": 1.0489, + "margin_dpo/margin_mean": 98.02427673339844, + "margin_dpo/margin_std": 122.67973327636719, + "step": 495 + }, + { + "KL/chosen_KL_mean": -186.36358642578125, + "KL/mean": -227.69281005859375, + "KL/rejected_KL_mean": -269.02203369140625, + "KL/std": 103.54518127441406, + "epoch": 0.7283406754772394, + "fcm_dpo/beta": 0.004865934606641531, + "fcm_dpo/delta": 0.04945981130003929, + "fcm_dpo/margin": 82.658447265625, + "fcm_dpo/q_t": 0.4063739478588104, + "grad_norm": 9.600869178771973, + "learning_rate": 1.0555654214793722e-07, + "logits/chosen": -2.279493808746338, + "logits/rejected": -1.647784948348999, + "logps/chosen": -294.2203369140625, + "logps/ref_chosen": -107.85675048828125, + "logps/ref_rejected": -92.77056121826172, + "logps/rejected": -361.7926025390625, + "loss": 1.0865, + "margin_dpo/margin_mean": 82.65843963623047, + "margin_dpo/margin_std": 103.96592712402344, + "step": 496 + }, + { + "KL/chosen_KL_mean": -200.2681884765625, + "KL/mean": -227.3326416015625, + "KL/rejected_KL_mean": -254.39710998535156, + "KL/std": 105.10977172851562, + "epoch": 0.7298091042584435, + "fcm_dpo/beta": 0.004926739726215601, + "fcm_dpo/delta": 0.06447851657867432, + "fcm_dpo/margin": 54.12892150878906, + "fcm_dpo/q_t": 0.43916094303131104, + "grad_norm": 9.265952110290527, + "learning_rate": 1.0451099960308374e-07, + "logits/chosen": -1.6075884103775024, + "logits/rejected": -1.146307349205017, + "logps/chosen": -292.3514099121094, + "logps/ref_chosen": -92.08322143554688, + "logps/ref_rejected": -81.79503631591797, + "logps/rejected": -336.192138671875, + "loss": 1.2122, + "margin_dpo/margin_mean": 54.12892150878906, + "margin_dpo/margin_std": 112.56846618652344, + "step": 497 + }, + { + "KL/chosen_KL_mean": -168.26063537597656, + "KL/mean": -213.84844970703125, + "KL/rejected_KL_mean": -259.436279296875, + "KL/std": 101.29026794433594, + "epoch": 0.7312775330396476, + "fcm_dpo/beta": 0.00492420420050621, + "fcm_dpo/delta": 0.0010725930333137512, + "fcm_dpo/margin": 91.17562103271484, + "fcm_dpo/q_t": 0.396063894033432, + "grad_norm": 11.956170082092285, + "learning_rate": 1.0346929082869641e-07, + "logits/chosen": -1.6750918626785278, + "logits/rejected": -1.1646183729171753, + "logps/chosen": -266.45501708984375, + "logps/ref_chosen": -98.19437408447266, + "logps/ref_rejected": -90.68745422363281, + "logps/rejected": -350.12371826171875, + "loss": 1.0544, + "margin_dpo/margin_mean": 91.17562103271484, + "margin_dpo/margin_std": 109.13645935058594, + "step": 498 + }, + { + "KL/chosen_KL_mean": -177.30838012695312, + "KL/mean": -224.7327423095703, + "KL/rejected_KL_mean": -272.1571044921875, + "KL/std": 111.17930603027344, + "epoch": 0.7327459618208517, + "fcm_dpo/beta": 0.004920176230370998, + "fcm_dpo/delta": -0.01749299466609955, + "fcm_dpo/margin": 94.84874725341797, + "fcm_dpo/q_t": 0.3935285806655884, + "grad_norm": 11.57456111907959, + "learning_rate": 1.0243144327477013e-07, + "logits/chosen": -2.0922465324401855, + "logits/rejected": -2.0417070388793945, + "logps/chosen": -258.3482666015625, + "logps/ref_chosen": -81.0399169921875, + "logps/ref_rejected": -106.92170715332031, + "logps/rejected": -379.07879638671875, + "loss": 1.0514, + "margin_dpo/margin_mean": 94.8487548828125, + "margin_dpo/margin_std": 118.310302734375, + "step": 499 + }, + { + "KL/chosen_KL_mean": -196.89361572265625, + "KL/mean": -237.7458953857422, + "KL/rejected_KL_mean": -278.5981750488281, + "KL/std": 107.0694580078125, + "epoch": 0.7342143906020558, + "fcm_dpo/beta": 0.004939177073538303, + "fcm_dpo/delta": 0.048427581787109375, + "fcm_dpo/margin": 81.70455169677734, + "fcm_dpo/q_t": 0.40973156690597534, + "grad_norm": 12.509004592895508, + "learning_rate": 1.0139748428955333e-07, + "logits/chosen": -1.6324963569641113, + "logits/rejected": -1.4601924419403076, + "logps/chosen": -286.14166259765625, + "logps/ref_chosen": -89.248046875, + "logps/ref_rejected": -100.41021728515625, + "logps/rejected": -379.0083923339844, + "loss": 1.1061, + "margin_dpo/margin_mean": 81.70455932617188, + "margin_dpo/margin_std": 121.18946075439453, + "step": 500 + }, + { + "KL/chosen_KL_mean": -172.25335693359375, + "KL/mean": -216.698974609375, + "KL/rejected_KL_mean": -261.14459228515625, + "KL/std": 116.15937805175781, + "epoch": 0.73568281938326, + "fcm_dpo/beta": 0.005002699792385101, + "fcm_dpo/delta": 0.004122689366340637, + "fcm_dpo/margin": 88.89126586914062, + "fcm_dpo/q_t": 0.40019088983535767, + "grad_norm": 10.078176498413086, + "learning_rate": 1.0036744111882672e-07, + "logits/chosen": -1.3775291442871094, + "logits/rejected": -0.9249249696731567, + "logps/chosen": -262.85028076171875, + "logps/ref_chosen": -90.596923828125, + "logps/ref_rejected": -85.2972640991211, + "logps/rejected": -346.4418640136719, + "loss": 1.0941, + "margin_dpo/margin_mean": 88.8912582397461, + "margin_dpo/margin_std": 129.65127563476562, + "step": 501 + }, + { + "KL/chosen_KL_mean": -173.2459716796875, + "KL/mean": -213.20974731445312, + "KL/rejected_KL_mean": -253.1735382080078, + "KL/std": 96.82359313964844, + "epoch": 0.737151248164464, + "fcm_dpo/beta": 0.0050225649029016495, + "fcm_dpo/delta": 0.049928393214941025, + "fcm_dpo/margin": 79.92756652832031, + "fcm_dpo/q_t": 0.40794628858566284, + "grad_norm": 10.836005210876465, + "learning_rate": 9.934134090518592e-08, + "logits/chosen": -1.8895217180252075, + "logits/rejected": -1.1493624448776245, + "logps/chosen": -276.7167053222656, + "logps/ref_chosen": -103.47073364257812, + "logps/ref_rejected": -89.04415893554688, + "logps/rejected": -342.21771240234375, + "loss": 1.1121, + "margin_dpo/margin_mean": 79.92756652832031, + "margin_dpo/margin_std": 119.62471008300781, + "step": 502 + }, + { + "KL/chosen_KL_mean": -170.5072021484375, + "KL/mean": -216.42495727539062, + "KL/rejected_KL_mean": -262.34271240234375, + "KL/std": 103.584228515625, + "epoch": 0.7386196769456681, + "fcm_dpo/beta": 0.005027182400226593, + "fcm_dpo/delta": -0.012657150626182556, + "fcm_dpo/margin": 91.83548736572266, + "fcm_dpo/q_t": 0.3943704068660736, + "grad_norm": 9.954883575439453, + "learning_rate": 9.831921068732571e-08, + "logits/chosen": -1.704979419708252, + "logits/rejected": -1.1992563009262085, + "logps/chosen": -264.2371826171875, + "logps/ref_chosen": -93.72998809814453, + "logps/ref_rejected": -87.60896301269531, + "logps/rejected": -349.95166015625, + "loss": 1.0565, + "margin_dpo/margin_mean": 91.83548736572266, + "margin_dpo/margin_std": 114.19137573242188, + "step": 503 + }, + { + "KL/chosen_KL_mean": -173.15093994140625, + "KL/mean": -231.43263244628906, + "KL/rejected_KL_mean": -289.7143249511719, + "KL/std": 107.3223648071289, + "epoch": 0.7400881057268722, + "fcm_dpo/beta": 0.004910448100417852, + "fcm_dpo/delta": -0.12967948615550995, + "fcm_dpo/margin": 116.56340026855469, + "fcm_dpo/q_t": 0.36714255809783936, + "grad_norm": 10.799518585205078, + "learning_rate": 9.730107739932805e-08, + "logits/chosen": -2.193516254425049, + "logits/rejected": -1.863387107849121, + "logps/chosen": -265.44281005859375, + "logps/ref_chosen": -92.2918701171875, + "logps/ref_rejected": -109.03765106201172, + "logps/rejected": -398.7519836425781, + "loss": 0.9587, + "margin_dpo/margin_mean": 116.56340026855469, + "margin_dpo/margin_std": 107.65071105957031, + "step": 504 + }, + { + "KL/chosen_KL_mean": -191.77728271484375, + "KL/mean": -233.93405151367188, + "KL/rejected_KL_mean": -276.0908203125, + "KL/std": 106.50577545166016, + "epoch": 0.7415565345080763, + "fcm_dpo/beta": 0.004897596314549446, + "fcm_dpo/delta": 0.03867579996585846, + "fcm_dpo/margin": 84.31356811523438, + "fcm_dpo/q_t": 0.4046417474746704, + "grad_norm": 9.086536407470703, + "learning_rate": 9.628696786995188e-08, + "logits/chosen": -2.4688057899475098, + "logits/rejected": -1.9173367023468018, + "logps/chosen": -298.05419921875, + "logps/ref_chosen": -106.27693176269531, + "logps/ref_rejected": -94.20611572265625, + "logps/rejected": -370.29693603515625, + "loss": 1.0777, + "margin_dpo/margin_mean": 84.31356811523438, + "margin_dpo/margin_std": 104.20630645751953, + "step": 505 + }, + { + "KL/chosen_KL_mean": -186.02706909179688, + "KL/mean": -232.8184051513672, + "KL/rejected_KL_mean": -279.6097717285156, + "KL/std": 99.92256164550781, + "epoch": 0.7430249632892805, + "fcm_dpo/beta": 0.004884321242570877, + "fcm_dpo/delta": -0.007781848311424255, + "fcm_dpo/margin": 93.58268737792969, + "fcm_dpo/q_t": 0.3927346169948578, + "grad_norm": 9.635473251342773, + "learning_rate": 9.527690882192635e-08, + "logits/chosen": -2.3033764362335205, + "logits/rejected": -1.9620904922485352, + "logps/chosen": -267.4679260253906, + "logps/ref_chosen": -81.44086456298828, + "logps/ref_rejected": -85.66439819335938, + "logps/rejected": -365.274169921875, + "loss": 1.0439, + "margin_dpo/margin_mean": 93.58268737792969, + "margin_dpo/margin_std": 104.6795654296875, + "step": 506 + }, + { + "KL/chosen_KL_mean": -193.2694091796875, + "KL/mean": -241.5196533203125, + "KL/rejected_KL_mean": -289.7698974609375, + "KL/std": 105.75428771972656, + "epoch": 0.7444933920704846, + "fcm_dpo/beta": 0.004886812996119261, + "fcm_dpo/delta": -0.022642912343144417, + "fcm_dpo/margin": 96.50050354003906, + "fcm_dpo/q_t": 0.39291203022003174, + "grad_norm": 10.023951530456543, + "learning_rate": 9.427092687124691e-08, + "logits/chosen": -2.1121931076049805, + "logits/rejected": -1.6658493280410767, + "logps/chosen": -292.69635009765625, + "logps/ref_chosen": -99.42694854736328, + "logps/ref_rejected": -102.25296020507812, + "logps/rejected": -392.02288818359375, + "loss": 1.0569, + "margin_dpo/margin_mean": 96.50050354003906, + "margin_dpo/margin_std": 124.43849182128906, + "step": 507 + }, + { + "KL/chosen_KL_mean": -188.124755859375, + "KL/mean": -228.52621459960938, + "KL/rejected_KL_mean": -268.92767333984375, + "KL/std": 113.13078308105469, + "epoch": 0.7459618208516887, + "fcm_dpo/beta": 0.004920968785881996, + "fcm_dpo/delta": 0.05452117323875427, + "fcm_dpo/margin": 80.8029556274414, + "fcm_dpo/q_t": 0.41117680072784424, + "grad_norm": 17.02985191345215, + "learning_rate": 9.326904852647344e-08, + "logits/chosen": -1.8558951616287231, + "logits/rejected": -1.3565008640289307, + "logps/chosen": -286.0404052734375, + "logps/ref_chosen": -97.9156494140625, + "logps/ref_rejected": -103.79782104492188, + "logps/rejected": -372.72552490234375, + "loss": 1.1276, + "margin_dpo/margin_mean": 80.8029556274414, + "margin_dpo/margin_std": 130.61660766601562, + "step": 508 + }, + { + "KL/chosen_KL_mean": -176.96371459960938, + "KL/mean": -214.24057006835938, + "KL/rejected_KL_mean": -251.5174102783203, + "KL/std": 87.99352264404297, + "epoch": 0.7474302496328928, + "fcm_dpo/beta": 0.005000775679945946, + "fcm_dpo/delta": 0.07996828854084015, + "fcm_dpo/margin": 74.55369567871094, + "fcm_dpo/q_t": 0.4130539000034332, + "grad_norm": 16.53234100341797, + "learning_rate": 9.227130018803195e-08, + "logits/chosen": -1.9470932483673096, + "logits/rejected": -1.2735739946365356, + "logps/chosen": -277.6935729980469, + "logps/ref_chosen": -100.72984313964844, + "logps/ref_rejected": -86.943359375, + "logps/rejected": -338.46075439453125, + "loss": 1.1296, + "margin_dpo/margin_mean": 74.55369567871094, + "margin_dpo/margin_std": 116.4517822265625, + "step": 509 + }, + { + "KL/chosen_KL_mean": -207.454833984375, + "KL/mean": -252.71673583984375, + "KL/rejected_KL_mean": -297.9786682128906, + "KL/std": 107.10193634033203, + "epoch": 0.748898678414097, + "fcm_dpo/beta": 0.0050178710371255875, + "fcm_dpo/delta": -0.004490138962864876, + "fcm_dpo/margin": 90.52381896972656, + "fcm_dpo/q_t": 0.39535173773765564, + "grad_norm": 12.655274391174316, + "learning_rate": 9.127770814751932e-08, + "logits/chosen": -1.4448351860046387, + "logits/rejected": -1.2549282312393188, + "logps/chosen": -296.62420654296875, + "logps/ref_chosen": -89.16938781738281, + "logps/ref_rejected": -107.91939544677734, + "logps/rejected": -405.8980712890625, + "loss": 1.0537, + "margin_dpo/margin_mean": 90.52381896972656, + "margin_dpo/margin_std": 107.55709838867188, + "step": 510 + }, + { + "KL/chosen_KL_mean": -189.1050262451172, + "KL/mean": -237.92193603515625, + "KL/rejected_KL_mean": -286.7388610839844, + "KL/std": 105.441650390625, + "epoch": 0.750367107195301, + "fcm_dpo/beta": 0.004991387017071247, + "fcm_dpo/delta": -0.039268966764211655, + "fcm_dpo/margin": 97.63385009765625, + "fcm_dpo/q_t": 0.3893653154373169, + "grad_norm": 11.885235786437988, + "learning_rate": 9.028829858700973e-08, + "logits/chosen": -2.0825769901275635, + "logits/rejected": -1.5090551376342773, + "logps/chosen": -278.85479736328125, + "logps/ref_chosen": -89.74979400634766, + "logps/ref_rejected": -97.73127746582031, + "logps/rejected": -384.47015380859375, + "loss": 1.0356, + "margin_dpo/margin_mean": 97.63385009765625, + "margin_dpo/margin_std": 116.70793151855469, + "step": 511 + }, + { + "KL/chosen_KL_mean": -180.0770263671875, + "KL/mean": -227.23626708984375, + "KL/rejected_KL_mean": -274.3955078125, + "KL/std": 101.90090942382812, + "epoch": 0.7518355359765051, + "fcm_dpo/beta": 0.004981291014701128, + "fcm_dpo/delta": -0.021333176642656326, + "fcm_dpo/margin": 94.3184814453125, + "fcm_dpo/q_t": 0.39237287640571594, + "grad_norm": 14.846067428588867, + "learning_rate": 8.930309757836516e-08, + "logits/chosen": -1.7191128730773926, + "logits/rejected": -1.2665915489196777, + "logps/chosen": -265.81219482421875, + "logps/ref_chosen": -85.73515319824219, + "logps/ref_rejected": -87.6248779296875, + "logps/rejected": -362.0203857421875, + "loss": 1.0597, + "margin_dpo/margin_mean": 94.31848907470703, + "margin_dpo/margin_std": 121.84278869628906, + "step": 512 + }, + { + "KL/chosen_KL_mean": -191.56814575195312, + "KL/mean": -231.19024658203125, + "KL/rejected_KL_mean": -270.8123779296875, + "KL/std": 98.29747772216797, + "epoch": 0.7533039647577092, + "fcm_dpo/beta": 0.004967710003256798, + "fcm_dpo/delta": 0.058465782552957535, + "fcm_dpo/margin": 79.24421691894531, + "fcm_dpo/q_t": 0.40833455324172974, + "grad_norm": 12.429883003234863, + "learning_rate": 8.832213108254863e-08, + "logits/chosen": -2.1757917404174805, + "logits/rejected": -1.460874080657959, + "logps/chosen": -287.7586669921875, + "logps/ref_chosen": -96.19050598144531, + "logps/ref_rejected": -82.91773986816406, + "logps/rejected": -353.7301025390625, + "loss": 1.1046, + "margin_dpo/margin_mean": 79.24421691894531, + "margin_dpo/margin_std": 109.68754577636719, + "step": 513 + }, + { + "KL/chosen_KL_mean": -185.51446533203125, + "KL/mean": -228.64682006835938, + "KL/rejected_KL_mean": -271.7791748046875, + "KL/std": 98.27638244628906, + "epoch": 0.7547723935389133, + "fcm_dpo/beta": 0.005019359290599823, + "fcm_dpo/delta": 0.01777580752968788, + "fcm_dpo/margin": 86.26473236083984, + "fcm_dpo/q_t": 0.40216636657714844, + "grad_norm": 12.15334415435791, + "learning_rate": 8.734542494893954e-08, + "logits/chosen": -1.8832634687423706, + "logits/rejected": -1.3261592388153076, + "logps/chosen": -283.3381042480469, + "logps/ref_chosen": -97.82363891601562, + "logps/ref_rejected": -85.36982727050781, + "logps/rejected": -357.14898681640625, + "loss": 1.0787, + "margin_dpo/margin_mean": 86.26473999023438, + "margin_dpo/margin_std": 116.4493408203125, + "step": 514 + }, + { + "KL/chosen_KL_mean": -178.60165405273438, + "KL/mean": -220.3987579345703, + "KL/rejected_KL_mean": -262.19586181640625, + "KL/std": 102.03836059570312, + "epoch": 0.7562408223201175, + "fcm_dpo/beta": 0.005045060068368912, + "fcm_dpo/delta": 0.02951134368777275, + "fcm_dpo/margin": 83.5942153930664, + "fcm_dpo/q_t": 0.4032410979270935, + "grad_norm": 13.163448333740234, + "learning_rate": 8.637300491465272e-08, + "logits/chosen": -1.918165683746338, + "logits/rejected": -1.6911985874176025, + "logps/chosen": -260.53521728515625, + "logps/ref_chosen": -81.93355560302734, + "logps/ref_rejected": -93.57656860351562, + "logps/rejected": -355.7724304199219, + "loss": 1.0837, + "margin_dpo/margin_mean": 83.59422302246094, + "margin_dpo/margin_std": 110.89984893798828, + "step": 515 + }, + { + "KL/chosen_KL_mean": -182.73809814453125, + "KL/mean": -223.7190399169922, + "KL/rejected_KL_mean": -264.6999816894531, + "KL/std": 117.15553283691406, + "epoch": 0.7577092511013216, + "fcm_dpo/beta": 0.005087654106318951, + "fcm_dpo/delta": 0.0343351885676384, + "fcm_dpo/margin": 81.96188354492188, + "fcm_dpo/q_t": 0.40796324610710144, + "grad_norm": 12.83326530456543, + "learning_rate": 8.540489660386064e-08, + "logits/chosen": -2.3145430088043213, + "logits/rejected": -2.1350057125091553, + "logps/chosen": -279.82977294921875, + "logps/ref_chosen": -97.09166717529297, + "logps/ref_rejected": -118.55667114257812, + "logps/rejected": -383.25665283203125, + "loss": 1.126, + "margin_dpo/margin_mean": 81.96188354492188, + "margin_dpo/margin_std": 138.1246337890625, + "step": 516 + }, + { + "KL/chosen_KL_mean": -194.80770874023438, + "KL/mean": -244.35067749023438, + "KL/rejected_KL_mean": -293.8936767578125, + "KL/std": 105.68379211425781, + "epoch": 0.7591776798825257, + "fcm_dpo/beta": 0.0050135329365730286, + "fcm_dpo/delta": -0.050679806619882584, + "fcm_dpo/margin": 99.08596801757812, + "fcm_dpo/q_t": 0.38733309507369995, + "grad_norm": 9.785173416137695, + "learning_rate": 8.444112552711752e-08, + "logits/chosen": -2.079163074493408, + "logits/rejected": -1.6662458181381226, + "logps/chosen": -287.9942626953125, + "logps/ref_chosen": -93.18656921386719, + "logps/ref_rejected": -95.5927963256836, + "logps/rejected": -389.4864501953125, + "loss": 1.041, + "margin_dpo/margin_mean": 99.08596801757812, + "margin_dpo/margin_std": 120.85176849365234, + "step": 517 + }, + { + "KL/chosen_KL_mean": -203.78955078125, + "KL/mean": -240.6020965576172, + "KL/rejected_KL_mean": -277.4146423339844, + "KL/std": 99.24911499023438, + "epoch": 0.7606461086637298, + "fcm_dpo/beta": 0.005094345659017563, + "fcm_dpo/delta": 0.07787081599235535, + "fcm_dpo/margin": 73.62509155273438, + "fcm_dpo/q_t": 0.4158647656440735, + "grad_norm": 11.506685256958008, + "learning_rate": 8.348171708068747e-08, + "logits/chosen": -1.72408926486969, + "logits/rejected": -1.3833379745483398, + "logps/chosen": -290.8109130859375, + "logps/ref_chosen": -87.0213623046875, + "logps/ref_rejected": -91.24456024169922, + "logps/rejected": -368.6592102050781, + "loss": 1.1378, + "margin_dpo/margin_mean": 73.62509155273438, + "margin_dpo/margin_std": 123.48959350585938, + "step": 518 + }, + { + "KL/chosen_KL_mean": -188.56741333007812, + "KL/mean": -228.94049072265625, + "KL/rejected_KL_mean": -269.3135681152344, + "KL/std": 93.03599548339844, + "epoch": 0.762114537444934, + "fcm_dpo/beta": 0.005131518002599478, + "fcm_dpo/delta": 0.03715251758694649, + "fcm_dpo/margin": 80.74616241455078, + "fcm_dpo/q_t": 0.4046279191970825, + "grad_norm": 11.094258308410645, + "learning_rate": 8.25266965458755e-08, + "logits/chosen": -1.780432939529419, + "logits/rejected": -1.3956313133239746, + "logps/chosen": -289.25494384765625, + "logps/ref_chosen": -100.68755340576172, + "logps/ref_rejected": -109.03042602539062, + "logps/rejected": -378.343994140625, + "loss": 1.0718, + "margin_dpo/margin_mean": 80.74616241455078, + "margin_dpo/margin_std": 96.15090942382812, + "step": 519 + }, + { + "KL/chosen_KL_mean": -174.54107666015625, + "KL/mean": -222.8872833251953, + "KL/rejected_KL_mean": -271.2334899902344, + "KL/std": 104.92181396484375, + "epoch": 0.7635829662261381, + "fcm_dpo/beta": 0.005110214464366436, + "fcm_dpo/delta": -0.04654761776328087, + "fcm_dpo/margin": 96.69241333007812, + "fcm_dpo/q_t": 0.38531070947647095, + "grad_norm": 10.005337715148926, + "learning_rate": 8.15760890883607e-08, + "logits/chosen": -1.5855275392532349, + "logits/rejected": -1.1460142135620117, + "logps/chosen": -273.3187255859375, + "logps/ref_chosen": -98.77762603759766, + "logps/ref_rejected": -106.9457015991211, + "logps/rejected": -378.17919921875, + "loss": 1.0157, + "margin_dpo/margin_mean": 96.69241333007812, + "margin_dpo/margin_std": 101.92752838134766, + "step": 520 + }, + { + "KL/chosen_KL_mean": -176.4241485595703, + "KL/mean": -225.93972778320312, + "KL/rejected_KL_mean": -275.4552917480469, + "KL/std": 98.50906372070312, + "epoch": 0.7650513950073421, + "fcm_dpo/beta": 0.00508786691352725, + "fcm_dpo/delta": -0.056909676641225815, + "fcm_dpo/margin": 99.03115844726562, + "fcm_dpo/q_t": 0.38355040550231934, + "grad_norm": 11.63822078704834, + "learning_rate": 8.062991975753378e-08, + "logits/chosen": -1.934647560119629, + "logits/rejected": -1.595494031906128, + "logps/chosen": -262.7752380371094, + "logps/ref_chosen": -86.35108947753906, + "logps/ref_rejected": -90.39317321777344, + "logps/rejected": -365.84844970703125, + "loss": 1.0063, + "margin_dpo/margin_mean": 99.03115844726562, + "margin_dpo/margin_std": 99.65690612792969, + "step": 521 + }, + { + "KL/chosen_KL_mean": -184.30136108398438, + "KL/mean": -229.09103393554688, + "KL/rejected_KL_mean": -273.8807373046875, + "KL/std": 106.60728454589844, + "epoch": 0.7665198237885462, + "fcm_dpo/beta": 0.005056227091699839, + "fcm_dpo/delta": -0.0031437724828720093, + "fcm_dpo/margin": 89.57937622070312, + "fcm_dpo/q_t": 0.3963169455528259, + "grad_norm": 11.193562507629395, + "learning_rate": 7.968821348583643e-08, + "logits/chosen": -1.5549134016036987, + "logits/rejected": -1.177118182182312, + "logps/chosen": -260.3094177246094, + "logps/ref_chosen": -76.008056640625, + "logps/ref_rejected": -74.44125366210938, + "logps/rejected": -348.3219909667969, + "loss": 1.0591, + "margin_dpo/margin_mean": 89.57936096191406, + "margin_dpo/margin_std": 112.14762878417969, + "step": 522 + }, + { + "KL/chosen_KL_mean": -203.88681030273438, + "KL/mean": -236.28988647460938, + "KL/rejected_KL_mean": -268.6929626464844, + "KL/std": 105.2451171875, + "epoch": 0.7679882525697503, + "fcm_dpo/beta": 0.005130934529006481, + "fcm_dpo/delta": 0.12165539711713791, + "fcm_dpo/margin": 64.80615234375, + "fcm_dpo/q_t": 0.42573124170303345, + "grad_norm": 12.027227401733398, + "learning_rate": 7.875099508810484e-08, + "logits/chosen": -1.9186699390411377, + "logits/rejected": -1.2036707401275635, + "logps/chosen": -300.3830871582031, + "logps/ref_chosen": -96.49627685546875, + "logps/ref_rejected": -91.0152816772461, + "logps/rejected": -359.708251953125, + "loss": 1.1731, + "margin_dpo/margin_mean": 64.80615234375, + "margin_dpo/margin_std": 122.15135192871094, + "step": 523 + }, + { + "KL/chosen_KL_mean": -185.89892578125, + "KL/mean": -231.63909912109375, + "KL/rejected_KL_mean": -277.37933349609375, + "KL/std": 105.57075500488281, + "epoch": 0.7694566813509545, + "fcm_dpo/beta": 0.005148780532181263, + "fcm_dpo/delta": -0.02208206057548523, + "fcm_dpo/margin": 91.48039245605469, + "fcm_dpo/q_t": 0.39336147904396057, + "grad_norm": 14.017488479614258, + "learning_rate": 7.781828926091535e-08, + "logits/chosen": -2.125394821166992, + "logits/rejected": -1.4248558282852173, + "logps/chosen": -289.304443359375, + "logps/ref_chosen": -103.40550994873047, + "logps/ref_rejected": -87.91015625, + "logps/rejected": -365.2894592285156, + "loss": 1.0462, + "margin_dpo/margin_mean": 91.48039245605469, + "margin_dpo/margin_std": 112.13156127929688, + "step": 524 + }, + { + "KL/chosen_KL_mean": -175.44296264648438, + "KL/mean": -226.84909057617188, + "KL/rejected_KL_mean": -278.2552490234375, + "KL/std": 106.01425170898438, + "epoch": 0.7709251101321586, + "fcm_dpo/beta": 0.005037224851548672, + "fcm_dpo/delta": -0.07464565336704254, + "fcm_dpo/margin": 102.8122787475586, + "fcm_dpo/q_t": 0.3849982023239136, + "grad_norm": 11.277276039123535, + "learning_rate": 7.689012058193384e-08, + "logits/chosen": -1.5974867343902588, + "logits/rejected": -1.5107810497283936, + "logps/chosen": -259.6494445800781, + "logps/ref_chosen": -84.20648193359375, + "logps/ref_rejected": -106.26158905029297, + "logps/rejected": -384.516845703125, + "loss": 1.0291, + "margin_dpo/margin_mean": 102.8122787475586, + "margin_dpo/margin_std": 122.57896423339844, + "step": 525 + }, + { + "KL/chosen_KL_mean": -181.08453369140625, + "KL/mean": -228.5096893310547, + "KL/rejected_KL_mean": -275.934814453125, + "KL/std": 100.11372375488281, + "epoch": 0.7723935389133627, + "fcm_dpo/beta": 0.005048790015280247, + "fcm_dpo/delta": -0.03032749518752098, + "fcm_dpo/margin": 94.85028839111328, + "fcm_dpo/q_t": 0.3906853199005127, + "grad_norm": 9.261003494262695, + "learning_rate": 7.596651350926836e-08, + "logits/chosen": -1.7839946746826172, + "logits/rejected": -1.1341900825500488, + "logps/chosen": -283.93865966796875, + "logps/ref_chosen": -102.85411071777344, + "logps/ref_rejected": -93.65145874023438, + "logps/rejected": -369.5863037109375, + "loss": 1.0454, + "margin_dpo/margin_mean": 94.85028839111328, + "margin_dpo/margin_std": 117.34310913085938, + "step": 526 + }, + { + "KL/chosen_KL_mean": -182.96224975585938, + "KL/mean": -230.83517456054688, + "KL/rejected_KL_mean": -278.70806884765625, + "KL/std": 105.57845306396484, + "epoch": 0.7738619676945668, + "fcm_dpo/beta": 0.005016027949750423, + "fcm_dpo/delta": -0.031792886555194855, + "fcm_dpo/margin": 95.7458267211914, + "fcm_dpo/q_t": 0.38758933544158936, + "grad_norm": 12.210872650146484, + "learning_rate": 7.504749238082414e-08, + "logits/chosen": -3.099153518676758, + "logits/rejected": -2.2796692848205566, + "logps/chosen": -279.7276916503906, + "logps/ref_chosen": -96.76543426513672, + "logps/ref_rejected": -85.39518737792969, + "logps/rejected": -364.103271484375, + "loss": 1.0137, + "margin_dpo/margin_mean": 95.74581909179688, + "margin_dpo/margin_std": 93.15516662597656, + "step": 527 + }, + { + "KL/chosen_KL_mean": -170.5989227294922, + "KL/mean": -217.8131103515625, + "KL/rejected_KL_mean": -265.02734375, + "KL/std": 106.9075927734375, + "epoch": 0.775330396475771, + "fcm_dpo/beta": 0.004999192431569099, + "fcm_dpo/delta": -0.023176094517111778, + "fcm_dpo/margin": 94.42839050292969, + "fcm_dpo/q_t": 0.3923468589782715, + "grad_norm": 10.352832794189453, + "learning_rate": 7.413308141366254e-08, + "logits/chosen": -2.2386622428894043, + "logits/rejected": -1.7365646362304688, + "logps/chosen": -264.76385498046875, + "logps/ref_chosen": -94.1649169921875, + "logps/ref_rejected": -100.372314453125, + "logps/rejected": -365.399658203125, + "loss": 1.0529, + "margin_dpo/margin_mean": 94.42839050292969, + "margin_dpo/margin_std": 120.28085327148438, + "step": 528 + }, + { + "KL/chosen_KL_mean": -198.13063049316406, + "KL/mean": -240.8045654296875, + "KL/rejected_KL_mean": -283.478515625, + "KL/std": 105.39470672607422, + "epoch": 0.7767988252569751, + "fcm_dpo/beta": 0.005012898240238428, + "fcm_dpo/delta": 0.023006722331047058, + "fcm_dpo/margin": 85.34791564941406, + "fcm_dpo/q_t": 0.40393248200416565, + "grad_norm": 11.763914108276367, + "learning_rate": 7.322330470336313e-08, + "logits/chosen": -1.6822190284729004, + "logits/rejected": -1.5023728609085083, + "logps/chosen": -278.90740966796875, + "logps/ref_chosen": -80.77680206298828, + "logps/ref_rejected": -97.191162109375, + "logps/rejected": -380.669677734375, + "loss": 1.0935, + "margin_dpo/margin_mean": 85.34791564941406, + "margin_dpo/margin_std": 124.56035614013672, + "step": 529 + }, + { + "KL/chosen_KL_mean": -181.14682006835938, + "KL/mean": -223.98953247070312, + "KL/rejected_KL_mean": -266.8322448730469, + "KL/std": 102.71823120117188, + "epoch": 0.7782672540381792, + "fcm_dpo/beta": 0.005006622523069382, + "fcm_dpo/delta": 0.02176816761493683, + "fcm_dpo/margin": 85.68543243408203, + "fcm_dpo/q_t": 0.4013262987136841, + "grad_norm": 10.490839958190918, + "learning_rate": 7.231818622338822e-08, + "logits/chosen": -1.5628221035003662, + "logits/rejected": -1.2402976751327515, + "logps/chosen": -260.92254638671875, + "logps/ref_chosen": -79.7757339477539, + "logps/ref_rejected": -92.05390167236328, + "logps/rejected": -358.8861389160156, + "loss": 1.0592, + "margin_dpo/margin_mean": 85.68543243408203, + "margin_dpo/margin_std": 98.09611511230469, + "step": 530 + }, + { + "KL/chosen_KL_mean": -175.47500610351562, + "KL/mean": -228.99588012695312, + "KL/rejected_KL_mean": -282.5167541503906, + "KL/std": 106.51791381835938, + "epoch": 0.7797356828193832, + "fcm_dpo/beta": 0.004961233586072922, + "fcm_dpo/delta": -0.08571291714906693, + "fcm_dpo/margin": 107.04176330566406, + "fcm_dpo/q_t": 0.3762897849082947, + "grad_norm": 11.941046714782715, + "learning_rate": 7.141774982445147e-08, + "logits/chosen": -1.834104299545288, + "logits/rejected": -1.229844570159912, + "logps/chosen": -254.794189453125, + "logps/ref_chosen": -79.31919860839844, + "logps/ref_rejected": -78.10832214355469, + "logps/rejected": -360.62506103515625, + "loss": 0.9892, + "margin_dpo/margin_mean": 107.04176330566406, + "margin_dpo/margin_std": 106.6690673828125, + "step": 531 + }, + { + "KL/chosen_KL_mean": -189.42636108398438, + "KL/mean": -247.23992919921875, + "KL/rejected_KL_mean": -305.0534973144531, + "KL/std": 102.47343444824219, + "epoch": 0.7812041116005873, + "fcm_dpo/beta": 0.004868891090154648, + "fcm_dpo/delta": -0.11954863369464874, + "fcm_dpo/margin": 115.62712097167969, + "fcm_dpo/q_t": 0.37052595615386963, + "grad_norm": 11.486917495727539, + "learning_rate": 7.052201923388953e-08, + "logits/chosen": -1.5253279209136963, + "logits/rejected": -0.8838618993759155, + "logps/chosen": -291.06329345703125, + "logps/ref_chosen": -101.63691711425781, + "logps/ref_rejected": -93.34539031982422, + "logps/rejected": -398.39886474609375, + "loss": 0.9723, + "margin_dpo/margin_mean": 115.62712860107422, + "margin_dpo/margin_std": 112.86474609375, + "step": 532 + }, + { + "KL/chosen_KL_mean": -183.12327575683594, + "KL/mean": -216.97735595703125, + "KL/rejected_KL_mean": -250.8314208984375, + "KL/std": 108.60610961914062, + "epoch": 0.7826725403817915, + "fcm_dpo/beta": 0.004921652842313051, + "fcm_dpo/delta": 0.12059812247753143, + "fcm_dpo/margin": 67.70816040039062, + "fcm_dpo/q_t": 0.42560046911239624, + "grad_norm": 11.506623268127441, + "learning_rate": 6.963101805503646e-08, + "logits/chosen": -2.447031259536743, + "logits/rejected": -1.8521575927734375, + "logps/chosen": -273.86993408203125, + "logps/ref_chosen": -90.74664306640625, + "logps/ref_rejected": -82.88848876953125, + "logps/rejected": -333.71990966796875, + "loss": 1.1571, + "margin_dpo/margin_mean": 67.70816040039062, + "margin_dpo/margin_std": 116.53468322753906, + "step": 533 + }, + { + "KL/chosen_KL_mean": -163.21951293945312, + "KL/mean": -213.57797241210938, + "KL/rejected_KL_mean": -263.93646240234375, + "KL/std": 102.16084289550781, + "epoch": 0.7841409691629956, + "fcm_dpo/beta": 0.004917072132229805, + "fcm_dpo/delta": -0.0475621297955513, + "fcm_dpo/margin": 100.7169418334961, + "fcm_dpo/q_t": 0.3843523859977722, + "grad_norm": 9.64101505279541, + "learning_rate": 6.874476976660184e-08, + "logits/chosen": -1.8881375789642334, + "logits/rejected": -1.487180233001709, + "logps/chosen": -249.850341796875, + "logps/ref_chosen": -86.6308364868164, + "logps/ref_rejected": -84.87196350097656, + "logps/rejected": -348.80841064453125, + "loss": 1.0065, + "margin_dpo/margin_mean": 100.71693420410156, + "margin_dpo/margin_std": 98.48625946044922, + "step": 534 + }, + { + "KL/chosen_KL_mean": -164.7309112548828, + "KL/mean": -216.9906768798828, + "KL/rejected_KL_mean": -269.2504577636719, + "KL/std": 102.95307922363281, + "epoch": 0.7856093979441997, + "fcm_dpo/beta": 0.0048759700730443, + "fcm_dpo/delta": -0.06299391388893127, + "fcm_dpo/margin": 104.51951599121094, + "fcm_dpo/q_t": 0.3836818337440491, + "grad_norm": 8.676311492919922, + "learning_rate": 6.786329772205246e-08, + "logits/chosen": -1.4965322017669678, + "logits/rejected": -1.3038082122802734, + "logps/chosen": -255.0762939453125, + "logps/ref_chosen": -90.34539031982422, + "logps/ref_rejected": -100.7779769897461, + "logps/rejected": -370.0284423828125, + "loss": 1.0178, + "margin_dpo/margin_mean": 104.51952362060547, + "margin_dpo/margin_std": 117.58296203613281, + "step": 535 + }, + { + "KL/chosen_KL_mean": -165.7255859375, + "KL/mean": -217.365478515625, + "KL/rejected_KL_mean": -269.00537109375, + "KL/std": 103.90904998779297, + "epoch": 0.7870778267254038, + "fcm_dpo/beta": 0.004792565479874611, + "fcm_dpo/delta": -0.04752416908740997, + "fcm_dpo/margin": 103.27981567382812, + "fcm_dpo/q_t": 0.38502782583236694, + "grad_norm": 9.45933723449707, + "learning_rate": 6.698662514899638e-08, + "logits/chosen": -1.9056837558746338, + "logits/rejected": -1.6558258533477783, + "logps/chosen": -244.68515014648438, + "logps/ref_chosen": -78.95956420898438, + "logps/ref_rejected": -93.08779907226562, + "logps/rejected": -362.09320068359375, + "loss": 1.0155, + "margin_dpo/margin_mean": 103.27983093261719, + "margin_dpo/margin_std": 108.91937255859375, + "step": 536 + }, + { + "KL/chosen_KL_mean": -142.94801330566406, + "KL/mean": -198.394775390625, + "KL/rejected_KL_mean": -253.84153747558594, + "KL/std": 107.80245971679688, + "epoch": 0.788546255506608, + "fcm_dpo/beta": 0.004764278419315815, + "fcm_dpo/delta": -0.08304021507501602, + "fcm_dpo/margin": 110.89352416992188, + "fcm_dpo/q_t": 0.37737858295440674, + "grad_norm": 9.695345878601074, + "learning_rate": 6.611477514857114e-08, + "logits/chosen": -1.4823005199432373, + "logits/rejected": -0.7776378393173218, + "logps/chosen": -229.65740966796875, + "logps/ref_chosen": -86.70939636230469, + "logps/ref_rejected": -78.90766906738281, + "logps/rejected": -332.74920654296875, + "loss": 0.9888, + "margin_dpo/margin_mean": 110.89352416992188, + "margin_dpo/margin_std": 104.5932846069336, + "step": 537 + }, + { + "KL/chosen_KL_mean": -194.40371704101562, + "KL/mean": -239.60638427734375, + "KL/rejected_KL_mean": -284.80908203125, + "KL/std": 95.72799682617188, + "epoch": 0.7900146842878121, + "fcm_dpo/beta": 0.004731073975563049, + "fcm_dpo/delta": 0.0232668649405241, + "fcm_dpo/margin": 90.4053726196289, + "fcm_dpo/q_t": 0.400529146194458, + "grad_norm": 9.736135482788086, + "learning_rate": 6.524777069483525e-08, + "logits/chosen": -1.6960458755493164, + "logits/rejected": -1.1449663639068604, + "logps/chosen": -292.8458557128906, + "logps/ref_chosen": -98.44214630126953, + "logps/ref_rejected": -91.09465026855469, + "logps/rejected": -375.90374755859375, + "loss": 1.0584, + "margin_dpo/margin_mean": 90.40536499023438, + "margin_dpo/margin_std": 101.99767303466797, + "step": 538 + }, + { + "KL/chosen_KL_mean": -170.5214080810547, + "KL/mean": -217.82464599609375, + "KL/rejected_KL_mean": -265.12786865234375, + "KL/std": 105.82832336425781, + "epoch": 0.7914831130690162, + "fcm_dpo/beta": 0.004737939685583115, + "fcm_dpo/delta": 0.0018382698763161898, + "fcm_dpo/margin": 94.60646057128906, + "fcm_dpo/q_t": 0.39711710810661316, + "grad_norm": 8.939530372619629, + "learning_rate": 6.438563463416221e-08, + "logits/chosen": -2.4470839500427246, + "logits/rejected": -1.9688284397125244, + "logps/chosen": -261.4535217285156, + "logps/ref_chosen": -90.93211364746094, + "logps/ref_rejected": -97.44546508789062, + "logps/rejected": -362.5733337402344, + "loss": 1.0634, + "margin_dpo/margin_mean": 94.60646057128906, + "margin_dpo/margin_std": 120.57238006591797, + "step": 539 + }, + { + "KL/chosen_KL_mean": -180.2310333251953, + "KL/mean": -237.65408325195312, + "KL/rejected_KL_mean": -295.0771484375, + "KL/std": 114.18052673339844, + "epoch": 0.7929515418502202, + "fcm_dpo/beta": 0.004691114649176598, + "fcm_dpo/delta": -0.09378757327795029, + "fcm_dpo/margin": 114.84609985351562, + "fcm_dpo/q_t": 0.37873169779777527, + "grad_norm": 10.716968536376953, + "learning_rate": 6.352838968463919e-08, + "logits/chosen": -2.0232460498809814, + "logits/rejected": -1.7843815088272095, + "logps/chosen": -288.1009521484375, + "logps/ref_chosen": -107.86990356445312, + "logps/ref_rejected": -121.39350128173828, + "logps/rejected": -416.47064208984375, + "loss": 1.0075, + "margin_dpo/margin_mean": 114.84609985351562, + "margin_dpo/margin_std": 132.90914916992188, + "step": 540 + }, + { + "KL/chosen_KL_mean": -200.82887268066406, + "KL/mean": -236.84466552734375, + "KL/rejected_KL_mean": -272.8604431152344, + "KL/std": 95.08665466308594, + "epoch": 0.7944199706314243, + "fcm_dpo/beta": 0.004642473068088293, + "fcm_dpo/delta": -0.006293103098869324, + "fcm_dpo/margin": 72.03156280517578, + "fcm_dpo/q_t": 0.42235273122787476, + "grad_norm": 10.101605415344238, + "learning_rate": 6.267605843546767e-08, + "logits/chosen": -2.330595016479492, + "logits/rejected": -1.8627632856369019, + "logps/chosen": -307.3963623046875, + "logps/ref_chosen": -106.56748962402344, + "logps/ref_rejected": -109.06633758544922, + "logps/rejected": -381.9267578125, + "loss": 1.1382, + "margin_dpo/margin_mean": 72.03156280517578, + "margin_dpo/margin_std": 101.48991394042969, + "step": 541 + }, + { + "KL/chosen_KL_mean": -170.8132781982422, + "KL/mean": -225.95448303222656, + "KL/rejected_KL_mean": -281.095703125, + "KL/std": 106.34501647949219, + "epoch": 0.7958883994126285, + "fcm_dpo/beta": 0.004601244814693928, + "fcm_dpo/delta": -0.060502491891384125, + "fcm_dpo/margin": 110.28240966796875, + "fcm_dpo/q_t": 0.38187992572784424, + "grad_norm": 10.427596092224121, + "learning_rate": 6.182866334636888e-08, + "logits/chosen": -2.3982412815093994, + "logits/rejected": -2.2766671180725098, + "logps/chosen": -258.18255615234375, + "logps/ref_chosen": -87.36929321289062, + "logps/ref_rejected": -102.82034301757812, + "logps/rejected": -383.9160461425781, + "loss": 1.0078, + "margin_dpo/margin_mean": 110.28240966796875, + "margin_dpo/margin_std": 114.96615600585938, + "step": 542 + }, + { + "KL/chosen_KL_mean": -195.9742431640625, + "KL/mean": -225.4713897705078, + "KL/rejected_KL_mean": -254.96853637695312, + "KL/std": 112.53939819335938, + "epoch": 0.7973568281938326, + "fcm_dpo/beta": 0.004633826203644276, + "fcm_dpo/delta": 0.05118121951818466, + "fcm_dpo/margin": 58.994293212890625, + "fcm_dpo/q_t": 0.43852630257606506, + "grad_norm": 12.674715042114258, + "learning_rate": 6.098622674699147e-08, + "logits/chosen": -1.7109944820404053, + "logits/rejected": -1.7543551921844482, + "logps/chosen": -283.1268310546875, + "logps/ref_chosen": -87.152587890625, + "logps/ref_rejected": -111.53172302246094, + "logps/rejected": -366.5002746582031, + "loss": 1.2258, + "margin_dpo/margin_mean": 58.994300842285156, + "margin_dpo/margin_std": 135.8321533203125, + "step": 543 + }, + { + "KL/chosen_KL_mean": -203.7451171875, + "KL/mean": -240.53851318359375, + "KL/rejected_KL_mean": -277.3319091796875, + "KL/std": 105.84512329101562, + "epoch": 0.7988252569750367, + "fcm_dpo/beta": 0.004706394858658314, + "fcm_dpo/delta": 0.10743935406208038, + "fcm_dpo/margin": 73.58678436279297, + "fcm_dpo/q_t": 0.4208550453186035, + "grad_norm": 9.800400733947754, + "learning_rate": 6.01487708363232e-08, + "logits/chosen": -1.6806445121765137, + "logits/rejected": -1.5842194557189941, + "logps/chosen": -289.89202880859375, + "logps/ref_chosen": -86.14691162109375, + "logps/ref_rejected": -106.3939208984375, + "logps/rejected": -383.725830078125, + "loss": 1.1315, + "margin_dpo/margin_mean": 73.58677673339844, + "margin_dpo/margin_std": 109.59210205078125, + "step": 544 + }, + { + "KL/chosen_KL_mean": -182.01885986328125, + "KL/mean": -227.2618408203125, + "KL/rejected_KL_mean": -272.50482177734375, + "KL/std": 104.80342102050781, + "epoch": 0.8002936857562408, + "fcm_dpo/beta": 0.004756860435009003, + "fcm_dpo/delta": 0.02035892754793167, + "fcm_dpo/margin": 90.48593139648438, + "fcm_dpo/q_t": 0.4016454219818115, + "grad_norm": 9.674954414367676, + "learning_rate": 5.9316317682106294e-08, + "logits/chosen": -1.644040584564209, + "logits/rejected": -1.2250263690948486, + "logps/chosen": -278.443115234375, + "logps/ref_chosen": -96.42424011230469, + "logps/ref_rejected": -100.73405456542969, + "logps/rejected": -373.2388610839844, + "loss": 1.0912, + "margin_dpo/margin_mean": 90.48593139648438, + "margin_dpo/margin_std": 129.81398010253906, + "step": 545 + }, + { + "KL/chosen_KL_mean": -187.50372314453125, + "KL/mean": -228.64816284179688, + "KL/rejected_KL_mean": -269.7926025390625, + "KL/std": 106.960693359375, + "epoch": 0.801762114537445, + "fcm_dpo/beta": 0.004786289297044277, + "fcm_dpo/delta": 0.0584806352853775, + "fcm_dpo/margin": 82.28887939453125, + "fcm_dpo/q_t": 0.4085407853126526, + "grad_norm": 9.608528137207031, + "learning_rate": 5.848888922025552e-08, + "logits/chosen": -1.3605380058288574, + "logits/rejected": -0.9943258762359619, + "logps/chosen": -273.52484130859375, + "logps/ref_chosen": -86.02112579345703, + "logps/ref_rejected": -87.37263488769531, + "logps/rejected": -357.16522216796875, + "loss": 1.1057, + "margin_dpo/margin_mean": 82.28887939453125, + "margin_dpo/margin_std": 117.43290710449219, + "step": 546 + }, + { + "KL/chosen_KL_mean": -180.25697326660156, + "KL/mean": -229.6207275390625, + "KL/rejected_KL_mean": -278.98443603515625, + "KL/std": 104.44168090820312, + "epoch": 0.8032305433186491, + "fcm_dpo/beta": 0.004806514363735914, + "fcm_dpo/delta": -0.02588154375553131, + "fcm_dpo/margin": 98.72744750976562, + "fcm_dpo/q_t": 0.3885681927204132, + "grad_norm": 9.542840957641602, + "learning_rate": 5.7666507254280265e-08, + "logits/chosen": -1.700653076171875, + "logits/rejected": -1.2276029586791992, + "logps/chosen": -280.907958984375, + "logps/ref_chosen": -100.6510009765625, + "logps/ref_rejected": -97.9117431640625, + "logps/rejected": -376.89617919921875, + "loss": 1.0144, + "margin_dpo/margin_mean": 98.72744750976562, + "margin_dpo/margin_std": 92.12249755859375, + "step": 547 + }, + { + "KL/chosen_KL_mean": -177.18148803710938, + "KL/mean": -216.97726440429688, + "KL/rejected_KL_mean": -256.77301025390625, + "KL/std": 106.55033874511719, + "epoch": 0.8046989720998532, + "fcm_dpo/beta": 0.00483623705804348, + "fcm_dpo/delta": 0.06763452291488647, + "fcm_dpo/margin": 79.59153747558594, + "fcm_dpo/q_t": 0.412605345249176, + "grad_norm": 11.051373481750488, + "learning_rate": 5.684919345471029e-08, + "logits/chosen": -2.2747554779052734, + "logits/rejected": -1.9185969829559326, + "logps/chosen": -273.21160888671875, + "logps/ref_chosen": -96.03010559082031, + "logps/ref_rejected": -98.41322326660156, + "logps/rejected": -355.1862487792969, + "loss": 1.1371, + "margin_dpo/margin_mean": 79.59153747558594, + "margin_dpo/margin_std": 133.7232666015625, + "step": 548 + }, + { + "KL/chosen_KL_mean": -166.967529296875, + "KL/mean": -208.77001953125, + "KL/rejected_KL_mean": -250.57254028320312, + "KL/std": 100.32382202148438, + "epoch": 0.8061674008810573, + "fcm_dpo/beta": 0.004903590306639671, + "fcm_dpo/delta": 0.04106505215167999, + "fcm_dpo/margin": 83.60503387451172, + "fcm_dpo/q_t": 0.40571844577789307, + "grad_norm": 11.730501174926758, + "learning_rate": 5.603696935852426e-08, + "logits/chosen": -1.8173472881317139, + "logits/rejected": -1.3947107791900635, + "logps/chosen": -250.41848754882812, + "logps/ref_chosen": -83.45095825195312, + "logps/ref_rejected": -81.38023376464844, + "logps/rejected": -331.9527587890625, + "loss": 1.1079, + "margin_dpo/margin_mean": 83.60504150390625, + "margin_dpo/margin_std": 123.45405578613281, + "step": 549 + }, + { + "KL/chosen_KL_mean": -186.98297119140625, + "KL/mean": -227.52963256835938, + "KL/rejected_KL_mean": -268.0762939453125, + "KL/std": 96.12370300292969, + "epoch": 0.8076358296622613, + "fcm_dpo/beta": 0.004938596859574318, + "fcm_dpo/delta": 0.051362667232751846, + "fcm_dpo/margin": 81.09334564208984, + "fcm_dpo/q_t": 0.4077901840209961, + "grad_norm": 9.947425842285156, + "learning_rate": 5.5229856368582376e-08, + "logits/chosen": -1.6357877254486084, + "logits/rejected": -1.4200901985168457, + "logps/chosen": -276.57733154296875, + "logps/ref_chosen": -89.59434509277344, + "logps/ref_rejected": -101.31549835205078, + "logps/rejected": -369.39178466796875, + "loss": 1.0886, + "margin_dpo/margin_mean": 81.09333801269531, + "margin_dpo/margin_std": 102.67247009277344, + "step": 550 + }, + { + "KL/chosen_KL_mean": -166.31878662109375, + "KL/mean": -215.8612518310547, + "KL/rejected_KL_mean": -265.40374755859375, + "KL/std": 109.61463928222656, + "epoch": 0.8091042584434655, + "fcm_dpo/beta": 0.00488814152777195, + "fcm_dpo/delta": -0.0368325300514698, + "fcm_dpo/margin": 99.08496856689453, + "fcm_dpo/q_t": 0.3900824189186096, + "grad_norm": 12.51354694366455, + "learning_rate": 5.4427875753062734e-08, + "logits/chosen": -1.5582746267318726, + "logits/rejected": -1.4134386777877808, + "logps/chosen": -261.40093994140625, + "logps/ref_chosen": -95.08216094970703, + "logps/ref_rejected": -117.7192611694336, + "logps/rejected": -383.1230163574219, + "loss": 1.0274, + "margin_dpo/margin_mean": 99.0849609375, + "margin_dpo/margin_std": 110.4867172241211, + "step": 551 + }, + { + "KL/chosen_KL_mean": -157.32858276367188, + "KL/mean": -227.43338012695312, + "KL/rejected_KL_mean": -297.5381774902344, + "KL/std": 118.01866912841797, + "epoch": 0.8105726872246696, + "fcm_dpo/beta": 0.004703568760305643, + "fcm_dpo/delta": -0.22745440900325775, + "fcm_dpo/margin": 140.20960998535156, + "fcm_dpo/q_t": 0.3480021357536316, + "grad_norm": 11.194291114807129, + "learning_rate": 5.363104864490034e-08, + "logits/chosen": -2.0567474365234375, + "logits/rejected": -1.6932432651519775, + "logps/chosen": -258.7377624511719, + "logps/ref_chosen": -101.40918731689453, + "logps/ref_rejected": -110.83843994140625, + "logps/rejected": -408.3766174316406, + "loss": 0.9101, + "margin_dpo/margin_mean": 140.20960998535156, + "margin_dpo/margin_std": 121.55499267578125, + "step": 552 + }, + { + "KL/chosen_KL_mean": -172.49346923828125, + "KL/mean": -214.55206298828125, + "KL/rejected_KL_mean": -256.61065673828125, + "KL/std": 109.88848114013672, + "epoch": 0.8120411160058737, + "fcm_dpo/beta": 0.0047313496470451355, + "fcm_dpo/delta": 0.054036304354667664, + "fcm_dpo/margin": 84.1171646118164, + "fcm_dpo/q_t": 0.4077776074409485, + "grad_norm": 9.854809761047363, + "learning_rate": 5.2839396041230415e-08, + "logits/chosen": -1.5239918231964111, + "logits/rejected": -1.1946778297424316, + "logps/chosen": -270.4573669433594, + "logps/ref_chosen": -97.96388244628906, + "logps/ref_rejected": -103.57719421386719, + "logps/rejected": -360.1878356933594, + "loss": 1.1011, + "margin_dpo/margin_mean": 84.1171646118164, + "margin_dpo/margin_std": 117.28703308105469, + "step": 553 + }, + { + "KL/chosen_KL_mean": -181.18539428710938, + "KL/mean": -227.14401245117188, + "KL/rejected_KL_mean": -273.1026611328125, + "KL/std": 110.34535217285156, + "epoch": 0.8135095447870778, + "fcm_dpo/beta": 0.0047657424584031105, + "fcm_dpo/delta": 0.012077197432518005, + "fcm_dpo/margin": 91.9172592163086, + "fcm_dpo/q_t": 0.4010778069496155, + "grad_norm": 10.021539688110352, + "learning_rate": 5.205293880283551e-08, + "logits/chosen": -1.9108518362045288, + "logits/rejected": -1.1373028755187988, + "logps/chosen": -284.8721923828125, + "logps/ref_chosen": -103.68680572509766, + "logps/ref_rejected": -92.99210357666016, + "logps/rejected": -366.0947570800781, + "loss": 1.1034, + "margin_dpo/margin_mean": 91.91726684570312, + "margin_dpo/margin_std": 141.96827697753906, + "step": 554 + }, + { + "KL/chosen_KL_mean": -188.48377990722656, + "KL/mean": -231.75, + "KL/rejected_KL_mean": -275.0162353515625, + "KL/std": 114.02422332763672, + "epoch": 0.8149779735682819, + "fcm_dpo/beta": 0.004774688743054867, + "fcm_dpo/delta": 0.038429953157901764, + "fcm_dpo/margin": 86.5324478149414, + "fcm_dpo/q_t": 0.4072245955467224, + "grad_norm": 10.103257179260254, + "learning_rate": 5.127169765359515e-08, + "logits/chosen": -1.7459869384765625, + "logits/rejected": -1.6526474952697754, + "logps/chosen": -274.6874084472656, + "logps/ref_chosen": -86.20362854003906, + "logps/ref_rejected": -114.54719543457031, + "logps/rejected": -389.5634460449219, + "loss": 1.1053, + "margin_dpo/margin_mean": 86.53244018554688, + "margin_dpo/margin_std": 130.62356567382812, + "step": 555 + }, + { + "KL/chosen_KL_mean": -169.15457153320312, + "KL/mean": -215.77145385742188, + "KL/rejected_KL_mean": -262.38836669921875, + "KL/std": 101.57382202148438, + "epoch": 0.8164464023494861, + "fcm_dpo/beta": 0.004775206092745066, + "fcm_dpo/delta": 0.0048639290034770966, + "fcm_dpo/margin": 93.2337646484375, + "fcm_dpo/q_t": 0.3975391983985901, + "grad_norm": 10.624995231628418, + "learning_rate": 5.049569317994012e-08, + "logits/chosen": -1.567216157913208, + "logits/rejected": -1.261490821838379, + "logps/chosen": -257.55804443359375, + "logps/ref_chosen": -88.40348815917969, + "logps/ref_rejected": -106.63911437988281, + "logps/rejected": -369.0274658203125, + "loss": 1.0616, + "margin_dpo/margin_mean": 93.2337646484375, + "margin_dpo/margin_std": 115.73711395263672, + "step": 556 + }, + { + "KL/chosen_KL_mean": -180.4205780029297, + "KL/mean": -234.20274353027344, + "KL/rejected_KL_mean": -287.98492431640625, + "KL/std": 116.70541381835938, + "epoch": 0.8179148311306902, + "fcm_dpo/beta": 0.004723517224192619, + "fcm_dpo/delta": -0.061944834887981415, + "fcm_dpo/margin": 107.5643539428711, + "fcm_dpo/q_t": 0.385007381439209, + "grad_norm": 8.434736251831055, + "learning_rate": 4.9724945830310144e-08, + "logits/chosen": -1.986304759979248, + "logits/rejected": -1.8124982118606567, + "logps/chosen": -279.25579833984375, + "logps/ref_chosen": -98.83523559570312, + "logps/ref_rejected": -115.64888763427734, + "logps/rejected": -403.6338195800781, + "loss": 1.025, + "margin_dpo/margin_mean": 107.5643539428711, + "margin_dpo/margin_std": 126.51203155517578, + "step": 557 + }, + { + "KL/chosen_KL_mean": -179.14883422851562, + "KL/mean": -235.4542999267578, + "KL/rejected_KL_mean": -291.7597351074219, + "KL/std": 100.75602722167969, + "epoch": 0.8193832599118943, + "fcm_dpo/beta": 0.004664961248636246, + "fcm_dpo/delta": -0.07977241277694702, + "fcm_dpo/margin": 112.61088562011719, + "fcm_dpo/q_t": 0.37782907485961914, + "grad_norm": 11.25912094116211, + "learning_rate": 4.8959475914614554e-08, + "logits/chosen": -2.3992674350738525, + "logits/rejected": -1.7925952672958374, + "logps/chosen": -285.1606750488281, + "logps/ref_chosen": -106.01183319091797, + "logps/ref_rejected": -108.39131927490234, + "logps/rejected": -400.15106201171875, + "loss": 0.9847, + "margin_dpo/margin_mean": 112.61088562011719, + "margin_dpo/margin_std": 104.0299072265625, + "step": 558 + }, + { + "KL/chosen_KL_mean": -193.26144409179688, + "KL/mean": -238.98703002929688, + "KL/rejected_KL_mean": -284.71258544921875, + "KL/std": 103.2884292602539, + "epoch": 0.8208516886930984, + "fcm_dpo/beta": 0.004676289856433868, + "fcm_dpo/delta": 0.02331012487411499, + "fcm_dpo/margin": 91.45114135742188, + "fcm_dpo/q_t": 0.400562584400177, + "grad_norm": 11.394143104553223, + "learning_rate": 4.8199303603697614e-08, + "logits/chosen": -2.356294631958008, + "logits/rejected": -1.8778878450393677, + "logps/chosen": -293.6861877441406, + "logps/ref_chosen": -100.42474365234375, + "logps/ref_rejected": -100.41270446777344, + "logps/rejected": -385.12530517578125, + "loss": 1.0747, + "margin_dpo/margin_mean": 91.45114135742188, + "margin_dpo/margin_std": 116.84442138671875, + "step": 559 + }, + { + "KL/chosen_KL_mean": -185.6719970703125, + "KL/mean": -229.6200714111328, + "KL/rejected_KL_mean": -273.568115234375, + "KL/std": 101.43780517578125, + "epoch": 0.8223201174743024, + "fcm_dpo/beta": 0.004700183868408203, + "fcm_dpo/delta": 0.03847907856106758, + "fcm_dpo/margin": 87.89613342285156, + "fcm_dpo/q_t": 0.404948353767395, + "grad_norm": 12.233609199523926, + "learning_rate": 4.7444448928806615e-08, + "logits/chosen": -1.284921407699585, + "logits/rejected": -0.8312946557998657, + "logps/chosen": -273.3075256347656, + "logps/ref_chosen": -87.63551330566406, + "logps/ref_rejected": -87.37985229492188, + "logps/rejected": -360.947998046875, + "loss": 1.0867, + "margin_dpo/margin_mean": 87.89612579345703, + "margin_dpo/margin_std": 117.25211334228516, + "step": 560 + }, + { + "KL/chosen_KL_mean": -193.4197540283203, + "KL/mean": -234.80206298828125, + "KL/rejected_KL_mean": -276.18438720703125, + "KL/std": 100.51805114746094, + "epoch": 0.8237885462555066, + "fcm_dpo/beta": 0.0047654202207922935, + "fcm_dpo/delta": 0.05763612687587738, + "fcm_dpo/margin": 82.7646255493164, + "fcm_dpo/q_t": 0.40828776359558105, + "grad_norm": 8.805339813232422, + "learning_rate": 4.669493178106432e-08, + "logits/chosen": -2.100630760192871, + "logits/rejected": -2.0700228214263916, + "logps/chosen": -268.3011474609375, + "logps/ref_chosen": -74.88140106201172, + "logps/ref_rejected": -106.04796600341797, + "logps/rejected": -382.23236083984375, + "loss": 1.0802, + "margin_dpo/margin_mean": 82.76463317871094, + "margin_dpo/margin_std": 96.5006103515625, + "step": 561 + }, + { + "KL/chosen_KL_mean": -175.75299072265625, + "KL/mean": -217.31918334960938, + "KL/rejected_KL_mean": -258.8854064941406, + "KL/std": 101.22796630859375, + "epoch": 0.8252569750367107, + "fcm_dpo/beta": 0.004790683276951313, + "fcm_dpo/delta": 0.05383138731122017, + "fcm_dpo/margin": 83.1324234008789, + "fcm_dpo/q_t": 0.40960556268692017, + "grad_norm": 13.344965934753418, + "learning_rate": 4.5950771910944596e-08, + "logits/chosen": -2.152010440826416, + "logits/rejected": -1.6993939876556396, + "logps/chosen": -276.41619873046875, + "logps/ref_chosen": -100.66322326660156, + "logps/ref_rejected": -102.24087524414062, + "logps/rejected": -361.12628173828125, + "loss": 1.1124, + "margin_dpo/margin_mean": 83.13243103027344, + "margin_dpo/margin_std": 126.220947265625, + "step": 562 + }, + { + "KL/chosen_KL_mean": -186.78192138671875, + "KL/mean": -221.53622436523438, + "KL/rejected_KL_mean": -256.29052734375, + "KL/std": 101.7255859375, + "epoch": 0.8267254038179148, + "fcm_dpo/beta": 0.004918321035802364, + "fcm_dpo/delta": 0.11130814254283905, + "fcm_dpo/margin": 69.50858306884766, + "fcm_dpo/q_t": 0.4213360846042633, + "grad_norm": 11.235312461853027, + "learning_rate": 4.521198892775202e-08, + "logits/chosen": -1.5184147357940674, + "logits/rejected": -1.170419454574585, + "logps/chosen": -283.0411376953125, + "logps/ref_chosen": -96.25921630859375, + "logps/ref_rejected": -101.20503997802734, + "logps/rejected": -357.49554443359375, + "loss": 1.1416, + "margin_dpo/margin_mean": 69.50858306884766, + "margin_dpo/margin_std": 108.3563232421875, + "step": 563 + }, + { + "KL/chosen_KL_mean": -181.8468780517578, + "KL/mean": -220.7062530517578, + "KL/rejected_KL_mean": -259.5656433105469, + "KL/std": 102.5434341430664, + "epoch": 0.8281938325991189, + "fcm_dpo/beta": 0.004989357199519873, + "fcm_dpo/delta": 0.06443215906620026, + "fcm_dpo/margin": 77.71874237060547, + "fcm_dpo/q_t": 0.4094778597354889, + "grad_norm": 12.664972305297852, + "learning_rate": 4.447860229910544e-08, + "logits/chosen": -2.2479751110076904, + "logits/rejected": -1.6669130325317383, + "logps/chosen": -294.8072814941406, + "logps/ref_chosen": -112.96040344238281, + "logps/ref_rejected": -99.52803802490234, + "logps/rejected": -359.09368896484375, + "loss": 1.0882, + "margin_dpo/margin_mean": 77.71875, + "margin_dpo/margin_std": 93.1473159790039, + "step": 564 + }, + { + "KL/chosen_KL_mean": -177.90982055664062, + "KL/mean": -226.44406127929688, + "KL/rejected_KL_mean": -274.9783020019531, + "KL/std": 110.97595977783203, + "epoch": 0.8296622613803231, + "fcm_dpo/beta": 0.0049600861966609955, + "fcm_dpo/delta": -0.03316520154476166, + "fcm_dpo/margin": 97.06849670410156, + "fcm_dpo/q_t": 0.39050543308258057, + "grad_norm": 9.854287147521973, + "learning_rate": 4.375063135042445e-08, + "logits/chosen": -1.5753204822540283, + "logits/rejected": -1.057618260383606, + "logps/chosen": -273.275390625, + "logps/ref_chosen": -95.36558532714844, + "logps/ref_rejected": -92.21737670898438, + "logps/rejected": -367.1956787109375, + "loss": 1.0441, + "margin_dpo/margin_mean": 97.06849670410156, + "margin_dpo/margin_std": 120.12101745605469, + "step": 565 + }, + { + "KL/chosen_KL_mean": -185.07666015625, + "KL/mean": -231.44873046875, + "KL/rejected_KL_mean": -277.82080078125, + "KL/std": 113.49041748046875, + "epoch": 0.8311306901615272, + "fcm_dpo/beta": 0.004964838735759258, + "fcm_dpo/delta": -0.011084163561463356, + "fcm_dpo/margin": 92.74412536621094, + "fcm_dpo/q_t": 0.39338457584381104, + "grad_norm": 9.84110164642334, + "learning_rate": 4.3028095264420525e-08, + "logits/chosen": -1.8871031999588013, + "logits/rejected": -1.452309489250183, + "logps/chosen": -284.04986572265625, + "logps/ref_chosen": -98.97320556640625, + "logps/ref_rejected": -110.82090759277344, + "logps/rejected": -388.6416931152344, + "loss": 1.0524, + "margin_dpo/margin_mean": 92.74413299560547, + "margin_dpo/margin_std": 112.541259765625, + "step": 566 + }, + { + "KL/chosen_KL_mean": -182.21493530273438, + "KL/mean": -223.1317138671875, + "KL/rejected_KL_mean": -264.0484924316406, + "KL/std": 102.00434112548828, + "epoch": 0.8325991189427313, + "fcm_dpo/beta": 0.004959041718393564, + "fcm_dpo/delta": 0.04581092298030853, + "fcm_dpo/margin": 81.83354187011719, + "fcm_dpo/q_t": 0.40543872117996216, + "grad_norm": 10.911598205566406, + "learning_rate": 4.231101308059165e-08, + "logits/chosen": -2.275144100189209, + "logits/rejected": -1.6030158996582031, + "logps/chosen": -274.91851806640625, + "logps/ref_chosen": -92.7035903930664, + "logps/ref_rejected": -91.22431945800781, + "logps/rejected": -355.2728271484375, + "loss": 1.0913, + "margin_dpo/margin_mean": 81.83353424072266, + "margin_dpo/margin_std": 107.32051086425781, + "step": 567 + }, + { + "KL/chosen_KL_mean": -173.46554565429688, + "KL/mean": -226.94647216796875, + "KL/rejected_KL_mean": -280.4273986816406, + "KL/std": 107.77141571044922, + "epoch": 0.8340675477239354, + "fcm_dpo/beta": 0.004917928948998451, + "fcm_dpo/delta": -0.08075903356075287, + "fcm_dpo/margin": 106.96184539794922, + "fcm_dpo/q_t": 0.3789040446281433, + "grad_norm": 9.328248977661133, + "learning_rate": 4.1599403694720145e-08, + "logits/chosen": -1.3077096939086914, + "logits/rejected": -1.1737713813781738, + "logps/chosen": -246.59780883789062, + "logps/ref_chosen": -73.13226318359375, + "logps/ref_rejected": -94.95265197753906, + "logps/rejected": -375.38006591796875, + "loss": 0.9935, + "margin_dpo/margin_mean": 106.96184539794922, + "margin_dpo/margin_std": 108.15916442871094, + "step": 568 + }, + { + "KL/chosen_KL_mean": -183.51248168945312, + "KL/mean": -226.96575927734375, + "KL/rejected_KL_mean": -270.4190673828125, + "KL/std": 108.67988586425781, + "epoch": 0.8355359765051396, + "fcm_dpo/beta": 0.004938815720379353, + "fcm_dpo/delta": 0.021612131968140602, + "fcm_dpo/margin": 86.90658569335938, + "fcm_dpo/q_t": 0.4017311930656433, + "grad_norm": 15.93490982055664, + "learning_rate": 4.089328585837512e-08, + "logits/chosen": -1.8864727020263672, + "logits/rejected": -1.5055018663406372, + "logps/chosen": -272.72955322265625, + "logps/ref_chosen": -89.21708679199219, + "logps/ref_rejected": -86.41105651855469, + "logps/rejected": -356.8301086425781, + "loss": 1.1006, + "margin_dpo/margin_mean": 86.90658569335938, + "margin_dpo/margin_std": 129.9566650390625, + "step": 569 + }, + { + "KL/chosen_KL_mean": -163.5234375, + "KL/mean": -220.2521514892578, + "KL/rejected_KL_mean": -276.9808654785156, + "KL/std": 103.90315246582031, + "epoch": 0.8370044052863436, + "fcm_dpo/beta": 0.0048536863178014755, + "fcm_dpo/delta": -0.10666178166866302, + "fcm_dpo/margin": 113.45742797851562, + "fcm_dpo/q_t": 0.3717441260814667, + "grad_norm": 9.970564842224121, + "learning_rate": 4.019267817841834e-08, + "logits/chosen": -2.5078535079956055, + "logits/rejected": -1.846949577331543, + "logps/chosen": -258.28619384765625, + "logps/ref_chosen": -94.76277160644531, + "logps/ref_rejected": -88.846923828125, + "logps/rejected": -365.8277893066406, + "loss": 0.9737, + "margin_dpo/margin_mean": 113.45742797851562, + "margin_dpo/margin_std": 108.15154266357422, + "step": 570 + }, + { + "KL/chosen_KL_mean": -170.45916748046875, + "KL/mean": -220.59762573242188, + "KL/rejected_KL_mean": -270.736083984375, + "KL/std": 102.62117004394531, + "epoch": 0.8384728340675477, + "fcm_dpo/beta": 0.004818316549062729, + "fcm_dpo/delta": -0.034890636801719666, + "fcm_dpo/margin": 100.27690887451172, + "fcm_dpo/q_t": 0.3898862898349762, + "grad_norm": 12.527082443237305, + "learning_rate": 3.9497599116513705e-08, + "logits/chosen": -1.6255197525024414, + "logits/rejected": -1.4647748470306396, + "logps/chosen": -251.28045654296875, + "logps/ref_chosen": -80.8212890625, + "logps/ref_rejected": -97.01528930664062, + "logps/rejected": -367.7513732910156, + "loss": 1.0439, + "margin_dpo/margin_mean": 100.27690887451172, + "margin_dpo/margin_std": 123.81684875488281, + "step": 571 + }, + { + "KL/chosen_KL_mean": -174.47024536132812, + "KL/mean": -225.146484375, + "KL/rejected_KL_mean": -275.82269287109375, + "KL/std": 108.68096160888672, + "epoch": 0.8399412628487518, + "fcm_dpo/beta": 0.004766533151268959, + "fcm_dpo/delta": -0.034862007945775986, + "fcm_dpo/margin": 101.35246276855469, + "fcm_dpo/q_t": 0.39039546251296997, + "grad_norm": 10.338237762451172, + "learning_rate": 3.880806698864086e-08, + "logits/chosen": -1.8114025592803955, + "logits/rejected": -1.6437764167785645, + "logps/chosen": -251.8501739501953, + "logps/ref_chosen": -77.37992858886719, + "logps/ref_rejected": -90.56597900390625, + "logps/rejected": -366.388671875, + "loss": 1.0323, + "margin_dpo/margin_mean": 101.35246276855469, + "margin_dpo/margin_std": 117.46690368652344, + "step": 572 + }, + { + "KL/chosen_KL_mean": -168.44476318359375, + "KL/mean": -216.23353576660156, + "KL/rejected_KL_mean": -264.0223388671875, + "KL/std": 108.87879943847656, + "epoch": 0.8414096916299559, + "fcm_dpo/beta": 0.0047918884083628654, + "fcm_dpo/delta": -0.009356088936328888, + "fcm_dpo/margin": 95.57757568359375, + "fcm_dpo/q_t": 0.3938070237636566, + "grad_norm": 9.493922233581543, + "learning_rate": 3.812409996461275e-08, + "logits/chosen": -2.253699541091919, + "logits/rejected": -2.0283048152923584, + "logps/chosen": -252.1508331298828, + "logps/ref_chosen": -83.7060775756836, + "logps/ref_rejected": -90.69746398925781, + "logps/rejected": -354.71978759765625, + "loss": 1.0547, + "margin_dpo/margin_mean": 95.57757568359375, + "margin_dpo/margin_std": 114.06787872314453, + "step": 573 + }, + { + "KL/chosen_KL_mean": -178.24514770507812, + "KL/mean": -228.97540283203125, + "KL/rejected_KL_mean": -279.7056884765625, + "KL/std": 107.54293823242188, + "epoch": 0.8428781204111601, + "fcm_dpo/beta": 0.0047480324283242226, + "fcm_dpo/delta": -0.03346884623169899, + "fcm_dpo/margin": 101.46055603027344, + "fcm_dpo/q_t": 0.38947343826293945, + "grad_norm": 10.1945161819458, + "learning_rate": 3.74457160675965e-08, + "logits/chosen": -2.303833484649658, + "logits/rejected": -1.9847769737243652, + "logps/chosen": -263.3746337890625, + "logps/ref_chosen": -85.12948608398438, + "logps/ref_rejected": -99.05403137207031, + "logps/rejected": -378.75970458984375, + "loss": 1.0407, + "margin_dpo/margin_mean": 101.46055603027344, + "margin_dpo/margin_std": 121.93394470214844, + "step": 574 + }, + { + "KL/chosen_KL_mean": -184.06289672851562, + "KL/mean": -231.4376220703125, + "KL/rejected_KL_mean": -278.8123779296875, + "KL/std": 109.81668090820312, + "epoch": 0.8443465491923642, + "fcm_dpo/beta": 0.00472361221909523, + "fcm_dpo/delta": 0.002552381716668606, + "fcm_dpo/margin": 94.74946594238281, + "fcm_dpo/q_t": 0.3979812562465668, + "grad_norm": 12.434805870056152, + "learning_rate": 3.677293317363864e-08, + "logits/chosen": -1.3767006397247314, + "logits/rejected": -0.9102492928504944, + "logps/chosen": -289.833740234375, + "logps/ref_chosen": -105.77084350585938, + "logps/ref_rejected": -100.94944763183594, + "logps/rejected": -379.7618103027344, + "loss": 1.0723, + "margin_dpo/margin_mean": 94.74946594238281, + "margin_dpo/margin_std": 127.75698852539062, + "step": 575 + }, + { + "KL/chosen_KL_mean": -182.92852783203125, + "KL/mean": -228.7220001220703, + "KL/rejected_KL_mean": -274.5154724121094, + "KL/std": 101.88496398925781, + "epoch": 0.8458149779735683, + "fcm_dpo/beta": 0.004745191428810358, + "fcm_dpo/delta": 0.016020432114601135, + "fcm_dpo/margin": 91.58695220947266, + "fcm_dpo/q_t": 0.4000887870788574, + "grad_norm": 11.390907287597656, + "learning_rate": 3.6105769011194224e-08, + "logits/chosen": -1.8920578956604004, + "logits/rejected": -1.8284693956375122, + "logps/chosen": -264.133056640625, + "logps/ref_chosen": -81.20452880859375, + "logps/ref_rejected": -105.40940856933594, + "logps/rejected": -379.92486572265625, + "loss": 1.078, + "margin_dpo/margin_mean": 91.58695983886719, + "margin_dpo/margin_std": 122.36265563964844, + "step": 576 + }, + { + "KL/chosen_KL_mean": -163.3242950439453, + "KL/mean": -220.65541076660156, + "KL/rejected_KL_mean": -277.98651123046875, + "KL/std": 110.72624206542969, + "epoch": 0.8472834067547724, + "fcm_dpo/beta": 0.004659945145249367, + "fcm_dpo/delta": -0.08975732326507568, + "fcm_dpo/margin": 114.6622085571289, + "fcm_dpo/q_t": 0.3772873878479004, + "grad_norm": 8.622326850891113, + "learning_rate": 3.5444241160659304e-08, + "logits/chosen": -1.8065013885498047, + "logits/rejected": -1.0841326713562012, + "logps/chosen": -264.7884826660156, + "logps/ref_chosen": -101.46419525146484, + "logps/ref_rejected": -89.32637023925781, + "logps/rejected": -367.3128662109375, + "loss": 1.0027, + "margin_dpo/margin_mean": 114.6622085571289, + "margin_dpo/margin_std": 126.03982543945312, + "step": 577 + }, + { + "KL/chosen_KL_mean": -195.02908325195312, + "KL/mean": -236.84829711914062, + "KL/rejected_KL_mean": -278.66754150390625, + "KL/std": 105.22526550292969, + "epoch": 0.8487518355359766, + "fcm_dpo/beta": 0.004698064178228378, + "fcm_dpo/delta": 0.05939852446317673, + "fcm_dpo/margin": 83.638427734375, + "fcm_dpo/q_t": 0.40966176986694336, + "grad_norm": 14.506484031677246, + "learning_rate": 3.478836705390808e-08, + "logits/chosen": -1.154599666595459, + "logits/rejected": -0.907585859298706, + "logps/chosen": -272.3215026855469, + "logps/ref_chosen": -77.29241943359375, + "logps/ref_rejected": -88.24701690673828, + "logps/rejected": -366.91455078125, + "loss": 1.1063, + "margin_dpo/margin_mean": 83.638427734375, + "margin_dpo/margin_std": 120.07523345947266, + "step": 578 + }, + { + "KL/chosen_KL_mean": -174.91900634765625, + "KL/mean": -217.19815063476562, + "KL/rejected_KL_mean": -259.4773254394531, + "KL/std": 105.2657699584961, + "epoch": 0.8502202643171806, + "fcm_dpo/beta": 0.004728085361421108, + "fcm_dpo/delta": 0.052112363278865814, + "fcm_dpo/margin": 84.5583267211914, + "fcm_dpo/q_t": 0.408389151096344, + "grad_norm": 8.933969497680664, + "learning_rate": 3.41381639738331e-08, + "logits/chosen": -1.7135779857635498, + "logits/rejected": -1.408900260925293, + "logps/chosen": -262.2620544433594, + "logps/ref_chosen": -87.34305572509766, + "logps/ref_rejected": -102.12100982666016, + "logps/rejected": -361.59832763671875, + "loss": 1.092, + "margin_dpo/margin_mean": 84.5583267211914, + "margin_dpo/margin_std": 107.45379638671875, + "step": 579 + }, + { + "KL/chosen_KL_mean": -167.37413024902344, + "KL/mean": -218.71963500976562, + "KL/rejected_KL_mean": -270.06512451171875, + "KL/std": 103.72488403320312, + "epoch": 0.8516886930983847, + "fcm_dpo/beta": 0.004716012626886368, + "fcm_dpo/delta": -0.036564163863658905, + "fcm_dpo/margin": 102.69097900390625, + "fcm_dpo/q_t": 0.39027202129364014, + "grad_norm": 10.67998218536377, + "learning_rate": 3.349364905389032e-08, + "logits/chosen": -1.3836811780929565, + "logits/rejected": -1.0624589920043945, + "logps/chosen": -256.30291748046875, + "logps/ref_chosen": -88.92879486083984, + "logps/ref_rejected": -88.69661712646484, + "logps/rejected": -358.76171875, + "loss": 1.0399, + "margin_dpo/margin_mean": 102.69097900390625, + "margin_dpo/margin_std": 124.42339324951172, + "step": 580 + }, + { + "KL/chosen_KL_mean": -177.17019653320312, + "KL/mean": -223.97393798828125, + "KL/rejected_KL_mean": -270.77764892578125, + "KL/std": 106.09624481201172, + "epoch": 0.8531571218795888, + "fcm_dpo/beta": 0.004737949930131435, + "fcm_dpo/delta": 0.006753142923116684, + "fcm_dpo/margin": 93.60745239257812, + "fcm_dpo/q_t": 0.39859747886657715, + "grad_norm": 11.960638046264648, + "learning_rate": 3.285483927764726e-08, + "logits/chosen": -2.027475357055664, + "logits/rejected": -1.6816093921661377, + "logps/chosen": -269.3934326171875, + "logps/ref_chosen": -92.22323608398438, + "logps/ref_rejected": -99.38943481445312, + "logps/rejected": -370.1670837402344, + "loss": 1.0847, + "margin_dpo/margin_mean": 93.60745239257812, + "margin_dpo/margin_std": 132.91754150390625, + "step": 581 + }, + { + "KL/chosen_KL_mean": -156.19517517089844, + "KL/mean": -198.79200744628906, + "KL/rejected_KL_mean": -241.3888397216797, + "KL/std": 109.99616241455078, + "epoch": 0.8546255506607929, + "fcm_dpo/beta": 0.004808083176612854, + "fcm_dpo/delta": 0.039822474122047424, + "fcm_dpo/margin": 85.19365692138672, + "fcm_dpo/q_t": 0.40805190801620483, + "grad_norm": 10.671210289001465, + "learning_rate": 3.222175147833556e-08, + "logits/chosen": -2.2726540565490723, + "logits/rejected": -2.2626798152923584, + "logps/chosen": -243.53079223632812, + "logps/ref_chosen": -87.33561706542969, + "logps/ref_rejected": -115.1544189453125, + "logps/rejected": -356.54327392578125, + "loss": 1.1027, + "margin_dpo/margin_mean": 85.19365692138672, + "margin_dpo/margin_std": 120.42509460449219, + "step": 582 + }, + { + "KL/chosen_KL_mean": -189.59402465820312, + "KL/mean": -236.19659423828125, + "KL/rejected_KL_mean": -282.7991943359375, + "KL/std": 103.25798034667969, + "epoch": 0.856093979441997, + "fcm_dpo/beta": 0.00475682970136404, + "fcm_dpo/delta": 0.006634358316659927, + "fcm_dpo/margin": 93.20515441894531, + "fcm_dpo/q_t": 0.3995400667190552, + "grad_norm": 11.125332832336426, + "learning_rate": 3.159440233840763e-08, + "logits/chosen": -1.5351135730743408, + "logits/rejected": -1.3023037910461426, + "logps/chosen": -273.31109619140625, + "logps/ref_chosen": -83.71707153320312, + "logps/ref_rejected": -94.58465576171875, + "logps/rejected": -377.38385009765625, + "loss": 1.0703, + "margin_dpo/margin_mean": 93.20515441894531, + "margin_dpo/margin_std": 121.99015045166016, + "step": 583 + }, + { + "KL/chosen_KL_mean": -167.42575073242188, + "KL/mean": -216.7974853515625, + "KL/rejected_KL_mean": -266.169189453125, + "KL/std": 107.61104583740234, + "epoch": 0.8575624082232012, + "fcm_dpo/beta": 0.0047964793629944324, + "fcm_dpo/delta": -0.025810755789279938, + "fcm_dpo/margin": 98.74342346191406, + "fcm_dpo/q_t": 0.3911040425300598, + "grad_norm": 8.54472827911377, + "learning_rate": 3.0972808389096635e-08, + "logits/chosen": -1.9103858470916748, + "logits/rejected": -1.4478344917297363, + "logps/chosen": -261.685302734375, + "logps/ref_chosen": -94.25955963134766, + "logps/ref_rejected": -104.48269653320312, + "logps/rejected": -370.6518859863281, + "loss": 1.054, + "margin_dpo/margin_mean": 98.7434310913086, + "margin_dpo/margin_std": 122.18767547607422, + "step": 584 + }, + { + "KL/chosen_KL_mean": -178.62423706054688, + "KL/mean": -234.43539428710938, + "KL/rejected_KL_mean": -290.2465515136719, + "KL/std": 122.98223876953125, + "epoch": 0.8590308370044053, + "fcm_dpo/beta": 0.0047151679173111916, + "fcm_dpo/delta": -0.08055678009986877, + "fcm_dpo/margin": 111.62234497070312, + "fcm_dpo/q_t": 0.38218066096305847, + "grad_norm": 9.863591194152832, + "learning_rate": 3.035698600998121e-08, + "logits/chosen": -2.1924378871917725, + "logits/rejected": -1.9288148880004883, + "logps/chosen": -264.260986328125, + "logps/ref_chosen": -85.63676452636719, + "logps/ref_rejected": -90.28547668457031, + "logps/rejected": -380.53204345703125, + "loss": 1.0326, + "margin_dpo/margin_mean": 111.62234497070312, + "margin_dpo/margin_std": 143.08486938476562, + "step": 585 + }, + { + "KL/chosen_KL_mean": -198.4064178466797, + "KL/mean": -236.27288818359375, + "KL/rejected_KL_mean": -274.1393737792969, + "KL/std": 107.932861328125, + "epoch": 0.8604992657856094, + "fcm_dpo/beta": 0.004743432626128197, + "fcm_dpo/delta": 0.0941477119922638, + "fcm_dpo/margin": 75.73295593261719, + "fcm_dpo/q_t": 0.4166196584701538, + "grad_norm": 11.513298988342285, + "learning_rate": 2.974695142855388e-08, + "logits/chosen": -1.8965815305709839, + "logits/rejected": -1.6332364082336426, + "logps/chosen": -282.0875549316406, + "logps/ref_chosen": -83.68114471435547, + "logps/ref_rejected": -98.35916137695312, + "logps/rejected": -372.49853515625, + "loss": 1.1148, + "margin_dpo/margin_mean": 75.73295593261719, + "margin_dpo/margin_std": 101.49545288085938, + "step": 586 + }, + { + "KL/chosen_KL_mean": -160.5615234375, + "KL/mean": -201.52297973632812, + "KL/rejected_KL_mean": -242.48443603515625, + "KL/std": 104.24542999267578, + "epoch": 0.8619676945668135, + "fcm_dpo/beta": 0.004813813604414463, + "fcm_dpo/delta": 0.057809218764305115, + "fcm_dpo/margin": 81.92289733886719, + "fcm_dpo/q_t": 0.4092627763748169, + "grad_norm": 11.267950057983398, + "learning_rate": 2.9142720719793122e-08, + "logits/chosen": -2.5131096839904785, + "logits/rejected": -2.232171058654785, + "logps/chosen": -238.1390380859375, + "logps/ref_chosen": -77.57750701904297, + "logps/ref_rejected": -89.32032775878906, + "logps/rejected": -331.80474853515625, + "loss": 1.0939, + "margin_dpo/margin_mean": 81.92289733886719, + "margin_dpo/margin_std": 108.14523315429688, + "step": 587 + }, + { + "KL/chosen_KL_mean": -174.6376953125, + "KL/mean": -212.6331329345703, + "KL/rejected_KL_mean": -250.62857055664062, + "KL/std": 106.32521057128906, + "epoch": 0.8634361233480177, + "fcm_dpo/beta": 0.004871168173849583, + "fcm_dpo/delta": 0.0829886794090271, + "fcm_dpo/margin": 75.9908676147461, + "fcm_dpo/q_t": 0.41710951924324036, + "grad_norm": 13.760732650756836, + "learning_rate": 2.8544309805740018e-08, + "logits/chosen": -1.6966710090637207, + "logits/rejected": -1.6548397541046143, + "logps/chosen": -258.44195556640625, + "logps/ref_chosen": -83.80426025390625, + "logps/ref_rejected": -113.11943054199219, + "logps/rejected": -363.74798583984375, + "loss": 1.1403, + "margin_dpo/margin_mean": 75.99087524414062, + "margin_dpo/margin_std": 127.1422119140625, + "step": 588 + }, + { + "KL/chosen_KL_mean": -187.17181396484375, + "KL/mean": -237.55117797851562, + "KL/rejected_KL_mean": -287.9305725097656, + "KL/std": 104.54691314697266, + "epoch": 0.8649045521292217, + "fcm_dpo/beta": 0.004872431047260761, + "fcm_dpo/delta": -0.043043069541454315, + "fcm_dpo/margin": 100.75877380371094, + "fcm_dpo/q_t": 0.38554611802101135, + "grad_norm": 9.566523551940918, + "learning_rate": 2.7951734455078786e-08, + "logits/chosen": -1.542040228843689, + "logits/rejected": -1.1396864652633667, + "logps/chosen": -283.6368408203125, + "logps/ref_chosen": -96.46501922607422, + "logps/ref_rejected": -116.03719329833984, + "logps/rejected": -403.9677734375, + "loss": 1.0104, + "margin_dpo/margin_mean": 100.75877380371094, + "margin_dpo/margin_std": 100.22196960449219, + "step": 589 + }, + { + "KL/chosen_KL_mean": -169.7957305908203, + "KL/mean": -220.34649658203125, + "KL/rejected_KL_mean": -270.89727783203125, + "KL/std": 97.92941284179688, + "epoch": 0.8663729809104258, + "fcm_dpo/beta": 0.004851190373301506, + "fcm_dpo/delta": -0.0427684523165226, + "fcm_dpo/margin": 101.10150146484375, + "fcm_dpo/q_t": 0.3867577016353607, + "grad_norm": 11.905020713806152, + "learning_rate": 2.736501028272095e-08, + "logits/chosen": -1.7764450311660767, + "logits/rejected": -1.6228710412979126, + "logps/chosen": -258.22357177734375, + "logps/ref_chosen": -88.4278564453125, + "logps/ref_rejected": -110.19876861572266, + "logps/rejected": -381.09600830078125, + "loss": 1.0131, + "margin_dpo/margin_mean": 101.10150909423828, + "margin_dpo/margin_std": 102.35348510742188, + "step": 590 + }, + { + "KL/chosen_KL_mean": -163.75808715820312, + "KL/mean": -216.78436279296875, + "KL/rejected_KL_mean": -269.81060791015625, + "KL/std": 105.61711120605469, + "epoch": 0.8678414096916299, + "fcm_dpo/beta": 0.004765205085277557, + "fcm_dpo/delta": -0.05858120694756508, + "fcm_dpo/margin": 106.05250549316406, + "fcm_dpo/q_t": 0.3826689124107361, + "grad_norm": 7.80182409286499, + "learning_rate": 2.678415274939408e-08, + "logits/chosen": -1.9163053035736084, + "logits/rejected": -1.2959301471710205, + "logps/chosen": -251.6077880859375, + "logps/ref_chosen": -87.84968566894531, + "logps/ref_rejected": -89.57919311523438, + "logps/rejected": -359.38983154296875, + "loss": 1.0001, + "margin_dpo/margin_mean": 106.05252075195312, + "margin_dpo/margin_std": 102.91415405273438, + "step": 591 + }, + { + "KL/chosen_KL_mean": -193.43685913085938, + "KL/mean": -235.18594360351562, + "KL/rejected_KL_mean": -276.93505859375, + "KL/std": 101.48924255371094, + "epoch": 0.869309838472834, + "fcm_dpo/beta": 0.004801520146429539, + "fcm_dpo/delta": 0.05110149085521698, + "fcm_dpo/margin": 83.4981689453125, + "fcm_dpo/q_t": 0.4080692529678345, + "grad_norm": 12.897024154663086, + "learning_rate": 2.6209177161234442e-08, + "logits/chosen": -1.7987284660339355, + "logits/rejected": -1.470049262046814, + "logps/chosen": -268.41259765625, + "logps/ref_chosen": -74.97573852539062, + "logps/ref_rejected": -81.55262756347656, + "logps/rejected": -358.4876708984375, + "loss": 1.1104, + "margin_dpo/margin_mean": 83.49818420410156, + "margin_dpo/margin_std": 125.49330139160156, + "step": 592 + }, + { + "KL/chosen_KL_mean": -177.61285400390625, + "KL/mean": -212.94195556640625, + "KL/rejected_KL_mean": -248.27108764648438, + "KL/std": 106.93603515625, + "epoch": 0.8707782672540382, + "fcm_dpo/beta": 0.004873002879321575, + "fcm_dpo/delta": 0.10957694798707962, + "fcm_dpo/margin": 70.65821075439453, + "fcm_dpo/q_t": 0.42214667797088623, + "grad_norm": 15.375000953674316, + "learning_rate": 2.564009866938349e-08, + "logits/chosen": -0.8637688159942627, + "logits/rejected": -0.5610991716384888, + "logps/chosen": -249.85189819335938, + "logps/ref_chosen": -72.23902893066406, + "logps/ref_rejected": -70.09652709960938, + "logps/rejected": -318.36761474609375, + "loss": 1.1502, + "margin_dpo/margin_mean": 70.65821075439453, + "margin_dpo/margin_std": 117.49882507324219, + "step": 593 + }, + { + "KL/chosen_KL_mean": -159.781005859375, + "KL/mean": -214.12362670898438, + "KL/rejected_KL_mean": -268.4662780761719, + "KL/std": 104.10148620605469, + "epoch": 0.8722466960352423, + "fcm_dpo/beta": 0.0048754652962088585, + "fcm_dpo/delta": -0.08433500677347183, + "fcm_dpo/margin": 108.68526458740234, + "fcm_dpo/q_t": 0.37713533639907837, + "grad_norm": 9.886177062988281, + "learning_rate": 2.5076932269588708e-08, + "logits/chosen": -2.007129192352295, + "logits/rejected": -1.487847924232483, + "logps/chosen": -250.21871948242188, + "logps/ref_chosen": -90.43771362304688, + "logps/ref_rejected": -92.267578125, + "logps/rejected": -360.7338562011719, + "loss": 0.9819, + "margin_dpo/margin_mean": 108.68526458740234, + "margin_dpo/margin_std": 100.19976043701172, + "step": 594 + }, + { + "KL/chosen_KL_mean": -171.06468200683594, + "KL/mean": -215.6085205078125, + "KL/rejected_KL_mean": -260.15234375, + "KL/std": 103.46780395507812, + "epoch": 0.8737151248164464, + "fcm_dpo/beta": 0.004862664267420769, + "fcm_dpo/delta": 0.017333049327135086, + "fcm_dpo/margin": 89.08767700195312, + "fcm_dpo/q_t": 0.4019402265548706, + "grad_norm": 10.475020408630371, + "learning_rate": 2.451969280180849e-08, + "logits/chosen": -1.879788875579834, + "logits/rejected": -1.472987413406372, + "logps/chosen": -252.57949829101562, + "logps/ref_chosen": -81.51480102539062, + "logps/ref_rejected": -86.52047729492188, + "logps/rejected": -346.6728210449219, + "loss": 1.0782, + "margin_dpo/margin_mean": 89.08767700195312, + "margin_dpo/margin_std": 119.22312927246094, + "step": 595 + }, + { + "KL/chosen_KL_mean": -178.04481506347656, + "KL/mean": -217.61434936523438, + "KL/rejected_KL_mean": -257.1838684082031, + "KL/std": 102.07959747314453, + "epoch": 0.8751835535976505, + "fcm_dpo/beta": 0.004903014749288559, + "fcm_dpo/delta": 0.06442783027887344, + "fcm_dpo/margin": 79.13906860351562, + "fcm_dpo/q_t": 0.41271764039993286, + "grad_norm": 13.890960693359375, + "learning_rate": 2.396839494982103e-08, + "logits/chosen": -1.5041656494140625, + "logits/rejected": -0.9086236953735352, + "logps/chosen": -266.25286865234375, + "logps/ref_chosen": -88.20805358886719, + "logps/ref_rejected": -86.09375, + "logps/rejected": -343.27764892578125, + "loss": 1.1292, + "margin_dpo/margin_mean": 79.13906860351562, + "margin_dpo/margin_std": 128.70896911621094, + "step": 596 + }, + { + "KL/chosen_KL_mean": -172.63336181640625, + "KL/mean": -225.65191650390625, + "KL/rejected_KL_mean": -278.67047119140625, + "KL/std": 110.42037963867188, + "epoch": 0.8766519823788547, + "fcm_dpo/beta": 0.004855680279433727, + "fcm_dpo/delta": -0.06863737106323242, + "fcm_dpo/margin": 106.037109375, + "fcm_dpo/q_t": 0.382799357175827, + "grad_norm": 9.550575256347656, + "learning_rate": 2.3423053240837514e-08, + "logits/chosen": -1.3084278106689453, + "logits/rejected": -1.1904325485229492, + "logps/chosen": -256.22332763671875, + "logps/ref_chosen": -83.58998107910156, + "logps/ref_rejected": -106.94439697265625, + "logps/rejected": -385.6148681640625, + "loss": 1.0145, + "margin_dpo/margin_mean": 106.037109375, + "margin_dpo/margin_std": 120.2269058227539, + "step": 597 + }, + { + "KL/chosen_KL_mean": -164.1527099609375, + "KL/mean": -208.7948760986328, + "KL/rejected_KL_mean": -253.43702697753906, + "KL/std": 103.03628540039062, + "epoch": 0.8781204111600588, + "fcm_dpo/beta": 0.004840881563723087, + "fcm_dpo/delta": 0.018280409276485443, + "fcm_dpo/margin": 89.28433227539062, + "fcm_dpo/q_t": 0.39983421564102173, + "grad_norm": 10.328418731689453, + "learning_rate": 2.2883682045119062e-08, + "logits/chosen": -1.920729398727417, + "logits/rejected": -1.6049250364303589, + "logps/chosen": -254.33534240722656, + "logps/ref_chosen": -90.18262481689453, + "logps/ref_rejected": -94.79856872558594, + "logps/rejected": -348.235595703125, + "loss": 1.078, + "margin_dpo/margin_mean": 89.28433227539062, + "margin_dpo/margin_std": 116.40673828125, + "step": 598 + }, + { + "KL/chosen_KL_mean": -167.89181518554688, + "KL/mean": -207.4963836669922, + "KL/rejected_KL_mean": -247.1009521484375, + "KL/std": 107.64773559570312, + "epoch": 0.8795888399412628, + "fcm_dpo/beta": 0.004888907074928284, + "fcm_dpo/delta": 0.06507787853479385, + "fcm_dpo/margin": 79.20914459228516, + "fcm_dpo/q_t": 0.411202609539032, + "grad_norm": 10.175901412963867, + "learning_rate": 2.2350295575598367e-08, + "logits/chosen": -1.636303186416626, + "logits/rejected": -1.5111571550369263, + "logps/chosen": -245.7340850830078, + "logps/ref_chosen": -77.84226989746094, + "logps/ref_rejected": -89.1976318359375, + "logps/rejected": -336.298583984375, + "loss": 1.1178, + "margin_dpo/margin_mean": 79.20914459228516, + "margin_dpo/margin_std": 117.85272216796875, + "step": 599 + }, + { + "KL/chosen_KL_mean": -186.15249633789062, + "KL/mean": -230.818115234375, + "KL/rejected_KL_mean": -275.48370361328125, + "KL/std": 101.63578033447266, + "epoch": 0.8810572687224669, + "fcm_dpo/beta": 0.004939359147101641, + "fcm_dpo/delta": 0.00915601383894682, + "fcm_dpo/margin": 89.33125305175781, + "fcm_dpo/q_t": 0.3990960717201233, + "grad_norm": 10.114336013793945, + "learning_rate": 2.1822907887504932e-08, + "logits/chosen": -2.3316235542297363, + "logits/rejected": -1.980991244316101, + "logps/chosen": -278.3403015136719, + "logps/ref_chosen": -92.18781280517578, + "logps/ref_rejected": -92.95429992675781, + "logps/rejected": -368.4380187988281, + "loss": 1.0634, + "margin_dpo/margin_mean": 89.33124542236328, + "margin_dpo/margin_std": 110.2298583984375, + "step": 600 + }, + { + "KL/chosen_KL_mean": -187.6708984375, + "KL/mean": -234.78570556640625, + "KL/rejected_KL_mean": -281.9005432128906, + "KL/std": 103.02455139160156, + "epoch": 0.882525697503671, + "fcm_dpo/beta": 0.004912571050226688, + "fcm_dpo/delta": -0.013815362006425858, + "fcm_dpo/margin": 94.22964477539062, + "fcm_dpo/q_t": 0.39269837737083435, + "grad_norm": 10.507804870605469, + "learning_rate": 2.1301532877994742e-08, + "logits/chosen": -1.7895655632019043, + "logits/rejected": -1.4660348892211914, + "logps/chosen": -276.88702392578125, + "logps/ref_chosen": -89.21614074707031, + "logps/ref_rejected": -100.17054748535156, + "logps/rejected": -382.07110595703125, + "loss": 1.0319, + "margin_dpo/margin_mean": 94.22964477539062, + "margin_dpo/margin_std": 98.8106460571289, + "step": 601 + }, + { + "KL/chosen_KL_mean": -148.8975830078125, + "KL/mean": -210.4900360107422, + "KL/rejected_KL_mean": -272.08245849609375, + "KL/std": 113.03854370117188, + "epoch": 0.8839941262848752, + "fcm_dpo/beta": 0.0048156240954995155, + "fcm_dpo/delta": -0.15224555134773254, + "fcm_dpo/margin": 123.18489074707031, + "fcm_dpo/q_t": 0.36536622047424316, + "grad_norm": 10.005538940429688, + "learning_rate": 2.0786184285784298e-08, + "logits/chosen": -2.169020652770996, + "logits/rejected": -1.9122167825698853, + "logps/chosen": -228.9552001953125, + "logps/ref_chosen": -80.05760192871094, + "logps/ref_rejected": -93.197509765625, + "logps/rejected": -365.27996826171875, + "loss": 0.965, + "margin_dpo/margin_mean": 123.18489074707031, + "margin_dpo/margin_std": 128.56874084472656, + "step": 602 + }, + { + "KL/chosen_KL_mean": -159.88824462890625, + "KL/mean": -210.6729278564453, + "KL/rejected_KL_mean": -261.4576110839844, + "KL/std": 102.02827453613281, + "epoch": 0.8854625550660793, + "fcm_dpo/beta": 0.00475830864161253, + "fcm_dpo/delta": -0.03497551381587982, + "fcm_dpo/margin": 101.5693359375, + "fcm_dpo/q_t": 0.3890826106071472, + "grad_norm": 9.110335350036621, + "learning_rate": 2.0276875690788204e-08, + "logits/chosen": -2.11802339553833, + "logits/rejected": -1.5342588424682617, + "logps/chosen": -262.1978454589844, + "logps/ref_chosen": -102.30958557128906, + "logps/ref_rejected": -108.06884765625, + "logps/rejected": -369.5264587402344, + "loss": 1.0311, + "margin_dpo/margin_mean": 101.56934356689453, + "margin_dpo/margin_std": 117.06777954101562, + "step": 603 + }, + { + "KL/chosen_KL_mean": -165.91738891601562, + "KL/mean": -218.85491943359375, + "KL/rejected_KL_mean": -271.79241943359375, + "KL/std": 107.22111511230469, + "epoch": 0.8869309838472834, + "fcm_dpo/beta": 0.0046936641447246075, + "fcm_dpo/delta": -0.049801453948020935, + "fcm_dpo/margin": 105.87506103515625, + "fcm_dpo/q_t": 0.3833632469177246, + "grad_norm": 9.115262031555176, + "learning_rate": 1.977362051376158e-08, + "logits/chosen": -1.6962742805480957, + "logits/rejected": -1.5405266284942627, + "logps/chosen": -244.0914764404297, + "logps/ref_chosen": -78.17408752441406, + "logps/ref_rejected": -99.49610900878906, + "logps/rejected": -371.28857421875, + "loss": 1.0123, + "margin_dpo/margin_mean": 105.87506103515625, + "margin_dpo/margin_std": 108.82525634765625, + "step": 604 + }, + { + "KL/chosen_KL_mean": -181.64035034179688, + "KL/mean": -224.87034606933594, + "KL/rejected_KL_mean": -268.100341796875, + "KL/std": 96.00192260742188, + "epoch": 0.8883994126284875, + "fcm_dpo/beta": 0.004725199192762375, + "fcm_dpo/delta": 0.043243639171123505, + "fcm_dpo/margin": 86.45996856689453, + "fcm_dpo/q_t": 0.40586990118026733, + "grad_norm": 10.408443450927734, + "learning_rate": 1.9276432015946446e-08, + "logits/chosen": -1.8498953580856323, + "logits/rejected": -1.3510491847991943, + "logps/chosen": -276.4136962890625, + "logps/ref_chosen": -94.77332305908203, + "logps/ref_rejected": -107.30490112304688, + "logps/rejected": -375.40521240234375, + "loss": 1.0805, + "margin_dpo/margin_mean": 86.45997619628906, + "margin_dpo/margin_std": 108.26947784423828, + "step": 605 + }, + { + "KL/chosen_KL_mean": -174.79664611816406, + "KL/mean": -219.79824829101562, + "KL/rejected_KL_mean": -264.7998352050781, + "KL/std": 104.88798522949219, + "epoch": 0.8898678414096917, + "fcm_dpo/beta": 0.004780953750014305, + "fcm_dpo/delta": 0.019732438027858734, + "fcm_dpo/margin": 90.00321960449219, + "fcm_dpo/q_t": 0.39998355507850647, + "grad_norm": 9.32400894165039, + "learning_rate": 1.8785323298722093e-08, + "logits/chosen": -1.779496192932129, + "logits/rejected": -1.3828221559524536, + "logps/chosen": -262.550048828125, + "logps/ref_chosen": -87.7533950805664, + "logps/ref_rejected": -104.2422103881836, + "logps/rejected": -369.04205322265625, + "loss": 1.0716, + "margin_dpo/margin_mean": 90.00321960449219, + "margin_dpo/margin_std": 111.46886444091797, + "step": 606 + }, + { + "KL/chosen_KL_mean": -170.30282592773438, + "KL/mean": -221.07574462890625, + "KL/rejected_KL_mean": -271.8486633300781, + "KL/std": 111.82902526855469, + "epoch": 0.8913362701908958, + "fcm_dpo/beta": 0.004737128037959337, + "fcm_dpo/delta": -0.03259171172976494, + "fcm_dpo/margin": 101.54586029052734, + "fcm_dpo/q_t": 0.3909189701080322, + "grad_norm": 12.469743728637695, + "learning_rate": 1.8300307303259904e-08, + "logits/chosen": -1.8413944244384766, + "logits/rejected": -1.3157618045806885, + "logps/chosen": -258.6318664550781, + "logps/ref_chosen": -88.32904052734375, + "logps/ref_rejected": -86.76810455322266, + "logps/rejected": -358.61676025390625, + "loss": 1.0608, + "margin_dpo/margin_mean": 101.54585266113281, + "margin_dpo/margin_std": 137.57362365722656, + "step": 607 + }, + { + "KL/chosen_KL_mean": -155.44029235839844, + "KL/mean": -208.53402709960938, + "KL/rejected_KL_mean": -261.6277770996094, + "KL/std": 110.51072692871094, + "epoch": 0.8928046989720999, + "fcm_dpo/beta": 0.004659214057028294, + "fcm_dpo/delta": -0.048452552407979965, + "fcm_dpo/margin": 106.18748474121094, + "fcm_dpo/q_t": 0.38495171070098877, + "grad_norm": 10.721970558166504, + "learning_rate": 1.7821396810182437e-08, + "logits/chosen": -1.8137047290802002, + "logits/rejected": -1.5158920288085938, + "logps/chosen": -241.20965576171875, + "logps/ref_chosen": -85.76937866210938, + "logps/ref_rejected": -100.23281860351562, + "logps/rejected": -361.860595703125, + "loss": 1.0298, + "margin_dpo/margin_mean": 106.18748474121094, + "margin_dpo/margin_std": 119.422119140625, + "step": 608 + }, + { + "KL/chosen_KL_mean": -172.4480438232422, + "KL/mean": -222.6395263671875, + "KL/rejected_KL_mean": -272.83099365234375, + "KL/std": 110.90241241455078, + "epoch": 0.8942731277533039, + "fcm_dpo/beta": 0.004647298250347376, + "fcm_dpo/delta": -0.017813783138990402, + "fcm_dpo/margin": 100.38296508789062, + "fcm_dpo/q_t": 0.3932035565376282, + "grad_norm": 11.612581253051758, + "learning_rate": 1.7348604439226617e-08, + "logits/chosen": -2.2478439807891846, + "logits/rejected": -1.8245940208435059, + "logps/chosen": -265.41461181640625, + "logps/ref_chosen": -92.9665756225586, + "logps/ref_rejected": -95.91818237304688, + "logps/rejected": -368.74920654296875, + "loss": 1.0486, + "margin_dpo/margin_mean": 100.3829574584961, + "margin_dpo/margin_std": 117.288818359375, + "step": 609 + }, + { + "KL/chosen_KL_mean": -158.64759826660156, + "KL/mean": -212.06503295898438, + "KL/rejected_KL_mean": -265.48248291015625, + "KL/std": 106.18761444091797, + "epoch": 0.895741556534508, + "fcm_dpo/beta": 0.004621714353561401, + "fcm_dpo/delta": -0.04623851925134659, + "fcm_dpo/margin": 106.83487701416016, + "fcm_dpo/q_t": 0.3845939040184021, + "grad_norm": 8.957317352294922, + "learning_rate": 1.6881942648911074e-08, + "logits/chosen": -1.76459801197052, + "logits/rejected": -1.2192771434783936, + "logps/chosen": -253.34788513183594, + "logps/ref_chosen": -94.70028686523438, + "logps/ref_rejected": -89.68739318847656, + "logps/rejected": -355.16986083984375, + "loss": 1.0159, + "margin_dpo/margin_mean": 106.83486938476562, + "margin_dpo/margin_std": 112.31874084472656, + "step": 610 + }, + { + "KL/chosen_KL_mean": -167.4735565185547, + "KL/mean": -219.63662719726562, + "KL/rejected_KL_mean": -271.7997131347656, + "KL/std": 110.14964294433594, + "epoch": 0.8972099853157122, + "fcm_dpo/beta": 0.004620520398020744, + "fcm_dpo/delta": -0.03399227187037468, + "fcm_dpo/margin": 104.3261489868164, + "fcm_dpo/q_t": 0.3893621563911438, + "grad_norm": 10.346162796020508, + "learning_rate": 1.6421423736208e-08, + "logits/chosen": -1.9958183765411377, + "logits/rejected": -1.6436913013458252, + "logps/chosen": -254.25689697265625, + "logps/ref_chosen": -86.78334045410156, + "logps/ref_rejected": -89.84307861328125, + "logps/rejected": -361.64276123046875, + "loss": 1.0256, + "margin_dpo/margin_mean": 104.3261489868164, + "margin_dpo/margin_std": 112.11479187011719, + "step": 611 + }, + { + "KL/chosen_KL_mean": -183.2674560546875, + "KL/mean": -236.60855102539062, + "KL/rejected_KL_mean": -289.9496765136719, + "KL/std": 104.09312438964844, + "epoch": 0.8986784140969163, + "fcm_dpo/beta": 0.004556679166853428, + "fcm_dpo/delta": -0.03807242587208748, + "fcm_dpo/margin": 106.6822280883789, + "fcm_dpo/q_t": 0.3870040774345398, + "grad_norm": 9.27346134185791, + "learning_rate": 1.5967059836219042e-08, + "logits/chosen": -2.1088085174560547, + "logits/rejected": -1.267046570777893, + "logps/chosen": -284.28759765625, + "logps/ref_chosen": -101.02015686035156, + "logps/ref_rejected": -93.78302764892578, + "logps/rejected": -383.7326965332031, + "loss": 1.014, + "margin_dpo/margin_mean": 106.6822280883789, + "margin_dpo/margin_std": 107.85067749023438, + "step": 612 + }, + { + "KL/chosen_KL_mean": -146.6988525390625, + "KL/mean": -206.68246459960938, + "KL/rejected_KL_mean": -266.66607666015625, + "KL/std": 106.4085693359375, + "epoch": 0.9001468428781204, + "fcm_dpo/beta": 0.004484009929001331, + "fcm_dpo/delta": -0.09317655861377716, + "fcm_dpo/margin": 119.96723175048828, + "fcm_dpo/q_t": 0.37405645847320557, + "grad_norm": 8.81155776977539, + "learning_rate": 1.551886292185553e-08, + "logits/chosen": -2.016073703765869, + "logits/rejected": -1.8203673362731934, + "logps/chosen": -235.68748474121094, + "logps/ref_chosen": -88.98861694335938, + "logps/ref_rejected": -109.99551391601562, + "logps/rejected": -376.6615905761719, + "loss": 0.9694, + "margin_dpo/margin_mean": 119.96722412109375, + "margin_dpo/margin_std": 104.03485107421875, + "step": 613 + }, + { + "KL/chosen_KL_mean": -185.25738525390625, + "KL/mean": -233.3673858642578, + "KL/rejected_KL_mean": -281.4773864746094, + "KL/std": 100.46821594238281, + "epoch": 0.9016152716593245, + "fcm_dpo/beta": 0.0044782571494579315, + "fcm_dpo/delta": 0.019953060895204544, + "fcm_dpo/margin": 96.2199935913086, + "fcm_dpo/q_t": 0.40007302165031433, + "grad_norm": 11.269189834594727, + "learning_rate": 1.507684480352292e-08, + "logits/chosen": -1.4590733051300049, + "logits/rejected": -1.4495806694030762, + "logps/chosen": -265.45745849609375, + "logps/ref_chosen": -80.20005798339844, + "logps/ref_rejected": -109.86239624023438, + "logps/rejected": -391.33978271484375, + "loss": 1.0738, + "margin_dpo/margin_mean": 96.2199935913086, + "margin_dpo/margin_std": 122.71536254882812, + "step": 614 + }, + { + "KL/chosen_KL_mean": -173.26766967773438, + "KL/mean": -213.72019958496094, + "KL/rejected_KL_mean": -254.17269897460938, + "KL/std": 101.97980499267578, + "epoch": 0.9030837004405287, + "fcm_dpo/beta": 0.004574804566800594, + "fcm_dpo/delta": 0.0814293697476387, + "fcm_dpo/margin": 80.9050521850586, + "fcm_dpo/q_t": 0.41295474767684937, + "grad_norm": 9.873544692993164, + "learning_rate": 1.4641017128809801e-08, + "logits/chosen": -1.590479850769043, + "logits/rejected": -1.1801893711090088, + "logps/chosen": -273.70294189453125, + "logps/ref_chosen": -100.43525695800781, + "logps/ref_rejected": -101.1800537109375, + "logps/rejected": -355.352783203125, + "loss": 1.1009, + "margin_dpo/margin_mean": 80.9050521850586, + "margin_dpo/margin_std": 96.39158630371094, + "step": 615 + }, + { + "KL/chosen_KL_mean": -175.1005401611328, + "KL/mean": -230.30368041992188, + "KL/rejected_KL_mean": -285.5068054199219, + "KL/std": 102.12892150878906, + "epoch": 0.9045521292217328, + "fcm_dpo/beta": 0.004543361719697714, + "fcm_dpo/delta": -0.054439835250377655, + "fcm_dpo/margin": 110.40625762939453, + "fcm_dpo/q_t": 0.38406243920326233, + "grad_norm": 9.328760147094727, + "learning_rate": 1.4211391382180637e-08, + "logits/chosen": -1.753568410873413, + "logits/rejected": -1.2510175704956055, + "logps/chosen": -267.5934753417969, + "logps/ref_chosen": -92.49292755126953, + "logps/ref_rejected": -82.06065368652344, + "logps/rejected": -367.56744384765625, + "loss": 1.0146, + "margin_dpo/margin_mean": 110.40625762939453, + "margin_dpo/margin_std": 118.6685791015625, + "step": 616 + }, + { + "KL/chosen_KL_mean": -182.56533813476562, + "KL/mean": -226.22821044921875, + "KL/rejected_KL_mean": -269.89111328125, + "KL/std": 99.32344055175781, + "epoch": 0.9060205580029369, + "fcm_dpo/beta": 0.004548664204776287, + "fcm_dpo/delta": 0.054983749985694885, + "fcm_dpo/margin": 87.32576751708984, + "fcm_dpo/q_t": 0.40742790699005127, + "grad_norm": 9.524311065673828, + "learning_rate": 1.378797888467345e-08, + "logits/chosen": -1.422196388244629, + "logits/rejected": -0.7640622854232788, + "logps/chosen": -273.6623229980469, + "logps/ref_chosen": -91.09699249267578, + "logps/ref_rejected": -70.41004943847656, + "logps/rejected": -340.3011474609375, + "loss": 1.0898, + "margin_dpo/margin_mean": 87.32576751708984, + "margin_dpo/margin_std": 111.27055358886719, + "step": 617 + }, + { + "KL/chosen_KL_mean": -184.1827392578125, + "KL/mean": -232.53775024414062, + "KL/rejected_KL_mean": -280.89276123046875, + "KL/std": 109.46292877197266, + "epoch": 0.9074889867841409, + "fcm_dpo/beta": 0.00456183310598135, + "fcm_dpo/delta": 0.009194123558700085, + "fcm_dpo/margin": 96.71001434326172, + "fcm_dpo/q_t": 0.3969927430152893, + "grad_norm": 9.151403427124023, + "learning_rate": 1.3370790793601371e-08, + "logits/chosen": -2.015610456466675, + "logits/rejected": -1.6541812419891357, + "logps/chosen": -286.2033386230469, + "logps/ref_chosen": -102.02059936523438, + "logps/ref_rejected": -99.80119323730469, + "logps/rejected": -380.6939697265625, + "loss": 1.0661, + "margin_dpo/margin_mean": 96.71001434326172, + "margin_dpo/margin_std": 121.06686401367188, + "step": 618 + }, + { + "KL/chosen_KL_mean": -178.2374267578125, + "KL/mean": -221.03900146484375, + "KL/rejected_KL_mean": -263.8406066894531, + "KL/std": 108.02735137939453, + "epoch": 0.908957415565345, + "fcm_dpo/beta": 0.004609095398336649, + "fcm_dpo/delta": 0.05774322897195816, + "fcm_dpo/margin": 85.60318756103516, + "fcm_dpo/q_t": 0.4113660454750061, + "grad_norm": 9.244808197021484, + "learning_rate": 1.2959838102258535e-08, + "logits/chosen": -1.4586336612701416, + "logits/rejected": -1.0880571603775024, + "logps/chosen": -267.9787902832031, + "logps/ref_chosen": -89.74136352539062, + "logps/ref_rejected": -99.90138244628906, + "logps/rejected": -363.74200439453125, + "loss": 1.1125, + "margin_dpo/margin_mean": 85.60319519042969, + "margin_dpo/margin_std": 129.63429260253906, + "step": 619 + }, + { + "KL/chosen_KL_mean": -170.60426330566406, + "KL/mean": -218.12603759765625, + "KL/rejected_KL_mean": -265.6478271484375, + "KL/std": 113.73348999023438, + "epoch": 0.9104258443465492, + "fcm_dpo/beta": 0.004587803967297077, + "fcm_dpo/delta": 0.012716904282569885, + "fcm_dpo/margin": 95.04354858398438, + "fcm_dpo/q_t": 0.4000408947467804, + "grad_norm": 10.134498596191406, + "learning_rate": 1.2555131639630567e-08, + "logits/chosen": -2.3685035705566406, + "logits/rejected": -1.856990098953247, + "logps/chosen": -255.72857666015625, + "logps/ref_chosen": -85.12431335449219, + "logps/ref_rejected": -85.41253662109375, + "logps/rejected": -351.06036376953125, + "loss": 1.081, + "margin_dpo/margin_mean": 95.04354858398438, + "margin_dpo/margin_std": 122.09893798828125, + "step": 620 + }, + { + "KL/chosen_KL_mean": -173.086669921875, + "KL/mean": -219.64083862304688, + "KL/rejected_KL_mean": -266.1950378417969, + "KL/std": 109.73408508300781, + "epoch": 0.9118942731277533, + "fcm_dpo/beta": 0.004689943976700306, + "fcm_dpo/delta": 0.01188938319683075, + "fcm_dpo/margin": 93.10836791992188, + "fcm_dpo/q_t": 0.4004325270652771, + "grad_norm": 9.9964017868042, + "learning_rate": 1.2156682070109086e-08, + "logits/chosen": -2.0491368770599365, + "logits/rejected": -1.7892625331878662, + "logps/chosen": -262.3350830078125, + "logps/ref_chosen": -89.24842071533203, + "logps/ref_rejected": -95.46463775634766, + "logps/rejected": -361.65966796875, + "loss": 1.09, + "margin_dpo/margin_mean": 93.10836791992188, + "margin_dpo/margin_std": 127.75167846679688, + "step": 621 + }, + { + "KL/chosen_KL_mean": -171.01986694335938, + "KL/mean": -220.12286376953125, + "KL/rejected_KL_mean": -269.225830078125, + "KL/std": 97.43096160888672, + "epoch": 0.9133627019089574, + "fcm_dpo/beta": 0.004632966592907906, + "fcm_dpo/delta": -0.005341339856386185, + "fcm_dpo/margin": 98.20596313476562, + "fcm_dpo/q_t": 0.39399614930152893, + "grad_norm": 11.154865264892578, + "learning_rate": 1.1764499893210878e-08, + "logits/chosen": -1.434091329574585, + "logits/rejected": -0.7677087783813477, + "logps/chosen": -270.8139953613281, + "logps/ref_chosen": -99.79412841796875, + "logps/ref_rejected": -90.82821655273438, + "logps/rejected": -360.0540466308594, + "loss": 1.0392, + "margin_dpo/margin_mean": 98.20596313476562, + "margin_dpo/margin_std": 105.42474365234375, + "step": 622 + }, + { + "KL/chosen_KL_mean": -178.2598876953125, + "KL/mean": -212.8975372314453, + "KL/rejected_KL_mean": -247.53517150878906, + "KL/std": 105.79509735107422, + "epoch": 0.9148311306901615, + "fcm_dpo/beta": 0.004716922529041767, + "fcm_dpo/delta": 0.1275968998670578, + "fcm_dpo/margin": 69.27526092529297, + "fcm_dpo/q_t": 0.4265623986721039, + "grad_norm": 10.9679594039917, + "learning_rate": 1.1378595443300998e-08, + "logits/chosen": -2.231057643890381, + "logits/rejected": -1.790826678276062, + "logps/chosen": -268.7154541015625, + "logps/ref_chosen": -90.45555114746094, + "logps/ref_rejected": -91.32276153564453, + "logps/rejected": -338.85791015625, + "loss": 1.1756, + "margin_dpo/margin_mean": 69.27526092529297, + "margin_dpo/margin_std": 129.77325439453125, + "step": 623 + }, + { + "KL/chosen_KL_mean": -182.99085998535156, + "KL/mean": -238.72291564941406, + "KL/rejected_KL_mean": -294.4549865722656, + "KL/std": 104.68896484375, + "epoch": 0.9162995594713657, + "fcm_dpo/beta": 0.004705128259956837, + "fcm_dpo/delta": -0.0786077231168747, + "fcm_dpo/margin": 111.46412658691406, + "fcm_dpo/q_t": 0.379744291305542, + "grad_norm": 10.653809547424316, + "learning_rate": 1.0998978889320582e-08, + "logits/chosen": -2.5369553565979004, + "logits/rejected": -1.6156611442565918, + "logps/chosen": -292.8660888671875, + "logps/ref_chosen": -109.87522888183594, + "logps/ref_rejected": -104.77320861816406, + "logps/rejected": -399.22821044921875, + "loss": 1.003, + "margin_dpo/margin_mean": 111.46412658691406, + "margin_dpo/margin_std": 120.23019409179688, + "step": 624 + }, + { + "KL/chosen_KL_mean": -191.24447631835938, + "KL/mean": -231.90008544921875, + "KL/rejected_KL_mean": -272.55572509765625, + "KL/std": 104.50779724121094, + "epoch": 0.9177679882525698, + "fcm_dpo/beta": 0.004745472222566605, + "fcm_dpo/delta": 0.06645552814006805, + "fcm_dpo/margin": 81.31123352050781, + "fcm_dpo/q_t": 0.4112913906574249, + "grad_norm": 9.284390449523926, + "learning_rate": 1.0625660234518913e-08, + "logits/chosen": -1.7679665088653564, + "logits/rejected": -1.3943238258361816, + "logps/chosen": -278.41265869140625, + "logps/ref_chosen": -87.16815948486328, + "logps/ref_rejected": -91.86148071289062, + "logps/rejected": -364.41717529296875, + "loss": 1.1024, + "margin_dpo/margin_mean": 81.31123352050781, + "margin_dpo/margin_std": 110.92190551757812, + "step": 625 + }, + { + "KL/chosen_KL_mean": -181.08592224121094, + "KL/mean": -221.35281372070312, + "KL/rejected_KL_mean": -261.61968994140625, + "KL/std": 115.74142456054688, + "epoch": 0.9192364170337739, + "fcm_dpo/beta": 0.004809065721929073, + "fcm_dpo/delta": 0.0649387314915657, + "fcm_dpo/margin": 80.53378295898438, + "fcm_dpo/q_t": 0.4128105342388153, + "grad_norm": 11.609611511230469, + "learning_rate": 1.0258649316189721e-08, + "logits/chosen": -1.4680793285369873, + "logits/rejected": -1.0361180305480957, + "logps/chosen": -285.31011962890625, + "logps/ref_chosen": -104.22421264648438, + "logps/ref_rejected": -104.1774673461914, + "logps/rejected": -365.79718017578125, + "loss": 1.1224, + "margin_dpo/margin_mean": 80.53378295898438, + "margin_dpo/margin_std": 126.5710220336914, + "step": 626 + }, + { + "KL/chosen_KL_mean": -161.65158081054688, + "KL/mean": -212.0081787109375, + "KL/rejected_KL_mean": -262.36474609375, + "KL/std": 118.21295166015625, + "epoch": 0.920704845814978, + "fcm_dpo/beta": 0.004788564518094063, + "fcm_dpo/delta": -0.033907003700733185, + "fcm_dpo/margin": 100.71316528320312, + "fcm_dpo/q_t": 0.3918336033821106, + "grad_norm": 8.047354698181152, + "learning_rate": 9.897955805412e-09, + "logits/chosen": -1.5927295684814453, + "logits/rejected": -1.737905740737915, + "logps/chosen": -236.58621215820312, + "logps/ref_chosen": -74.93461608886719, + "logps/ref_rejected": -112.57289123535156, + "logps/rejected": -374.9376525878906, + "loss": 1.0484, + "margin_dpo/margin_mean": 100.71316528320312, + "margin_dpo/margin_std": 129.9547576904297, + "step": 627 + }, + { + "KL/chosen_KL_mean": -169.31735229492188, + "KL/mean": -222.12353515625, + "KL/rejected_KL_mean": -274.9296875, + "KL/std": 106.74333953857422, + "epoch": 0.922173274596182, + "fcm_dpo/beta": 0.004742040298879147, + "fcm_dpo/delta": -0.053501617163419724, + "fcm_dpo/margin": 105.61235046386719, + "fcm_dpo/q_t": 0.38509994745254517, + "grad_norm": 10.507683753967285, + "learning_rate": 9.543589206795238e-09, + "logits/chosen": -1.8934911489486694, + "logits/rejected": -1.6267330646514893, + "logps/chosen": -263.0084228515625, + "logps/ref_chosen": -93.69107818603516, + "logps/ref_rejected": -107.34395599365234, + "logps/rejected": -382.2736511230469, + "loss": 1.0197, + "margin_dpo/margin_mean": 105.61235046386719, + "margin_dpo/margin_std": 117.55007934570312, + "step": 628 + }, + { + "KL/chosen_KL_mean": -182.3243408203125, + "KL/mean": -229.99411010742188, + "KL/rejected_KL_mean": -277.66387939453125, + "KL/std": 106.87051391601562, + "epoch": 0.9236417033773862, + "fcm_dpo/beta": 0.004714585840702057, + "fcm_dpo/delta": 0.00033011287450790405, + "fcm_dpo/margin": 95.33956909179688, + "fcm_dpo/q_t": 0.39539289474487305, + "grad_norm": 10.074275970458984, + "learning_rate": 9.19555885822887e-09, + "logits/chosen": -2.1480307579040527, + "logits/rejected": -1.7198066711425781, + "logps/chosen": -285.5547180175781, + "logps/ref_chosen": -103.23037719726562, + "logps/ref_rejected": -97.16841888427734, + "logps/rejected": -374.83233642578125, + "loss": 1.0553, + "margin_dpo/margin_mean": 95.33956909179688, + "margin_dpo/margin_std": 113.55947875976562, + "step": 629 + }, + { + "KL/chosen_KL_mean": -166.311279296875, + "KL/mean": -197.57908630371094, + "KL/rejected_KL_mean": -228.8468780517578, + "KL/std": 99.85243225097656, + "epoch": 0.9251101321585903, + "fcm_dpo/beta": 0.004856232088059187, + "fcm_dpo/delta": 0.14996640384197235, + "fcm_dpo/margin": 62.53557586669922, + "fcm_dpo/q_t": 0.43007415533065796, + "grad_norm": 13.54204273223877, + "learning_rate": 8.85387393063622e-09, + "logits/chosen": -1.973757266998291, + "logits/rejected": -1.4395477771759033, + "logps/chosen": -260.2088317871094, + "logps/ref_chosen": -93.89755249023438, + "logps/ref_rejected": -89.3743896484375, + "logps/rejected": -318.22125244140625, + "loss": 1.1781, + "margin_dpo/margin_mean": 62.53557586669922, + "margin_dpo/margin_std": 112.05850219726562, + "step": 630 + }, + { + "KL/chosen_KL_mean": -164.4595489501953, + "KL/mean": -210.38082885742188, + "KL/rejected_KL_mean": -256.3020935058594, + "KL/std": 98.89360046386719, + "epoch": 0.9265785609397944, + "fcm_dpo/beta": 0.004870392847806215, + "fcm_dpo/delta": 0.0027950862422585487, + "fcm_dpo/margin": 91.842529296875, + "fcm_dpo/q_t": 0.39640283584594727, + "grad_norm": 9.133491516113281, + "learning_rate": 8.518543427732949e-09, + "logits/chosen": -2.471451759338379, + "logits/rejected": -2.041480779647827, + "logps/chosen": -252.23037719726562, + "logps/ref_chosen": -87.77082061767578, + "logps/ref_rejected": -88.68241882324219, + "logps/rejected": -344.9844970703125, + "loss": 1.059, + "margin_dpo/margin_mean": 91.842529296875, + "margin_dpo/margin_std": 112.66258239746094, + "step": 631 + }, + { + "KL/chosen_KL_mean": -161.1116180419922, + "KL/mean": -211.72911071777344, + "KL/rejected_KL_mean": -262.34661865234375, + "KL/std": 104.71236419677734, + "epoch": 0.9280469897209985, + "fcm_dpo/beta": 0.004819872789084911, + "fcm_dpo/delta": -0.040760889649391174, + "fcm_dpo/margin": 101.23500061035156, + "fcm_dpo/q_t": 0.3892359733581543, + "grad_norm": 12.062975883483887, + "learning_rate": 8.189576185789637e-09, + "logits/chosen": -2.054488182067871, + "logits/rejected": -1.6294389963150024, + "logps/chosen": -249.7381591796875, + "logps/ref_chosen": -88.62652587890625, + "logps/ref_rejected": -91.45091247558594, + "logps/rejected": -353.79754638671875, + "loss": 1.0482, + "margin_dpo/margin_mean": 101.2349853515625, + "margin_dpo/margin_std": 128.82888793945312, + "step": 632 + }, + { + "KL/chosen_KL_mean": -200.8941650390625, + "KL/mean": -230.75674438476562, + "KL/rejected_KL_mean": -260.6193542480469, + "KL/std": 106.32029724121094, + "epoch": 0.9295154185022027, + "fcm_dpo/beta": 0.004847935400903225, + "fcm_dpo/delta": 0.022854354232549667, + "fcm_dpo/margin": 59.72520446777344, + "fcm_dpo/q_t": 0.4360605776309967, + "grad_norm": 11.445959091186523, + "learning_rate": 7.866980873399015e-09, + "logits/chosen": -2.161868095397949, + "logits/rejected": -1.9704504013061523, + "logps/chosen": -282.2685852050781, + "logps/ref_chosen": -81.37442016601562, + "logps/ref_rejected": -98.62571716308594, + "logps/rejected": -359.2450866699219, + "loss": 1.2294, + "margin_dpo/margin_mean": 59.72520446777344, + "margin_dpo/margin_std": 141.68458557128906, + "step": 633 + }, + { + "KL/chosen_KL_mean": -184.75729370117188, + "KL/mean": -223.542236328125, + "KL/rejected_KL_mean": -262.3271484375, + "KL/std": 95.78713989257812, + "epoch": 0.9309838472834068, + "fcm_dpo/beta": 0.0048352256417274475, + "fcm_dpo/delta": -0.04920076206326485, + "fcm_dpo/margin": 77.5698471069336, + "fcm_dpo/q_t": 0.41228485107421875, + "grad_norm": 11.62212085723877, + "learning_rate": 7.550765991247654e-09, + "logits/chosen": -1.3433046340942383, + "logits/rejected": -1.166331171989441, + "logps/chosen": -280.880126953125, + "logps/ref_chosen": -96.12284851074219, + "logps/ref_rejected": -112.84780883789062, + "logps/rejected": -375.1749572753906, + "loss": 1.105, + "margin_dpo/margin_mean": 77.5698471069336, + "margin_dpo/margin_std": 97.00047302246094, + "step": 634 + }, + { + "KL/chosen_KL_mean": -192.25848388671875, + "KL/mean": -222.4342803955078, + "KL/rejected_KL_mean": -252.61007690429688, + "KL/std": 103.95018005371094, + "epoch": 0.9324522760646109, + "fcm_dpo/beta": 0.004827727563679218, + "fcm_dpo/delta": 0.033745329827070236, + "fcm_dpo/margin": 60.351585388183594, + "fcm_dpo/q_t": 0.43377456068992615, + "grad_norm": 13.313103675842285, + "learning_rate": 7.240939871891699e-09, + "logits/chosen": -1.741532564163208, + "logits/rejected": -1.2016403675079346, + "logps/chosen": -290.9425964355469, + "logps/ref_chosen": -98.68411254882812, + "logps/ref_rejected": -89.89920043945312, + "logps/rejected": -342.50927734375, + "loss": 1.2027, + "margin_dpo/margin_mean": 60.351585388183594, + "margin_dpo/margin_std": 124.63910675048828, + "step": 635 + }, + { + "KL/chosen_KL_mean": -172.7763671875, + "KL/mean": -223.48602294921875, + "KL/rejected_KL_mean": -274.1956787109375, + "KL/std": 107.63565826416016, + "epoch": 0.933920704845815, + "fcm_dpo/beta": 0.004821810871362686, + "fcm_dpo/delta": -0.04101572185754776, + "fcm_dpo/margin": 101.41928100585938, + "fcm_dpo/q_t": 0.38723403215408325, + "grad_norm": 10.1853609085083, + "learning_rate": 6.937510679537628e-09, + "logits/chosen": -1.5749695301055908, + "logits/rejected": -1.09731924533844, + "logps/chosen": -263.1943359375, + "logps/ref_chosen": -90.41796112060547, + "logps/ref_rejected": -87.70687866210938, + "logps/rejected": -361.90252685546875, + "loss": 1.019, + "margin_dpo/margin_mean": 101.41928100585938, + "margin_dpo/margin_std": 107.80775451660156, + "step": 636 + }, + { + "KL/chosen_KL_mean": -174.0252685546875, + "KL/mean": -228.01861572265625, + "KL/rejected_KL_mean": -282.011962890625, + "KL/std": 119.41990661621094, + "epoch": 0.9353891336270191, + "fcm_dpo/beta": 0.004764461889863014, + "fcm_dpo/delta": -0.06794070452451706, + "fcm_dpo/margin": 107.98666381835938, + "fcm_dpo/q_t": 0.38107621669769287, + "grad_norm": 8.683420181274414, + "learning_rate": 6.640486409826785e-09, + "logits/chosen": -2.010817050933838, + "logits/rejected": -1.9075961112976074, + "logps/chosen": -256.4749755859375, + "logps/ref_chosen": -82.44971466064453, + "logps/ref_rejected": -104.02860260009766, + "logps/rejected": -386.04052734375, + "loss": 1.0056, + "margin_dpo/margin_mean": 107.98666381835938, + "margin_dpo/margin_std": 112.98416137695312, + "step": 637 + }, + { + "KL/chosen_KL_mean": -161.8271484375, + "KL/mean": -215.07395935058594, + "KL/rejected_KL_mean": -268.3207702636719, + "KL/std": 105.83755493164062, + "epoch": 0.9368575624082232, + "fcm_dpo/beta": 0.004725996404886246, + "fcm_dpo/delta": -0.056419726461172104, + "fcm_dpo/margin": 106.49361419677734, + "fcm_dpo/q_t": 0.3823361396789551, + "grad_norm": 8.91274642944336, + "learning_rate": 6.349874889624962e-09, + "logits/chosen": -1.4914363622665405, + "logits/rejected": -0.7988900542259216, + "logps/chosen": -253.75213623046875, + "logps/ref_chosen": -91.92498779296875, + "logps/ref_rejected": -86.28703308105469, + "logps/rejected": -354.6077880859375, + "loss": 0.9972, + "margin_dpo/margin_mean": 106.49362182617188, + "margin_dpo/margin_std": 95.94612121582031, + "step": 638 + }, + { + "KL/chosen_KL_mean": -214.31985473632812, + "KL/mean": -239.8204345703125, + "KL/rejected_KL_mean": -265.321044921875, + "KL/std": 99.06515502929688, + "epoch": 0.9383259911894273, + "fcm_dpo/beta": 0.004721669480204582, + "fcm_dpo/delta": 0.07464981079101562, + "fcm_dpo/margin": 51.00115203857422, + "fcm_dpo/q_t": 0.4439978003501892, + "grad_norm": 10.234736442565918, + "learning_rate": 6.065683776815933e-09, + "logits/chosen": -1.3730717897415161, + "logits/rejected": -0.6060410141944885, + "logps/chosen": -318.847412109375, + "logps/ref_chosen": -104.52755737304688, + "logps/ref_rejected": -81.4803466796875, + "logps/rejected": -346.8013916015625, + "loss": 1.2413, + "margin_dpo/margin_mean": 51.00115203857422, + "margin_dpo/margin_std": 123.33016967773438, + "step": 639 + }, + { + "KL/chosen_KL_mean": -180.89990234375, + "KL/mean": -228.81802368164062, + "KL/rejected_KL_mean": -276.73614501953125, + "KL/std": 106.22747802734375, + "epoch": 0.9397944199706314, + "fcm_dpo/beta": 0.004792827181518078, + "fcm_dpo/delta": -0.011392727494239807, + "fcm_dpo/margin": 95.83621978759766, + "fcm_dpo/q_t": 0.39340639114379883, + "grad_norm": 11.994951248168945, + "learning_rate": 5.7879205600998296e-09, + "logits/chosen": -1.8697707653045654, + "logits/rejected": -1.3506278991699219, + "logps/chosen": -278.78515625, + "logps/ref_chosen": -97.88526916503906, + "logps/ref_rejected": -112.70502471923828, + "logps/rejected": -389.441162109375, + "loss": 1.0486, + "margin_dpo/margin_mean": 95.83621978759766, + "margin_dpo/margin_std": 105.78228759765625, + "step": 640 + }, + { + "KL/chosen_KL_mean": -166.28106689453125, + "KL/mean": -216.40504455566406, + "KL/rejected_KL_mean": -266.529052734375, + "KL/std": 123.32797241210938, + "epoch": 0.9412628487518355, + "fcm_dpo/beta": 0.004731766879558563, + "fcm_dpo/delta": -0.025555146858096123, + "fcm_dpo/margin": 100.24797058105469, + "fcm_dpo/q_t": 0.3950076103210449, + "grad_norm": 9.189568519592285, + "learning_rate": 5.516592558795746e-09, + "logits/chosen": -2.4536731243133545, + "logits/rejected": -1.9285693168640137, + "logps/chosen": -262.72674560546875, + "logps/ref_chosen": -96.4456787109375, + "logps/ref_rejected": -95.13568878173828, + "logps/rejected": -361.66473388671875, + "loss": 1.058, + "margin_dpo/margin_mean": 100.24797058105469, + "margin_dpo/margin_std": 135.9686737060547, + "step": 641 + }, + { + "KL/chosen_KL_mean": -184.8050537109375, + "KL/mean": -231.78439331054688, + "KL/rejected_KL_mean": -278.76373291015625, + "KL/std": 106.00778198242188, + "epoch": 0.9427312775330396, + "fcm_dpo/beta": 0.004737743176519871, + "fcm_dpo/delta": 0.004920683801174164, + "fcm_dpo/margin": 93.9587173461914, + "fcm_dpo/q_t": 0.40042415261268616, + "grad_norm": 11.429861068725586, + "learning_rate": 5.251706922648868e-09, + "logits/chosen": -1.5664832592010498, + "logits/rejected": -1.2332630157470703, + "logps/chosen": -285.56488037109375, + "logps/ref_chosen": -100.75984954833984, + "logps/ref_rejected": -114.7076416015625, + "logps/rejected": -393.47137451171875, + "loss": 1.0766, + "margin_dpo/margin_mean": 93.9587173461914, + "margin_dpo/margin_std": 131.00064086914062, + "step": 642 + }, + { + "KL/chosen_KL_mean": -173.32237243652344, + "KL/mean": -218.06170654296875, + "KL/rejected_KL_mean": -262.801025390625, + "KL/std": 98.66541290283203, + "epoch": 0.9441997063142438, + "fcm_dpo/beta": 0.004710989538580179, + "fcm_dpo/delta": 0.028752140700817108, + "fcm_dpo/margin": 89.47865295410156, + "fcm_dpo/q_t": 0.4023323059082031, + "grad_norm": 11.671807289123535, + "learning_rate": 4.993270631642038e-09, + "logits/chosen": -2.3143203258514404, + "logits/rejected": -1.9429935216903687, + "logps/chosen": -258.0660400390625, + "logps/ref_chosen": -84.74365997314453, + "logps/ref_rejected": -94.31842041015625, + "logps/rejected": -357.11944580078125, + "loss": 1.083, + "margin_dpo/margin_mean": 89.47865295410156, + "margin_dpo/margin_std": 114.07974243164062, + "step": 643 + }, + { + "KL/chosen_KL_mean": -179.53750610351562, + "KL/mean": -216.66766357421875, + "KL/rejected_KL_mean": -253.79782104492188, + "KL/std": 99.16422271728516, + "epoch": 0.9456681350954479, + "fcm_dpo/beta": 0.004822202026844025, + "fcm_dpo/delta": 0.09530861675739288, + "fcm_dpo/margin": 74.26031494140625, + "fcm_dpo/q_t": 0.4172069728374481, + "grad_norm": 12.127134323120117, + "learning_rate": 4.741290495811873e-09, + "logits/chosen": -1.798338770866394, + "logits/rejected": -1.526397466659546, + "logps/chosen": -264.8602600097656, + "logps/ref_chosen": -85.32274627685547, + "logps/ref_rejected": -94.60861206054688, + "logps/rejected": -348.40643310546875, + "loss": 1.1223, + "margin_dpo/margin_mean": 74.26031494140625, + "margin_dpo/margin_std": 106.45948791503906, + "step": 644 + }, + { + "KL/chosen_KL_mean": -192.14700317382812, + "KL/mean": -216.12217712402344, + "KL/rejected_KL_mean": -240.0973358154297, + "KL/std": 108.17072296142578, + "epoch": 0.947136563876652, + "fcm_dpo/beta": 0.004887172486633062, + "fcm_dpo/delta": 0.08570849895477295, + "fcm_dpo/margin": 47.950321197509766, + "fcm_dpo/q_t": 0.4466143846511841, + "grad_norm": 11.438575744628906, + "learning_rate": 4.495773155069299e-09, + "logits/chosen": -1.275451898574829, + "logits/rejected": -1.1009612083435059, + "logps/chosen": -274.73724365234375, + "logps/ref_chosen": -82.59024047851562, + "logps/ref_rejected": -103.00375366210938, + "logps/rejected": -343.10107421875, + "loss": 1.2523, + "margin_dpo/margin_mean": 47.950321197509766, + "margin_dpo/margin_std": 122.57881164550781, + "step": 645 + }, + { + "KL/chosen_KL_mean": -167.93702697753906, + "KL/mean": -215.34378051757812, + "KL/rejected_KL_mean": -262.75054931640625, + "KL/std": 94.5502700805664, + "epoch": 0.9486049926578561, + "fcm_dpo/beta": 0.004896007478237152, + "fcm_dpo/delta": -0.015372931957244873, + "fcm_dpo/margin": 94.81353759765625, + "fcm_dpo/q_t": 0.392168253660202, + "grad_norm": 9.645984649658203, + "learning_rate": 4.256725079024553e-09, + "logits/chosen": -1.759879231452942, + "logits/rejected": -1.1478692293167114, + "logps/chosen": -259.1290588378906, + "logps/ref_chosen": -91.1920394897461, + "logps/ref_rejected": -83.77833557128906, + "logps/rejected": -346.52886962890625, + "loss": 1.0355, + "margin_dpo/margin_mean": 94.81353759765625, + "margin_dpo/margin_std": 102.04331970214844, + "step": 646 + }, + { + "KL/chosen_KL_mean": -183.03317260742188, + "KL/mean": -224.60198974609375, + "KL/rejected_KL_mean": -266.1707763671875, + "KL/std": 114.03018188476562, + "epoch": 0.9500734214390602, + "fcm_dpo/beta": 0.0049431947991251945, + "fcm_dpo/delta": 0.04071224108338356, + "fcm_dpo/margin": 83.13764190673828, + "fcm_dpo/q_t": 0.40789031982421875, + "grad_norm": 10.713689804077148, + "learning_rate": 4.024152566816791e-09, + "logits/chosen": -1.5028190612792969, + "logits/rejected": -1.3653578758239746, + "logps/chosen": -271.87762451171875, + "logps/ref_chosen": -88.8444595336914, + "logps/ref_rejected": -99.49833679199219, + "logps/rejected": -365.66912841796875, + "loss": 1.1079, + "margin_dpo/margin_mean": 83.13764953613281, + "margin_dpo/margin_std": 126.62890625, + "step": 647 + }, + { + "KL/chosen_KL_mean": -169.91302490234375, + "KL/mean": -227.63442993164062, + "KL/rejected_KL_mean": -285.3558349609375, + "KL/std": 116.15348815917969, + "epoch": 0.9515418502202643, + "fcm_dpo/beta": 0.004880198277533054, + "fcm_dpo/delta": -0.11997868120670319, + "fcm_dpo/margin": 115.44281005859375, + "fcm_dpo/q_t": 0.3711271584033966, + "grad_norm": 10.906291961669922, + "learning_rate": 3.798061746947995e-09, + "logits/chosen": -1.8949060440063477, + "logits/rejected": -1.8207893371582031, + "logps/chosen": -257.7611389160156, + "logps/ref_chosen": -87.84810638427734, + "logps/ref_rejected": -104.67005920410156, + "logps/rejected": -390.02587890625, + "loss": 0.983, + "margin_dpo/margin_mean": 115.44281005859375, + "margin_dpo/margin_std": 122.52619171142578, + "step": 648 + }, + { + "KL/chosen_KL_mean": -155.88717651367188, + "KL/mean": -204.36273193359375, + "KL/rejected_KL_mean": -252.83828735351562, + "KL/std": 101.7880859375, + "epoch": 0.9530102790014684, + "fcm_dpo/beta": 0.00479974877089262, + "fcm_dpo/delta": -0.0167253278195858, + "fcm_dpo/margin": 96.95110321044922, + "fcm_dpo/q_t": 0.39167076349258423, + "grad_norm": 11.536678314208984, + "learning_rate": 3.5784585771215235e-09, + "logits/chosen": -2.4893527030944824, + "logits/rejected": -2.091926097869873, + "logps/chosen": -245.57968139648438, + "logps/ref_chosen": -89.6925048828125, + "logps/ref_rejected": -88.70658111572266, + "logps/rejected": -341.54486083984375, + "loss": 1.0292, + "margin_dpo/margin_mean": 96.95109558105469, + "margin_dpo/margin_std": 97.85920715332031, + "step": 649 + }, + { + "KL/chosen_KL_mean": -184.9298553466797, + "KL/mean": -230.22842407226562, + "KL/rejected_KL_mean": -275.5269775390625, + "KL/std": 109.57231140136719, + "epoch": 0.9544787077826725, + "fcm_dpo/beta": 0.004835030063986778, + "fcm_dpo/delta": 0.012466302141547203, + "fcm_dpo/margin": 90.59712982177734, + "fcm_dpo/q_t": 0.40001872181892395, + "grad_norm": 10.516207695007324, + "learning_rate": 3.3653488440851253e-09, + "logits/chosen": -1.15730881690979, + "logits/rejected": -0.9545855522155762, + "logps/chosen": -274.8604736328125, + "logps/ref_chosen": -89.93060302734375, + "logps/ref_rejected": -102.61282348632812, + "logps/rejected": -378.1398010253906, + "loss": 1.0848, + "margin_dpo/margin_mean": 90.59713745117188, + "margin_dpo/margin_std": 129.09523010253906, + "step": 650 + }, + { + "KL/chosen_KL_mean": -153.58432006835938, + "KL/mean": -209.2176971435547, + "KL/rejected_KL_mean": -264.85107421875, + "KL/std": 106.31681823730469, + "epoch": 0.9559471365638766, + "fcm_dpo/beta": 0.00476833013817668, + "fcm_dpo/delta": -0.08514226973056793, + "fcm_dpo/margin": 111.2667236328125, + "fcm_dpo/q_t": 0.37607014179229736, + "grad_norm": 9.568754196166992, + "learning_rate": 3.158738163478475e-09, + "logits/chosen": -2.108980655670166, + "logits/rejected": -1.9345710277557373, + "logps/chosen": -232.7716522216797, + "logps/ref_chosen": -79.18731689453125, + "logps/ref_rejected": -105.93333435058594, + "logps/rejected": -370.7843933105469, + "loss": 0.9835, + "margin_dpo/margin_mean": 111.2667236328125, + "margin_dpo/margin_std": 104.88247680664062, + "step": 651 + }, + { + "KL/chosen_KL_mean": -160.6580047607422, + "KL/mean": -207.53530883789062, + "KL/rejected_KL_mean": -254.41259765625, + "KL/std": 98.65438079833984, + "epoch": 0.9574155653450808, + "fcm_dpo/beta": 0.0047644879668951035, + "fcm_dpo/delta": 0.003341231495141983, + "fcm_dpo/margin": 93.75457763671875, + "fcm_dpo/q_t": 0.39559128880500793, + "grad_norm": 9.580215454101562, + "learning_rate": 2.9586319796851555e-09, + "logits/chosen": -2.5949487686157227, + "logits/rejected": -2.302154541015625, + "logps/chosen": -262.4482421875, + "logps/ref_chosen": -101.79022216796875, + "logps/ref_rejected": -116.3245849609375, + "logps/rejected": -370.7371826171875, + "loss": 1.0354, + "margin_dpo/margin_mean": 93.75457000732422, + "margin_dpo/margin_std": 93.6308822631836, + "step": 652 + }, + { + "KL/chosen_KL_mean": -184.45904541015625, + "KL/mean": -229.0028076171875, + "KL/rejected_KL_mean": -273.54656982421875, + "KL/std": 107.53547668457031, + "epoch": 0.9588839941262849, + "fcm_dpo/beta": 0.004780316725373268, + "fcm_dpo/delta": 0.025099385529756546, + "fcm_dpo/margin": 89.08750915527344, + "fcm_dpo/q_t": 0.3997681736946106, + "grad_norm": 10.551021575927734, + "learning_rate": 2.7650355656892166e-09, + "logits/chosen": -2.2270355224609375, + "logits/rejected": -1.978057622909546, + "logps/chosen": -277.8126220703125, + "logps/ref_chosen": -93.35359191894531, + "logps/ref_rejected": -109.12324523925781, + "logps/rejected": -382.6697998046875, + "loss": 1.066, + "margin_dpo/margin_mean": 89.08751678466797, + "margin_dpo/margin_std": 105.1566390991211, + "step": 653 + }, + { + "KL/chosen_KL_mean": -184.26467895507812, + "KL/mean": -223.2406005859375, + "KL/rejected_KL_mean": -262.2165222167969, + "KL/std": 104.10440826416016, + "epoch": 0.960352422907489, + "fcm_dpo/beta": 0.004833141341805458, + "fcm_dpo/delta": 0.07610604166984558, + "fcm_dpo/margin": 77.95184326171875, + "fcm_dpo/q_t": 0.4165007472038269, + "grad_norm": 13.57054615020752, + "learning_rate": 2.577954022936174e-09, + "logits/chosen": -2.2561111450195312, + "logits/rejected": -2.0729568004608154, + "logps/chosen": -273.3802185058594, + "logps/ref_chosen": -89.11553955078125, + "logps/ref_rejected": -104.91995239257812, + "logps/rejected": -367.136474609375, + "loss": 1.1369, + "margin_dpo/margin_mean": 77.95184326171875, + "margin_dpo/margin_std": 131.37371826171875, + "step": 654 + }, + { + "KL/chosen_KL_mean": -168.5019073486328, + "KL/mean": -215.442138671875, + "KL/rejected_KL_mean": -262.38238525390625, + "KL/std": 112.25436401367188, + "epoch": 0.9618208516886931, + "fcm_dpo/beta": 0.004854114260524511, + "fcm_dpo/delta": -0.006014498881995678, + "fcm_dpo/margin": 93.88047790527344, + "fcm_dpo/q_t": 0.394818514585495, + "grad_norm": 9.471657752990723, + "learning_rate": 2.397392281198729e-09, + "logits/chosen": -1.9707748889923096, + "logits/rejected": -1.971581220626831, + "logps/chosen": -249.5380096435547, + "logps/ref_chosen": -81.03610229492188, + "logps/ref_rejected": -102.80233764648438, + "logps/rejected": -365.1847229003906, + "loss": 1.0617, + "margin_dpo/margin_mean": 93.88047790527344, + "margin_dpo/margin_std": 119.57864379882812, + "step": 655 + }, + { + "KL/chosen_KL_mean": -168.82737731933594, + "KL/mean": -228.6781005859375, + "KL/rejected_KL_mean": -288.5288391113281, + "KL/std": 113.01327514648438, + "epoch": 0.9632892804698973, + "fcm_dpo/beta": 0.004747756756842136, + "fcm_dpo/delta": -0.125717893242836, + "fcm_dpo/margin": 119.70145416259766, + "fcm_dpo/q_t": 0.3699204921722412, + "grad_norm": 10.317848205566406, + "learning_rate": 2.223355098446622e-09, + "logits/chosen": -1.5403485298156738, + "logits/rejected": -1.6090104579925537, + "logps/chosen": -254.15272521972656, + "logps/ref_chosen": -85.32534790039062, + "logps/ref_rejected": -118.33866119384766, + "logps/rejected": -406.86749267578125, + "loss": 0.965, + "margin_dpo/margin_mean": 119.70146179199219, + "margin_dpo/margin_std": 116.03785705566406, + "step": 656 + }, + { + "KL/chosen_KL_mean": -184.0067138671875, + "KL/mean": -227.7652587890625, + "KL/rejected_KL_mean": -271.5237731933594, + "KL/std": 103.92236328125, + "epoch": 0.9647577092511013, + "fcm_dpo/beta": 0.004729727283120155, + "fcm_dpo/delta": 0.03736189752817154, + "fcm_dpo/margin": 87.51704406738281, + "fcm_dpo/q_t": 0.4064503312110901, + "grad_norm": 12.323017120361328, + "learning_rate": 2.055847060721566e-09, + "logits/chosen": -1.886023998260498, + "logits/rejected": -1.7156312465667725, + "logps/chosen": -264.2044372558594, + "logps/ref_chosen": -80.19772338867188, + "logps/ref_rejected": -102.581298828125, + "logps/rejected": -374.1050720214844, + "loss": 1.099, + "margin_dpo/margin_mean": 87.51704406738281, + "margin_dpo/margin_std": 125.48452758789062, + "step": 657 + }, + { + "KL/chosen_KL_mean": -183.02366638183594, + "KL/mean": -223.7471923828125, + "KL/rejected_KL_mean": -264.470703125, + "KL/std": 98.57740020751953, + "epoch": 0.9662261380323054, + "fcm_dpo/beta": 0.0048185959458351135, + "fcm_dpo/delta": 0.0594821460545063, + "fcm_dpo/margin": 81.4470443725586, + "fcm_dpo/q_t": 0.4092613160610199, + "grad_norm": 9.629891395568848, + "learning_rate": 1.8948725820160662e-09, + "logits/chosen": -2.125157356262207, + "logits/rejected": -1.6637425422668457, + "logps/chosen": -277.658203125, + "logps/ref_chosen": -94.634521484375, + "logps/ref_rejected": -101.6316146850586, + "logps/rejected": -366.1023254394531, + "loss": 1.0951, + "margin_dpo/margin_mean": 81.4470443725586, + "margin_dpo/margin_std": 106.2724380493164, + "step": 658 + }, + { + "KL/chosen_KL_mean": -165.2835693359375, + "KL/mean": -214.57528686523438, + "KL/rejected_KL_mean": -263.86700439453125, + "KL/std": 106.11569213867188, + "epoch": 0.9676945668135095, + "fcm_dpo/beta": 0.004780924879014492, + "fcm_dpo/delta": -0.022840596735477448, + "fcm_dpo/margin": 98.58344268798828, + "fcm_dpo/q_t": 0.39388275146484375, + "grad_norm": 9.142254829406738, + "learning_rate": 1.7404359041573723e-09, + "logits/chosen": -2.28271484375, + "logits/rejected": -1.3397457599639893, + "logps/chosen": -277.8394470214844, + "logps/ref_chosen": -112.55587005615234, + "logps/ref_rejected": -93.9216079711914, + "logps/rejected": -357.78863525390625, + "loss": 1.075, + "margin_dpo/margin_mean": 98.58345031738281, + "margin_dpo/margin_std": 138.21209716796875, + "step": 659 + }, + { + "KL/chosen_KL_mean": -172.90357971191406, + "KL/mean": -223.55447387695312, + "KL/rejected_KL_mean": -274.20538330078125, + "KL/std": 103.32798767089844, + "epoch": 0.9691629955947136, + "fcm_dpo/beta": 0.00476967915892601, + "fcm_dpo/delta": -0.034870874136686325, + "fcm_dpo/margin": 101.30177307128906, + "fcm_dpo/q_t": 0.38880249857902527, + "grad_norm": 12.850919723510742, + "learning_rate": 1.592541096695571e-09, + "logits/chosen": -1.628364086151123, + "logits/rejected": -0.9756385087966919, + "logps/chosen": -266.281005859375, + "logps/ref_chosen": -93.37742614746094, + "logps/ref_rejected": -81.39482116699219, + "logps/rejected": -355.6001892089844, + "loss": 1.0288, + "margin_dpo/margin_mean": 101.30177307128906, + "margin_dpo/margin_std": 114.35575866699219, + "step": 660 + }, + { + "KL/chosen_KL_mean": -172.91943359375, + "KL/mean": -230.09494018554688, + "KL/rejected_KL_mean": -287.27044677734375, + "KL/std": 115.08345031738281, + "epoch": 0.9706314243759178, + "fcm_dpo/beta": 0.004706782754510641, + "fcm_dpo/delta": -0.09313911199569702, + "fcm_dpo/margin": 114.35101318359375, + "fcm_dpo/q_t": 0.3779047727584839, + "grad_norm": 10.328747749328613, + "learning_rate": 1.4511920567963908e-09, + "logits/chosen": -1.8205652236938477, + "logits/rejected": -1.3034087419509888, + "logps/chosen": -260.77459716796875, + "logps/ref_chosen": -87.85516357421875, + "logps/ref_rejected": -92.40330505371094, + "logps/rejected": -379.67376708984375, + "loss": 1.003, + "margin_dpo/margin_mean": 114.35101318359375, + "margin_dpo/margin_std": 127.89285278320312, + "step": 661 + }, + { + "KL/chosen_KL_mean": -168.3148193359375, + "KL/mean": -220.00526428222656, + "KL/rejected_KL_mean": -271.6957092285156, + "KL/std": 100.7904052734375, + "epoch": 0.9720998531571219, + "fcm_dpo/beta": 0.0046593910083174706, + "fcm_dpo/delta": -0.033299483358860016, + "fcm_dpo/margin": 103.380859375, + "fcm_dpo/q_t": 0.3881474733352661, + "grad_norm": 10.51960563659668, + "learning_rate": 1.3163925091384532e-09, + "logits/chosen": -1.7792580127716064, + "logits/rejected": -1.2135748863220215, + "logps/chosen": -271.0946044921875, + "logps/ref_chosen": -102.77980041503906, + "logps/ref_rejected": -95.22531127929688, + "logps/rejected": -366.9210205078125, + "loss": 1.0198, + "margin_dpo/margin_mean": 103.38087463378906, + "margin_dpo/margin_std": 105.8804702758789, + "step": 662 + }, + { + "KL/chosen_KL_mean": -169.60647583007812, + "KL/mean": -216.4501953125, + "KL/rejected_KL_mean": -263.2939147949219, + "KL/std": 104.20580291748047, + "epoch": 0.973568281938326, + "fcm_dpo/beta": 0.004663016647100449, + "fcm_dpo/delta": 0.01360559556633234, + "fcm_dpo/margin": 93.68742370605469, + "fcm_dpo/q_t": 0.39977845549583435, + "grad_norm": 11.331018447875977, + "learning_rate": 1.1881460058152382e-09, + "logits/chosen": -2.1741013526916504, + "logits/rejected": -1.9527440071105957, + "logps/chosen": -265.95306396484375, + "logps/ref_chosen": -96.34658813476562, + "logps/ref_rejected": -120.52645874023438, + "logps/rejected": -383.82037353515625, + "loss": 1.0722, + "margin_dpo/margin_mean": 93.68743133544922, + "margin_dpo/margin_std": 121.16324615478516, + "step": 663 + }, + { + "KL/chosen_KL_mean": -187.57080078125, + "KL/mean": -230.36416625976562, + "KL/rejected_KL_mean": -273.15753173828125, + "KL/std": 106.96551513671875, + "epoch": 0.9750367107195301, + "fcm_dpo/beta": 0.0047010621055960655, + "fcm_dpo/delta": 0.04943276569247246, + "fcm_dpo/margin": 85.58675384521484, + "fcm_dpo/q_t": 0.40661299228668213, + "grad_norm": 9.259799003601074, + "learning_rate": 1.066455926241383e-09, + "logits/chosen": -1.527221441268921, + "logits/rejected": -1.3484258651733398, + "logps/chosen": -279.4132080078125, + "logps/ref_chosen": -91.84242248535156, + "logps/ref_rejected": -111.8366928100586, + "logps/rejected": -384.9942321777344, + "loss": 1.0844, + "margin_dpo/margin_mean": 85.58674621582031, + "margin_dpo/margin_std": 105.78905487060547, + "step": 664 + }, + { + "KL/chosen_KL_mean": -161.17608642578125, + "KL/mean": -208.01638793945312, + "KL/rejected_KL_mean": -254.85671997070312, + "KL/std": 103.48817443847656, + "epoch": 0.9765051395007343, + "fcm_dpo/beta": 0.004718531854450703, + "fcm_dpo/delta": 0.00821390375494957, + "fcm_dpo/margin": 93.68061828613281, + "fcm_dpo/q_t": 0.39649200439453125, + "grad_norm": 8.78691291809082, + "learning_rate": 9.513254770636137e-10, + "logits/chosen": -2.5400588512420654, + "logits/rejected": -2.1195805072784424, + "logps/chosen": -249.36227416992188, + "logps/ref_chosen": -88.18618774414062, + "logps/ref_rejected": -91.9120101928711, + "logps/rejected": -346.76873779296875, + "loss": 1.0511, + "margin_dpo/margin_mean": 93.68061828613281, + "margin_dpo/margin_std": 105.55032348632812, + "step": 665 + }, + { + "KL/chosen_KL_mean": -177.7834014892578, + "KL/mean": -225.45401000976562, + "KL/rejected_KL_mean": -273.12457275390625, + "KL/std": 101.31594848632812, + "epoch": 0.9779735682819384, + "fcm_dpo/beta": 0.004697824828326702, + "fcm_dpo/delta": 0.00203627347946167, + "fcm_dpo/margin": 95.34117126464844, + "fcm_dpo/q_t": 0.39769503474235535, + "grad_norm": 11.992183685302734, + "learning_rate": 8.427576920763956e-10, + "logits/chosen": -1.56244695186615, + "logits/rejected": -1.1785318851470947, + "logps/chosen": -278.75799560546875, + "logps/ref_chosen": -100.97459411621094, + "logps/ref_rejected": -101.24992370605469, + "logps/rejected": -374.37451171875, + "loss": 1.0639, + "margin_dpo/margin_mean": 95.34117126464844, + "margin_dpo/margin_std": 119.4270248413086, + "step": 666 + }, + { + "KL/chosen_KL_mean": -192.60086059570312, + "KL/mean": -241.48414611816406, + "KL/rejected_KL_mean": -290.3674621582031, + "KL/std": 105.29362487792969, + "epoch": 0.9794419970631424, + "fcm_dpo/beta": 0.004698245320469141, + "fcm_dpo/delta": -0.009837804362177849, + "fcm_dpo/margin": 97.76659393310547, + "fcm_dpo/q_t": 0.3927449584007263, + "grad_norm": 10.372552871704102, + "learning_rate": 7.407554321417764e-10, + "logits/chosen": -1.4981027841567993, + "logits/rejected": -0.839004635810852, + "logps/chosen": -290.1720275878906, + "logps/ref_chosen": -97.5711669921875, + "logps/ref_rejected": -93.58476257324219, + "logps/rejected": -383.95220947265625, + "loss": 1.0336, + "margin_dpo/margin_mean": 97.76659393310547, + "margin_dpo/margin_std": 102.42034912109375, + "step": 667 + }, + { + "KL/chosen_KL_mean": -170.088623046875, + "KL/mean": -227.27218627929688, + "KL/rejected_KL_mean": -284.45574951171875, + "KL/std": 113.23973083496094, + "epoch": 0.9809104258443465, + "fcm_dpo/beta": 0.004648497793823481, + "fcm_dpo/delta": -0.08614526689052582, + "fcm_dpo/margin": 114.36713409423828, + "fcm_dpo/q_t": 0.37957486510276794, + "grad_norm": 11.999695777893066, + "learning_rate": 6.453213851142225e-10, + "logits/chosen": -2.3595683574676514, + "logits/rejected": -1.8413339853286743, + "logps/chosen": -272.6636657714844, + "logps/ref_chosen": -102.5750503540039, + "logps/ref_rejected": -108.81768798828125, + "logps/rejected": -393.2734375, + "loss": 0.9981, + "margin_dpo/margin_mean": 114.36714172363281, + "margin_dpo/margin_std": 123.38951110839844, + "step": 668 + }, + { + "KL/chosen_KL_mean": -155.29486083984375, + "KL/mean": -210.3527374267578, + "KL/rejected_KL_mean": -265.41064453125, + "KL/std": 109.57008361816406, + "epoch": 0.9823788546255506, + "fcm_dpo/beta": 0.004584559705108404, + "fcm_dpo/delta": -0.0577290877699852, + "fcm_dpo/margin": 110.11578369140625, + "fcm_dpo/q_t": 0.3834017217159271, + "grad_norm": 10.260527610778809, + "learning_rate": 5.564580657695939e-10, + "logits/chosen": -1.5053339004516602, + "logits/rejected": -1.0941473245620728, + "logps/chosen": -244.78964233398438, + "logps/ref_chosen": -89.49478149414062, + "logps/ref_rejected": -82.51950073242188, + "logps/rejected": -347.93011474609375, + "loss": 1.0095, + "margin_dpo/margin_mean": 110.11578369140625, + "margin_dpo/margin_std": 115.93708801269531, + "step": 669 + }, + { + "KL/chosen_KL_mean": -155.6254119873047, + "KL/mean": -216.21726989746094, + "KL/rejected_KL_mean": -276.80914306640625, + "KL/std": 112.19461822509766, + "epoch": 0.9838472834067548, + "fcm_dpo/beta": 0.004506596364080906, + "fcm_dpo/delta": -0.10158378630876541, + "fcm_dpo/margin": 121.18373107910156, + "fcm_dpo/q_t": 0.3722996711730957, + "grad_norm": 8.068968772888184, + "learning_rate": 4.741678157389739e-10, + "logits/chosen": -1.727135419845581, + "logits/rejected": -1.1860239505767822, + "logps/chosen": -251.08001708984375, + "logps/ref_chosen": -95.45459747314453, + "logps/ref_rejected": -101.53292846679688, + "logps/rejected": -378.342041015625, + "loss": 0.9783, + "margin_dpo/margin_mean": 121.18373107910156, + "margin_dpo/margin_std": 116.87356567382812, + "step": 670 + }, + { + "KL/chosen_KL_mean": -190.99586486816406, + "KL/mean": -231.87551879882812, + "KL/rejected_KL_mean": -272.7551574707031, + "KL/std": 103.49320983886719, + "epoch": 0.9853157121879589, + "fcm_dpo/beta": 0.004556992091238499, + "fcm_dpo/delta": 0.07927247136831284, + "fcm_dpo/margin": 81.7593002319336, + "fcm_dpo/q_t": 0.4144529700279236, + "grad_norm": 10.071465492248535, + "learning_rate": 3.9845280344705245e-10, + "logits/chosen": -1.6471831798553467, + "logits/rejected": -1.4217215776443481, + "logps/chosen": -273.1189880371094, + "logps/ref_chosen": -82.12312316894531, + "logps/ref_rejected": -90.21969604492188, + "logps/rejected": -362.974853515625, + "loss": 1.1147, + "margin_dpo/margin_mean": 81.7593002319336, + "margin_dpo/margin_std": 114.15276336669922, + "step": 671 + }, + { + "KL/chosen_KL_mean": -189.8311767578125, + "KL/mean": -225.12796020507812, + "KL/rejected_KL_mean": -260.42474365234375, + "KL/std": 99.92588806152344, + "epoch": 0.986784140969163, + "fcm_dpo/beta": 0.004630607552826405, + "fcm_dpo/delta": 0.127395898103714, + "fcm_dpo/margin": 70.59355926513672, + "fcm_dpo/q_t": 0.42451173067092896, + "grad_norm": 12.29115104675293, + "learning_rate": 3.293150240547549e-10, + "logits/chosen": -1.893174648284912, + "logits/rejected": -1.5534356832504272, + "logps/chosen": -279.8930969238281, + "logps/ref_chosen": -90.0619125366211, + "logps/ref_rejected": -100.45323181152344, + "logps/rejected": -360.87799072265625, + "loss": 1.1456, + "margin_dpo/margin_mean": 70.59355926513672, + "margin_dpo/margin_std": 104.85562133789062, + "step": 672 + }, + { + "KL/chosen_KL_mean": -170.20123291015625, + "KL/mean": -210.61099243164062, + "KL/rejected_KL_mean": -251.02072143554688, + "KL/std": 102.75851440429688, + "epoch": 0.9882525697503671, + "fcm_dpo/beta": 0.004727584309875965, + "fcm_dpo/delta": 0.0703030526638031, + "fcm_dpo/margin": 80.8194808959961, + "fcm_dpo/q_t": 0.4116077125072479, + "grad_norm": 11.87732982635498, + "learning_rate": 2.6675629940689504e-10, + "logits/chosen": -1.9416654109954834, + "logits/rejected": -1.5728824138641357, + "logps/chosen": -249.46438598632812, + "logps/ref_chosen": -79.26315307617188, + "logps/ref_rejected": -91.34925079345703, + "logps/rejected": -342.3699645996094, + "loss": 1.0986, + "margin_dpo/margin_mean": 80.81948852539062, + "margin_dpo/margin_std": 104.83180236816406, + "step": 673 + }, + { + "KL/chosen_KL_mean": -159.9476318359375, + "KL/mean": -216.77557373046875, + "KL/rejected_KL_mean": -273.60357666015625, + "KL/std": 103.0810546875, + "epoch": 0.9897209985315712, + "fcm_dpo/beta": 0.004667171277105808, + "fcm_dpo/delta": -0.08509520441293716, + "fcm_dpo/margin": 113.65593719482422, + "fcm_dpo/q_t": 0.37813296914100647, + "grad_norm": 8.982342720031738, + "learning_rate": 2.1077827798404725e-10, + "logits/chosen": -1.3266725540161133, + "logits/rejected": -0.9054714441299438, + "logps/chosen": -235.40594482421875, + "logps/ref_chosen": -75.45831298828125, + "logps/ref_rejected": -76.20362854003906, + "logps/rejected": -349.80718994140625, + "loss": 0.9866, + "margin_dpo/margin_mean": 113.65594482421875, + "margin_dpo/margin_std": 112.28579711914062, + "step": 674 + }, + { + "KL/chosen_KL_mean": -175.3204803466797, + "KL/mean": -225.6072235107422, + "KL/rejected_KL_mean": -275.8939514160156, + "KL/std": 111.3902359008789, + "epoch": 0.9911894273127754, + "fcm_dpo/beta": 0.004610551055520773, + "fcm_dpo/delta": -0.015415102243423462, + "fcm_dpo/margin": 100.573486328125, + "fcm_dpo/q_t": 0.3937044143676758, + "grad_norm": 10.685237884521484, + "learning_rate": 1.6138243485910863e-10, + "logits/chosen": -1.6183019876480103, + "logits/rejected": -1.2845849990844727, + "logps/chosen": -255.23001098632812, + "logps/ref_chosen": -79.90953063964844, + "logps/ref_rejected": -81.21824645996094, + "logps/rejected": -357.1121826171875, + "loss": 1.0705, + "margin_dpo/margin_mean": 100.573486328125, + "margin_dpo/margin_std": 135.77354431152344, + "step": 675 + }, + { + "KL/chosen_KL_mean": -188.84844970703125, + "KL/mean": -238.12591552734375, + "KL/rejected_KL_mean": -287.40338134765625, + "KL/std": 106.96812438964844, + "epoch": 0.9926578560939795, + "fcm_dpo/beta": 0.0046224407851696014, + "fcm_dpo/delta": -0.005975460633635521, + "fcm_dpo/margin": 98.5549545288086, + "fcm_dpo/q_t": 0.3946434259414673, + "grad_norm": 9.166580200195312, + "learning_rate": 1.1857007165852472e-10, + "logits/chosen": -2.1155519485473633, + "logits/rejected": -1.5424535274505615, + "logps/chosen": -287.0195617675781, + "logps/ref_chosen": -98.17111206054688, + "logps/ref_rejected": -95.024658203125, + "logps/rejected": -382.42803955078125, + "loss": 1.0555, + "margin_dpo/margin_mean": 98.55494689941406, + "margin_dpo/margin_std": 119.53730010986328, + "step": 676 + }, + { + "KL/chosen_KL_mean": -168.68438720703125, + "KL/mean": -218.90933227539062, + "KL/rejected_KL_mean": -269.13427734375, + "KL/std": 105.43244934082031, + "epoch": 0.9941262848751835, + "fcm_dpo/beta": 0.00461141113191843, + "fcm_dpo/delta": -0.01402292400598526, + "fcm_dpo/margin": 100.44989013671875, + "fcm_dpo/q_t": 0.39210790395736694, + "grad_norm": 8.679218292236328, + "learning_rate": 8.23423165278725e-11, + "logits/chosen": -1.9051822423934937, + "logits/rejected": -1.2468986511230469, + "logps/chosen": -260.06365966796875, + "logps/ref_chosen": -91.37928771972656, + "logps/ref_rejected": -82.87776947021484, + "logps/rejected": -352.01202392578125, + "loss": 1.0304, + "margin_dpo/margin_mean": 100.44989013671875, + "margin_dpo/margin_std": 102.97470092773438, + "step": 677 + }, + { + "KL/chosen_KL_mean": -159.788330078125, + "KL/mean": -219.01699829101562, + "KL/rejected_KL_mean": -278.24566650390625, + "KL/std": 112.53511047363281, + "epoch": 0.9955947136563876, + "fcm_dpo/beta": 0.004575745202600956, + "fcm_dpo/delta": -0.09731646627187729, + "fcm_dpo/margin": 118.45732116699219, + "fcm_dpo/q_t": 0.3743739724159241, + "grad_norm": 10.344512939453125, + "learning_rate": 5.270012410216185e-11, + "logits/chosen": -1.8819496631622314, + "logits/rejected": -1.5630353689193726, + "logps/chosen": -235.4342041015625, + "logps/ref_chosen": -75.6458740234375, + "logps/ref_rejected": -86.96611022949219, + "logps/rejected": -365.2117919921875, + "loss": 0.9814, + "margin_dpo/margin_mean": 118.45732116699219, + "margin_dpo/margin_std": 113.800537109375, + "step": 678 + }, + { + "KL/chosen_KL_mean": -183.46322631835938, + "KL/mean": -232.7026824951172, + "KL/rejected_KL_mean": -281.942138671875, + "KL/std": 93.56512451171875, + "epoch": 0.9970631424375918, + "fcm_dpo/beta": 0.0045309788547456264, + "fcm_dpo/delta": 0.003967920318245888, + "fcm_dpo/margin": 98.47888946533203, + "fcm_dpo/q_t": 0.3962134122848511, + "grad_norm": 9.133407592773438, + "learning_rate": 2.9644275480772416e-11, + "logits/chosen": -2.3770525455474854, + "logits/rejected": -1.8721058368682861, + "logps/chosen": -264.2366943359375, + "logps/ref_chosen": -80.77344512939453, + "logps/ref_rejected": -82.87850189208984, + "logps/rejected": -364.82061767578125, + "loss": 1.0359, + "margin_dpo/margin_mean": 98.4788818359375, + "margin_dpo/margin_std": 99.60039520263672, + "step": 679 + }, + { + "KL/chosen_KL_mean": -185.55514526367188, + "KL/mean": -238.05531311035156, + "KL/rejected_KL_mean": -290.55548095703125, + "KL/std": 111.19891357421875, + "epoch": 0.9985315712187959, + "fcm_dpo/beta": 0.00454926211386919, + "fcm_dpo/delta": -0.030054979026317596, + "fcm_dpo/margin": 105.0003433227539, + "fcm_dpo/q_t": 0.39070942997932434, + "grad_norm": 7.670646667480469, + "learning_rate": 1.31753782067201e-11, + "logits/chosen": -2.0947909355163574, + "logits/rejected": -1.77396559715271, + "logps/chosen": -293.23809814453125, + "logps/ref_chosen": -107.68292999267578, + "logps/ref_rejected": -116.0948715209961, + "logps/rejected": -406.6503601074219, + "loss": 1.0388, + "margin_dpo/margin_mean": 105.00033569335938, + "margin_dpo/margin_std": 119.4825668334961, + "step": 680 + }, + { + "KL/chosen_KL_mean": -168.48878479003906, + "KL/mean": -214.45407104492188, + "KL/rejected_KL_mean": -260.41937255859375, + "KL/std": 105.0776138305664, + "epoch": 1.0, + "fcm_dpo/beta": 0.0045382184907794, + "fcm_dpo/delta": 0.03409893810749054, + "fcm_dpo/margin": 91.9305648803711, + "fcm_dpo/q_t": 0.40387219190597534, + "grad_norm": 10.59050464630127, + "learning_rate": 3.2938662507808745e-12, + "logits/chosen": -2.4693641662597656, + "logits/rejected": -2.1671481132507324, + "logps/chosen": -261.4998474121094, + "logps/ref_chosen": -93.01106262207031, + "logps/ref_rejected": -94.8221664428711, + "logps/rejected": -355.24151611328125, + "loss": 1.0761, + "margin_dpo/margin_mean": 91.9305648803711, + "margin_dpo/margin_std": 114.75448608398438, + "step": 681 + }, + { + "epoch": 1.0, + "step": 681, + "total_flos": 0.0, + "train_loss": 1.0783974358513841, + "train_runtime": 2022.8503, + "train_samples_per_second": 21.553, + "train_steps_per_second": 0.337 + } + ], + "logging_steps": 1, + "max_steps": 681, + "num_input_tokens_seen": 0, + "num_train_epochs": 1, + "save_steps": 50, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": false, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 0.0, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +}