{ "best_global_step": null, "best_metric": null, "best_model_checkpoint": null, "epoch": 1.0, "eval_steps": 100, "global_step": 681, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "KL/chosen_KL_mean": 0.05419158935546875, "KL/mean": 0.023573115468025208, "KL/rejected_KL_mean": -0.007045745849609375, "KL/std": 0.33097243309020996, "epoch": 0.0014684287812041115, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.06123492121696472, "fcm_dpo/q_t": 0.4984707534313202, "grad_norm": 38.8179931640625, "learning_rate": 0.0, "logits/chosen": 0.2677389979362488, "logits/rejected": 0.6191319227218628, "logps/chosen": -85.37664031982422, "logps/ref_chosen": -85.43083190917969, "logps/ref_rejected": -79.90458679199219, "logps/rejected": -79.91163635253906, "loss": 1.3807, "margin_dpo/margin_mean": 0.061235010623931885, "margin_dpo/margin_std": 0.4424324035644531, "step": 1 }, { "KL/chosen_KL_mean": -0.044902801513671875, "KL/mean": 0.017303764820098877, "KL/rejected_KL_mean": 0.07951736450195312, "KL/std": 0.2933448553085327, "epoch": 0.002936857562408223, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.12442219257354736, "fcm_dpo/q_t": 0.5031089186668396, "grad_norm": 29.757017135620117, "learning_rate": 7.246376811594203e-09, "logits/chosen": 0.47771453857421875, "logits/rejected": 0.5908545851707458, "logps/chosen": -82.11383056640625, "logps/ref_chosen": -82.06893157958984, "logps/ref_rejected": -81.65457916259766, "logps/rejected": -81.57505798339844, "loss": 1.3992, "margin_dpo/margin_mean": -0.12442246079444885, "margin_dpo/margin_std": 0.41358208656311035, "step": 2 }, { "KL/chosen_KL_mean": 0.024188995361328125, "KL/mean": 0.006642557680606842, "KL/rejected_KL_mean": -0.010898590087890625, "KL/std": 0.3097880482673645, "epoch": 0.004405286343612335, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.03509153425693512, "fcm_dpo/q_t": 0.4991232752799988, "grad_norm": 26.000652313232422, "learning_rate": 1.4492753623188406e-08, "logits/chosen": 0.42709434032440186, "logits/rejected": 0.650235652923584, "logps/chosen": -93.78679656982422, "logps/ref_chosen": -93.81098937988281, "logps/ref_rejected": -74.22950744628906, "logps/rejected": -74.24040222167969, "loss": 1.3834, "margin_dpo/margin_mean": 0.03509168326854706, "margin_dpo/margin_std": 0.48292967677116394, "step": 3 }, { "KL/chosen_KL_mean": -0.07156753540039062, "KL/mean": -0.0424887016415596, "KL/rejected_KL_mean": -0.013408660888671875, "KL/std": 0.33623313903808594, "epoch": 0.005873715124816446, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.05815790593624115, "fcm_dpo/q_t": 0.5014516115188599, "grad_norm": 32.81612777709961, "learning_rate": 2.1739130434782606e-08, "logits/chosen": 0.6472669839859009, "logits/rejected": 0.7786778211593628, "logps/chosen": -87.3640365600586, "logps/ref_chosen": -87.29246520996094, "logps/ref_rejected": -93.82425689697266, "logps/rejected": -93.83766174316406, "loss": 1.3926, "margin_dpo/margin_mean": -0.058158084750175476, "margin_dpo/margin_std": 0.43809592723846436, "step": 4 }, { "KL/chosen_KL_mean": 0.0205230712890625, "KL/mean": -0.0029112771153450012, "KL/rejected_KL_mean": -0.026348114013671875, "KL/std": 0.3122915029525757, "epoch": 0.007342143906020558, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.046868130564689636, "fcm_dpo/q_t": 0.4988270699977875, "grad_norm": 35.3472900390625, "learning_rate": 2.898550724637681e-08, "logits/chosen": 0.38373515009880066, "logits/rejected": 0.5904780030250549, "logps/chosen": -89.31623840332031, "logps/ref_chosen": -89.33676147460938, "logps/ref_rejected": -88.74783325195312, "logps/rejected": -88.77418518066406, "loss": 1.3822, "margin_dpo/margin_mean": 0.046868354082107544, "margin_dpo/margin_std": 0.4555869698524475, "step": 5 }, { "KL/chosen_KL_mean": -0.04218292236328125, "KL/mean": -0.04054884612560272, "KL/rejected_KL_mean": -0.03891754150390625, "KL/std": 0.30800819396972656, "epoch": 0.00881057268722467, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.003269016742706299, "fcm_dpo/q_t": 0.5000814199447632, "grad_norm": 36.064876556396484, "learning_rate": 3.6231884057971014e-08, "logits/chosen": 0.4867173433303833, "logits/rejected": 0.8018932342529297, "logps/chosen": -97.36366271972656, "logps/ref_chosen": -97.32147216796875, "logps/ref_rejected": -97.88345336914062, "logps/rejected": -97.92236328125, "loss": 1.3871, "margin_dpo/margin_mean": -0.0032696127891540527, "margin_dpo/margin_std": 0.4106166958808899, "step": 6 }, { "KL/chosen_KL_mean": -0.018283843994140625, "KL/mean": -0.032152727246284485, "KL/rejected_KL_mean": -0.04602813720703125, "KL/std": 0.31660744547843933, "epoch": 0.010279001468428781, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.027742892503738403, "fcm_dpo/q_t": 0.4993075728416443, "grad_norm": 37.842254638671875, "learning_rate": 4.347826086956521e-08, "logits/chosen": 0.48803120851516724, "logits/rejected": 0.5360611081123352, "logps/chosen": -86.66681671142578, "logps/ref_chosen": -86.64852905273438, "logps/ref_rejected": -109.61618041992188, "logps/rejected": -109.6622085571289, "loss": 1.384, "margin_dpo/margin_mean": 0.02774253487586975, "margin_dpo/margin_std": 0.42086201906204224, "step": 7 }, { "KL/chosen_KL_mean": -0.0385284423828125, "KL/mean": -0.018659211695194244, "KL/rejected_KL_mean": 0.00121307373046875, "KL/std": 0.34195733070373535, "epoch": 0.011747430249632892, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.03973837196826935, "fcm_dpo/q_t": 0.5009934902191162, "grad_norm": 32.732177734375, "learning_rate": 5.0724637681159424e-08, "logits/chosen": 0.4075879454612732, "logits/rejected": 0.719414234161377, "logps/chosen": -89.96221160888672, "logps/ref_chosen": -89.9236831665039, "logps/ref_rejected": -86.22804260253906, "logps/rejected": -86.22682189941406, "loss": 1.3908, "margin_dpo/margin_mean": -0.03973822295665741, "margin_dpo/margin_std": 0.46964970231056213, "step": 8 }, { "KL/chosen_KL_mean": 0.0368194580078125, "KL/mean": -0.006641313433647156, "KL/rejected_KL_mean": -0.0500946044921875, "KL/std": 0.36104485392570496, "epoch": 0.013215859030837005, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.08690676093101501, "fcm_dpo/q_t": 0.49782997369766235, "grad_norm": 36.08580780029297, "learning_rate": 5.797101449275362e-08, "logits/chosen": 0.719907820224762, "logits/rejected": 0.903343677520752, "logps/chosen": -103.8203125, "logps/ref_chosen": -103.85713195800781, "logps/ref_rejected": -104.31932067871094, "logps/rejected": -104.36941528320312, "loss": 1.3783, "margin_dpo/margin_mean": 0.08690647780895233, "margin_dpo/margin_std": 0.4995291531085968, "step": 9 }, { "KL/chosen_KL_mean": -0.005512237548828125, "KL/mean": 0.04146455228328705, "KL/rejected_KL_mean": 0.08843612670898438, "KL/std": 0.26931285858154297, "epoch": 0.014684287812041116, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.09395936131477356, "fcm_dpo/q_t": 0.5023483037948608, "grad_norm": 33.644287109375, "learning_rate": 6.521739130434782e-08, "logits/chosen": 0.5238694548606873, "logits/rejected": 0.7369919419288635, "logps/chosen": -76.20987701416016, "logps/ref_chosen": -76.20436096191406, "logps/ref_rejected": -87.152099609375, "logps/rejected": -87.06366729736328, "loss": 1.396, "margin_dpo/margin_mean": -0.09395924210548401, "margin_dpo/margin_std": 0.3398798406124115, "step": 10 }, { "KL/chosen_KL_mean": 0.0365447998046875, "KL/mean": 0.018243655562400818, "KL/rejected_KL_mean": -5.340576171875e-05, "KL/std": 0.306723952293396, "epoch": 0.016152716593245228, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.03659513592720032, "fcm_dpo/q_t": 0.499085009098053, "grad_norm": 37.03773880004883, "learning_rate": 7.246376811594203e-08, "logits/chosen": 0.3368200361728668, "logits/rejected": 0.555517315864563, "logps/chosen": -82.3299560546875, "logps/ref_chosen": -82.36650085449219, "logps/ref_rejected": -94.26461791992188, "logps/rejected": -94.26467895507812, "loss": 1.3831, "margin_dpo/margin_mean": 0.03659576177597046, "margin_dpo/margin_std": 0.42015379667282104, "step": 11 }, { "KL/chosen_KL_mean": 0.016056060791015625, "KL/mean": -0.013466224074363708, "KL/rejected_KL_mean": -0.042987823486328125, "KL/std": 0.36298543214797974, "epoch": 0.01762114537444934, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.05904069542884827, "fcm_dpo/q_t": 0.4985245168209076, "grad_norm": 40.121910095214844, "learning_rate": 7.971014492753623e-08, "logits/chosen": 0.5381792783737183, "logits/rejected": 0.670903205871582, "logps/chosen": -99.08944702148438, "logps/ref_chosen": -99.10549926757812, "logps/ref_rejected": -110.27140808105469, "logps/rejected": -110.31439971923828, "loss": 1.381, "margin_dpo/margin_mean": 0.05904114246368408, "margin_dpo/margin_std": 0.5055064558982849, "step": 12 }, { "KL/chosen_KL_mean": 0.023166656494140625, "KL/mean": 0.023870334029197693, "KL/rejected_KL_mean": 0.02457427978515625, "KL/std": 0.29374513030052185, "epoch": 0.01908957415565345, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.0014038383960723877, "fcm_dpo/q_t": 0.5000355243682861, "grad_norm": 41.807979583740234, "learning_rate": 8.695652173913042e-08, "logits/chosen": 0.362546443939209, "logits/rejected": 0.6090709567070007, "logps/chosen": -90.53656005859375, "logps/ref_chosen": -90.55972290039062, "logps/ref_rejected": -93.69110107421875, "logps/rejected": -93.66651916503906, "loss": 1.3869, "margin_dpo/margin_mean": -0.0014039874076843262, "margin_dpo/margin_std": 0.4222813546657562, "step": 13 }, { "KL/chosen_KL_mean": -0.068084716796875, "KL/mean": -0.022164784371852875, "KL/rejected_KL_mean": 0.0237579345703125, "KL/std": 0.34052860736846924, "epoch": 0.020558002936857563, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.09185294806957245, "fcm_dpo/q_t": 0.5022954940795898, "grad_norm": 36.01750183105469, "learning_rate": 9.420289855072464e-08, "logits/chosen": 0.9169459939002991, "logits/rejected": 1.0052374601364136, "logps/chosen": -99.895263671875, "logps/ref_chosen": -99.82717895507812, "logps/ref_rejected": -108.94200134277344, "logps/rejected": -108.91824340820312, "loss": 1.396, "margin_dpo/margin_mean": -0.09185254573822021, "margin_dpo/margin_std": 0.45488160848617554, "step": 14 }, { "KL/chosen_KL_mean": -0.009136199951171875, "KL/mean": -0.028981618583202362, "KL/rejected_KL_mean": -0.048828125, "KL/std": 0.3243560492992401, "epoch": 0.022026431718061675, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.03969092667102814, "fcm_dpo/q_t": 0.49900883436203003, "grad_norm": 31.644763946533203, "learning_rate": 1.0144927536231885e-07, "logits/chosen": 0.3826875388622284, "logits/rejected": 0.5501496195793152, "logps/chosen": -78.91910552978516, "logps/ref_chosen": -78.90997314453125, "logps/ref_rejected": -90.06234741210938, "logps/rejected": -90.11117553710938, "loss": 1.3829, "margin_dpo/margin_mean": 0.03969104588031769, "margin_dpo/margin_std": 0.4570927619934082, "step": 15 }, { "KL/chosen_KL_mean": 0.02811431884765625, "KL/mean": 0.016169600188732147, "KL/rejected_KL_mean": 0.00421905517578125, "KL/std": 0.3299415111541748, "epoch": 0.023494860499265784, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.023894861340522766, "fcm_dpo/q_t": 0.49940139055252075, "grad_norm": 34.61058807373047, "learning_rate": 1.0869565217391303e-07, "logits/chosen": 0.5568860769271851, "logits/rejected": 0.8136681318283081, "logps/chosen": -97.3951644897461, "logps/ref_chosen": -97.42327880859375, "logps/ref_rejected": -90.59945678710938, "logps/rejected": -90.59524536132812, "loss": 1.3844, "margin_dpo/margin_mean": 0.023894816637039185, "margin_dpo/margin_std": 0.4568118453025818, "step": 16 }, { "KL/chosen_KL_mean": -0.007831573486328125, "KL/mean": -0.02137354016304016, "KL/rejected_KL_mean": -0.03491973876953125, "KL/std": 0.29245567321777344, "epoch": 0.024963289280469897, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.02709352970123291, "fcm_dpo/q_t": 0.4993228316307068, "grad_norm": 35.51238250732422, "learning_rate": 1.1594202898550725e-07, "logits/chosen": 0.5117985010147095, "logits/rejected": 0.7446095943450928, "logps/chosen": -104.37213897705078, "logps/ref_chosen": -104.36430358886719, "logps/ref_rejected": -90.46772766113281, "logps/rejected": -90.50263977050781, "loss": 1.3841, "margin_dpo/margin_mean": 0.027093231678009033, "margin_dpo/margin_std": 0.4270169138908386, "step": 17 }, { "KL/chosen_KL_mean": 0.024341583251953125, "KL/mean": 0.0127788707613945, "KL/rejected_KL_mean": 0.001224517822265625, "KL/std": 0.31795647740364075, "epoch": 0.02643171806167401, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.02311234176158905, "fcm_dpo/q_t": 0.49942252039909363, "grad_norm": 41.704132080078125, "learning_rate": 1.2318840579710146e-07, "logits/chosen": 0.34284430742263794, "logits/rejected": 0.6835780143737793, "logps/chosen": -87.06761169433594, "logps/ref_chosen": -87.09195709228516, "logps/ref_rejected": -81.85072326660156, "logps/rejected": -81.84949493408203, "loss": 1.3845, "margin_dpo/margin_mean": 0.023112982511520386, "margin_dpo/margin_std": 0.4300282597541809, "step": 18 }, { "KL/chosen_KL_mean": 0.020893096923828125, "KL/mean": -0.011265799403190613, "KL/rejected_KL_mean": -0.043422698974609375, "KL/std": 0.31101712584495544, "epoch": 0.027900146842878122, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.064310222864151, "fcm_dpo/q_t": 0.4983947277069092, "grad_norm": 31.057170867919922, "learning_rate": 1.3043478260869563e-07, "logits/chosen": 0.6548206806182861, "logits/rejected": 0.9354963302612305, "logps/chosen": -105.85264587402344, "logps/ref_chosen": -105.87354278564453, "logps/ref_rejected": -96.93023681640625, "logps/rejected": -96.9736557006836, "loss": 1.3805, "margin_dpo/margin_mean": 0.06431007385253906, "margin_dpo/margin_std": 0.47644898295402527, "step": 19 }, { "KL/chosen_KL_mean": 0.016139984130859375, "KL/mean": 0.009934380650520325, "KL/rejected_KL_mean": 0.00372314453125, "KL/std": 0.3077821433544159, "epoch": 0.02936857562408223, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.012421995401382446, "fcm_dpo/q_t": 0.49968940019607544, "grad_norm": 32.689388275146484, "learning_rate": 1.3768115942028986e-07, "logits/chosen": 0.1846219301223755, "logits/rejected": 0.4561445116996765, "logps/chosen": -90.74198150634766, "logps/ref_chosen": -90.75811767578125, "logps/ref_rejected": -85.91232299804688, "logps/rejected": -85.90859985351562, "loss": 1.3854, "margin_dpo/margin_mean": 0.012422174215316772, "margin_dpo/margin_std": 0.3584854006767273, "step": 20 }, { "KL/chosen_KL_mean": 0.018489837646484375, "KL/mean": -0.019078142940998077, "KL/rejected_KL_mean": -0.056652069091796875, "KL/std": 0.3066573143005371, "epoch": 0.030837004405286344, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.07514522969722748, "fcm_dpo/q_t": 0.4981222450733185, "grad_norm": 31.387516021728516, "learning_rate": 1.4492753623188405e-07, "logits/chosen": 0.45054763555526733, "logits/rejected": 0.6211205124855042, "logps/chosen": -80.31497192382812, "logps/ref_chosen": -80.33345794677734, "logps/ref_rejected": -83.93373107910156, "logps/rejected": -83.99038696289062, "loss": 1.3794, "margin_dpo/margin_mean": 0.07514509558677673, "margin_dpo/margin_std": 0.47712084650993347, "step": 21 }, { "KL/chosen_KL_mean": 0.062107086181640625, "KL/mean": 0.018313273787498474, "KL/rejected_KL_mean": -0.025478363037109375, "KL/std": 0.3286937475204468, "epoch": 0.032305433186490456, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.08757480978965759, "fcm_dpo/q_t": 0.49781161546707153, "grad_norm": 42.74930191040039, "learning_rate": 1.5217391304347825e-07, "logits/chosen": 0.7168225049972534, "logits/rejected": 0.8415087461471558, "logps/chosen": -95.33319091796875, "logps/ref_chosen": -95.39530181884766, "logps/ref_rejected": -103.47351837158203, "logps/rejected": -103.49899291992188, "loss": 1.378, "margin_dpo/margin_mean": 0.08757475018501282, "margin_dpo/margin_std": 0.4211307764053345, "step": 22 }, { "KL/chosen_KL_mean": -0.020757675170898438, "KL/mean": 0.00687447190284729, "KL/rejected_KL_mean": 0.034511566162109375, "KL/std": 0.3154371380805969, "epoch": 0.033773861967694566, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.055268943309783936, "fcm_dpo/q_t": 0.5013798475265503, "grad_norm": 31.475515365600586, "learning_rate": 1.5942028985507245e-07, "logits/chosen": 0.7378020286560059, "logits/rejected": 0.917765736579895, "logps/chosen": -90.65827178955078, "logps/ref_chosen": -90.63751220703125, "logps/ref_rejected": -86.59425354003906, "logps/rejected": -86.55974578857422, "loss": 1.3923, "margin_dpo/margin_mean": -0.05526900291442871, "margin_dpo/margin_std": 0.4065503180027008, "step": 23 }, { "KL/chosen_KL_mean": 0.015657424926757812, "KL/mean": -0.00726298987865448, "KL/rejected_KL_mean": -0.030185699462890625, "KL/std": 0.304160475730896, "epoch": 0.03524229074889868, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.04584333300590515, "fcm_dpo/q_t": 0.4988551437854767, "grad_norm": 44.36646270751953, "learning_rate": 1.6666666666666665e-07, "logits/chosen": 0.4017108678817749, "logits/rejected": 0.4368298053741455, "logps/chosen": -69.9016342163086, "logps/ref_chosen": -69.91728973388672, "logps/ref_rejected": -106.60990142822266, "logps/rejected": -106.64009094238281, "loss": 1.3822, "margin_dpo/margin_mean": 0.04584340751171112, "margin_dpo/margin_std": 0.4237392544746399, "step": 24 }, { "KL/chosen_KL_mean": -0.021209716796875, "KL/mean": -0.019529297947883606, "KL/rejected_KL_mean": -0.017848968505859375, "KL/std": 0.3543895184993744, "epoch": 0.03671071953010279, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.0033625662326812744, "fcm_dpo/q_t": 0.5000829696655273, "grad_norm": 37.1446418762207, "learning_rate": 1.7391304347826085e-07, "logits/chosen": 0.3448099195957184, "logits/rejected": 0.5056936740875244, "logps/chosen": -80.84669494628906, "logps/ref_chosen": -80.82548522949219, "logps/ref_rejected": -95.95710754394531, "logps/rejected": -95.97496032714844, "loss": 1.3873, "margin_dpo/margin_mean": -0.0033624768257141113, "margin_dpo/margin_std": 0.5277082920074463, "step": 25 }, { "KL/chosen_KL_mean": -0.018627166748046875, "KL/mean": -0.02942410111427307, "KL/rejected_KL_mean": -0.040218353271484375, "KL/std": 0.3203151822090149, "epoch": 0.0381791483113069, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.02159792184829712, "fcm_dpo/q_t": 0.49945974349975586, "grad_norm": 41.023773193359375, "learning_rate": 1.8115942028985507e-07, "logits/chosen": 0.3951873481273651, "logits/rejected": 0.60643470287323, "logps/chosen": -88.9197998046875, "logps/ref_chosen": -88.90116882324219, "logps/ref_rejected": -109.82819366455078, "logps/rejected": -109.868408203125, "loss": 1.3846, "margin_dpo/margin_mean": 0.021598011255264282, "margin_dpo/margin_std": 0.4305553734302521, "step": 26 }, { "KL/chosen_KL_mean": 0.010311126708984375, "KL/mean": -0.030757389962673187, "KL/rejected_KL_mean": -0.07183074951171875, "KL/std": 0.34087151288986206, "epoch": 0.039647577092511016, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.08214427530765533, "fcm_dpo/q_t": 0.49794501066207886, "grad_norm": 44.443389892578125, "learning_rate": 1.8840579710144927e-07, "logits/chosen": 0.17309018969535828, "logits/rejected": 0.36056604981422424, "logps/chosen": -77.58570098876953, "logps/ref_chosen": -77.59600830078125, "logps/ref_rejected": -103.93850708007812, "logps/rejected": -104.01033782958984, "loss": 1.3787, "margin_dpo/margin_mean": 0.08214372396469116, "margin_dpo/margin_std": 0.4637642502784729, "step": 27 }, { "KL/chosen_KL_mean": 0.11543655395507812, "KL/mean": 0.025208979845046997, "KL/rejected_KL_mean": -0.06502151489257812, "KL/std": 0.30955493450164795, "epoch": 0.041116005873715125, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.1804606318473816, "fcm_dpo/q_t": 0.49549025297164917, "grad_norm": 36.613136291503906, "learning_rate": 1.9565217391304347e-07, "logits/chosen": 0.5012938380241394, "logits/rejected": 0.7116572856903076, "logps/chosen": -102.11312866210938, "logps/ref_chosen": -102.22856903076172, "logps/ref_rejected": -96.9594955444336, "logps/rejected": -97.02452087402344, "loss": 1.3688, "margin_dpo/margin_mean": 0.18046081066131592, "margin_dpo/margin_std": 0.42057356238365173, "step": 28 }, { "KL/chosen_KL_mean": -0.0085296630859375, "KL/mean": -0.08421530574560165, "KL/rejected_KL_mean": -0.15990066528320312, "KL/std": 0.37015920877456665, "epoch": 0.042584434654919234, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.15137828886508942, "fcm_dpo/q_t": 0.49621906876564026, "grad_norm": 42.10970687866211, "learning_rate": 2.028985507246377e-07, "logits/chosen": 0.11546057462692261, "logits/rejected": 0.3588726222515106, "logps/chosen": -88.65557861328125, "logps/ref_chosen": -88.64704895019531, "logps/ref_rejected": -102.96011352539062, "logps/rejected": -103.12001037597656, "loss": 1.3719, "margin_dpo/margin_mean": 0.15137861669063568, "margin_dpo/margin_std": 0.5257235765457153, "step": 29 }, { "KL/chosen_KL_mean": 0.03961181640625, "KL/mean": -0.024579517543315887, "KL/rejected_KL_mean": -0.088775634765625, "KL/std": 0.32620030641555786, "epoch": 0.04405286343612335, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.12838895618915558, "fcm_dpo/q_t": 0.4967923164367676, "grad_norm": 39.2256965637207, "learning_rate": 2.1014492753623187e-07, "logits/chosen": 0.5374374389648438, "logits/rejected": 0.6512571573257446, "logps/chosen": -88.34877014160156, "logps/ref_chosen": -88.38838195800781, "logps/ref_rejected": -102.31889343261719, "logps/rejected": -102.40766906738281, "loss": 1.374, "margin_dpo/margin_mean": 0.12838923931121826, "margin_dpo/margin_std": 0.4373652935028076, "step": 30 }, { "KL/chosen_KL_mean": -0.039501190185546875, "KL/mean": -0.05637046694755554, "KL/rejected_KL_mean": -0.07323837280273438, "KL/std": 0.3393365740776062, "epoch": 0.04552129221732746, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.03373754024505615, "fcm_dpo/q_t": 0.4991576671600342, "grad_norm": 31.257678985595703, "learning_rate": 2.1739130434782607e-07, "logits/chosen": 0.8123838305473328, "logits/rejected": 1.068558692932129, "logps/chosen": -101.1651611328125, "logps/ref_chosen": -101.12565612792969, "logps/ref_rejected": -79.8584213256836, "logps/rejected": -79.93165588378906, "loss": 1.3834, "margin_dpo/margin_mean": 0.033737361431121826, "margin_dpo/margin_std": 0.4375351071357727, "step": 31 }, { "KL/chosen_KL_mean": -0.059062957763671875, "KL/mean": -0.09265387058258057, "KL/rejected_KL_mean": -0.12624359130859375, "KL/std": 0.3508501350879669, "epoch": 0.04698972099853157, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.06718266010284424, "fcm_dpo/q_t": 0.4983203411102295, "grad_norm": 35.197174072265625, "learning_rate": 2.2463768115942027e-07, "logits/chosen": 0.4451717138290405, "logits/rejected": 0.5880470871925354, "logps/chosen": -96.67608642578125, "logps/ref_chosen": -96.61702728271484, "logps/ref_rejected": -96.224365234375, "logps/rejected": -96.3506088256836, "loss": 1.3802, "margin_dpo/margin_mean": 0.06718306243419647, "margin_dpo/margin_std": 0.4782736897468567, "step": 32 }, { "KL/chosen_KL_mean": 0.01537322998046875, "KL/mean": -0.033518873155117035, "KL/rejected_KL_mean": -0.08240890502929688, "KL/std": 0.2934386730194092, "epoch": 0.048458149779735685, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.09778077900409698, "fcm_dpo/q_t": 0.49755772948265076, "grad_norm": 33.55974197387695, "learning_rate": 2.318840579710145e-07, "logits/chosen": 0.6998616456985474, "logits/rejected": 0.8830456733703613, "logps/chosen": -81.50564575195312, "logps/ref_chosen": -81.52101135253906, "logps/ref_rejected": -93.80595397949219, "logps/rejected": -93.88836669921875, "loss": 1.377, "margin_dpo/margin_mean": 0.09778153896331787, "margin_dpo/margin_std": 0.40736132860183716, "step": 33 }, { "KL/chosen_KL_mean": 0.0535430908203125, "KL/mean": -0.0476461797952652, "KL/rejected_KL_mean": -0.14883804321289062, "KL/std": 0.3794667422771454, "epoch": 0.049926578560939794, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.20238515734672546, "fcm_dpo/q_t": 0.4949512183666229, "grad_norm": 40.8648796081543, "learning_rate": 2.391304347826087e-07, "logits/chosen": 0.660446047782898, "logits/rejected": 0.8321809768676758, "logps/chosen": -77.14849853515625, "logps/ref_chosen": -77.20204162597656, "logps/ref_rejected": -106.71876525878906, "logps/rejected": -106.86759948730469, "loss": 1.3669, "margin_dpo/margin_mean": 0.20238518714904785, "margin_dpo/margin_std": 0.5540159940719604, "step": 34 }, { "KL/chosen_KL_mean": 0.008678436279296875, "KL/mean": -0.11909741163253784, "KL/rejected_KL_mean": -0.2468719482421875, "KL/std": 0.4011994004249573, "epoch": 0.0513950073421439, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.2555462121963501, "fcm_dpo/q_t": 0.49361878633499146, "grad_norm": 42.36187744140625, "learning_rate": 2.463768115942029e-07, "logits/chosen": 0.3200201094150543, "logits/rejected": 0.4755074381828308, "logps/chosen": -77.56167602539062, "logps/ref_chosen": -77.57035064697266, "logps/ref_rejected": -112.18855285644531, "logps/rejected": -112.4354248046875, "loss": 1.3617, "margin_dpo/margin_mean": 0.25554654002189636, "margin_dpo/margin_std": 0.5492804646492004, "step": 35 }, { "KL/chosen_KL_mean": 0.00325775146484375, "KL/mean": -0.0820009633898735, "KL/rejected_KL_mean": -0.16726303100585938, "KL/std": 0.4162295162677765, "epoch": 0.05286343612334802, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.17051883041858673, "fcm_dpo/q_t": 0.4957442283630371, "grad_norm": 32.467105865478516, "learning_rate": 2.536231884057971e-07, "logits/chosen": 0.4792647957801819, "logits/rejected": 0.6717060804367065, "logps/chosen": -83.28498840332031, "logps/ref_chosen": -83.28823852539062, "logps/ref_rejected": -91.75741577148438, "logps/rejected": -91.9246826171875, "loss": 1.3701, "margin_dpo/margin_mean": 0.1705191731452942, "margin_dpo/margin_std": 0.5299290418624878, "step": 36 }, { "KL/chosen_KL_mean": -0.028217315673828125, "KL/mean": -0.14107736945152283, "KL/rejected_KL_mean": -0.253936767578125, "KL/std": 0.43520498275756836, "epoch": 0.05433186490455213, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.22571538388729095, "fcm_dpo/q_t": 0.4943671226501465, "grad_norm": 35.993526458740234, "learning_rate": 2.6086956521739126e-07, "logits/chosen": 0.5401657819747925, "logits/rejected": 0.6905125975608826, "logps/chosen": -94.79930877685547, "logps/ref_chosen": -94.77108764648438, "logps/ref_rejected": -85.71726989746094, "logps/rejected": -85.97120666503906, "loss": 1.3648, "margin_dpo/margin_mean": 0.2257155328989029, "margin_dpo/margin_std": 0.5866572856903076, "step": 37 }, { "KL/chosen_KL_mean": 0.08259963989257812, "KL/mean": -0.07191865891218185, "KL/rejected_KL_mean": -0.22642898559570312, "KL/std": 0.45193982124328613, "epoch": 0.055800293685756244, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.3090275526046753, "fcm_dpo/q_t": 0.492290735244751, "grad_norm": 36.67866516113281, "learning_rate": 2.681159420289855e-07, "logits/chosen": 0.681549072265625, "logits/rejected": 0.8211994171142578, "logps/chosen": -75.89590454101562, "logps/ref_chosen": -75.97850036621094, "logps/ref_rejected": -104.10401916503906, "logps/rejected": -104.3304443359375, "loss": 1.3567, "margin_dpo/margin_mean": 0.30902788043022156, "margin_dpo/margin_std": 0.6416710615158081, "step": 38 }, { "KL/chosen_KL_mean": -0.011474609375, "KL/mean": -0.1568700671195984, "KL/rejected_KL_mean": -0.3022727966308594, "KL/std": 0.38306891918182373, "epoch": 0.05726872246696035, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.29079556465148926, "fcm_dpo/q_t": 0.49273791909217834, "grad_norm": 36.345123291015625, "learning_rate": 2.753623188405797e-07, "logits/chosen": 0.6276674270629883, "logits/rejected": 0.8255791664123535, "logps/chosen": -81.19725036621094, "logps/ref_chosen": -81.18577575683594, "logps/ref_rejected": -84.1959228515625, "logps/rejected": -84.4981918334961, "loss": 1.3582, "margin_dpo/margin_mean": 0.2907947897911072, "margin_dpo/margin_std": 0.5327550768852234, "step": 39 }, { "KL/chosen_KL_mean": 0.026821136474609375, "KL/mean": -0.07124792039394379, "KL/rejected_KL_mean": -0.16931533813476562, "KL/std": 0.33206266164779663, "epoch": 0.05873715124816446, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.19613954424858093, "fcm_dpo/q_t": 0.4951009154319763, "grad_norm": 29.182266235351562, "learning_rate": 2.8260869565217386e-07, "logits/chosen": 0.6739002466201782, "logits/rejected": 0.8751500844955444, "logps/chosen": -83.30575561523438, "logps/ref_chosen": -83.33257293701172, "logps/ref_rejected": -80.25591278076172, "logps/rejected": -80.42523193359375, "loss": 1.3673, "margin_dpo/margin_mean": 0.1961391270160675, "margin_dpo/margin_std": 0.4317895174026489, "step": 40 }, { "KL/chosen_KL_mean": -0.0812530517578125, "KL/mean": -0.2116875797510147, "KL/rejected_KL_mean": -0.3421211242675781, "KL/std": 0.4825252592563629, "epoch": 0.06020558002936858, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.26086869835853577, "fcm_dpo/q_t": 0.4934980273246765, "grad_norm": 33.234317779541016, "learning_rate": 2.898550724637681e-07, "logits/chosen": 0.6371617317199707, "logits/rejected": 0.8245461583137512, "logps/chosen": -93.22991943359375, "logps/ref_chosen": -93.14866638183594, "logps/ref_rejected": -102.07920837402344, "logps/rejected": -102.42132568359375, "loss": 1.3617, "margin_dpo/margin_mean": 0.2608690559864044, "margin_dpo/margin_std": 0.7232835292816162, "step": 41 }, { "KL/chosen_KL_mean": -0.00684356689453125, "KL/mean": -0.2057129293680191, "KL/rejected_KL_mean": -0.4045829772949219, "KL/std": 0.4727141261100769, "epoch": 0.06167400881057269, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.39773795008659363, "fcm_dpo/q_t": 0.4900670647621155, "grad_norm": 40.909812927246094, "learning_rate": 2.971014492753623e-07, "logits/chosen": 0.48165372014045715, "logits/rejected": 0.6244504451751709, "logps/chosen": -90.70846557617188, "logps/ref_chosen": -90.70162200927734, "logps/ref_rejected": -113.8646469116211, "logps/rejected": -114.26922607421875, "loss": 1.3478, "margin_dpo/margin_mean": 0.39773809909820557, "margin_dpo/margin_std": 0.5912743806838989, "step": 42 }, { "KL/chosen_KL_mean": 0.033107757568359375, "KL/mean": -0.21173666417598724, "KL/rejected_KL_mean": -0.4565849304199219, "KL/std": 0.5068866014480591, "epoch": 0.0631424375917768, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.489694744348526, "fcm_dpo/q_t": 0.4877869188785553, "grad_norm": 37.69319534301758, "learning_rate": 3.043478260869565e-07, "logits/chosen": 0.5057786703109741, "logits/rejected": 0.6928372979164124, "logps/chosen": -89.61092376708984, "logps/ref_chosen": -89.64402770996094, "logps/ref_rejected": -103.95185852050781, "logps/rejected": -104.40843963623047, "loss": 1.3391, "margin_dpo/margin_mean": 0.48969531059265137, "margin_dpo/margin_std": 0.6913731694221497, "step": 43 }, { "KL/chosen_KL_mean": -0.05744171142578125, "KL/mean": -0.23107439279556274, "KL/rejected_KL_mean": -0.4047088623046875, "KL/std": 0.4452526271343231, "epoch": 0.06461086637298091, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.3472658693790436, "fcm_dpo/q_t": 0.49132949113845825, "grad_norm": 34.49655532836914, "learning_rate": 3.115942028985507e-07, "logits/chosen": 0.9959243535995483, "logits/rejected": 1.1132128238677979, "logps/chosen": -81.93576049804688, "logps/ref_chosen": -81.8783187866211, "logps/ref_rejected": -113.421630859375, "logps/rejected": -113.82633972167969, "loss": 1.3527, "margin_dpo/margin_mean": 0.3472673296928406, "margin_dpo/margin_std": 0.5616138577461243, "step": 44 }, { "KL/chosen_KL_mean": -0.06879806518554688, "KL/mean": -0.19089767336845398, "KL/rejected_KL_mean": -0.31299591064453125, "KL/std": 0.4350392818450928, "epoch": 0.06607929515418502, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.24419839680194855, "fcm_dpo/q_t": 0.49390164017677307, "grad_norm": 24.39482307434082, "learning_rate": 3.188405797101449e-07, "logits/chosen": 0.722280740737915, "logits/rejected": 0.835638165473938, "logps/chosen": -77.41339111328125, "logps/ref_chosen": -77.34459686279297, "logps/ref_rejected": -84.24774169921875, "logps/rejected": -84.56074523925781, "loss": 1.3628, "margin_dpo/margin_mean": 0.24419908225536346, "margin_dpo/margin_std": 0.5641046762466431, "step": 45 }, { "KL/chosen_KL_mean": -0.01519012451171875, "KL/mean": -0.22886362671852112, "KL/rejected_KL_mean": -0.4425392150878906, "KL/std": 0.5174437165260315, "epoch": 0.06754772393538913, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.42735230922698975, "fcm_dpo/q_t": 0.4893287420272827, "grad_norm": 33.09678649902344, "learning_rate": 3.260869565217391e-07, "logits/chosen": 0.5283759832382202, "logits/rejected": 0.7729748487472534, "logps/chosen": -90.35601806640625, "logps/ref_chosen": -90.3408203125, "logps/ref_rejected": -100.676513671875, "logps/rejected": -101.11905670166016, "loss": 1.3448, "margin_dpo/margin_mean": 0.4273521900177002, "margin_dpo/margin_std": 0.561310887336731, "step": 46 }, { "KL/chosen_KL_mean": -0.07775497436523438, "KL/mean": -0.28309375047683716, "KL/rejected_KL_mean": -0.488433837890625, "KL/std": 0.5084018111228943, "epoch": 0.06901615271659324, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.41067540645599365, "fcm_dpo/q_t": 0.4897534251213074, "grad_norm": 33.92341995239258, "learning_rate": 3.333333333333333e-07, "logits/chosen": 0.5192316770553589, "logits/rejected": 0.7062826156616211, "logps/chosen": -104.48905944824219, "logps/ref_chosen": -104.41130065917969, "logps/ref_rejected": -101.25489807128906, "logps/rejected": -101.74333190917969, "loss": 1.347, "margin_dpo/margin_mean": 0.4106749892234802, "margin_dpo/margin_std": 0.7300633192062378, "step": 47 }, { "KL/chosen_KL_mean": -0.061695098876953125, "KL/mean": -0.3085932731628418, "KL/rejected_KL_mean": -0.5554962158203125, "KL/std": 0.6243237257003784, "epoch": 0.07048458149779736, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.49380332231521606, "fcm_dpo/q_t": 0.48767757415771484, "grad_norm": 39.03615188598633, "learning_rate": 3.4057971014492755e-07, "logits/chosen": 0.19509437680244446, "logits/rejected": 0.3942684829235077, "logps/chosen": -91.16197204589844, "logps/ref_chosen": -91.10027313232422, "logps/ref_rejected": -95.08058166503906, "logps/rejected": -95.63607788085938, "loss": 1.339, "margin_dpo/margin_mean": 0.49380412697792053, "margin_dpo/margin_std": 0.7588241100311279, "step": 48 }, { "KL/chosen_KL_mean": 0.033519744873046875, "KL/mean": -0.3305027484893799, "KL/rejected_KL_mean": -0.6945266723632812, "KL/std": 0.7838465571403503, "epoch": 0.07195301027900147, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.728039026260376, "fcm_dpo/q_t": 0.4818696677684784, "grad_norm": 39.64306640625, "learning_rate": 3.478260869565217e-07, "logits/chosen": 0.2486877143383026, "logits/rejected": 0.7374652624130249, "logps/chosen": -92.97016143798828, "logps/ref_chosen": -93.00367736816406, "logps/ref_rejected": -91.74899291992188, "logps/rejected": -92.44352722167969, "loss": 1.3171, "margin_dpo/margin_mean": 0.7280387878417969, "margin_dpo/margin_std": 0.9538377523422241, "step": 49 }, { "KL/chosen_KL_mean": -0.038837432861328125, "KL/mean": -0.40678519010543823, "KL/rejected_KL_mean": -0.7747268676757812, "KL/std": 0.7677117586135864, "epoch": 0.07342143906020558, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.7358890771865845, "fcm_dpo/q_t": 0.48165565729141235, "grad_norm": 33.14576721191406, "learning_rate": 3.5507246376811595e-07, "logits/chosen": 0.45622560381889343, "logits/rejected": 0.6242513656616211, "logps/chosen": -94.6656494140625, "logps/ref_chosen": -94.62681579589844, "logps/ref_rejected": -103.57435607910156, "logps/rejected": -104.34909057617188, "loss": 1.3161, "margin_dpo/margin_mean": 0.7358888387680054, "margin_dpo/margin_std": 0.9077410697937012, "step": 50 }, { "KL/chosen_KL_mean": -0.06505584716796875, "KL/mean": -0.3700931966304779, "KL/rejected_KL_mean": -0.6751251220703125, "KL/std": 0.8226094245910645, "epoch": 0.07488986784140969, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.6100632548332214, "fcm_dpo/q_t": 0.4848117232322693, "grad_norm": 30.207778930664062, "learning_rate": 3.6231884057971015e-07, "logits/chosen": 0.7389651536941528, "logits/rejected": 0.8867373466491699, "logps/chosen": -87.57233428955078, "logps/ref_chosen": -87.50727844238281, "logps/ref_rejected": -83.47235107421875, "logps/rejected": -84.14747619628906, "loss": 1.3287, "margin_dpo/margin_mean": 0.6100629568099976, "margin_dpo/margin_std": 0.9975903630256653, "step": 51 }, { "KL/chosen_KL_mean": 0.039501190185546875, "KL/mean": -0.41753852367401123, "KL/rejected_KL_mean": -0.8745880126953125, "KL/std": 0.7825153470039368, "epoch": 0.0763582966226138, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.9140852689743042, "fcm_dpo/q_t": 0.4772176146507263, "grad_norm": 31.828645706176758, "learning_rate": 3.695652173913043e-07, "logits/chosen": 0.379444420337677, "logits/rejected": 0.5888497829437256, "logps/chosen": -90.59075927734375, "logps/ref_chosen": -90.63026428222656, "logps/ref_rejected": -87.0390625, "logps/rejected": -87.91365051269531, "loss": 1.2989, "margin_dpo/margin_mean": 0.9140851497650146, "margin_dpo/margin_std": 0.8735721111297607, "step": 52 }, { "KL/chosen_KL_mean": 0.00278472900390625, "KL/mean": -0.54905104637146, "KL/rejected_KL_mean": -1.1008872985839844, "KL/std": 1.1362104415893555, "epoch": 0.07782672540381791, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.1036782264709473, "fcm_dpo/q_t": 0.4727194905281067, "grad_norm": 39.239871978759766, "learning_rate": 3.7681159420289855e-07, "logits/chosen": 0.23845317959785461, "logits/rejected": 0.3551448881626129, "logps/chosen": -81.58029174804688, "logps/ref_chosen": -81.58307647705078, "logps/ref_rejected": -95.66152954101562, "logps/rejected": -96.76242065429688, "loss": 1.2844, "margin_dpo/margin_mean": 1.1036782264709473, "margin_dpo/margin_std": 1.4056880474090576, "step": 53 }, { "KL/chosen_KL_mean": 0.00627899169921875, "KL/mean": -0.5997223854064941, "KL/rejected_KL_mean": -1.2057304382324219, "KL/std": 1.359240174293518, "epoch": 0.07929515418502203, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.212003469467163, "fcm_dpo/q_t": 0.469951868057251, "grad_norm": 39.345611572265625, "learning_rate": 3.8405797101449274e-07, "logits/chosen": 0.3955309987068176, "logits/rejected": 0.575438380241394, "logps/chosen": -88.90388488769531, "logps/ref_chosen": -88.91017150878906, "logps/ref_rejected": -99.1175537109375, "logps/rejected": -100.32328796386719, "loss": 1.2736, "margin_dpo/margin_mean": 1.2120035886764526, "margin_dpo/margin_std": 1.3928658962249756, "step": 54 }, { "KL/chosen_KL_mean": 0.14455795288085938, "KL/mean": -0.46645084023475647, "KL/rejected_KL_mean": -1.07745361328125, "KL/std": 1.2257211208343506, "epoch": 0.08076358296622614, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.2220126390457153, "fcm_dpo/q_t": 0.469801127910614, "grad_norm": 35.018096923828125, "learning_rate": 3.9130434782608694e-07, "logits/chosen": 0.09322050958871841, "logits/rejected": 0.4001709818840027, "logps/chosen": -92.31136322021484, "logps/ref_chosen": -92.45591735839844, "logps/ref_rejected": -92.97093200683594, "logps/rejected": -94.04838562011719, "loss": 1.2746, "margin_dpo/margin_mean": 1.2220122814178467, "margin_dpo/margin_std": 1.6103571653366089, "step": 55 }, { "KL/chosen_KL_mean": -0.046146392822265625, "KL/mean": -0.6011971235275269, "KL/rejected_KL_mean": -1.1562461853027344, "KL/std": 1.2121695280075073, "epoch": 0.08223201174743025, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.1101038455963135, "fcm_dpo/q_t": 0.4725731313228607, "grad_norm": 33.058006286621094, "learning_rate": 3.9855072463768114e-07, "logits/chosen": 0.2968568503856659, "logits/rejected": 0.6376580595970154, "logps/chosen": -87.28280639648438, "logps/ref_chosen": -87.23665618896484, "logps/ref_rejected": -100.64553833007812, "logps/rejected": -101.80178833007812, "loss": 1.2846, "margin_dpo/margin_mean": 1.110103726387024, "margin_dpo/margin_std": 1.578169584274292, "step": 56 }, { "KL/chosen_KL_mean": -0.12443923950195312, "KL/mean": -0.6088008880615234, "KL/rejected_KL_mean": -1.0931587219238281, "KL/std": 1.192524790763855, "epoch": 0.08370044052863436, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.968733549118042, "fcm_dpo/q_t": 0.4759920537471771, "grad_norm": 34.070369720458984, "learning_rate": 4.057971014492754e-07, "logits/chosen": 0.44745880365371704, "logits/rejected": 0.6691286563873291, "logps/chosen": -98.27517700195312, "logps/ref_chosen": -98.15074157714844, "logps/ref_rejected": -101.05284118652344, "logps/rejected": -102.14599609375, "loss": 1.2974, "margin_dpo/margin_mean": 0.9687336683273315, "margin_dpo/margin_std": 1.5028915405273438, "step": 57 }, { "KL/chosen_KL_mean": 0.2667388916015625, "KL/mean": -0.49684733152389526, "KL/rejected_KL_mean": -1.2604255676269531, "KL/std": 1.4403879642486572, "epoch": 0.08516886930983847, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.5271626710891724, "fcm_dpo/q_t": 0.46250051259994507, "grad_norm": 37.94102096557617, "learning_rate": 4.1304347826086954e-07, "logits/chosen": 0.08838152885437012, "logits/rejected": 0.4025894105434418, "logps/chosen": -99.3142318725586, "logps/ref_chosen": -99.58097076416016, "logps/ref_rejected": -91.22227478027344, "logps/rejected": -92.48269653320312, "loss": 1.2493, "margin_dpo/margin_mean": 1.5271626710891724, "margin_dpo/margin_std": 1.8847169876098633, "step": 58 }, { "KL/chosen_KL_mean": 0.021205902099609375, "KL/mean": -0.57396399974823, "KL/rejected_KL_mean": -1.1691322326660156, "KL/std": 1.1711525917053223, "epoch": 0.08663729809104258, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.1903326511383057, "fcm_dpo/q_t": 0.4704800248146057, "grad_norm": 34.30070495605469, "learning_rate": 4.2028985507246374e-07, "logits/chosen": 0.11888501048088074, "logits/rejected": 0.30946826934814453, "logps/chosen": -89.78112030029297, "logps/ref_chosen": -89.80232238769531, "logps/ref_rejected": -94.27667236328125, "logps/rejected": -95.44580841064453, "loss": 1.2759, "margin_dpo/margin_mean": 1.190332293510437, "margin_dpo/margin_std": 1.4320528507232666, "step": 59 }, { "KL/chosen_KL_mean": 0.04380035400390625, "KL/mean": -0.558405876159668, "KL/rejected_KL_mean": -1.1606216430664062, "KL/std": 1.2293000221252441, "epoch": 0.0881057268722467, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.2044206857681274, "fcm_dpo/q_t": 0.4701216220855713, "grad_norm": 28.800195693969727, "learning_rate": 4.2753623188405794e-07, "logits/chosen": 0.34888800978660583, "logits/rejected": 0.6163915395736694, "logps/chosen": -95.11190795898438, "logps/ref_chosen": -95.15571594238281, "logps/ref_rejected": -91.3724365234375, "logps/rejected": -92.5330581665039, "loss": 1.2744, "margin_dpo/margin_mean": 1.2044206857681274, "margin_dpo/margin_std": 1.4062446355819702, "step": 60 }, { "KL/chosen_KL_mean": -0.2785377502441406, "KL/mean": -0.8395862579345703, "KL/rejected_KL_mean": -1.400634765625, "KL/std": 1.4459847211837769, "epoch": 0.08957415565345081, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.122102975845337, "fcm_dpo/q_t": 0.47252535820007324, "grad_norm": 28.436363220214844, "learning_rate": 4.3478260869565214e-07, "logits/chosen": 0.20631279051303864, "logits/rejected": 0.3471667468547821, "logps/chosen": -86.01084899902344, "logps/ref_chosen": -85.73230743408203, "logps/ref_rejected": -97.96575927734375, "logps/rejected": -99.36639404296875, "loss": 1.287, "margin_dpo/margin_mean": 1.1221034526824951, "margin_dpo/margin_std": 1.9247691631317139, "step": 61 }, { "KL/chosen_KL_mean": -0.11776351928710938, "KL/mean": -0.7294133901596069, "KL/rejected_KL_mean": -1.3410682678222656, "KL/std": 1.4079499244689941, "epoch": 0.09104258443465492, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.223311185836792, "fcm_dpo/q_t": 0.46987247467041016, "grad_norm": 28.47466468811035, "learning_rate": 4.420289855072464e-07, "logits/chosen": 0.10442984104156494, "logits/rejected": 0.2933581471443176, "logps/chosen": -81.75314331054688, "logps/ref_chosen": -81.63538360595703, "logps/ref_rejected": -84.03831481933594, "logps/rejected": -85.37937927246094, "loss": 1.2769, "margin_dpo/margin_mean": 1.2233108282089233, "margin_dpo/margin_std": 1.7898602485656738, "step": 62 }, { "KL/chosen_KL_mean": -0.06484222412109375, "KL/mean": -0.8377429246902466, "KL/rejected_KL_mean": -1.6106376647949219, "KL/std": 1.4270102977752686, "epoch": 0.09251101321585903, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.5457981824874878, "fcm_dpo/q_t": 0.4617375433444977, "grad_norm": 34.867984771728516, "learning_rate": 4.4927536231884053e-07, "logits/chosen": 0.21503260731697083, "logits/rejected": 0.418557345867157, "logps/chosen": -103.68890380859375, "logps/ref_chosen": -103.62405395507812, "logps/ref_rejected": -103.40303039550781, "logps/rejected": -105.013671875, "loss": 1.2442, "margin_dpo/margin_mean": 1.54579758644104, "margin_dpo/margin_std": 1.6060906648635864, "step": 63 }, { "KL/chosen_KL_mean": -0.09991836547851562, "KL/mean": -0.9311131238937378, "KL/rejected_KL_mean": -1.7623176574707031, "KL/std": 1.8329956531524658, "epoch": 0.09397944199706314, "fcm_dpo/beta": 0.10238748043775558, "fcm_dpo/delta": 0.11797122657299042, "fcm_dpo/margin": 1.6624014377593994, "fcm_dpo/q_t": 0.4589458703994751, "grad_norm": 35.67680740356445, "learning_rate": 4.5652173913043473e-07, "logits/chosen": 0.8073688745498657, "logits/rejected": 1.0393321514129639, "logps/chosen": -87.10150146484375, "logps/ref_chosen": -87.0015869140625, "logps/ref_rejected": -100.5854721069336, "logps/rejected": -102.34779357910156, "loss": 1.2353, "margin_dpo/margin_mean": 1.6624014377593994, "margin_dpo/margin_std": 2.069509267807007, "step": 64 }, { "KL/chosen_KL_mean": -0.029155731201171875, "KL/mean": -0.9522594809532166, "KL/rejected_KL_mean": -1.8753700256347656, "KL/std": 1.6730495691299438, "epoch": 0.09544787077826726, "fcm_dpo/beta": 0.10238748043775558, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.8462045192718506, "fcm_dpo/q_t": 0.45357537269592285, "grad_norm": 38.958038330078125, "learning_rate": 4.63768115942029e-07, "logits/chosen": -0.022555742412805557, "logits/rejected": 0.12171991169452667, "logps/chosen": -91.25106811523438, "logps/ref_chosen": -91.22191619873047, "logps/ref_rejected": -115.33553314208984, "logps/rejected": -117.21090698242188, "loss": 1.217, "margin_dpo/margin_mean": 1.8462042808532715, "margin_dpo/margin_std": 2.042468786239624, "step": 65 }, { "KL/chosen_KL_mean": -0.283233642578125, "KL/mean": -0.9032148122787476, "KL/rejected_KL_mean": -1.523193359375, "KL/std": 1.5640734434127808, "epoch": 0.09691629955947137, "fcm_dpo/beta": 0.10238748043775558, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.239959478378296, "fcm_dpo/q_t": 0.46878179907798767, "grad_norm": 26.243160247802734, "learning_rate": 4.7101449275362313e-07, "logits/chosen": 0.1345851719379425, "logits/rejected": 0.3374342918395996, "logps/chosen": -84.06745910644531, "logps/ref_chosen": -83.78422546386719, "logps/ref_rejected": -82.7520980834961, "logps/rejected": -84.2752914428711, "loss": 1.2733, "margin_dpo/margin_mean": 1.239959716796875, "margin_dpo/margin_std": 1.9022430181503296, "step": 66 }, { "KL/chosen_KL_mean": -0.8461761474609375, "KL/mean": -1.401423692703247, "KL/rejected_KL_mean": -1.9566688537597656, "KL/std": 2.0096323490142822, "epoch": 0.09838472834067548, "fcm_dpo/beta": 0.10238748043775558, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.1104950904846191, "fcm_dpo/q_t": 0.47256746888160706, "grad_norm": 24.86586570739746, "learning_rate": 4.782608695652174e-07, "logits/chosen": 0.28954365849494934, "logits/rejected": 0.5612607598304749, "logps/chosen": -88.51913452148438, "logps/ref_chosen": -87.67295837402344, "logps/ref_rejected": -79.08674621582031, "logps/rejected": -81.04341888427734, "loss": 1.2953, "margin_dpo/margin_mean": 1.1104950904846191, "margin_dpo/margin_std": 2.7410237789154053, "step": 67 }, { "KL/chosen_KL_mean": -0.21075057983398438, "KL/mean": -1.270378589630127, "KL/rejected_KL_mean": -2.3300018310546875, "KL/std": 2.157897472381592, "epoch": 0.09985315712187959, "fcm_dpo/beta": 0.10602904856204987, "fcm_dpo/delta": 0.23036563396453857, "fcm_dpo/margin": 2.1192517280578613, "fcm_dpo/q_t": 0.4469185769557953, "grad_norm": 29.703475952148438, "learning_rate": 4.855072463768116e-07, "logits/chosen": -0.051017943769693375, "logits/rejected": 0.24151284992694855, "logps/chosen": -97.28959655761719, "logps/ref_chosen": -97.07884216308594, "logps/ref_rejected": -84.11872863769531, "logps/rejected": -86.44873046875, "loss": 1.1931, "margin_dpo/margin_mean": 2.1192517280578613, "margin_dpo/margin_std": 2.628087043762207, "step": 68 }, { "KL/chosen_KL_mean": -0.26189422607421875, "KL/mean": -1.4020657539367676, "KL/rejected_KL_mean": -2.542236328125, "KL/std": 2.4624533653259277, "epoch": 0.1013215859030837, "fcm_dpo/beta": 0.11060892045497894, "fcm_dpo/delta": 0.20270594954490662, "fcm_dpo/margin": 2.2803425788879395, "fcm_dpo/q_t": 0.44102779030799866, "grad_norm": 32.46053695678711, "learning_rate": 4.927536231884058e-07, "logits/chosen": 0.1682801991701126, "logits/rejected": 0.2794885039329529, "logps/chosen": -85.98160552978516, "logps/ref_chosen": -85.71971130371094, "logps/ref_rejected": -109.48020935058594, "logps/rejected": -112.02244567871094, "loss": 1.1768, "margin_dpo/margin_mean": 2.2803428173065186, "margin_dpo/margin_std": 3.052189350128174, "step": 69 }, { "KL/chosen_KL_mean": -0.6263198852539062, "KL/mean": -1.899319052696228, "KL/rejected_KL_mean": -3.172313690185547, "KL/std": 2.73360013961792, "epoch": 0.1027900146842878, "fcm_dpo/beta": 0.11450241506099701, "fcm_dpo/delta": 0.16308385133743286, "fcm_dpo/margin": 2.545989513397217, "fcm_dpo/q_t": 0.43262767791748047, "grad_norm": 31.931215286254883, "learning_rate": 5e-07, "logits/chosen": 0.21033185720443726, "logits/rejected": 0.38455793261528015, "logps/chosen": -95.63626861572266, "logps/ref_chosen": -95.00994873046875, "logps/ref_rejected": -96.21272277832031, "logps/rejected": -99.38504028320312, "loss": 1.1523, "margin_dpo/margin_mean": 2.545989751815796, "margin_dpo/margin_std": 3.3493716716766357, "step": 70 }, { "KL/chosen_KL_mean": 0.001708984375, "KL/mean": -1.8702661991119385, "KL/rejected_KL_mean": -3.7422447204589844, "KL/std": 3.5025901794433594, "epoch": 0.10425844346549193, "fcm_dpo/beta": 0.11530126631259918, "fcm_dpo/delta": 0.01898610219359398, "fcm_dpo/margin": 3.743950128555298, "fcm_dpo/q_t": 0.400613009929657, "grad_norm": 37.1617317199707, "learning_rate": 4.999967061337492e-07, "logits/chosen": -0.19057126343250275, "logits/rejected": 0.1714491844177246, "logps/chosen": -90.97564697265625, "logps/ref_chosen": -90.97735595703125, "logps/ref_rejected": -102.59103393554688, "logps/rejected": -106.3332748413086, "loss": 1.0462, "margin_dpo/margin_mean": 3.7439494132995605, "margin_dpo/margin_std": 3.907170295715332, "step": 71 }, { "KL/chosen_KL_mean": -0.5383262634277344, "KL/mean": -1.9858675003051758, "KL/rejected_KL_mean": -3.43341064453125, "KL/std": 2.893824577331543, "epoch": 0.10572687224669604, "fcm_dpo/beta": 0.11765197664499283, "fcm_dpo/delta": 0.1133369654417038, "fcm_dpo/margin": 2.895083427429199, "fcm_dpo/q_t": 0.4203723669052124, "grad_norm": 32.836151123046875, "learning_rate": 4.999868246217933e-07, "logits/chosen": -0.24647319316864014, "logits/rejected": 0.07103689014911652, "logps/chosen": -98.42985534667969, "logps/ref_chosen": -97.89152526855469, "logps/ref_rejected": -100.19171142578125, "logps/rejected": -103.6251220703125, "loss": 1.1114, "margin_dpo/margin_mean": 2.895082712173462, "margin_dpo/margin_std": 3.351975679397583, "step": 72 }, { "KL/chosen_KL_mean": -0.7371978759765625, "KL/mean": -2.3258001804351807, "KL/rejected_KL_mean": -3.9144020080566406, "KL/std": 3.848926067352295, "epoch": 0.10719530102790015, "fcm_dpo/beta": 0.11976398527622223, "fcm_dpo/delta": 0.07221053540706635, "fcm_dpo/margin": 3.1772067546844482, "fcm_dpo/q_t": 0.4168751835823059, "grad_norm": 30.715152740478516, "learning_rate": 4.999703557245192e-07, "logits/chosen": -0.25766175985336304, "logits/rejected": 0.05361177772283554, "logps/chosen": -96.50621795654297, "logps/ref_chosen": -95.7690200805664, "logps/ref_rejected": -95.93243408203125, "logps/rejected": -99.84683990478516, "loss": 1.1183, "margin_dpo/margin_mean": 3.177206516265869, "margin_dpo/margin_std": 4.9230732917785645, "step": 73 }, { "KL/chosen_KL_mean": -0.98834228515625, "KL/mean": -2.741278648376465, "KL/rejected_KL_mean": -4.494209289550781, "KL/std": 3.77854585647583, "epoch": 0.10866372980910426, "fcm_dpo/beta": 0.1204468160867691, "fcm_dpo/delta": 0.02892753668129444, "fcm_dpo/margin": 3.505868434906006, "fcm_dpo/q_t": 0.40502387285232544, "grad_norm": 33.72903823852539, "learning_rate": 4.999472998758977e-07, "logits/chosen": -0.35920172929763794, "logits/rejected": -0.21716484427452087, "logps/chosen": -79.79673767089844, "logps/ref_chosen": -78.80839538574219, "logps/ref_rejected": -101.64677429199219, "logps/rejected": -106.14097595214844, "loss": 1.0762, "margin_dpo/margin_mean": 3.5058681964874268, "margin_dpo/margin_std": 4.543154716491699, "step": 74 }, { "KL/chosen_KL_mean": -1.028656005859375, "KL/mean": -3.3344218730926514, "KL/rejected_KL_mean": -5.640190124511719, "KL/std": 4.607555866241455, "epoch": 0.11013215859030837, "fcm_dpo/beta": 0.11946292221546173, "fcm_dpo/delta": -0.10670093446969986, "fcm_dpo/margin": 4.611537933349609, "fcm_dpo/q_t": 0.3799246549606323, "grad_norm": 34.582496643066406, "learning_rate": 4.999176576834721e-07, "logits/chosen": -0.2411120980978012, "logits/rejected": -0.039592910557985306, "logps/chosen": -79.31050872802734, "logps/ref_chosen": -78.28185272216797, "logps/ref_rejected": -115.40311431884766, "logps/rejected": -121.04330444335938, "loss": 0.999, "margin_dpo/margin_mean": 4.611536979675293, "margin_dpo/margin_std": 5.436832427978516, "step": 75 }, { "KL/chosen_KL_mean": -1.4529953002929688, "KL/mean": -3.011784791946411, "KL/rejected_KL_mean": -4.570579528808594, "KL/std": 3.8695666790008545, "epoch": 0.11160058737151249, "fcm_dpo/beta": 0.11969182640314102, "fcm_dpo/delta": 0.07985444366931915, "fcm_dpo/margin": 3.117582321166992, "fcm_dpo/q_t": 0.41714996099472046, "grad_norm": 26.333070755004883, "learning_rate": 4.998814299283415e-07, "logits/chosen": -0.6458679437637329, "logits/rejected": -0.27020227909088135, "logps/chosen": -89.33013916015625, "logps/ref_chosen": -87.87713623046875, "logps/ref_rejected": -85.71968078613281, "logps/rejected": -90.2902603149414, "loss": 1.1185, "margin_dpo/margin_mean": 3.117582321166992, "margin_dpo/margin_std": 4.7150468826293945, "step": 76 }, { "KL/chosen_KL_mean": -1.6510658264160156, "KL/mean": -3.3294880390167236, "KL/rejected_KL_mean": -5.007907867431641, "KL/std": 4.483570098876953, "epoch": 0.1130690161527166, "fcm_dpo/beta": 0.12075696885585785, "fcm_dpo/delta": 0.046513475477695465, "fcm_dpo/margin": 3.3568429946899414, "fcm_dpo/q_t": 0.4093211591243744, "grad_norm": 31.489459991455078, "learning_rate": 4.998386175651409e-07, "logits/chosen": -0.2016545534133911, "logits/rejected": 0.16604572534561157, "logps/chosen": -101.35140991210938, "logps/ref_chosen": -99.70034790039062, "logps/ref_rejected": -98.20576477050781, "logps/rejected": -103.21366882324219, "loss": 1.109, "margin_dpo/margin_mean": 3.356842517852783, "margin_dpo/margin_std": 5.0975823402404785, "step": 77 }, { "KL/chosen_KL_mean": -0.7189369201660156, "KL/mean": -2.515406608581543, "KL/rejected_KL_mean": -4.3118743896484375, "KL/std": 3.712735652923584, "epoch": 0.1145374449339207, "fcm_dpo/beta": 0.1215987429022789, "fcm_dpo/delta": 0.013700693845748901, "fcm_dpo/margin": 3.592939853668213, "fcm_dpo/q_t": 0.40087586641311646, "grad_norm": 28.228641510009766, "learning_rate": 4.997892217220159e-07, "logits/chosen": -0.07256608456373215, "logits/rejected": 0.1554630994796753, "logps/chosen": -91.0156478881836, "logps/ref_chosen": -90.29670715332031, "logps/ref_rejected": -91.13772583007812, "logps/rejected": -95.44960021972656, "loss": 1.0586, "margin_dpo/margin_mean": 3.5929393768310547, "margin_dpo/margin_std": 4.356261253356934, "step": 78 }, { "KL/chosen_KL_mean": -0.8799591064453125, "KL/mean": -3.268341541290283, "KL/rejected_KL_mean": -5.656734466552734, "KL/std": 5.356631278991699, "epoch": 0.11600587371512482, "fcm_dpo/beta": 0.11956053972244263, "fcm_dpo/delta": -0.128304123878479, "fcm_dpo/margin": 4.776772499084473, "fcm_dpo/q_t": 0.37577077746391296, "grad_norm": 28.946622848510742, "learning_rate": 4.997332437005931e-07, "logits/chosen": -0.3763652741909027, "logits/rejected": 0.024601154029369354, "logps/chosen": -87.25828552246094, "logps/ref_chosen": -86.37832641601562, "logps/ref_rejected": -94.10777282714844, "logps/rejected": -99.7645034790039, "loss": 0.9994, "margin_dpo/margin_mean": 4.776772499084473, "margin_dpo/margin_std": 5.929773330688477, "step": 79 }, { "KL/chosen_KL_mean": -1.5266952514648438, "KL/mean": -3.4288413524627686, "KL/rejected_KL_mean": -5.330986022949219, "KL/std": 4.7091450691223145, "epoch": 0.11747430249632893, "fcm_dpo/beta": 0.11858350038528442, "fcm_dpo/delta": -0.001179170561954379, "fcm_dpo/margin": 3.8042891025543213, "fcm_dpo/q_t": 0.4012908935546875, "grad_norm": 28.268394470214844, "learning_rate": 4.996706849759452e-07, "logits/chosen": -0.5914753675460815, "logits/rejected": -0.2243952751159668, "logps/chosen": -95.49700927734375, "logps/ref_chosen": -93.97032165527344, "logps/ref_rejected": -92.57441711425781, "logps/rejected": -97.90540313720703, "loss": 1.0689, "margin_dpo/margin_mean": 3.804288864135742, "margin_dpo/margin_std": 5.219730377197266, "step": 80 }, { "KL/chosen_KL_mean": -1.0590744018554688, "KL/mean": -3.745607852935791, "KL/rejected_KL_mean": -6.432132720947266, "KL/std": 5.830990314483643, "epoch": 0.11894273127753303, "fcm_dpo/beta": 0.11468231678009033, "fcm_dpo/delta": -0.17923633754253387, "fcm_dpo/margin": 5.373061180114746, "fcm_dpo/q_t": 0.3717917203903198, "grad_norm": 29.017410278320312, "learning_rate": 4.996015471965529e-07, "logits/chosen": -0.5799692273139954, "logits/rejected": -0.1557232141494751, "logps/chosen": -100.88919830322266, "logps/ref_chosen": -99.83012390136719, "logps/ref_rejected": -133.67247009277344, "logps/rejected": -140.10459899902344, "loss": 0.9883, "margin_dpo/margin_mean": 5.373061180114746, "margin_dpo/margin_std": 6.918301582336426, "step": 81 }, { "KL/chosen_KL_mean": -1.9359970092773438, "KL/mean": -3.785255193710327, "KL/rejected_KL_mean": -5.6345062255859375, "KL/std": 4.574352264404297, "epoch": 0.12041116005873716, "fcm_dpo/beta": 0.11558075249195099, "fcm_dpo/delta": 0.022369179874658585, "fcm_dpo/margin": 3.6985130310058594, "fcm_dpo/q_t": 0.4048781394958496, "grad_norm": 28.42616081237793, "learning_rate": 4.995258321842611e-07, "logits/chosen": -0.1761595904827118, "logits/rejected": 0.04140507057309151, "logps/chosen": -84.98197937011719, "logps/ref_chosen": -83.04598236083984, "logps/ref_rejected": -94.52595520019531, "logps/rejected": -100.16046142578125, "loss": 1.0848, "margin_dpo/margin_mean": 3.698512554168701, "margin_dpo/margin_std": 5.055111885070801, "step": 82 }, { "KL/chosen_KL_mean": -2.3091964721679688, "KL/mean": -4.382638931274414, "KL/rejected_KL_mean": -6.456077575683594, "KL/std": 5.10935115814209, "epoch": 0.12187958883994127, "fcm_dpo/beta": 0.11439836025238037, "fcm_dpo/delta": -0.02563435398042202, "fcm_dpo/margin": 4.146886825561523, "fcm_dpo/q_t": 0.3988873362541199, "grad_norm": 30.49053382873535, "learning_rate": 4.994435419342304e-07, "logits/chosen": -0.5838781595230103, "logits/rejected": -0.16465207934379578, "logps/chosen": -94.48541259765625, "logps/ref_chosen": -92.17621612548828, "logps/ref_rejected": -107.74464416503906, "logps/rejected": -114.20072174072266, "loss": 1.0859, "margin_dpo/margin_mean": 4.146885871887207, "margin_dpo/margin_std": 6.455161094665527, "step": 83 }, { "KL/chosen_KL_mean": -2.8626060485839844, "KL/mean": -4.194946765899658, "KL/rejected_KL_mean": -5.527286529541016, "KL/std": 4.59609842300415, "epoch": 0.12334801762114538, "fcm_dpo/beta": 0.1138247549533844, "fcm_dpo/delta": -0.021521486341953278, "fcm_dpo/margin": 2.664675235748291, "fcm_dpo/q_t": 0.433022141456604, "grad_norm": 32.54677963256836, "learning_rate": 4.993546786148857e-07, "logits/chosen": -0.4822829067707062, "logits/rejected": -0.09913340210914612, "logps/chosen": -104.38909912109375, "logps/ref_chosen": -101.5264892578125, "logps/ref_rejected": -92.42608642578125, "logps/rejected": -97.953369140625, "loss": 1.1864, "margin_dpo/margin_mean": 2.66467547416687, "margin_dpo/margin_std": 4.930209159851074, "step": 84 }, { "KL/chosen_KL_mean": -2.5848312377929688, "KL/mean": -4.304393768310547, "KL/rejected_KL_mean": -6.023960113525391, "KL/std": 5.050421237945557, "epoch": 0.12481644640234948, "fcm_dpo/beta": 0.11529316753149033, "fcm_dpo/delta": 0.05501023307442665, "fcm_dpo/margin": 3.4391307830810547, "fcm_dpo/q_t": 0.41383618116378784, "grad_norm": 31.917598724365234, "learning_rate": 4.992592445678582e-07, "logits/chosen": -0.2992115318775177, "logits/rejected": 0.03511541709303856, "logps/chosen": -98.71220397949219, "logps/ref_chosen": -96.12738037109375, "logps/ref_rejected": -85.05519104003906, "logps/rejected": -91.07914733886719, "loss": 1.1438, "margin_dpo/margin_mean": 3.439131736755371, "margin_dpo/margin_std": 6.141880035400391, "step": 85 }, { "KL/chosen_KL_mean": -2.005725860595703, "KL/mean": -4.218888759613037, "KL/rejected_KL_mean": -6.432056427001953, "KL/std": 5.8354387283325195, "epoch": 0.1262848751835536, "fcm_dpo/beta": 0.1139456182718277, "fcm_dpo/delta": -0.05737065523862839, "fcm_dpo/margin": 4.426335334777832, "fcm_dpo/q_t": 0.3978298604488373, "grad_norm": 24.41836929321289, "learning_rate": 4.991572423079235e-07, "logits/chosen": -0.5035943984985352, "logits/rejected": -0.258251428604126, "logps/chosen": -83.70999145507812, "logps/ref_chosen": -81.70426177978516, "logps/ref_rejected": -93.65550231933594, "logps/rejected": -100.08755493164062, "loss": 1.0791, "margin_dpo/margin_mean": 4.426335334777832, "margin_dpo/margin_std": 7.23484992980957, "step": 86 }, { "KL/chosen_KL_mean": -2.282470703125, "KL/mean": -4.64469051361084, "KL/rejected_KL_mean": -7.006904602050781, "KL/std": 6.405066013336182, "epoch": 0.1277533039647577, "fcm_dpo/beta": 0.112020343542099, "fcm_dpo/delta": -0.08409902453422546, "fcm_dpo/margin": 4.724432945251465, "fcm_dpo/q_t": 0.38618338108062744, "grad_norm": 29.623733520507812, "learning_rate": 4.990486745229364e-07, "logits/chosen": -0.7301775217056274, "logits/rejected": -0.3559700846672058, "logps/chosen": -94.96843719482422, "logps/ref_chosen": -92.68596649169922, "logps/ref_rejected": -102.91818237304688, "logps/rejected": -109.92507934570312, "loss": 1.0774, "margin_dpo/margin_mean": 4.724432945251465, "margin_dpo/margin_std": 7.575509071350098, "step": 87 }, { "KL/chosen_KL_mean": -2.8142852783203125, "KL/mean": -4.718484878540039, "KL/rejected_KL_mean": -6.622676849365234, "KL/std": 6.107346534729004, "epoch": 0.12922173274596183, "fcm_dpo/beta": 0.11150763928890228, "fcm_dpo/delta": 0.0256040021777153, "fcm_dpo/margin": 3.8083901405334473, "fcm_dpo/q_t": 0.4089120030403137, "grad_norm": 27.321096420288086, "learning_rate": 4.989335440737586e-07, "logits/chosen": -0.1671774983406067, "logits/rejected": 0.020983021706342697, "logps/chosen": -103.5772705078125, "logps/ref_chosen": -100.76298522949219, "logps/ref_rejected": -113.15037536621094, "logps/rejected": -119.7730484008789, "loss": 1.1211, "margin_dpo/margin_mean": 3.808389902114868, "margin_dpo/margin_std": 6.349940299987793, "step": 88 }, { "KL/chosen_KL_mean": -2.3073387145996094, "KL/mean": -4.367650985717773, "KL/rejected_KL_mean": -6.4279632568359375, "KL/std": 5.265946388244629, "epoch": 0.13069016152716592, "fcm_dpo/beta": 0.11189480125904083, "fcm_dpo/delta": -0.011854719370603561, "fcm_dpo/margin": 4.120621204376221, "fcm_dpo/q_t": 0.4049159288406372, "grad_norm": 25.438533782958984, "learning_rate": 4.988118539941847e-07, "logits/chosen": -0.8401930332183838, "logits/rejected": -0.5166289806365967, "logps/chosen": -91.99842071533203, "logps/ref_chosen": -89.69108581542969, "logps/ref_rejected": -88.56832885742188, "logps/rejected": -94.99629211425781, "loss": 1.0967, "margin_dpo/margin_mean": 4.1206207275390625, "margin_dpo/margin_std": 6.960460186004639, "step": 89 }, { "KL/chosen_KL_mean": -2.1090965270996094, "KL/mean": -4.7426629066467285, "KL/rejected_KL_mean": -7.376224517822266, "KL/std": 6.103512763977051, "epoch": 0.13215859030837004, "fcm_dpo/beta": 0.11034879833459854, "fcm_dpo/delta": -0.13909685611724854, "fcm_dpo/margin": 5.2671403884887695, "fcm_dpo/q_t": 0.37826719880104065, "grad_norm": 27.8148136138916, "learning_rate": 4.986836074908615e-07, "logits/chosen": -0.35358595848083496, "logits/rejected": -0.2331031858921051, "logps/chosen": -83.49165344238281, "logps/ref_chosen": -81.38255310058594, "logps/ref_rejected": -117.77714538574219, "logps/rejected": -125.15336608886719, "loss": 1.0209, "margin_dpo/margin_mean": 5.267140865325928, "margin_dpo/margin_std": 7.285268783569336, "step": 90 }, { "KL/chosen_KL_mean": -2.2910118103027344, "KL/mean": -4.56092643737793, "KL/rejected_KL_mean": -6.830844879150391, "KL/std": 6.004371166229248, "epoch": 0.13362701908957417, "fcm_dpo/beta": 0.10842406004667282, "fcm_dpo/delta": -0.04439987987279892, "fcm_dpo/margin": 4.539830207824707, "fcm_dpo/q_t": 0.39736443758010864, "grad_norm": 28.05781364440918, "learning_rate": 4.985488079432037e-07, "logits/chosen": -0.3208407461643219, "logits/rejected": 0.09519103169441223, "logps/chosen": -99.51289367675781, "logps/ref_chosen": -97.22187805175781, "logps/ref_rejected": -92.97674560546875, "logps/rejected": -99.80758666992188, "loss": 1.0725, "margin_dpo/margin_mean": 4.539829730987549, "margin_dpo/margin_std": 6.988130569458008, "step": 91 }, { "KL/chosen_KL_mean": -2.587505340576172, "KL/mean": -4.551479816436768, "KL/rejected_KL_mean": -6.515453338623047, "KL/std": 5.919102668762207, "epoch": 0.13509544787077826, "fcm_dpo/beta": 0.10871409624814987, "fcm_dpo/delta": 0.023897580802440643, "fcm_dpo/margin": 3.9279491901397705, "fcm_dpo/q_t": 0.40844058990478516, "grad_norm": 25.167949676513672, "learning_rate": 4.984074589033043e-07, "logits/chosen": -0.8719907999038696, "logits/rejected": -0.512382447719574, "logps/chosen": -87.11778259277344, "logps/ref_chosen": -84.5302734375, "logps/ref_rejected": -84.5013198852539, "logps/rejected": -91.01676940917969, "loss": 1.1075, "margin_dpo/margin_mean": 3.9279487133026123, "margin_dpo/margin_std": 6.353695392608643, "step": 92 }, { "KL/chosen_KL_mean": -2.7571678161621094, "KL/mean": -4.692116737365723, "KL/rejected_KL_mean": -6.627071380615234, "KL/std": 5.639936447143555, "epoch": 0.13656387665198239, "fcm_dpo/beta": 0.1093287318944931, "fcm_dpo/delta": 0.02795940451323986, "fcm_dpo/margin": 3.869906425476074, "fcm_dpo/q_t": 0.4095788300037384, "grad_norm": 29.23712158203125, "learning_rate": 4.982595640958425e-07, "logits/chosen": -0.984565794467926, "logits/rejected": -0.43060898780822754, "logps/chosen": -93.00759887695312, "logps/ref_chosen": -90.25043487548828, "logps/ref_rejected": -84.0942153930664, "logps/rejected": -90.72128295898438, "loss": 1.1339, "margin_dpo/margin_mean": 3.869905948638916, "margin_dpo/margin_std": 6.868067741394043, "step": 93 }, { "KL/chosen_KL_mean": -2.8524551391601562, "KL/mean": -5.7169413566589355, "KL/rejected_KL_mean": -8.581424713134766, "KL/std": 6.534344673156738, "epoch": 0.13803230543318648, "fcm_dpo/beta": 0.10555772483348846, "fcm_dpo/delta": -0.16819366812705994, "fcm_dpo/margin": 5.72897481918335, "fcm_dpo/q_t": 0.3694302439689636, "grad_norm": 27.815052032470703, "learning_rate": 4.98105127417984e-07, "logits/chosen": -0.7305547595024109, "logits/rejected": -0.43267369270324707, "logps/chosen": -95.30667114257812, "logps/ref_chosen": -92.4542236328125, "logps/ref_rejected": -105.2472915649414, "logps/rejected": -113.82872009277344, "loss": 1.0087, "margin_dpo/margin_mean": 5.728974342346191, "margin_dpo/margin_std": 7.491952896118164, "step": 94 }, { "KL/chosen_KL_mean": -2.2870559692382812, "KL/mean": -4.574873447418213, "KL/rejected_KL_mean": -6.862682342529297, "KL/std": 5.51740837097168, "epoch": 0.1395007342143906, "fcm_dpo/beta": 0.10563249886035919, "fcm_dpo/delta": -0.0353727787733078, "fcm_dpo/margin": 4.575626850128174, "fcm_dpo/q_t": 0.3979508578777313, "grad_norm": 23.662145614624023, "learning_rate": 4.979441529392784e-07, "logits/chosen": -0.7484630942344666, "logits/rejected": -0.4083735942840576, "logps/chosen": -81.1607666015625, "logps/ref_chosen": -78.87370300292969, "logps/ref_rejected": -83.59121704101562, "logps/rejected": -90.45389556884766, "loss": 1.0736, "margin_dpo/margin_mean": 4.575626373291016, "margin_dpo/margin_std": 6.794055938720703, "step": 95 }, { "KL/chosen_KL_mean": -1.9295196533203125, "KL/mean": -4.763896465301514, "KL/rejected_KL_mean": -7.5982666015625, "KL/std": 6.609318733215332, "epoch": 0.14096916299559473, "fcm_dpo/beta": 0.10252609103918076, "fcm_dpo/delta": -0.1397034227848053, "fcm_dpo/margin": 5.668753623962402, "fcm_dpo/q_t": 0.37637150287628174, "grad_norm": 23.820087432861328, "learning_rate": 4.977766449015534e-07, "logits/chosen": -0.7242352962493896, "logits/rejected": -0.2868039608001709, "logps/chosen": -108.52168273925781, "logps/ref_chosen": -106.5921630859375, "logps/ref_rejected": -101.76802825927734, "logps/rejected": -109.36629486083984, "loss": 1.0104, "margin_dpo/margin_mean": 5.668753623962402, "margin_dpo/margin_std": 7.402196884155273, "step": 96 }, { "KL/chosen_KL_mean": -2.9581756591796875, "KL/mean": -5.535565376281738, "KL/rejected_KL_mean": -8.112953186035156, "KL/std": 5.711134433746338, "epoch": 0.14243759177679882, "fcm_dpo/beta": 0.10165230184793472, "fcm_dpo/delta": -0.07878708839416504, "fcm_dpo/margin": 5.154780387878418, "fcm_dpo/q_t": 0.38353899121284485, "grad_norm": 26.42591667175293, "learning_rate": 4.976026077188012e-07, "logits/chosen": -0.6754403114318848, "logits/rejected": -0.29090479016304016, "logps/chosen": -91.6380615234375, "logps/ref_chosen": -88.67988586425781, "logps/ref_rejected": -84.81229400634766, "logps/rejected": -92.92524719238281, "loss": 1.017, "margin_dpo/margin_mean": 5.15477991104126, "margin_dpo/margin_std": 5.9855170249938965, "step": 97 }, { "KL/chosen_KL_mean": -2.1783065795898438, "KL/mean": -5.075955390930176, "KL/rejected_KL_mean": -7.973602294921875, "KL/std": 5.512714385986328, "epoch": 0.14390602055800295, "fcm_dpo/beta": 0.09801796078681946, "fcm_dpo/delta": -0.12645207345485687, "fcm_dpo/margin": 5.795301914215088, "fcm_dpo/q_t": 0.37369024753570557, "grad_norm": 24.648876190185547, "learning_rate": 4.974220459770639e-07, "logits/chosen": -0.7000224590301514, "logits/rejected": -0.4411921799182892, "logps/chosen": -94.42080688476562, "logps/ref_chosen": -92.24249267578125, "logps/ref_rejected": -101.51948547363281, "logps/rejected": -109.49308776855469, "loss": 0.9792, "margin_dpo/margin_mean": 5.79530143737793, "margin_dpo/margin_std": 6.1865949630737305, "step": 98 }, { "KL/chosen_KL_mean": -1.6604843139648438, "KL/mean": -4.779559135437012, "KL/rejected_KL_mean": -7.8986358642578125, "KL/std": 6.573156833648682, "epoch": 0.14537444933920704, "fcm_dpo/beta": 0.0955786406993866, "fcm_dpo/delta": -0.15569189190864563, "fcm_dpo/margin": 6.238154411315918, "fcm_dpo/q_t": 0.37738341093063354, "grad_norm": 23.617198944091797, "learning_rate": 4.972349644343108e-07, "logits/chosen": -0.4310641884803772, "logits/rejected": -0.21981525421142578, "logps/chosen": -73.84513854980469, "logps/ref_chosen": -72.18464660644531, "logps/ref_rejected": -91.88131713867188, "logps/rejected": -99.77995300292969, "loss": 1.0005, "margin_dpo/margin_mean": 6.238154411315918, "margin_dpo/margin_std": 8.400813102722168, "step": 99 }, { "KL/chosen_KL_mean": -1.9705314636230469, "KL/mean": -4.247406005859375, "KL/rejected_KL_mean": -6.524280548095703, "KL/std": 4.973667144775391, "epoch": 0.14684287812041116, "fcm_dpo/beta": 0.09482486546039581, "fcm_dpo/delta": 0.018798498436808586, "fcm_dpo/margin": 4.553745269775391, "fcm_dpo/q_t": 0.4016741216182709, "grad_norm": 21.318092346191406, "learning_rate": 4.970413680203148e-07, "logits/chosen": -0.7095170021057129, "logits/rejected": -0.2382434904575348, "logps/chosen": -91.48435974121094, "logps/ref_chosen": -89.51382446289062, "logps/ref_rejected": -81.21713256835938, "logps/rejected": -87.74140930175781, "loss": 1.0581, "margin_dpo/margin_mean": 4.553744792938232, "margin_dpo/margin_std": 5.306511878967285, "step": 100 }, { "KL/chosen_KL_mean": -3.2380905151367188, "KL/mean": -5.267632961273193, "KL/rejected_KL_mean": -7.297176361083984, "KL/std": 6.405241012573242, "epoch": 0.14831130690161526, "fcm_dpo/beta": 0.09572748839855194, "fcm_dpo/delta": 0.0637625902891159, "fcm_dpo/margin": 4.059075832366943, "fcm_dpo/q_t": 0.41352343559265137, "grad_norm": 26.655681610107422, "learning_rate": 4.968412618365215e-07, "logits/chosen": -0.9653269052505493, "logits/rejected": -0.714327335357666, "logps/chosen": -97.64128112792969, "logps/ref_chosen": -94.4031982421875, "logps/ref_rejected": -89.91001892089844, "logps/rejected": -97.20719146728516, "loss": 1.1382, "margin_dpo/margin_mean": 4.059075355529785, "margin_dpo/margin_std": 6.960895538330078, "step": 101 }, { "KL/chosen_KL_mean": -3.7383384704589844, "KL/mean": -5.492098808288574, "KL/rejected_KL_mean": -7.245857238769531, "KL/std": 6.08369255065918, "epoch": 0.14977973568281938, "fcm_dpo/beta": 0.09616687148809433, "fcm_dpo/delta": -0.03382991999387741, "fcm_dpo/margin": 3.5075225830078125, "fcm_dpo/q_t": 0.4243978261947632, "grad_norm": 31.925813674926758, "learning_rate": 4.966346511559149e-07, "logits/chosen": -0.734827995300293, "logits/rejected": -0.3681231737136841, "logps/chosen": -95.34776306152344, "logps/ref_chosen": -91.60942077636719, "logps/ref_rejected": -75.7689208984375, "logps/rejected": -83.01477813720703, "loss": 1.1806, "margin_dpo/margin_mean": 3.507523536682129, "margin_dpo/margin_std": 6.865536689758301, "step": 102 }, { "KL/chosen_KL_mean": -2.8404159545898438, "KL/mean": -5.731388092041016, "KL/rejected_KL_mean": -8.622367858886719, "KL/std": 6.446306228637695, "epoch": 0.1512481644640235, "fcm_dpo/beta": 0.0942225307226181, "fcm_dpo/delta": -0.10055366903543472, "fcm_dpo/margin": 5.781953811645508, "fcm_dpo/q_t": 0.37916356325149536, "grad_norm": 25.041677474975586, "learning_rate": 4.964215414228785e-07, "logits/chosen": -0.7607011198997498, "logits/rejected": -0.3788653612136841, "logps/chosen": -100.2427978515625, "logps/ref_chosen": -97.40238952636719, "logps/ref_rejected": -98.981689453125, "logps/rejected": -107.60404968261719, "loss": 1.0197, "margin_dpo/margin_mean": 5.781954288482666, "margin_dpo/margin_std": 7.250732421875, "step": 103 }, { "KL/chosen_KL_mean": -2.1470069885253906, "KL/mean": -5.268563270568848, "KL/rejected_KL_mean": -8.390117645263672, "KL/std": 7.010969161987305, "epoch": 0.1527165932452276, "fcm_dpo/beta": 0.09195356070995331, "fcm_dpo/delta": -0.13192078471183777, "fcm_dpo/margin": 6.24310827255249, "fcm_dpo/q_t": 0.3788053095340729, "grad_norm": 20.705169677734375, "learning_rate": 4.96201938253052e-07, "logits/chosen": -0.9428690671920776, "logits/rejected": -0.6052085757255554, "logps/chosen": -97.00337219238281, "logps/ref_chosen": -94.85636138916016, "logps/ref_rejected": -97.02669525146484, "logps/rejected": -105.41680908203125, "loss": 1.006, "margin_dpo/margin_mean": 6.243109703063965, "margin_dpo/margin_std": 8.268304824829102, "step": 104 }, { "KL/chosen_KL_mean": -2.795276641845703, "KL/mean": -5.931557655334473, "KL/rejected_KL_mean": -9.06783676147461, "KL/std": 6.289705276489258, "epoch": 0.15418502202643172, "fcm_dpo/beta": 0.09026394784450531, "fcm_dpo/delta": -0.1230391189455986, "fcm_dpo/margin": 6.272557735443115, "fcm_dpo/q_t": 0.37542030215263367, "grad_norm": 23.900367736816406, "learning_rate": 4.959758474331832e-07, "logits/chosen": -0.8815001249313354, "logits/rejected": -0.41239750385284424, "logps/chosen": -99.55317687988281, "logps/ref_chosen": -96.75790405273438, "logps/ref_rejected": -101.5774917602539, "logps/rejected": -110.64532470703125, "loss": 1.0079, "margin_dpo/margin_mean": 6.272557735443115, "margin_dpo/margin_std": 7.820766448974609, "step": 105 }, { "KL/chosen_KL_mean": -2.7666282653808594, "KL/mean": -5.792070388793945, "KL/rejected_KL_mean": -8.817512512207031, "KL/std": 6.627781867980957, "epoch": 0.15565345080763582, "fcm_dpo/beta": 0.0884179174900055, "fcm_dpo/delta": -0.08976646512746811, "fcm_dpo/margin": 6.05088472366333, "fcm_dpo/q_t": 0.3848966062068939, "grad_norm": 21.478395462036133, "learning_rate": 4.957432749209755e-07, "logits/chosen": -0.8322691321372986, "logits/rejected": -0.41586387157440186, "logps/chosen": -95.20453643798828, "logps/ref_chosen": -92.43791198730469, "logps/ref_rejected": -91.00970458984375, "logps/rejected": -99.82720947265625, "loss": 1.025, "margin_dpo/margin_mean": 6.050884246826172, "margin_dpo/margin_std": 7.906839370727539, "step": 106 }, { "KL/chosen_KL_mean": -1.926666259765625, "KL/mean": -5.380865097045898, "KL/rejected_KL_mean": -8.835067749023438, "KL/std": 8.281155586242676, "epoch": 0.15712187958883994, "fcm_dpo/beta": 0.08573895692825317, "fcm_dpo/delta": -0.1512652188539505, "fcm_dpo/margin": 6.90839958190918, "fcm_dpo/q_t": 0.3809913694858551, "grad_norm": 21.756074905395508, "learning_rate": 4.955042268449307e-07, "logits/chosen": -0.6202389001846313, "logits/rejected": -0.10301902145147324, "logps/chosen": -107.05167388916016, "logps/ref_chosen": -105.12500762939453, "logps/ref_rejected": -101.1824951171875, "logps/rejected": -110.01756286621094, "loss": 1.0173, "margin_dpo/margin_mean": 6.908400058746338, "margin_dpo/margin_std": 10.279182434082031, "step": 107 }, { "KL/chosen_KL_mean": -3.0530052185058594, "KL/mean": -6.2047247886657715, "KL/rejected_KL_mean": -9.356449127197266, "KL/std": 7.492901802062988, "epoch": 0.15859030837004406, "fcm_dpo/beta": 0.08362379670143127, "fcm_dpo/delta": -0.08228491991758347, "fcm_dpo/margin": 6.303438186645508, "fcm_dpo/q_t": 0.38573557138442993, "grad_norm": 21.562427520751953, "learning_rate": 4.952587095041881e-07, "logits/chosen": -0.8234080672264099, "logits/rejected": -0.31772857904434204, "logps/chosen": -92.26236724853516, "logps/ref_chosen": -89.20936584472656, "logps/ref_rejected": -101.83784484863281, "logps/rejected": -111.19429779052734, "loss": 1.0301, "margin_dpo/margin_mean": 6.303438186645508, "margin_dpo/margin_std": 8.144371032714844, "step": 108 }, { "KL/chosen_KL_mean": -2.3106842041015625, "KL/mean": -6.337649345397949, "KL/rejected_KL_mean": -10.364620208740234, "KL/std": 7.792704105377197, "epoch": 0.16005873715124816, "fcm_dpo/beta": 0.08079753816127777, "fcm_dpo/delta": -0.21555772423744202, "fcm_dpo/margin": 8.053934097290039, "fcm_dpo/q_t": 0.36050570011138916, "grad_norm": 20.57724380493164, "learning_rate": 4.95006729368358e-07, "logits/chosen": -0.5087655186653137, "logits/rejected": -0.2829950153827667, "logps/chosen": -102.49777221679688, "logps/ref_chosen": -100.18708801269531, "logps/ref_rejected": -103.53739929199219, "logps/rejected": -113.90201568603516, "loss": 0.9607, "margin_dpo/margin_mean": 8.053934097290039, "margin_dpo/margin_std": 9.591377258300781, "step": 109 }, { "KL/chosen_KL_mean": -3.0492591857910156, "KL/mean": -5.415680408477783, "KL/rejected_KL_mean": -7.782108306884766, "KL/std": 6.225885391235352, "epoch": 0.16152716593245228, "fcm_dpo/beta": 0.08057047426700592, "fcm_dpo/delta": 0.0711936354637146, "fcm_dpo/margin": 4.732847690582275, "fcm_dpo/q_t": 0.4142294228076935, "grad_norm": 22.01658821105957, "learning_rate": 4.947482930773511e-07, "logits/chosen": -0.26160767674446106, "logits/rejected": 0.1310604214668274, "logps/chosen": -109.94497680664062, "logps/ref_chosen": -106.89572143554688, "logps/ref_rejected": -85.81120300292969, "logps/rejected": -93.59330749511719, "loss": 1.1182, "margin_dpo/margin_mean": 4.732847213745117, "margin_dpo/margin_std": 7.055660724639893, "step": 110 }, { "KL/chosen_KL_mean": -2.6614913940429688, "KL/mean": -7.284836292266846, "KL/rejected_KL_mean": -11.908180236816406, "KL/std": 8.842585563659668, "epoch": 0.16299559471365638, "fcm_dpo/beta": 0.07738985121250153, "fcm_dpo/delta": -0.2887975573539734, "fcm_dpo/margin": 9.246686935424805, "fcm_dpo/q_t": 0.3454093933105469, "grad_norm": 19.24679183959961, "learning_rate": 4.944834074412042e-07, "logits/chosen": -0.8856979608535767, "logits/rejected": -0.6032519340515137, "logps/chosen": -95.99085998535156, "logps/ref_chosen": -93.3293685913086, "logps/ref_rejected": -104.177490234375, "logps/rejected": -116.08566284179688, "loss": 0.9146, "margin_dpo/margin_mean": 9.246686935424805, "margin_dpo/margin_std": 9.710762977600098, "step": 111 }, { "KL/chosen_KL_mean": -3.026447296142578, "KL/mean": -5.440064430236816, "KL/rejected_KL_mean": -7.8536834716796875, "KL/std": 5.122225761413574, "epoch": 0.1644640234948605, "fcm_dpo/beta": 0.07755783945322037, "fcm_dpo/delta": 0.07860900461673737, "fcm_dpo/margin": 4.827238082885742, "fcm_dpo/q_t": 0.41483229398727417, "grad_norm": 19.261314392089844, "learning_rate": 4.942120794399002e-07, "logits/chosen": -1.0721787214279175, "logits/rejected": -0.5804400444030762, "logps/chosen": -85.77349090576172, "logps/ref_chosen": -82.74703979492188, "logps/ref_rejected": -71.5584487915039, "logps/rejected": -79.4121322631836, "loss": 1.1127, "margin_dpo/margin_mean": 4.8272385597229, "margin_dpo/margin_std": 6.858962059020996, "step": 112 }, { "KL/chosen_KL_mean": -3.902820587158203, "KL/mean": -6.444548606872559, "KL/rejected_KL_mean": -8.986282348632812, "KL/std": 6.719594478607178, "epoch": 0.16593245227606462, "fcm_dpo/beta": 0.07859328389167786, "fcm_dpo/delta": 0.05254785716533661, "fcm_dpo/margin": 5.083460807800293, "fcm_dpo/q_t": 0.40949755907058716, "grad_norm": 22.944673538208008, "learning_rate": 4.939343162231841e-07, "logits/chosen": -0.8270721435546875, "logits/rejected": -0.30553609132766724, "logps/chosen": -101.80231475830078, "logps/ref_chosen": -97.89949035644531, "logps/ref_rejected": -84.41960144042969, "logps/rejected": -93.4058837890625, "loss": 1.0913, "margin_dpo/margin_mean": 5.083460807800293, "margin_dpo/margin_std": 6.480808734893799, "step": 113 }, { "KL/chosen_KL_mean": -3.5971412658691406, "KL/mean": -7.166137218475342, "KL/rejected_KL_mean": -10.735137939453125, "KL/std": 7.941897869110107, "epoch": 0.16740088105726872, "fcm_dpo/beta": 0.07726357132196426, "fcm_dpo/delta": -0.1080031767487526, "fcm_dpo/margin": 7.137995719909668, "fcm_dpo/q_t": 0.3839123249053955, "grad_norm": 22.6549129486084, "learning_rate": 4.936501251103751e-07, "logits/chosen": -0.8440374135971069, "logits/rejected": -0.2608720660209656, "logps/chosen": -100.09225463867188, "logps/ref_chosen": -96.4951171875, "logps/ref_rejected": -92.01536560058594, "logps/rejected": -102.75050354003906, "loss": 1.0423, "margin_dpo/margin_mean": 7.137994766235352, "margin_dpo/margin_std": 10.222505569458008, "step": 114 }, { "KL/chosen_KL_mean": -3.5465164184570312, "KL/mean": -7.230834007263184, "KL/rejected_KL_mean": -10.915145874023438, "KL/std": 8.279781341552734, "epoch": 0.16886930983847284, "fcm_dpo/beta": 0.07585737854242325, "fcm_dpo/delta": -0.11531846225261688, "fcm_dpo/margin": 7.36862325668335, "fcm_dpo/q_t": 0.37747281789779663, "grad_norm": 19.451929092407227, "learning_rate": 4.933595135901732e-07, "logits/chosen": -0.8900731801986694, "logits/rejected": -0.4940794110298157, "logps/chosen": -106.72663879394531, "logps/ref_chosen": -103.18013000488281, "logps/ref_rejected": -105.71595764160156, "logps/rejected": -116.631103515625, "loss": 1.0032, "margin_dpo/margin_mean": 7.368623733520508, "margin_dpo/margin_std": 8.848615646362305, "step": 115 }, { "KL/chosen_KL_mean": -3.2444305419921875, "KL/mean": -6.25869083404541, "KL/rejected_KL_mean": -9.272945404052734, "KL/std": 6.523632049560547, "epoch": 0.17033773861967694, "fcm_dpo/beta": 0.07550537586212158, "fcm_dpo/delta": -0.0056202951818704605, "fcm_dpo/margin": 6.028512954711914, "fcm_dpo/q_t": 0.3978078067302704, "grad_norm": 18.775808334350586, "learning_rate": 4.930624893204624e-07, "logits/chosen": -0.978033185005188, "logits/rejected": -0.5550253391265869, "logps/chosen": -84.40280151367188, "logps/ref_chosen": -81.15837097167969, "logps/ref_rejected": -86.30586242675781, "logps/rejected": -95.57880401611328, "loss": 1.056, "margin_dpo/margin_mean": 6.028513431549072, "margin_dpo/margin_std": 7.502448558807373, "step": 116 }, { "KL/chosen_KL_mean": -4.012409210205078, "KL/mean": -6.524020671844482, "KL/rejected_KL_mean": -9.035633087158203, "KL/std": 6.570239067077637, "epoch": 0.17180616740088106, "fcm_dpo/beta": 0.0756017416715622, "fcm_dpo/delta": 0.07268474996089935, "fcm_dpo/margin": 5.023226737976074, "fcm_dpo/q_t": 0.41434288024902344, "grad_norm": 26.403039932250977, "learning_rate": 4.927590601281083e-07, "logits/chosen": -0.5369125008583069, "logits/rejected": -0.12467953562736511, "logps/chosen": -101.09336853027344, "logps/ref_chosen": -97.0809555053711, "logps/ref_rejected": -75.1399154663086, "logps/rejected": -84.17555236816406, "loss": 1.1271, "margin_dpo/margin_mean": 5.023226261138916, "margin_dpo/margin_std": 7.87533712387085, "step": 117 }, { "KL/chosen_KL_mean": -2.8037071228027344, "KL/mean": -6.297224998474121, "KL/rejected_KL_mean": -9.790740966796875, "KL/std": 7.187756538391113, "epoch": 0.17327459618208516, "fcm_dpo/beta": 0.07533556967973709, "fcm_dpo/delta": -0.08075873553752899, "fcm_dpo/margin": 6.987035751342773, "fcm_dpo/q_t": 0.3795852065086365, "grad_norm": 18.19528579711914, "learning_rate": 4.924492340087524e-07, "logits/chosen": -0.9829877614974976, "logits/rejected": -0.6505037546157837, "logps/chosen": -89.3399429321289, "logps/ref_chosen": -86.53623962402344, "logps/ref_rejected": -82.93563842773438, "logps/rejected": -92.72637939453125, "loss": 0.9982, "margin_dpo/margin_mean": 6.987035751342773, "margin_dpo/margin_std": 7.422086238861084, "step": 118 }, { "KL/chosen_KL_mean": -4.230094909667969, "KL/mean": -7.863368034362793, "KL/rejected_KL_mean": -11.496639251708984, "KL/std": 8.438607215881348, "epoch": 0.17474302496328928, "fcm_dpo/beta": 0.07391680032014847, "fcm_dpo/delta": -0.09241504222154617, "fcm_dpo/margin": 7.266541957855225, "fcm_dpo/q_t": 0.3817199468612671, "grad_norm": 20.986663818359375, "learning_rate": 4.92133019126601e-07, "logits/chosen": -1.16860830783844, "logits/rejected": -0.8600578308105469, "logps/chosen": -108.83917236328125, "logps/ref_chosen": -104.60908508300781, "logps/ref_rejected": -109.15780639648438, "logps/rejected": -120.65444946289062, "loss": 1.0506, "margin_dpo/margin_mean": 7.266542434692383, "margin_dpo/margin_std": 10.496493339538574, "step": 119 }, { "KL/chosen_KL_mean": -3.249011993408203, "KL/mean": -8.116844177246094, "KL/rejected_KL_mean": -12.984676361083984, "KL/std": 10.410438537597656, "epoch": 0.1762114537444934, "fcm_dpo/beta": 0.07091405242681503, "fcm_dpo/delta": -0.2584834694862366, "fcm_dpo/margin": 9.735661506652832, "fcm_dpo/q_t": 0.3579162359237671, "grad_norm": 23.868812561035156, "learning_rate": 4.918104238142103e-07, "logits/chosen": -1.3024057149887085, "logits/rejected": -0.7799468040466309, "logps/chosen": -117.93643951416016, "logps/ref_chosen": -114.68742370605469, "logps/ref_rejected": -111.18586730957031, "logps/rejected": -124.17054748535156, "loss": 0.9846, "margin_dpo/margin_mean": 9.735660552978516, "margin_dpo/margin_std": 13.09634780883789, "step": 120 }, { "KL/chosen_KL_mean": -2.951152801513672, "KL/mean": -7.305971145629883, "KL/rejected_KL_mean": -11.660797119140625, "KL/std": 8.056767463684082, "epoch": 0.1776798825256975, "fcm_dpo/beta": 0.06817714869976044, "fcm_dpo/delta": -0.1538236141204834, "fcm_dpo/margin": 8.709644317626953, "fcm_dpo/q_t": 0.3668941259384155, "grad_norm": 20.163454055786133, "learning_rate": 4.91481456572267e-07, "logits/chosen": -1.070889949798584, "logits/rejected": -0.749244213104248, "logps/chosen": -95.06781005859375, "logps/ref_chosen": -92.11666107177734, "logps/ref_rejected": -114.972900390625, "logps/rejected": -126.63369750976562, "loss": 0.9593, "margin_dpo/margin_mean": 8.70964527130127, "margin_dpo/margin_std": 8.971741676330566, "step": 121 }, { "KL/chosen_KL_mean": -3.3295440673828125, "KL/mean": -8.211607933044434, "KL/rejected_KL_mean": -13.093673706054688, "KL/std": 8.696558952331543, "epoch": 0.17914831130690162, "fcm_dpo/beta": 0.06610635668039322, "fcm_dpo/delta": -0.20845989882946014, "fcm_dpo/margin": 9.764131546020508, "fcm_dpo/q_t": 0.3557118773460388, "grad_norm": 20.708974838256836, "learning_rate": 4.911461260693638e-07, "logits/chosen": -0.9383471012115479, "logits/rejected": -0.8294496536254883, "logps/chosen": -88.20711517333984, "logps/ref_chosen": -84.87757110595703, "logps/ref_rejected": -111.0829849243164, "logps/rejected": -124.1766586303711, "loss": 0.9401, "margin_dpo/margin_mean": 9.764131546020508, "margin_dpo/margin_std": 10.03492546081543, "step": 122 }, { "KL/chosen_KL_mean": -3.7645301818847656, "KL/mean": -7.370052337646484, "KL/rejected_KL_mean": -10.975570678710938, "KL/std": 8.020940780639648, "epoch": 0.18061674008810572, "fcm_dpo/beta": 0.06455057859420776, "fcm_dpo/delta": -0.017208658158779144, "fcm_dpo/margin": 7.211041450500488, "fcm_dpo/q_t": 0.39527085423469543, "grad_norm": 17.73777198791504, "learning_rate": 4.908044411417711e-07, "logits/chosen": -1.1302647590637207, "logits/rejected": -0.8233848810195923, "logps/chosen": -102.24327850341797, "logps/ref_chosen": -98.47874450683594, "logps/ref_rejected": -93.85417175292969, "logps/rejected": -104.82974243164062, "loss": 1.056, "margin_dpo/margin_mean": 7.211041450500488, "margin_dpo/margin_std": 9.136102676391602, "step": 123 }, { "KL/chosen_KL_mean": -3.4926681518554688, "KL/mean": -7.897808074951172, "KL/rejected_KL_mean": -12.30295181274414, "KL/std": 10.150493621826172, "epoch": 0.18208516886930984, "fcm_dpo/beta": 0.06381617486476898, "fcm_dpo/delta": -0.11886176466941833, "fcm_dpo/margin": 8.810285568237305, "fcm_dpo/q_t": 0.3854610323905945, "grad_norm": 18.02562141418457, "learning_rate": 4.904564107932048e-07, "logits/chosen": -0.7194161415100098, "logits/rejected": -0.4542362093925476, "logps/chosen": -106.25556945800781, "logps/ref_chosen": -102.76290130615234, "logps/ref_rejected": -123.005615234375, "logps/rejected": -135.30856323242188, "loss": 1.041, "margin_dpo/margin_mean": 8.810285568237305, "margin_dpo/margin_std": 13.129676818847656, "step": 124 }, { "KL/chosen_KL_mean": -4.987255096435547, "KL/mean": -8.224781036376953, "KL/rejected_KL_mean": -11.462310791015625, "KL/std": 8.092391967773438, "epoch": 0.18355359765051396, "fcm_dpo/beta": 0.0637354701757431, "fcm_dpo/delta": 0.03892642632126808, "fcm_dpo/margin": 6.4750494956970215, "fcm_dpo/q_t": 0.40852275490760803, "grad_norm": 18.638343811035156, "learning_rate": 4.90102044194588e-07, "logits/chosen": -0.6182900667190552, "logits/rejected": -0.40189093351364136, "logps/chosen": -95.41352844238281, "logps/ref_chosen": -90.42627716064453, "logps/ref_rejected": -89.24901580810547, "logps/rejected": -100.7113265991211, "loss": 1.1068, "margin_dpo/margin_mean": 6.4750494956970215, "margin_dpo/margin_std": 9.89763355255127, "step": 125 }, { "KL/chosen_KL_mean": -3.870624542236328, "KL/mean": -8.586780548095703, "KL/rejected_KL_mean": -13.302947998046875, "KL/std": 8.751096725463867, "epoch": 0.18502202643171806, "fcm_dpo/beta": 0.06251658499240875, "fcm_dpo/delta": -0.14837688207626343, "fcm_dpo/margin": 9.432323455810547, "fcm_dpo/q_t": 0.3651265799999237, "grad_norm": 17.993650436401367, "learning_rate": 4.897413506838102e-07, "logits/chosen": -0.7394363284111023, "logits/rejected": -0.45875149965286255, "logps/chosen": -98.29350280761719, "logps/ref_chosen": -94.42288208007812, "logps/ref_rejected": -103.88156127929688, "logps/rejected": -117.18450927734375, "loss": 0.9524, "margin_dpo/margin_mean": 9.432323455810547, "margin_dpo/margin_std": 9.009821891784668, "step": 126 }, { "KL/chosen_KL_mean": -3.8602333068847656, "KL/mean": -7.447772979736328, "KL/rejected_KL_mean": -11.03531265258789, "KL/std": 8.205819129943848, "epoch": 0.18649045521292218, "fcm_dpo/beta": 0.06226357817649841, "fcm_dpo/delta": 0.003141850233078003, "fcm_dpo/margin": 7.175076484680176, "fcm_dpo/q_t": 0.4015953242778778, "grad_norm": 18.24762535095215, "learning_rate": 4.89374339765481e-07, "logits/chosen": -0.9668242335319519, "logits/rejected": -0.6396434903144836, "logps/chosen": -88.61128234863281, "logps/ref_chosen": -84.75105285644531, "logps/ref_rejected": -83.54523468017578, "logps/rejected": -94.58055114746094, "loss": 1.0801, "margin_dpo/margin_mean": 7.175076484680176, "margin_dpo/margin_std": 10.24130916595459, "step": 127 }, { "KL/chosen_KL_mean": -3.8696937561035156, "KL/mean": -7.551077365875244, "KL/rejected_KL_mean": -11.232461929321289, "KL/std": 9.975054740905762, "epoch": 0.18795888399412627, "fcm_dpo/beta": 0.06226147711277008, "fcm_dpo/delta": -0.009228337556123734, "fcm_dpo/margin": 7.362769603729248, "fcm_dpo/q_t": 0.39746347069740295, "grad_norm": 20.354082107543945, "learning_rate": 4.890010211106795e-07, "logits/chosen": -0.7365471124649048, "logits/rejected": -0.28447097539901733, "logps/chosen": -94.48933410644531, "logps/ref_chosen": -90.61964416503906, "logps/ref_rejected": -83.10347747802734, "logps/rejected": -94.3359375, "loss": 1.0974, "margin_dpo/margin_mean": 7.362770080566406, "margin_dpo/margin_std": 12.026878356933594, "step": 128 }, { "KL/chosen_KL_mean": -4.365577697753906, "KL/mean": -8.200631141662598, "KL/rejected_KL_mean": -12.035690307617188, "KL/std": 8.236621856689453, "epoch": 0.1894273127753304, "fcm_dpo/beta": 0.06190914288163185, "fcm_dpo/delta": -0.026235422119498253, "fcm_dpo/margin": 7.670107364654541, "fcm_dpo/q_t": 0.39817970991134644, "grad_norm": 16.664791107177734, "learning_rate": 4.88621404556699e-07, "logits/chosen": -0.7015722990036011, "logits/rejected": -0.3987889587879181, "logps/chosen": -103.28428649902344, "logps/ref_chosen": -98.91870880126953, "logps/ref_rejected": -103.28419494628906, "logps/rejected": -115.31988525390625, "loss": 1.0725, "margin_dpo/margin_mean": 7.670106887817383, "margin_dpo/margin_std": 10.961736679077148, "step": 129 }, { "KL/chosen_KL_mean": -3.4742355346679688, "KL/mean": -8.451114654541016, "KL/rejected_KL_mean": -13.427993774414062, "KL/std": 9.836666107177734, "epoch": 0.19089574155653452, "fcm_dpo/beta": 0.06065458431839943, "fcm_dpo/delta": -0.16370603442192078, "fcm_dpo/margin": 9.953760147094727, "fcm_dpo/q_t": 0.3699725866317749, "grad_norm": 15.890512466430664, "learning_rate": 4.882355001067891e-07, "logits/chosen": -1.00321364402771, "logits/rejected": -0.7537674307823181, "logps/chosen": -76.65792846679688, "logps/ref_chosen": -73.18368530273438, "logps/ref_rejected": -88.02154541015625, "logps/rejected": -101.44953918457031, "loss": 0.9811, "margin_dpo/margin_mean": 9.953760147094727, "margin_dpo/margin_std": 11.440141677856445, "step": 130 }, { "KL/chosen_KL_mean": -2.8554420471191406, "KL/mean": -7.318256378173828, "KL/rejected_KL_mean": -11.781074523925781, "KL/std": 9.620319366455078, "epoch": 0.19236417033773862, "fcm_dpo/beta": 0.059274956583976746, "fcm_dpo/delta": -0.08378984034061432, "fcm_dpo/margin": 8.92563533782959, "fcm_dpo/q_t": 0.3831636905670166, "grad_norm": 17.686756134033203, "learning_rate": 4.878433179298909e-07, "logits/chosen": -1.0559083223342896, "logits/rejected": -0.8412282466888428, "logps/chosen": -76.8118896484375, "logps/ref_chosen": -73.9564437866211, "logps/ref_rejected": -94.45582580566406, "logps/rejected": -106.23690032958984, "loss": 1.0228, "margin_dpo/margin_mean": 8.92563533782959, "margin_dpo/margin_std": 10.949589729309082, "step": 131 }, { "KL/chosen_KL_mean": -4.320400238037109, "KL/mean": -8.87709903717041, "KL/rejected_KL_mean": -13.433792114257812, "KL/std": 10.098474502563477, "epoch": 0.19383259911894274, "fcm_dpo/beta": 0.05771227926015854, "fcm_dpo/delta": -0.080808125436306, "fcm_dpo/margin": 9.113384246826172, "fcm_dpo/q_t": 0.3815780580043793, "grad_norm": 18.335140228271484, "learning_rate": 4.874448683603694e-07, "logits/chosen": -1.1837594509124756, "logits/rejected": -0.7761355042457581, "logps/chosen": -96.7598876953125, "logps/ref_chosen": -92.43948364257812, "logps/ref_rejected": -94.13252258300781, "logps/rejected": -107.56631469726562, "loss": 1.0207, "margin_dpo/margin_mean": 9.113384246826172, "margin_dpo/margin_std": 11.179327964782715, "step": 132 }, { "KL/chosen_KL_mean": -3.47509765625, "KL/mean": -8.17091178894043, "KL/rejected_KL_mean": -12.86672592163086, "KL/std": 10.175192832946777, "epoch": 0.19530102790014683, "fcm_dpo/beta": 0.0570124089717865, "fcm_dpo/delta": -0.09018716216087341, "fcm_dpo/margin": 9.391630172729492, "fcm_dpo/q_t": 0.3806670606136322, "grad_norm": 16.736326217651367, "learning_rate": 4.870401618977415e-07, "logits/chosen": -1.52720308303833, "logits/rejected": -1.1478118896484375, "logps/chosen": -96.59175872802734, "logps/ref_chosen": -93.11666107177734, "logps/ref_rejected": -102.09490966796875, "logps/rejected": -114.96163940429688, "loss": 1.0005, "margin_dpo/margin_mean": 9.391630172729492, "margin_dpo/margin_std": 10.577903747558594, "step": 133 }, { "KL/chosen_KL_mean": -3.762378692626953, "KL/mean": -8.813262939453125, "KL/rejected_KL_mean": -13.864151000976562, "KL/std": 9.70240592956543, "epoch": 0.19676945668135096, "fcm_dpo/beta": 0.05603744089603424, "fcm_dpo/delta": -0.1231902688741684, "fcm_dpo/margin": 10.10176944732666, "fcm_dpo/q_t": 0.3723108172416687, "grad_norm": 15.872366905212402, "learning_rate": 4.866292092063986e-07, "logits/chosen": -1.6060965061187744, "logits/rejected": -1.2849180698394775, "logps/chosen": -87.3125, "logps/ref_chosen": -83.55012512207031, "logps/ref_rejected": -93.8815689086914, "logps/rejected": -107.74571990966797, "loss": 0.9789, "margin_dpo/margin_mean": 10.101770401000977, "margin_dpo/margin_std": 10.374430656433105, "step": 134 }, { "KL/chosen_KL_mean": -3.5816001892089844, "KL/mean": -8.457603454589844, "KL/rejected_KL_mean": -13.333610534667969, "KL/std": 10.658963203430176, "epoch": 0.19823788546255505, "fcm_dpo/beta": 0.054719895124435425, "fcm_dpo/delta": -0.08825526386499405, "fcm_dpo/margin": 9.752004623413086, "fcm_dpo/q_t": 0.3835597038269043, "grad_norm": 21.14170265197754, "learning_rate": 4.862120211153265e-07, "logits/chosen": -1.071568489074707, "logits/rejected": -0.942071795463562, "logps/chosen": -81.16423034667969, "logps/ref_chosen": -77.58262634277344, "logps/ref_rejected": -121.05648040771484, "logps/rejected": -134.3900909423828, "loss": 1.015, "margin_dpo/margin_mean": 9.752005577087402, "margin_dpo/margin_std": 11.867852210998535, "step": 135 }, { "KL/chosen_KL_mean": -3.9735069274902344, "KL/mean": -8.918501853942871, "KL/rejected_KL_mean": -13.86349868774414, "KL/std": 10.705445289611816, "epoch": 0.19970631424375918, "fcm_dpo/beta": 0.05359436944127083, "fcm_dpo/delta": -0.08462314307689667, "fcm_dpo/margin": 9.889991760253906, "fcm_dpo/q_t": 0.37985050678253174, "grad_norm": 15.652995109558105, "learning_rate": 4.857886086178193e-07, "logits/chosen": -1.3203887939453125, "logits/rejected": -0.8870661854743958, "logps/chosen": -98.146240234375, "logps/ref_chosen": -94.1727294921875, "logps/ref_rejected": -103.26428985595703, "logps/rejected": -117.12779235839844, "loss": 0.9944, "margin_dpo/margin_mean": 9.889989852905273, "margin_dpo/margin_std": 10.464974403381348, "step": 136 }, { "KL/chosen_KL_mean": -3.146820068359375, "KL/mean": -10.815319061279297, "KL/rejected_KL_mean": -18.483814239501953, "KL/std": 13.656440734863281, "epoch": 0.2011747430249633, "fcm_dpo/beta": 0.05032534897327423, "fcm_dpo/delta": -0.35249799489974976, "fcm_dpo/margin": 15.336999893188477, "fcm_dpo/q_t": 0.3331122398376465, "grad_norm": 18.101778030395508, "learning_rate": 4.853589828711902e-07, "logits/chosen": -1.2057225704193115, "logits/rejected": -0.9818614721298218, "logps/chosen": -81.26556396484375, "logps/ref_chosen": -78.11874389648438, "logps/ref_rejected": -117.92237091064453, "logps/rejected": -136.40618896484375, "loss": 0.8664, "margin_dpo/margin_mean": 15.336999893188477, "margin_dpo/margin_std": 14.805900573730469, "step": 137 }, { "KL/chosen_KL_mean": -5.702308654785156, "KL/mean": -10.494245529174805, "KL/rejected_KL_mean": -15.286182403564453, "KL/std": 10.31187629699707, "epoch": 0.2026431718061674, "fcm_dpo/beta": 0.04949317127466202, "fcm_dpo/delta": -0.02566656284034252, "fcm_dpo/margin": 9.583876609802246, "fcm_dpo/q_t": 0.3929169178009033, "grad_norm": 14.387374877929688, "learning_rate": 4.849231551964771e-07, "logits/chosen": -0.6397614479064941, "logits/rejected": -0.3746693730354309, "logps/chosen": -98.53225708007812, "logps/ref_chosen": -92.82994079589844, "logps/ref_rejected": -98.64840698242188, "logps/rejected": -113.93458557128906, "loss": 1.0379, "margin_dpo/margin_mean": 9.583876609802246, "margin_dpo/margin_std": 11.318662643432617, "step": 138 }, { "KL/chosen_KL_mean": -3.7344131469726562, "KL/mean": -9.354459762573242, "KL/rejected_KL_mean": -14.974510192871094, "KL/std": 11.291858673095703, "epoch": 0.20411160058737152, "fcm_dpo/beta": 0.048662759363651276, "fcm_dpo/delta": -0.10290683060884476, "fcm_dpo/margin": 11.240102767944336, "fcm_dpo/q_t": 0.3744027018547058, "grad_norm": 14.602463722229004, "learning_rate": 4.844811370781446e-07, "logits/chosen": -1.3200509548187256, "logits/rejected": -1.003678560256958, "logps/chosen": -82.7374267578125, "logps/ref_chosen": -79.00301361083984, "logps/ref_rejected": -85.82733154296875, "logps/rejected": -100.80184173583984, "loss": 1.0031, "margin_dpo/margin_mean": 11.240104675292969, "margin_dpo/margin_std": 13.213805198669434, "step": 139 }, { "KL/chosen_KL_mean": -4.216518402099609, "KL/mean": -9.270885467529297, "KL/rejected_KL_mean": -14.325252532958984, "KL/std": 10.672473907470703, "epoch": 0.2055800293685756, "fcm_dpo/beta": 0.04824919253587723, "fcm_dpo/delta": -0.039659008383750916, "fcm_dpo/margin": 10.108734130859375, "fcm_dpo/q_t": 0.3884885907173157, "grad_norm": 14.080016136169434, "learning_rate": 4.840329401637809e-07, "logits/chosen": -1.6188607215881348, "logits/rejected": -1.2256109714508057, "logps/chosen": -92.5689697265625, "logps/ref_chosen": -88.35244750976562, "logps/ref_rejected": -90.57476806640625, "logps/rejected": -104.9000244140625, "loss": 1.0231, "margin_dpo/margin_mean": 10.108734130859375, "margin_dpo/margin_std": 11.146553993225098, "step": 140 }, { "KL/chosen_KL_mean": -5.553092956542969, "KL/mean": -11.420736312866211, "KL/rejected_KL_mean": -17.288379669189453, "KL/std": 13.659140586853027, "epoch": 0.20704845814977973, "fcm_dpo/beta": 0.047365203499794006, "fcm_dpo/delta": -0.11200984567403793, "fcm_dpo/margin": 11.735284805297852, "fcm_dpo/q_t": 0.3748253881931305, "grad_norm": 16.944189071655273, "learning_rate": 4.83578576263792e-07, "logits/chosen": -1.092730164527893, "logits/rejected": -0.704372227191925, "logps/chosen": -112.77325439453125, "logps/ref_chosen": -107.22016143798828, "logps/ref_rejected": -104.71223449707031, "logps/rejected": -122.0006103515625, "loss": 0.9987, "margin_dpo/margin_mean": 11.735283851623535, "margin_dpo/margin_std": 13.693428039550781, "step": 141 }, { "KL/chosen_KL_mean": -6.127674102783203, "KL/mean": -11.769968032836914, "KL/rejected_KL_mean": -17.412261962890625, "KL/std": 12.742448806762695, "epoch": 0.20851688693098386, "fcm_dpo/beta": 0.0464097335934639, "fcm_dpo/delta": -0.07808461785316467, "fcm_dpo/margin": 11.284584045410156, "fcm_dpo/q_t": 0.3839187026023865, "grad_norm": 14.27704906463623, "learning_rate": 4.83118057351089e-07, "logits/chosen": -1.030910849571228, "logits/rejected": -0.7692275047302246, "logps/chosen": -94.85662078857422, "logps/ref_chosen": -88.72894287109375, "logps/ref_rejected": -99.73038482666016, "logps/rejected": -117.14264678955078, "loss": 1.0091, "margin_dpo/margin_mean": 11.284584045410156, "margin_dpo/margin_std": 13.189620971679688, "step": 142 }, { "KL/chosen_KL_mean": -4.1996307373046875, "KL/mean": -8.925056457519531, "KL/rejected_KL_mean": -13.650474548339844, "KL/std": 10.892311096191406, "epoch": 0.20998531571218795, "fcm_dpo/beta": 0.04621533304452896, "fcm_dpo/delta": 0.013468053191900253, "fcm_dpo/margin": 9.450836181640625, "fcm_dpo/q_t": 0.4005667269229889, "grad_norm": 14.099587440490723, "learning_rate": 4.826513955607734e-07, "logits/chosen": -1.5993086099624634, "logits/rejected": -1.1545026302337646, "logps/chosen": -89.70420837402344, "logps/ref_chosen": -85.50457763671875, "logps/ref_rejected": -85.8056640625, "logps/rejected": -99.45613098144531, "loss": 1.0679, "margin_dpo/margin_mean": 9.450836181640625, "margin_dpo/margin_std": 11.928359985351562, "step": 143 }, { "KL/chosen_KL_mean": -5.068080902099609, "KL/mean": -9.908295631408691, "KL/rejected_KL_mean": -14.748512268066406, "KL/std": 11.455781936645508, "epoch": 0.21145374449339208, "fcm_dpo/beta": 0.04674536734819412, "fcm_dpo/delta": -0.0035240501165390015, "fcm_dpo/margin": 9.680429458618164, "fcm_dpo/q_t": 0.39962923526763916, "grad_norm": 18.266056060791016, "learning_rate": 4.821786031898176e-07, "logits/chosen": -1.4986419677734375, "logits/rejected": -0.9835749864578247, "logps/chosen": -109.75517272949219, "logps/ref_chosen": -104.68708801269531, "logps/ref_rejected": -87.71908569335938, "logps/rejected": -102.46759033203125, "loss": 1.085, "margin_dpo/margin_mean": 9.680429458618164, "margin_dpo/margin_std": 14.155384063720703, "step": 144 }, { "KL/chosen_KL_mean": -4.3048095703125, "KL/mean": -9.987184524536133, "KL/rejected_KL_mean": -15.669563293457031, "KL/std": 11.09414291381836, "epoch": 0.21292217327459617, "fcm_dpo/beta": 0.045895203948020935, "fcm_dpo/delta": -0.07554689049720764, "fcm_dpo/margin": 11.364750862121582, "fcm_dpo/q_t": 0.3813377916812897, "grad_norm": 14.497042655944824, "learning_rate": 4.816996926967401e-07, "logits/chosen": -1.2646183967590332, "logits/rejected": -0.707251787185669, "logps/chosen": -92.82469940185547, "logps/ref_chosen": -88.51988983154297, "logps/ref_rejected": -83.76513671875, "logps/rejected": -99.43470764160156, "loss": 1.0046, "margin_dpo/margin_mean": 11.364751815795898, "margin_dpo/margin_std": 12.453365325927734, "step": 145 }, { "KL/chosen_KL_mean": -4.763313293457031, "KL/mean": -11.054548263549805, "KL/rejected_KL_mean": -17.345783233642578, "KL/std": 12.090798377990723, "epoch": 0.2143906020558003, "fcm_dpo/beta": 0.044664401561021805, "fcm_dpo/delta": -0.12001601606607437, "fcm_dpo/margin": 12.582468032836914, "fcm_dpo/q_t": 0.3720228672027588, "grad_norm": 14.317770004272461, "learning_rate": 4.812146767012779e-07, "logits/chosen": -1.3493354320526123, "logits/rejected": -0.7054879665374756, "logps/chosen": -107.66917419433594, "logps/ref_chosen": -102.90586853027344, "logps/ref_rejected": -88.9493637084961, "logps/rejected": -106.29515075683594, "loss": 0.9807, "margin_dpo/margin_mean": 12.582468032836914, "margin_dpo/margin_std": 13.1751708984375, "step": 146 }, { "KL/chosen_KL_mean": -4.699851989746094, "KL/mean": -11.160320281982422, "KL/rejected_KL_mean": -17.62078857421875, "KL/std": 11.679289817810059, "epoch": 0.21585903083700442, "fcm_dpo/beta": 0.04384419322013855, "fcm_dpo/delta": -0.1234760582447052, "fcm_dpo/margin": 12.920938491821289, "fcm_dpo/q_t": 0.3704032301902771, "grad_norm": 13.309486389160156, "learning_rate": 4.807235679840536e-07, "logits/chosen": -1.292638897895813, "logits/rejected": -0.7031066417694092, "logps/chosen": -93.9620361328125, "logps/ref_chosen": -89.26219177246094, "logps/ref_rejected": -76.50011444091797, "logps/rejected": -94.12090301513672, "loss": 0.9648, "margin_dpo/margin_mean": 12.920938491821289, "margin_dpo/margin_std": 12.50632095336914, "step": 147 }, { "KL/chosen_KL_mean": -3.066070556640625, "KL/mean": -8.936927795410156, "KL/rejected_KL_mean": -14.807792663574219, "KL/std": 11.961725234985352, "epoch": 0.2173274596182085, "fcm_dpo/beta": 0.04350690543651581, "fcm_dpo/delta": -0.06505133211612701, "fcm_dpo/margin": 11.741720199584961, "fcm_dpo/q_t": 0.3875482678413391, "grad_norm": 14.381302833557129, "learning_rate": 4.802263794862384e-07, "logits/chosen": -1.5446524620056152, "logits/rejected": -1.1513751745224, "logps/chosen": -94.82894897460938, "logps/ref_chosen": -91.76287841796875, "logps/ref_rejected": -110.38552856445312, "logps/rejected": -125.19332122802734, "loss": 1.0254, "margin_dpo/margin_mean": 11.741720199584961, "margin_dpo/margin_std": 13.526103973388672, "step": 148 }, { "KL/chosen_KL_mean": -6.082599639892578, "KL/mean": -11.758880615234375, "KL/rejected_KL_mean": -17.43515396118164, "KL/std": 12.793182373046875, "epoch": 0.21879588839941264, "fcm_dpo/beta": 0.04260088503360748, "fcm_dpo/delta": -0.03577341139316559, "fcm_dpo/margin": 11.352545738220215, "fcm_dpo/q_t": 0.396129846572876, "grad_norm": 13.894159317016602, "learning_rate": 4.797231243092118e-07, "logits/chosen": -1.283447265625, "logits/rejected": -0.9648770689964294, "logps/chosen": -99.56661987304688, "logps/ref_chosen": -93.48401641845703, "logps/ref_rejected": -105.6436996459961, "logps/rejected": -123.078857421875, "loss": 1.0569, "margin_dpo/margin_mean": 11.352546691894531, "margin_dpo/margin_std": 15.676870346069336, "step": 149 }, { "KL/chosen_KL_mean": -4.868541717529297, "KL/mean": -10.874984741210938, "KL/rejected_KL_mean": -16.881431579589844, "KL/std": 13.660394668579102, "epoch": 0.22026431718061673, "fcm_dpo/beta": 0.042201243340969086, "fcm_dpo/delta": -0.060271378606557846, "fcm_dpo/margin": 12.01289176940918, "fcm_dpo/q_t": 0.38769829273223877, "grad_norm": 12.558541297912598, "learning_rate": 4.792138157142157e-07, "logits/chosen": -1.2181215286254883, "logits/rejected": -0.9505946636199951, "logps/chosen": -80.64723205566406, "logps/ref_chosen": -75.7786865234375, "logps/ref_rejected": -90.02676391601562, "logps/rejected": -106.90819549560547, "loss": 1.0293, "margin_dpo/margin_mean": 12.01289176940918, "margin_dpo/margin_std": 15.103593826293945, "step": 150 }, { "KL/chosen_KL_mean": -6.614654541015625, "KL/mean": -12.416755676269531, "KL/rejected_KL_mean": -18.21884536743164, "KL/std": 13.398006439208984, "epoch": 0.22173274596182085, "fcm_dpo/beta": 0.04193378984928131, "fcm_dpo/delta": -0.03847986459732056, "fcm_dpo/margin": 11.604190826416016, "fcm_dpo/q_t": 0.3916553854942322, "grad_norm": 15.315407752990723, "learning_rate": 4.786984671220053e-07, "logits/chosen": -2.1024904251098633, "logits/rejected": -1.5320215225219727, "logps/chosen": -109.348876953125, "logps/ref_chosen": -102.73422241210938, "logps/ref_rejected": -106.03781127929688, "logps/rejected": -124.25665283203125, "loss": 1.0433, "margin_dpo/margin_mean": 11.604188919067383, "margin_dpo/margin_std": 14.949684143066406, "step": 151 }, { "KL/chosen_KL_mean": -3.824298858642578, "KL/mean": -10.796260833740234, "KL/rejected_KL_mean": -17.768226623535156, "KL/std": 13.248649597167969, "epoch": 0.22320117474302498, "fcm_dpo/beta": 0.04098113998770714, "fcm_dpo/delta": -0.1291038691997528, "fcm_dpo/margin": 13.943924903869629, "fcm_dpo/q_t": 0.3707083761692047, "grad_norm": 13.648688316345215, "learning_rate": 4.78177092112495e-07, "logits/chosen": -1.5742216110229492, "logits/rejected": -1.20149564743042, "logps/chosen": -101.4150390625, "logps/ref_chosen": -97.59074401855469, "logps/ref_rejected": -110.49913024902344, "logps/rejected": -128.26736450195312, "loss": 0.9782, "margin_dpo/margin_mean": 13.943923950195312, "margin_dpo/margin_std": 15.15260124206543, "step": 152 }, { "KL/chosen_KL_mean": -6.445198059082031, "KL/mean": -11.51744270324707, "KL/rejected_KL_mean": -16.58969497680664, "KL/std": 12.89784049987793, "epoch": 0.22466960352422907, "fcm_dpo/beta": 0.040995821356773376, "fcm_dpo/delta": 0.03560230880975723, "fcm_dpo/margin": 10.144502639770508, "fcm_dpo/q_t": 0.40656381845474243, "grad_norm": 13.640460968017578, "learning_rate": 4.776497044244016e-07, "logits/chosen": -1.5161995887756348, "logits/rejected": -1.1859508752822876, "logps/chosen": -93.47079467773438, "logps/ref_chosen": -87.02560424804688, "logps/ref_rejected": -91.06854248046875, "logps/rejected": -107.65823364257812, "loss": 1.0979, "margin_dpo/margin_mean": 10.144502639770508, "margin_dpo/margin_std": 14.666120529174805, "step": 153 }, { "KL/chosen_KL_mean": -4.714118957519531, "KL/mean": -12.884841918945312, "KL/rejected_KL_mean": -21.05556869506836, "KL/std": 14.184276580810547, "epoch": 0.2261380323054332, "fcm_dpo/beta": 0.040085356682538986, "fcm_dpo/delta": -0.21898622810840607, "fcm_dpo/margin": 16.341453552246094, "fcm_dpo/q_t": 0.3526090979576111, "grad_norm": 13.28144359588623, "learning_rate": 4.771163179548808e-07, "logits/chosen": -1.4826412200927734, "logits/rejected": -1.1600244045257568, "logps/chosen": -95.81978607177734, "logps/ref_chosen": -91.10566711425781, "logps/ref_rejected": -111.37238311767578, "logps/rejected": -132.42794799804688, "loss": 0.9173, "margin_dpo/margin_mean": 16.341453552246094, "margin_dpo/margin_std": 14.729225158691406, "step": 154 }, { "KL/chosen_KL_mean": -6.760845184326172, "KL/mean": -12.999425888061523, "KL/rejected_KL_mean": -19.237995147705078, "KL/std": 14.634838104248047, "epoch": 0.2276064610866373, "fcm_dpo/beta": 0.03888711705803871, "fcm_dpo/delta": -0.037780579179525375, "fcm_dpo/margin": 12.477148056030273, "fcm_dpo/q_t": 0.38871341943740845, "grad_norm": 16.313806533813477, "learning_rate": 4.7657694675916247e-07, "logits/chosen": -1.1022429466247559, "logits/rejected": -0.7117606401443481, "logps/chosen": -101.32182312011719, "logps/ref_chosen": -94.56097412109375, "logps/ref_rejected": -84.77359008789062, "logps/rejected": -104.01158142089844, "loss": 1.0683, "margin_dpo/margin_mean": 12.477148056030273, "margin_dpo/margin_std": 17.88715362548828, "step": 155 }, { "KL/chosen_KL_mean": -8.382980346679688, "KL/mean": -13.902236938476562, "KL/rejected_KL_mean": -19.421493530273438, "KL/std": 12.998016357421875, "epoch": 0.2290748898678414, "fcm_dpo/beta": 0.03896789997816086, "fcm_dpo/delta": 0.02044248953461647, "fcm_dpo/margin": 11.0385103225708, "fcm_dpo/q_t": 0.4047124683856964, "grad_norm": 14.493485450744629, "learning_rate": 4.7603160505017893e-07, "logits/chosen": -1.380091905593872, "logits/rejected": -0.8696132898330688, "logps/chosen": -103.03836822509766, "logps/ref_chosen": -94.65538787841797, "logps/ref_rejected": -84.41934204101562, "logps/rejected": -103.84083557128906, "loss": 1.0885, "margin_dpo/margin_mean": 11.038509368896484, "margin_dpo/margin_std": 15.625204086303711, "step": 156 }, { "KL/chosen_KL_mean": -7.839717864990234, "KL/mean": -15.43508529663086, "KL/rejected_KL_mean": -23.03044891357422, "KL/std": 13.96629810333252, "epoch": 0.2305433186490455, "fcm_dpo/beta": 0.038414839655160904, "fcm_dpo/delta": -0.1416759192943573, "fcm_dpo/margin": 15.19072437286377, "fcm_dpo/q_t": 0.36670780181884766, "grad_norm": 14.359320640563965, "learning_rate": 4.7548030719819154e-07, "logits/chosen": -1.9664299488067627, "logits/rejected": -1.5126819610595703, "logps/chosen": -104.85211181640625, "logps/ref_chosen": -97.01239013671875, "logps/ref_rejected": -111.70133972167969, "logps/rejected": -134.73179626464844, "loss": 0.9577, "margin_dpo/margin_mean": 15.19072437286377, "margin_dpo/margin_std": 14.695119857788086, "step": 157 }, { "KL/chosen_KL_mean": -7.228157043457031, "KL/mean": -15.251038551330566, "KL/rejected_KL_mean": -23.27391815185547, "KL/std": 14.975448608398438, "epoch": 0.23201174743024963, "fcm_dpo/beta": 0.037267886102199554, "fcm_dpo/delta": -0.15720441937446594, "fcm_dpo/margin": 16.0457706451416, "fcm_dpo/q_t": 0.3649120628833771, "grad_norm": 13.368614196777344, "learning_rate": 4.7492306773041136e-07, "logits/chosen": -1.1402028799057007, "logits/rejected": -0.8593883514404297, "logps/chosen": -101.94111633300781, "logps/ref_chosen": -94.71296691894531, "logps/ref_rejected": -120.93733978271484, "logps/rejected": -144.2112579345703, "loss": 0.9518, "margin_dpo/margin_mean": 16.04576873779297, "margin_dpo/margin_std": 15.876977920532227, "step": 158 }, { "KL/chosen_KL_mean": -6.558082580566406, "KL/mean": -13.74714469909668, "KL/rejected_KL_mean": -20.93621063232422, "KL/std": 14.688760757446289, "epoch": 0.23348017621145375, "fcm_dpo/beta": 0.03646499663591385, "fcm_dpo/delta": -0.07844078540802002, "fcm_dpo/margin": 14.378135681152344, "fcm_dpo/q_t": 0.381900817155838, "grad_norm": 12.028036117553711, "learning_rate": 4.743599013306165e-07, "logits/chosen": -1.9074044227600098, "logits/rejected": -1.418100357055664, "logps/chosen": -114.77632141113281, "logps/ref_chosen": -108.2182388305664, "logps/ref_rejected": -94.83901977539062, "logps/rejected": -115.77523803710938, "loss": 1.012, "margin_dpo/margin_mean": 14.378135681152344, "margin_dpo/margin_std": 16.724992752075195, "step": 159 }, { "KL/chosen_KL_mean": -5.5895233154296875, "KL/mean": -12.679683685302734, "KL/rejected_KL_mean": -19.769851684570312, "KL/std": 13.713754653930664, "epoch": 0.23494860499265785, "fcm_dpo/beta": 0.03592487424612045, "fcm_dpo/delta": -0.06281135976314545, "fcm_dpo/margin": 14.18032455444336, "fcm_dpo/q_t": 0.3823738992214203, "grad_norm": 12.476420402526855, "learning_rate": 4.737908228387656e-07, "logits/chosen": -1.5746586322784424, "logits/rejected": -1.0254801511764526, "logps/chosen": -103.7529296875, "logps/ref_chosen": -98.16340637207031, "logps/ref_rejected": -102.28097534179688, "logps/rejected": -122.05082702636719, "loss": 1.004, "margin_dpo/margin_mean": 14.180325508117676, "margin_dpo/margin_std": 14.713062286376953, "step": 160 }, { "KL/chosen_KL_mean": -7.667057037353516, "KL/mean": -14.123154640197754, "KL/rejected_KL_mean": -20.579246520996094, "KL/std": 12.620899200439453, "epoch": 0.23641703377386197, "fcm_dpo/beta": 0.035670921206474304, "fcm_dpo/delta": -0.01148887351155281, "fcm_dpo/margin": 12.912195205688477, "fcm_dpo/q_t": 0.39526236057281494, "grad_norm": 12.445168495178223, "learning_rate": 4.7321584725060594e-07, "logits/chosen": -1.8524129390716553, "logits/rejected": -1.3672497272491455, "logps/chosen": -94.25892639160156, "logps/ref_chosen": -86.59187316894531, "logps/ref_rejected": -89.01643371582031, "logps/rejected": -109.5956802368164, "loss": 1.0558, "margin_dpo/margin_mean": 12.912195205688477, "margin_dpo/margin_std": 16.13299560546875, "step": 161 }, { "KL/chosen_KL_mean": -6.879505157470703, "KL/mean": -13.900179862976074, "KL/rejected_KL_mean": -20.920848846435547, "KL/std": 14.575180053710938, "epoch": 0.23788546255506607, "fcm_dpo/beta": 0.035436250269412994, "fcm_dpo/delta": -0.050333380699157715, "fcm_dpo/margin": 14.041352272033691, "fcm_dpo/q_t": 0.38521867990493774, "grad_norm": 11.967451095581055, "learning_rate": 4.7263498971727905e-07, "logits/chosen": -1.1731939315795898, "logits/rejected": -0.7678380012512207, "logps/chosen": -104.28239440917969, "logps/ref_chosen": -97.40288543701172, "logps/ref_rejected": -95.86923217773438, "logps/rejected": -116.79008483886719, "loss": 1.018, "margin_dpo/margin_mean": 14.041352272033691, "margin_dpo/margin_std": 15.449185371398926, "step": 162 }, { "KL/chosen_KL_mean": -6.612934112548828, "KL/mean": -15.126335144042969, "KL/rejected_KL_mean": -23.639739990234375, "KL/std": 16.71318244934082, "epoch": 0.2393538913362702, "fcm_dpo/beta": 0.034580446779727936, "fcm_dpo/delta": -0.1479780077934265, "fcm_dpo/margin": 17.02680206298828, "fcm_dpo/q_t": 0.36851170659065247, "grad_norm": 10.702917098999023, "learning_rate": 4.720482655449212e-07, "logits/chosen": -1.5282377004623413, "logits/rejected": -1.1244807243347168, "logps/chosen": -88.788330078125, "logps/ref_chosen": -82.1753921508789, "logps/ref_rejected": -82.86473846435547, "logps/rejected": -106.50447845458984, "loss": 0.9625, "margin_dpo/margin_mean": 17.02680206298828, "margin_dpo/margin_std": 17.83454132080078, "step": 163 }, { "KL/chosen_KL_mean": -10.24361801147461, "KL/mean": -15.968530654907227, "KL/rejected_KL_mean": -21.693431854248047, "KL/std": 13.200516700744629, "epoch": 0.24082232011747431, "fcm_dpo/beta": 0.03443962335586548, "fcm_dpo/delta": 0.05723126232624054, "fcm_dpo/margin": 11.449810981750488, "fcm_dpo/q_t": 0.4088016152381897, "grad_norm": 12.716316223144531, "learning_rate": 4.714556901942599e-07, "logits/chosen": -1.2501944303512573, "logits/rejected": -0.7770636677742004, "logps/chosen": -100.71788787841797, "logps/ref_chosen": -90.47427368164062, "logps/ref_rejected": -87.02809143066406, "logps/rejected": -108.72152709960938, "loss": 1.0932, "margin_dpo/margin_mean": 11.449811935424805, "margin_dpo/margin_std": 14.289133071899414, "step": 164 }, { "KL/chosen_KL_mean": -6.868404388427734, "KL/mean": -15.295869827270508, "KL/rejected_KL_mean": -23.72333526611328, "KL/std": 15.947157859802246, "epoch": 0.2422907488986784, "fcm_dpo/beta": 0.03420202434062958, "fcm_dpo/delta": -0.13400211930274963, "fcm_dpo/margin": 16.85493278503418, "fcm_dpo/q_t": 0.3720381259918213, "grad_norm": 10.454062461853027, "learning_rate": 4.708572792802069e-07, "logits/chosen": -1.6756125688552856, "logits/rejected": -1.2562980651855469, "logps/chosen": -94.18163299560547, "logps/ref_chosen": -87.313232421875, "logps/ref_rejected": -81.902587890625, "logps/rejected": -105.62591552734375, "loss": 0.9691, "margin_dpo/margin_mean": 16.854930877685547, "margin_dpo/margin_std": 18.01761817932129, "step": 165 }, { "KL/chosen_KL_mean": -11.347457885742188, "KL/mean": -19.54320526123047, "KL/rejected_KL_mean": -27.73895263671875, "KL/std": 17.023427963256836, "epoch": 0.24375917767988253, "fcm_dpo/beta": 0.03336024656891823, "fcm_dpo/delta": -0.1024494618177414, "fcm_dpo/margin": 16.391490936279297, "fcm_dpo/q_t": 0.38189011812210083, "grad_norm": 11.76332950592041, "learning_rate": 4.702530485714461e-07, "logits/chosen": -1.5171177387237549, "logits/rejected": -1.311524510383606, "logps/chosen": -92.93352508544922, "logps/ref_chosen": -81.58606719970703, "logps/ref_rejected": -103.16517639160156, "logps/rejected": -130.9041290283203, "loss": 1.0065, "margin_dpo/margin_mean": 16.391490936279297, "margin_dpo/margin_std": 20.421768188476562, "step": 166 }, { "KL/chosen_KL_mean": -10.083026885986328, "KL/mean": -18.919815063476562, "KL/rejected_KL_mean": -27.756610870361328, "KL/std": 15.828325271606445, "epoch": 0.24522760646108663, "fcm_dpo/beta": 0.03236294537782669, "fcm_dpo/delta": -0.13084493577480316, "fcm_dpo/margin": 17.673582077026367, "fcm_dpo/q_t": 0.36864155530929565, "grad_norm": 11.045381546020508, "learning_rate": 4.6964301399001877e-07, "logits/chosen": -1.0328116416931152, "logits/rejected": -0.8548140525817871, "logps/chosen": -93.64469909667969, "logps/ref_chosen": -83.56167602539062, "logps/ref_rejected": -99.05305480957031, "logps/rejected": -126.80966186523438, "loss": 0.9549, "margin_dpo/margin_mean": 17.673582077026367, "margin_dpo/margin_std": 15.923975944519043, "step": 167 }, { "KL/chosen_KL_mean": -9.740535736083984, "KL/mean": -17.327659606933594, "KL/rejected_KL_mean": -24.91478729248047, "KL/std": 15.80506706237793, "epoch": 0.24669603524229075, "fcm_dpo/beta": 0.032199449837207794, "fcm_dpo/delta": -0.04057988524436951, "fcm_dpo/margin": 15.174245834350586, "fcm_dpo/q_t": 0.38755232095718384, "grad_norm": 12.069757461547852, "learning_rate": 4.690271916109034e-07, "logits/chosen": -1.6021510362625122, "logits/rejected": -1.2641432285308838, "logps/chosen": -89.67176818847656, "logps/ref_chosen": -79.93123626708984, "logps/ref_rejected": -80.10216522216797, "logps/rejected": -105.01695251464844, "loss": 1.0162, "margin_dpo/margin_mean": 15.174247741699219, "margin_dpo/margin_std": 15.809932708740234, "step": 168 }, { "KL/chosen_KL_mean": -8.248973846435547, "KL/mean": -16.996950149536133, "KL/rejected_KL_mean": -25.74492645263672, "KL/std": 16.920543670654297, "epoch": 0.24816446402349487, "fcm_dpo/beta": 0.03156084194779396, "fcm_dpo/delta": -0.10816548019647598, "fcm_dpo/margin": 17.495952606201172, "fcm_dpo/q_t": 0.375136137008667, "grad_norm": 11.632828712463379, "learning_rate": 4.6840559766159235e-07, "logits/chosen": -1.4784125089645386, "logits/rejected": -1.2528705596923828, "logps/chosen": -91.15267944335938, "logps/ref_chosen": -82.90370178222656, "logps/ref_rejected": -90.12191009521484, "logps/rejected": -115.86683654785156, "loss": 0.9796, "margin_dpo/margin_mean": 17.495952606201172, "margin_dpo/margin_std": 17.988906860351562, "step": 169 }, { "KL/chosen_KL_mean": -9.879325866699219, "KL/mean": -17.5045166015625, "KL/rejected_KL_mean": -25.12970733642578, "KL/std": 14.28281021118164, "epoch": 0.24963289280469897, "fcm_dpo/beta": 0.031117241829633713, "fcm_dpo/delta": -0.026353713124990463, "fcm_dpo/margin": 15.250386238098145, "fcm_dpo/q_t": 0.3886600434780121, "grad_norm": 11.964139938354492, "learning_rate": 4.6777824852166437e-07, "logits/chosen": -1.4093681573867798, "logits/rejected": -1.190025806427002, "logps/chosen": -84.86129760742188, "logps/ref_chosen": -74.98197174072266, "logps/ref_rejected": -82.16560363769531, "logps/rejected": -107.2953109741211, "loss": 1.0143, "margin_dpo/margin_mean": 15.250385284423828, "margin_dpo/margin_std": 14.157035827636719, "step": 170 }, { "KL/chosen_KL_mean": -12.461994171142578, "KL/mean": -20.921245574951172, "KL/rejected_KL_mean": -29.380496978759766, "KL/std": 19.991989135742188, "epoch": 0.2511013215859031, "fcm_dpo/beta": 0.03089229390025139, "fcm_dpo/delta": -0.07661177217960358, "fcm_dpo/margin": 16.91851043701172, "fcm_dpo/q_t": 0.38857120275497437, "grad_norm": 13.428890228271484, "learning_rate": 4.6714516072235273e-07, "logits/chosen": -1.5971887111663818, "logits/rejected": -1.0483250617980957, "logps/chosen": -131.44052124023438, "logps/ref_chosen": -118.97853088378906, "logps/ref_rejected": -115.76930236816406, "logps/rejected": -145.14981079101562, "loss": 1.0393, "margin_dpo/margin_mean": 16.918508529663086, "margin_dpo/margin_std": 23.625953674316406, "step": 171 }, { "KL/chosen_KL_mean": -13.391273498535156, "KL/mean": -21.045547485351562, "KL/rejected_KL_mean": -28.699817657470703, "KL/std": 14.668384552001953, "epoch": 0.2525697503671072, "fcm_dpo/beta": 0.030571138486266136, "fcm_dpo/delta": -0.019111724570393562, "fcm_dpo/margin": 15.308544158935547, "fcm_dpo/q_t": 0.3887174129486084, "grad_norm": 11.000654220581055, "learning_rate": 4.6650635094610966e-07, "logits/chosen": -2.1800696849823, "logits/rejected": -1.4203687906265259, "logps/chosen": -114.54505920410156, "logps/ref_chosen": -101.15379333496094, "logps/ref_rejected": -91.00568389892578, "logps/rejected": -119.70550537109375, "loss": 1.0056, "margin_dpo/margin_mean": 15.308544158935547, "margin_dpo/margin_std": 12.103874206542969, "step": 172 }, { "KL/chosen_KL_mean": -12.206268310546875, "KL/mean": -18.89590072631836, "KL/rejected_KL_mean": -25.585529327392578, "KL/std": 14.004446983337402, "epoch": 0.2540381791483113, "fcm_dpo/beta": 0.03084501624107361, "fcm_dpo/delta": 0.03884591534733772, "fcm_dpo/margin": 13.379262924194336, "fcm_dpo/q_t": 0.4037604331970215, "grad_norm": 10.733778953552246, "learning_rate": 4.6586183602616687e-07, "logits/chosen": -1.5182697772979736, "logits/rejected": -1.044596552848816, "logps/chosen": -111.26295471191406, "logps/ref_chosen": -99.05668640136719, "logps/ref_rejected": -83.8507308959961, "logps/rejected": -109.43626403808594, "loss": 1.065, "margin_dpo/margin_mean": 13.379262924194336, "margin_dpo/margin_std": 14.841697692871094, "step": 173 }, { "KL/chosen_KL_mean": -11.841604232788086, "KL/mean": -20.907365798950195, "KL/rejected_KL_mean": -29.973129272460938, "KL/std": 18.7991886138916, "epoch": 0.2555066079295154, "fcm_dpo/beta": 0.030422156676650047, "fcm_dpo/delta": -0.10742342472076416, "fcm_dpo/margin": 18.13152313232422, "fcm_dpo/q_t": 0.3739241361618042, "grad_norm": 12.568257331848145, "learning_rate": 4.652116329460919e-07, "logits/chosen": -1.813513159751892, "logits/rejected": -1.7639669179916382, "logps/chosen": -92.2505111694336, "logps/ref_chosen": -80.40890502929688, "logps/ref_rejected": -108.20171356201172, "logps/rejected": -138.17483520507812, "loss": 0.98, "margin_dpo/margin_mean": 18.13152313232422, "margin_dpo/margin_std": 18.54364776611328, "step": 174 }, { "KL/chosen_KL_mean": -15.296707153320312, "KL/mean": -24.05184555053711, "KL/rejected_KL_mean": -32.806983947753906, "KL/std": 16.83472442626953, "epoch": 0.25697503671071953, "fcm_dpo/beta": 0.029946405440568924, "fcm_dpo/delta": -0.07842611521482468, "fcm_dpo/margin": 17.510276794433594, "fcm_dpo/q_t": 0.3779052495956421, "grad_norm": 12.207069396972656, "learning_rate": 4.645557588393406e-07, "logits/chosen": -1.1669551134109497, "logits/rejected": -0.8998315334320068, "logps/chosen": -98.01319885253906, "logps/ref_chosen": -82.71649169921875, "logps/ref_rejected": -96.66712951660156, "logps/rejected": -129.47412109375, "loss": 0.9836, "margin_dpo/margin_mean": 17.510276794433594, "margin_dpo/margin_std": 15.949636459350586, "step": 175 }, { "KL/chosen_KL_mean": -12.60012435913086, "KL/mean": -21.525157928466797, "KL/rejected_KL_mean": -30.4501953125, "KL/std": 18.355226516723633, "epoch": 0.25844346549192365, "fcm_dpo/beta": 0.029383402317762375, "fcm_dpo/delta": -0.07866019010543823, "fcm_dpo/margin": 17.85007095336914, "fcm_dpo/q_t": 0.3824155926704407, "grad_norm": 11.708728790283203, "learning_rate": 4.638942309888058e-07, "logits/chosen": -1.534851312637329, "logits/rejected": -1.3692426681518555, "logps/chosen": -92.22943115234375, "logps/ref_chosen": -79.62930297851562, "logps/ref_rejected": -102.6885986328125, "logps/rejected": -133.1387939453125, "loss": 1.0142, "margin_dpo/margin_mean": 17.85007095336914, "margin_dpo/margin_std": 20.96698760986328, "step": 176 }, { "KL/chosen_KL_mean": -13.994693756103516, "KL/mean": -24.452960968017578, "KL/rejected_KL_mean": -34.91122817993164, "KL/std": 20.982011795043945, "epoch": 0.2599118942731278, "fcm_dpo/beta": 0.028721442446112633, "fcm_dpo/delta": -0.16008484363555908, "fcm_dpo/margin": 20.916528701782227, "fcm_dpo/q_t": 0.36762043833732605, "grad_norm": 10.58277702331543, "learning_rate": 4.6322706682636137e-07, "logits/chosen": -1.4493741989135742, "logits/rejected": -1.0991737842559814, "logps/chosen": -106.55579376220703, "logps/ref_chosen": -92.56109619140625, "logps/ref_rejected": -103.33839416503906, "logps/rejected": -138.24961853027344, "loss": 0.9556, "margin_dpo/margin_mean": 20.91653060913086, "margin_dpo/margin_std": 21.6851806640625, "step": 177 }, { "KL/chosen_KL_mean": -12.885887145996094, "KL/mean": -25.311311721801758, "KL/rejected_KL_mean": -37.736717224121094, "KL/std": 20.406723022460938, "epoch": 0.26138032305433184, "fcm_dpo/beta": 0.02731304243206978, "fcm_dpo/delta": -0.2457062304019928, "fcm_dpo/margin": 24.850833892822266, "fcm_dpo/q_t": 0.347068727016449, "grad_norm": 11.803115844726562, "learning_rate": 4.6255428393240354e-07, "logits/chosen": -1.683790683746338, "logits/rejected": -1.457137107849121, "logps/chosen": -97.15168762207031, "logps/ref_chosen": -84.26579284667969, "logps/ref_rejected": -110.79730987548828, "logps/rejected": -148.53402709960938, "loss": 0.8943, "margin_dpo/margin_mean": 24.850830078125, "margin_dpo/margin_std": 22.01804542541504, "step": 178 }, { "KL/chosen_KL_mean": -16.510086059570312, "KL/mean": -24.588722229003906, "KL/rejected_KL_mean": -32.667362213134766, "KL/std": 19.8905029296875, "epoch": 0.26284875183553597, "fcm_dpo/beta": 0.026959318667650223, "fcm_dpo/delta": 0.014857767149806023, "fcm_dpo/margin": 16.15727996826172, "fcm_dpo/q_t": 0.402656614780426, "grad_norm": 11.33733081817627, "learning_rate": 4.6187590003538724e-07, "logits/chosen": -1.4103319644927979, "logits/rejected": -0.9745727181434631, "logps/chosen": -109.95806121826172, "logps/ref_chosen": -93.4479751586914, "logps/ref_rejected": -94.25541687011719, "logps/rejected": -126.92277526855469, "loss": 1.0857, "margin_dpo/margin_mean": 16.15727996826172, "margin_dpo/margin_std": 22.873689651489258, "step": 179 }, { "KL/chosen_KL_mean": -13.921092987060547, "KL/mean": -24.237394332885742, "KL/rejected_KL_mean": -34.55370330810547, "KL/std": 19.22161102294922, "epoch": 0.2643171806167401, "fcm_dpo/beta": 0.02657276764512062, "fcm_dpo/delta": -0.10452157258987427, "fcm_dpo/margin": 20.632612228393555, "fcm_dpo/q_t": 0.3743304908275604, "grad_norm": 13.59306526184082, "learning_rate": 4.611919330113591e-07, "logits/chosen": -1.1502811908721924, "logits/rejected": -0.901892364025116, "logps/chosen": -102.75663757324219, "logps/ref_chosen": -88.83554077148438, "logps/ref_rejected": -103.90696716308594, "logps/rejected": -138.46066284179688, "loss": 0.9707, "margin_dpo/margin_mean": 20.632612228393555, "margin_dpo/margin_std": 19.694652557373047, "step": 180 }, { "KL/chosen_KL_mean": -13.958858489990234, "KL/mean": -21.948686599731445, "KL/rejected_KL_mean": -29.938507080078125, "KL/std": 18.260334014892578, "epoch": 0.2657856093979442, "fcm_dpo/beta": 0.026639170944690704, "fcm_dpo/delta": 0.025360099971294403, "fcm_dpo/margin": 15.979652404785156, "fcm_dpo/q_t": 0.40332111716270447, "grad_norm": 10.452695846557617, "learning_rate": 4.605024008834863e-07, "logits/chosen": -1.4390556812286377, "logits/rejected": -0.9684023261070251, "logps/chosen": -93.19282531738281, "logps/ref_chosen": -79.23396301269531, "logps/ref_rejected": -69.17442321777344, "logps/rejected": -99.11293029785156, "loss": 1.0657, "margin_dpo/margin_mean": 15.979652404785156, "margin_dpo/margin_std": 19.490962982177734, "step": 181 }, { "KL/chosen_KL_mean": -17.644939422607422, "KL/mean": -27.92449951171875, "KL/rejected_KL_mean": -38.20405578613281, "KL/std": 21.12765121459961, "epoch": 0.26725403817914833, "fcm_dpo/beta": 0.02617749758064747, "fcm_dpo/delta": -0.09390002489089966, "fcm_dpo/margin": 20.55911636352539, "fcm_dpo/q_t": 0.37889379262924194, "grad_norm": 10.89322280883789, "learning_rate": 4.598073218215817e-07, "logits/chosen": -1.104196548461914, "logits/rejected": -0.8755874633789062, "logps/chosen": -93.72712707519531, "logps/ref_chosen": -76.08218383789062, "logps/ref_rejected": -94.9686050415039, "logps/rejected": -133.17266845703125, "loss": 1.0015, "margin_dpo/margin_mean": 20.559114456176758, "margin_dpo/margin_std": 23.262847900390625, "step": 182 }, { "KL/chosen_KL_mean": -18.160797119140625, "KL/mean": -28.280651092529297, "KL/rejected_KL_mean": -38.40050506591797, "KL/std": 19.251916885375977, "epoch": 0.2687224669603524, "fcm_dpo/beta": 0.02576177939772606, "fcm_dpo/delta": -0.0759400725364685, "fcm_dpo/margin": 20.23970603942871, "fcm_dpo/q_t": 0.3797719180583954, "grad_norm": 11.388766288757324, "learning_rate": 4.5910671414162484e-07, "logits/chosen": -1.658898115158081, "logits/rejected": -1.0657981634140015, "logps/chosen": -112.0687026977539, "logps/ref_chosen": -93.90790557861328, "logps/ref_rejected": -83.56061553955078, "logps/rejected": -121.96112060546875, "loss": 0.992, "margin_dpo/margin_mean": 20.23970603942871, "margin_dpo/margin_std": 19.79551887512207, "step": 183 }, { "KL/chosen_KL_mean": -16.416858673095703, "KL/mean": -23.260700225830078, "KL/rejected_KL_mean": -30.104541778564453, "KL/std": 19.949066162109375, "epoch": 0.2701908957415565, "fcm_dpo/beta": 0.026149462908506393, "fcm_dpo/delta": 0.09541141986846924, "fcm_dpo/margin": 13.687676429748535, "fcm_dpo/q_t": 0.417415976524353, "grad_norm": 10.762918472290039, "learning_rate": 4.5840059630527985e-07, "logits/chosen": -1.542266845703125, "logits/rejected": -1.2651708126068115, "logps/chosen": -103.2847671508789, "logps/ref_chosen": -86.86790466308594, "logps/ref_rejected": -84.28456115722656, "logps/rejected": -114.38909912109375, "loss": 1.1357, "margin_dpo/margin_mean": 13.687676429748535, "margin_dpo/margin_std": 21.762523651123047, "step": 184 }, { "KL/chosen_KL_mean": -17.35260009765625, "KL/mean": -25.63970184326172, "KL/rejected_KL_mean": -33.92680358886719, "KL/std": 19.54541778564453, "epoch": 0.27165932452276065, "fcm_dpo/beta": 0.02639647200703621, "fcm_dpo/delta": 0.012794554233551025, "fcm_dpo/margin": 16.57420539855957, "fcm_dpo/q_t": 0.40176862478256226, "grad_norm": 12.104183197021484, "learning_rate": 4.5768898691940836e-07, "logits/chosen": -2.1059458255767822, "logits/rejected": -1.4902725219726562, "logps/chosen": -111.3740234375, "logps/ref_chosen": -94.02142333984375, "logps/ref_rejected": -81.45668029785156, "logps/rejected": -115.38348388671875, "loss": 1.0775, "margin_dpo/margin_mean": 16.574207305908203, "margin_dpo/margin_std": 22.642166137695312, "step": 185 }, { "KL/chosen_KL_mean": -21.38967514038086, "KL/mean": -30.390981674194336, "KL/rejected_KL_mean": -39.39229202270508, "KL/std": 18.92312240600586, "epoch": 0.27312775330396477, "fcm_dpo/beta": 0.02614632062613964, "fcm_dpo/delta": -0.022055618464946747, "fcm_dpo/margin": 18.00262451171875, "fcm_dpo/q_t": 0.39177870750427246, "grad_norm": 12.5892915725708, "learning_rate": 4.5697190473557947e-07, "logits/chosen": -2.2961578369140625, "logits/rejected": -1.4019124507904053, "logps/chosen": -135.58135986328125, "logps/ref_chosen": -114.19168090820312, "logps/ref_rejected": -93.33912658691406, "logps/rejected": -132.73141479492188, "loss": 1.0364, "margin_dpo/margin_mean": 18.00262451171875, "margin_dpo/margin_std": 20.395862579345703, "step": 186 }, { "KL/chosen_KL_mean": -18.42379379272461, "KL/mean": -26.974987030029297, "KL/rejected_KL_mean": -35.526180267333984, "KL/std": 21.24410629272461, "epoch": 0.2745961820851689, "fcm_dpo/beta": 0.02628886140882969, "fcm_dpo/delta": 0.00022946670651435852, "fcm_dpo/margin": 17.10239028930664, "fcm_dpo/q_t": 0.39897704124450684, "grad_norm": 11.098430633544922, "learning_rate": 4.5624936864957555e-07, "logits/chosen": -1.6193220615386963, "logits/rejected": -1.3785688877105713, "logps/chosen": -95.29334259033203, "logps/ref_chosen": -76.86955261230469, "logps/ref_rejected": -88.16494750976562, "logps/rejected": -123.69112396240234, "loss": 1.071, "margin_dpo/margin_mean": 17.10239028930664, "margin_dpo/margin_std": 23.12689208984375, "step": 187 }, { "KL/chosen_KL_mean": -14.091022491455078, "KL/mean": -23.916259765625, "KL/rejected_KL_mean": -33.74150848388672, "KL/std": 20.944185256958008, "epoch": 0.27606461086637296, "fcm_dpo/beta": 0.025859780609607697, "fcm_dpo/delta": -0.061903007328510284, "fcm_dpo/margin": 19.650480270385742, "fcm_dpo/q_t": 0.3814808130264282, "grad_norm": 11.369409561157227, "learning_rate": 4.5552139770089454e-07, "logits/chosen": -1.9536044597625732, "logits/rejected": -1.7075963020324707, "logps/chosen": -93.28582763671875, "logps/ref_chosen": -79.1948013305664, "logps/ref_rejected": -96.82889556884766, "logps/rejected": -130.57040405273438, "loss": 1.0053, "margin_dpo/margin_mean": 19.65047836303711, "margin_dpo/margin_std": 19.69561767578125, "step": 188 }, { "KL/chosen_KL_mean": -18.532405853271484, "KL/mean": -28.625364303588867, "KL/rejected_KL_mean": -38.718326568603516, "KL/std": 21.00287628173828, "epoch": 0.2775330396475771, "fcm_dpo/beta": 0.025762559846043587, "fcm_dpo/delta": -0.07420355081558228, "fcm_dpo/margin": 20.18592071533203, "fcm_dpo/q_t": 0.38445261120796204, "grad_norm": 10.222798347473145, "learning_rate": 4.5478801107224794e-07, "logits/chosen": -1.5331592559814453, "logits/rejected": -0.865401029586792, "logps/chosen": -102.42520141601562, "logps/ref_chosen": -83.8927993774414, "logps/ref_rejected": -79.46589660644531, "logps/rejected": -118.18421936035156, "loss": 1.0148, "margin_dpo/margin_mean": 20.18592071533203, "margin_dpo/margin_std": 23.509119033813477, "step": 189 }, { "KL/chosen_KL_mean": -14.602851867675781, "KL/mean": -26.681779861450195, "KL/rejected_KL_mean": -38.760711669921875, "KL/std": 23.119483947753906, "epoch": 0.2790014684287812, "fcm_dpo/beta": 0.02488422766327858, "fcm_dpo/delta": -0.16096192598342896, "fcm_dpo/margin": 24.157867431640625, "fcm_dpo/q_t": 0.36203956604003906, "grad_norm": 11.011933326721191, "learning_rate": 4.5404922808905543e-07, "logits/chosen": -1.7000792026519775, "logits/rejected": -1.091031789779663, "logps/chosen": -118.04431915283203, "logps/ref_chosen": -103.44146728515625, "logps/ref_rejected": -109.21502685546875, "logps/rejected": -147.97573852539062, "loss": 0.9398, "margin_dpo/margin_mean": 24.157869338989258, "margin_dpo/margin_std": 21.921974182128906, "step": 190 }, { "KL/chosen_KL_mean": -13.981407165527344, "KL/mean": -27.475727081298828, "KL/rejected_KL_mean": -40.97004699707031, "KL/std": 24.972049713134766, "epoch": 0.28046989720998533, "fcm_dpo/beta": 0.0239790640771389, "fcm_dpo/delta": -0.21051949262619019, "fcm_dpo/margin": 26.98863983154297, "fcm_dpo/q_t": 0.35482460260391235, "grad_norm": 11.797168731689453, "learning_rate": 4.5330506821893565e-07, "logits/chosen": -1.5692925453186035, "logits/rejected": -1.2564661502838135, "logps/chosen": -113.76692199707031, "logps/ref_chosen": -99.78551483154297, "logps/ref_rejected": -116.00028228759766, "logps/rejected": -156.9703369140625, "loss": 0.9215, "margin_dpo/margin_mean": 26.98863983154297, "margin_dpo/margin_std": 25.576162338256836, "step": 191 }, { "KL/chosen_KL_mean": -23.067813873291016, "KL/mean": -32.510894775390625, "KL/rejected_KL_mean": -41.9539680480957, "KL/std": 20.102333068847656, "epoch": 0.28193832599118945, "fcm_dpo/beta": 0.02366233989596367, "fcm_dpo/delta": 0.003228548914194107, "fcm_dpo/margin": 18.886154174804688, "fcm_dpo/q_t": 0.3986278176307678, "grad_norm": 12.40970230102539, "learning_rate": 4.5255555107119336e-07, "logits/chosen": -1.56754469871521, "logits/rejected": -1.313439130783081, "logps/chosen": -123.92813873291016, "logps/ref_chosen": -100.86032104492188, "logps/ref_rejected": -109.44500732421875, "logps/rejected": -151.3989715576172, "loss": 1.0588, "margin_dpo/margin_mean": 18.886154174804688, "margin_dpo/margin_std": 23.490009307861328, "step": 192 }, { "KL/chosen_KL_mean": -18.929611206054688, "KL/mean": -26.77655792236328, "KL/rejected_KL_mean": -34.623504638671875, "KL/std": 21.295259475708008, "epoch": 0.2834067547723935, "fcm_dpo/beta": 0.02403874322772026, "fcm_dpo/delta": 0.07514029741287231, "fcm_dpo/margin": 15.693893432617188, "fcm_dpo/q_t": 0.41458457708358765, "grad_norm": 11.779868125915527, "learning_rate": 4.5180069639630236e-07, "logits/chosen": -2.00793719291687, "logits/rejected": -1.4219706058502197, "logps/chosen": -111.86792755126953, "logps/ref_chosen": -92.93831634521484, "logps/ref_rejected": -83.10917663574219, "logps/rejected": -117.73268127441406, "loss": 1.1358, "margin_dpo/margin_mean": 15.693893432617188, "margin_dpo/margin_std": 25.76551055908203, "step": 193 }, { "KL/chosen_KL_mean": -19.625965118408203, "KL/mean": -28.691383361816406, "KL/rejected_KL_mean": -37.75680160522461, "KL/std": 21.91200828552246, "epoch": 0.28487518355359764, "fcm_dpo/beta": 0.024184411391615868, "fcm_dpo/delta": 0.011726578697562218, "fcm_dpo/margin": 18.130844116210938, "fcm_dpo/q_t": 0.40050068497657776, "grad_norm": 9.958355903625488, "learning_rate": 4.510405240853854e-07, "logits/chosen": -2.278360605239868, "logits/rejected": -1.8964219093322754, "logps/chosen": -83.41754150390625, "logps/ref_chosen": -63.79157257080078, "logps/ref_rejected": -68.72665405273438, "logps/rejected": -106.48345947265625, "loss": 1.0675, "margin_dpo/margin_mean": 18.130842208862305, "margin_dpo/margin_std": 23.099246978759766, "step": 194 }, { "KL/chosen_KL_mean": -18.747093200683594, "KL/mean": -30.734756469726562, "KL/rejected_KL_mean": -42.7224235534668, "KL/std": 22.262435913085938, "epoch": 0.28634361233480177, "fcm_dpo/beta": 0.02373277023434639, "fcm_dpo/delta": -0.12598851323127747, "fcm_dpo/margin": 23.975330352783203, "fcm_dpo/q_t": 0.3703756332397461, "grad_norm": 10.308981895446777, "learning_rate": 4.5027505416968985e-07, "logits/chosen": -1.4115322828292847, "logits/rejected": -1.2881722450256348, "logps/chosen": -102.63255310058594, "logps/ref_chosen": -83.88545989990234, "logps/ref_rejected": -104.00125122070312, "logps/rejected": -146.7236785888672, "loss": 0.9804, "margin_dpo/margin_mean": 23.975330352783203, "margin_dpo/margin_std": 25.573593139648438, "step": 195 }, { "KL/chosen_KL_mean": -21.64429473876953, "KL/mean": -33.10361099243164, "KL/rejected_KL_mean": -44.562931060791016, "KL/std": 23.96672821044922, "epoch": 0.2878120411160059, "fcm_dpo/beta": 0.023208342492580414, "fcm_dpo/delta": -0.08651156723499298, "fcm_dpo/margin": 22.918628692626953, "fcm_dpo/q_t": 0.37599894404411316, "grad_norm": 9.788215637207031, "learning_rate": 4.495043068200599e-07, "logits/chosen": -1.357054591178894, "logits/rejected": -0.9101194143295288, "logps/chosen": -111.045654296875, "logps/ref_chosen": -89.4013671875, "logps/ref_rejected": -83.34735107421875, "logps/rejected": -127.9102783203125, "loss": 0.981, "margin_dpo/margin_mean": 22.918628692626953, "margin_dpo/margin_std": 21.413265228271484, "step": 196 }, { "KL/chosen_KL_mean": -22.132862091064453, "KL/mean": -29.120304107666016, "KL/rejected_KL_mean": -36.10774612426758, "KL/std": 19.36894989013672, "epoch": 0.28928046989721, "fcm_dpo/beta": 0.023539701476693153, "fcm_dpo/delta": 0.1251697987318039, "fcm_dpo/margin": 13.97488021850586, "fcm_dpo/q_t": 0.4233154058456421, "grad_norm": 10.3875150680542, "learning_rate": 4.4872830234640493e-07, "logits/chosen": -1.2650811672210693, "logits/rejected": -0.9297800064086914, "logps/chosen": -108.97357177734375, "logps/ref_chosen": -86.84070587158203, "logps/ref_rejected": -82.79289245605469, "logps/rejected": -118.90064239501953, "loss": 1.1384, "margin_dpo/margin_mean": 13.974882125854492, "margin_dpo/margin_std": 20.425912857055664, "step": 197 }, { "KL/chosen_KL_mean": -25.586280822753906, "KL/mean": -36.8443603515625, "KL/rejected_KL_mean": -48.102439880371094, "KL/std": 22.67544937133789, "epoch": 0.2907488986784141, "fcm_dpo/beta": 0.023422207683324814, "fcm_dpo/delta": -0.0816422700881958, "fcm_dpo/margin": 22.516151428222656, "fcm_dpo/q_t": 0.3797655999660492, "grad_norm": 12.176416397094727, "learning_rate": 4.479470611971645e-07, "logits/chosen": -1.6032984256744385, "logits/rejected": -1.4243170022964478, "logps/chosen": -110.634521484375, "logps/ref_chosen": -85.04824829101562, "logps/ref_rejected": -104.10365295410156, "logps/rejected": -152.20608520507812, "loss": 0.994, "margin_dpo/margin_mean": 22.516151428222656, "margin_dpo/margin_std": 23.426759719848633, "step": 198 }, { "KL/chosen_KL_mean": -25.030391693115234, "KL/mean": -34.420745849609375, "KL/rejected_KL_mean": -43.811100006103516, "KL/std": 22.346843719482422, "epoch": 0.2922173274596182, "fcm_dpo/beta": 0.0233822762966156, "fcm_dpo/delta": 0.011303754523396492, "fcm_dpo/margin": 18.780710220336914, "fcm_dpo/q_t": 0.3995858430862427, "grad_norm": 10.67126750946045, "learning_rate": 4.471606039587695e-07, "logits/chosen": -1.5150001049041748, "logits/rejected": -1.250755786895752, "logps/chosen": -106.81709289550781, "logps/ref_chosen": -81.78669738769531, "logps/ref_rejected": -90.790771484375, "logps/rejected": -134.60186767578125, "loss": 1.0646, "margin_dpo/margin_mean": 18.78070831298828, "margin_dpo/margin_std": 23.448162078857422, "step": 199 }, { "KL/chosen_KL_mean": -24.523277282714844, "KL/mean": -35.61384582519531, "KL/rejected_KL_mean": -46.70440673828125, "KL/std": 23.738174438476562, "epoch": 0.2936857562408223, "fcm_dpo/beta": 0.023026688024401665, "fcm_dpo/delta": -0.06481599807739258, "fcm_dpo/margin": 22.181123733520508, "fcm_dpo/q_t": 0.38517606258392334, "grad_norm": 10.47972583770752, "learning_rate": 4.4636895135509966e-07, "logits/chosen": -1.624913215637207, "logits/rejected": -1.0862584114074707, "logps/chosen": -108.79136657714844, "logps/ref_chosen": -84.26809692382812, "logps/ref_rejected": -86.96786499023438, "logps/rejected": -133.67227172851562, "loss": 1.0313, "margin_dpo/margin_mean": 22.181121826171875, "margin_dpo/margin_std": 27.287662506103516, "step": 200 }, { "KL/chosen_KL_mean": -26.327220916748047, "KL/mean": -37.032928466796875, "KL/rejected_KL_mean": -47.7386474609375, "KL/std": 25.674232482910156, "epoch": 0.29515418502202645, "fcm_dpo/beta": 0.022968199104070663, "fcm_dpo/delta": -0.0439612939953804, "fcm_dpo/margin": 21.411434173583984, "fcm_dpo/q_t": 0.3894343376159668, "grad_norm": 10.895137786865234, "learning_rate": 4.455721242469372e-07, "logits/chosen": -1.4008371829986572, "logits/rejected": -1.0123724937438965, "logps/chosen": -134.09971618652344, "logps/ref_chosen": -107.77249145507812, "logps/ref_rejected": -119.79248046875, "logps/rejected": -167.5311279296875, "loss": 1.0482, "margin_dpo/margin_mean": 21.411434173583984, "margin_dpo/margin_std": 28.15359115600586, "step": 201 }, { "KL/chosen_KL_mean": -24.698772430419922, "KL/mean": -34.60642623901367, "KL/rejected_KL_mean": -44.51408004760742, "KL/std": 21.26959228515625, "epoch": 0.2966226138032305, "fcm_dpo/beta": 0.022876489907503128, "fcm_dpo/delta": -0.0034904824569821358, "fcm_dpo/margin": 19.81529998779297, "fcm_dpo/q_t": 0.39716821908950806, "grad_norm": 11.087641716003418, "learning_rate": 4.4477014363141755e-07, "logits/chosen": -1.5747380256652832, "logits/rejected": -1.4448705911636353, "logps/chosen": -100.67122650146484, "logps/ref_chosen": -75.97245788574219, "logps/ref_rejected": -94.4599838256836, "logps/rejected": -138.97406005859375, "loss": 1.0554, "margin_dpo/margin_mean": 19.81529998779297, "margin_dpo/margin_std": 24.043787002563477, "step": 202 }, { "KL/chosen_KL_mean": -22.10192108154297, "KL/mean": -33.389915466308594, "KL/rejected_KL_mean": -44.67791748046875, "KL/std": 22.240814208984375, "epoch": 0.29809104258443464, "fcm_dpo/beta": 0.022657740861177444, "fcm_dpo/delta": -0.06478673964738846, "fcm_dpo/margin": 22.57599639892578, "fcm_dpo/q_t": 0.38190820813179016, "grad_norm": 9.961379051208496, "learning_rate": 4.439630306414758e-07, "logits/chosen": -1.8847124576568604, "logits/rejected": -1.4328787326812744, "logps/chosen": -117.06907653808594, "logps/ref_chosen": -94.96715545654297, "logps/ref_rejected": -92.8876724243164, "logps/rejected": -137.56558227539062, "loss": 1.0021, "margin_dpo/margin_mean": 22.57599639892578, "margin_dpo/margin_std": 23.117809295654297, "step": 203 }, { "KL/chosen_KL_mean": -27.695480346679688, "KL/mean": -37.42096710205078, "KL/rejected_KL_mean": -47.146453857421875, "KL/std": 22.132984161376953, "epoch": 0.29955947136563876, "fcm_dpo/beta": 0.02265116758644581, "fcm_dpo/delta": 0.009622307494282722, "fcm_dpo/margin": 19.45098304748535, "fcm_dpo/q_t": 0.4003278613090515, "grad_norm": 11.085248947143555, "learning_rate": 4.431508065452897e-07, "logits/chosen": -1.8878350257873535, "logits/rejected": -1.2349971532821655, "logps/chosen": -144.05267333984375, "logps/ref_chosen": -116.35719299316406, "logps/ref_rejected": -93.39759063720703, "logps/rejected": -140.54403686523438, "loss": 1.0745, "margin_dpo/margin_mean": 19.45098114013672, "margin_dpo/margin_std": 25.756441116333008, "step": 204 }, { "KL/chosen_KL_mean": -25.701007843017578, "KL/mean": -38.21492004394531, "KL/rejected_KL_mean": -50.72883605957031, "KL/std": 24.512920379638672, "epoch": 0.3010279001468429, "fcm_dpo/beta": 0.022171659395098686, "fcm_dpo/delta": -0.11142764240503311, "fcm_dpo/margin": 25.02783203125, "fcm_dpo/q_t": 0.36982864141464233, "grad_norm": 9.582030296325684, "learning_rate": 4.4233349274571974e-07, "logits/chosen": -2.1238603591918945, "logits/rejected": -1.6858125925064087, "logps/chosen": -114.56035614013672, "logps/ref_chosen": -88.85934448242188, "logps/ref_rejected": -91.8544921875, "logps/rejected": -142.5833282470703, "loss": 0.9569, "margin_dpo/margin_mean": 25.027830123901367, "margin_dpo/margin_std": 21.043460845947266, "step": 205 }, { "KL/chosen_KL_mean": -26.342140197753906, "KL/mean": -37.848567962646484, "KL/rejected_KL_mean": -49.35499954223633, "KL/std": 23.870223999023438, "epoch": 0.302496328928047, "fcm_dpo/beta": 0.02183135598897934, "fcm_dpo/delta": -0.05566246807575226, "fcm_dpo/margin": 23.012861251831055, "fcm_dpo/q_t": 0.3818650245666504, "grad_norm": 11.548815727233887, "learning_rate": 4.415111107797445e-07, "logits/chosen": -1.4087283611297607, "logits/rejected": -1.2552969455718994, "logps/chosen": -102.88848876953125, "logps/ref_chosen": -76.54634857177734, "logps/ref_rejected": -102.95314025878906, "logps/rejected": -152.30813598632812, "loss": 0.9927, "margin_dpo/margin_mean": 23.012861251831055, "margin_dpo/margin_std": 20.281452178955078, "step": 206 }, { "KL/chosen_KL_mean": -27.59354019165039, "KL/mean": -40.39479064941406, "KL/rejected_KL_mean": -53.196048736572266, "KL/std": 29.01898765563965, "epoch": 0.3039647577092511, "fcm_dpo/beta": 0.021515972912311554, "fcm_dpo/delta": -0.106700599193573, "fcm_dpo/margin": 25.602506637573242, "fcm_dpo/q_t": 0.3801648020744324, "grad_norm": 11.600892066955566, "learning_rate": 4.4068368231789365e-07, "logits/chosen": -2.1540088653564453, "logits/rejected": -1.7385635375976562, "logps/chosen": -113.82518768310547, "logps/ref_chosen": -86.23164367675781, "logps/ref_rejected": -90.65512084960938, "logps/rejected": -143.85116577148438, "loss": 1.0211, "margin_dpo/margin_mean": 25.60250473022461, "margin_dpo/margin_std": 33.49293518066406, "step": 207 }, { "KL/chosen_KL_mean": -28.55429458618164, "KL/mean": -41.814109802246094, "KL/rejected_KL_mean": -55.07392883300781, "KL/std": 26.690162658691406, "epoch": 0.3054331864904552, "fcm_dpo/beta": 0.021119004115462303, "fcm_dpo/delta": -0.11649561673402786, "fcm_dpo/margin": 26.519634246826172, "fcm_dpo/q_t": 0.3698372542858124, "grad_norm": 10.919686317443848, "learning_rate": 4.398512291636768e-07, "logits/chosen": -1.7398350238800049, "logits/rejected": -1.3810641765594482, "logps/chosen": -122.71383666992188, "logps/ref_chosen": -94.1595458984375, "logps/ref_rejected": -100.96233367919922, "logps/rejected": -156.0362548828125, "loss": 0.967, "margin_dpo/margin_mean": 26.519634246826172, "margin_dpo/margin_std": 24.634014129638672, "step": 208 }, { "KL/chosen_KL_mean": -24.12353515625, "KL/mean": -35.77311706542969, "KL/rejected_KL_mean": -47.42270278930664, "KL/std": 23.735620498657227, "epoch": 0.3069016152716593, "fcm_dpo/beta": 0.02069961279630661, "fcm_dpo/delta": -0.03445186838507652, "fcm_dpo/margin": 23.29916763305664, "fcm_dpo/q_t": 0.3899701237678528, "grad_norm": 9.853930473327637, "learning_rate": 4.3901377325300857e-07, "logits/chosen": -2.117767810821533, "logits/rejected": -1.647392749786377, "logps/chosen": -108.29409790039062, "logps/ref_chosen": -84.17056274414062, "logps/ref_rejected": -87.61955261230469, "logps/rejected": -135.04225158691406, "loss": 1.034, "margin_dpo/margin_mean": 23.29916763305664, "margin_dpo/margin_std": 26.88469696044922, "step": 209 }, { "KL/chosen_KL_mean": -27.380931854248047, "KL/mean": -38.50312042236328, "KL/rejected_KL_mean": -49.62531661987305, "KL/std": 25.168006896972656, "epoch": 0.30837004405286345, "fcm_dpo/beta": 0.020676523447036743, "fcm_dpo/delta": -0.010528551414608955, "fcm_dpo/margin": 22.244384765625, "fcm_dpo/q_t": 0.39471542835235596, "grad_norm": 9.15931224822998, "learning_rate": 4.381713366536311e-07, "logits/chosen": -1.8130569458007812, "logits/rejected": -1.417151689529419, "logps/chosen": -108.55210876464844, "logps/ref_chosen": -81.17117309570312, "logps/ref_rejected": -84.17478942871094, "logps/rejected": -133.80010986328125, "loss": 1.0421, "margin_dpo/margin_mean": 22.244382858276367, "margin_dpo/margin_std": 25.39181137084961, "step": 210 }, { "KL/chosen_KL_mean": -30.4791259765625, "KL/mean": -41.428985595703125, "KL/rejected_KL_mean": -52.378849029541016, "KL/std": 26.59711456298828, "epoch": 0.30983847283406757, "fcm_dpo/beta": 0.020739322528243065, "fcm_dpo/delta": -0.00446801632642746, "fcm_dpo/margin": 21.899728775024414, "fcm_dpo/q_t": 0.39676520228385925, "grad_norm": 11.158641815185547, "learning_rate": 4.373239415645323e-07, "logits/chosen": -1.9388091564178467, "logits/rejected": -1.3017692565917969, "logps/chosen": -139.19183349609375, "logps/ref_chosen": -108.71270751953125, "logps/ref_rejected": -93.55564880371094, "logps/rejected": -145.93450927734375, "loss": 1.0539, "margin_dpo/margin_mean": 21.899728775024414, "margin_dpo/margin_std": 26.995410919189453, "step": 211 }, { "KL/chosen_KL_mean": -21.315662384033203, "KL/mean": -37.14613723754883, "KL/rejected_KL_mean": -52.97661590576172, "KL/std": 29.989168167114258, "epoch": 0.31130690161527164, "fcm_dpo/beta": 0.02004014141857624, "fcm_dpo/delta": -0.1976533830165863, "fcm_dpo/margin": 31.66094970703125, "fcm_dpo/q_t": 0.3564627468585968, "grad_norm": 9.618505477905273, "learning_rate": 4.3647161031536086e-07, "logits/chosen": -1.7199327945709229, "logits/rejected": -1.2956655025482178, "logps/chosen": -119.6776123046875, "logps/ref_chosen": -98.36195373535156, "logps/ref_rejected": -109.88999938964844, "logps/rejected": -162.86660766601562, "loss": 0.9275, "margin_dpo/margin_mean": 31.660951614379883, "margin_dpo/margin_std": 29.891117095947266, "step": 212 }, { "KL/chosen_KL_mean": -25.102352142333984, "KL/mean": -37.12311935424805, "KL/rejected_KL_mean": -49.14387512207031, "KL/std": 27.50717544555664, "epoch": 0.31277533039647576, "fcm_dpo/beta": 0.019762098789215088, "fcm_dpo/delta": -0.02654850482940674, "fcm_dpo/margin": 24.041526794433594, "fcm_dpo/q_t": 0.3912399411201477, "grad_norm": 12.485479354858398, "learning_rate": 4.3561436536583774e-07, "logits/chosen": -1.7787501811981201, "logits/rejected": -0.9773513078689575, "logps/chosen": -133.15765380859375, "logps/ref_chosen": -108.05531311035156, "logps/ref_rejected": -100.14414978027344, "logps/rejected": -149.28802490234375, "loss": 1.0464, "margin_dpo/margin_mean": 24.041526794433594, "margin_dpo/margin_std": 29.77020835876465, "step": 213 }, { "KL/chosen_KL_mean": -26.931499481201172, "KL/mean": -38.158233642578125, "KL/rejected_KL_mean": -49.384971618652344, "KL/std": 29.45815658569336, "epoch": 0.3142437591776799, "fcm_dpo/beta": 0.01982264406979084, "fcm_dpo/delta": 0.005104076117277145, "fcm_dpo/margin": 22.45347785949707, "fcm_dpo/q_t": 0.4010830521583557, "grad_norm": 9.625554084777832, "learning_rate": 4.3475222930516473e-07, "logits/chosen": -1.8941937685012817, "logits/rejected": -1.77357816696167, "logps/chosen": -104.73623657226562, "logps/ref_chosen": -77.80473327636719, "logps/ref_rejected": -84.27578735351562, "logps/rejected": -133.6607666015625, "loss": 1.084, "margin_dpo/margin_mean": 22.45347785949707, "margin_dpo/margin_std": 32.474544525146484, "step": 214 }, { "KL/chosen_KL_mean": -23.280712127685547, "KL/mean": -38.35309600830078, "KL/rejected_KL_mean": -53.425479888916016, "KL/std": 29.038955688476562, "epoch": 0.315712187958884, "fcm_dpo/beta": 0.01939084380865097, "fcm_dpo/delta": -0.14286500215530396, "fcm_dpo/margin": 30.144752502441406, "fcm_dpo/q_t": 0.3668321669101715, "grad_norm": 9.348302841186523, "learning_rate": 4.3388522485142885e-07, "logits/chosen": -1.4726921319961548, "logits/rejected": -0.9769058227539062, "logps/chosen": -108.39460754394531, "logps/ref_chosen": -85.1138916015625, "logps/ref_rejected": -96.86151123046875, "logps/rejected": -150.2869873046875, "loss": 0.9528, "margin_dpo/margin_mean": 30.144752502441406, "margin_dpo/margin_std": 28.670223236083984, "step": 215 }, { "KL/chosen_KL_mean": -25.23155975341797, "KL/mean": -38.3830451965332, "KL/rejected_KL_mean": -51.53453826904297, "KL/std": 26.145912170410156, "epoch": 0.31718061674008813, "fcm_dpo/beta": 0.019167980179190636, "fcm_dpo/delta": -0.05720216780900955, "fcm_dpo/margin": 26.30297088623047, "fcm_dpo/q_t": 0.38458746671676636, "grad_norm": 8.929350852966309, "learning_rate": 4.330133748510036e-07, "logits/chosen": -1.9628796577453613, "logits/rejected": -1.5714607238769531, "logps/chosen": -105.82386779785156, "logps/ref_chosen": -80.59230041503906, "logps/ref_rejected": -81.41983795166016, "logps/rejected": -132.95437622070312, "loss": 1.0071, "margin_dpo/margin_mean": 26.3029727935791, "margin_dpo/margin_std": 27.05829620361328, "step": 216 }, { "KL/chosen_KL_mean": -30.480587005615234, "KL/mean": -41.486656188964844, "KL/rejected_KL_mean": -52.492733001708984, "KL/std": 26.829389572143555, "epoch": 0.3186490455212922, "fcm_dpo/beta": 0.019149091094732285, "fcm_dpo/delta": 0.02959912270307541, "fcm_dpo/margin": 22.012144088745117, "fcm_dpo/q_t": 0.4052965044975281, "grad_norm": 10.928380012512207, "learning_rate": 4.3213670227794757e-07, "logits/chosen": -1.852818489074707, "logits/rejected": -1.5402162075042725, "logps/chosen": -123.95317077636719, "logps/ref_chosen": -93.47257995605469, "logps/ref_rejected": -103.488525390625, "logps/rejected": -155.98126220703125, "loss": 1.0859, "margin_dpo/margin_mean": 22.012142181396484, "margin_dpo/margin_std": 30.480266571044922, "step": 217 }, { "KL/chosen_KL_mean": -29.74217987060547, "KL/mean": -42.884925842285156, "KL/rejected_KL_mean": -56.027671813964844, "KL/std": 27.209476470947266, "epoch": 0.3201174743024963, "fcm_dpo/beta": 0.018940910696983337, "fcm_dpo/delta": -0.05069791525602341, "fcm_dpo/margin": 26.285490036010742, "fcm_dpo/q_t": 0.3873566687107086, "grad_norm": 9.492467880249023, "learning_rate": 4.3125523023339815e-07, "logits/chosen": -1.284709095954895, "logits/rejected": -0.893244743347168, "logps/chosen": -118.80101013183594, "logps/ref_chosen": -89.05883026123047, "logps/ref_rejected": -94.30680847167969, "logps/rejected": -150.33447265625, "loss": 1.0359, "margin_dpo/margin_mean": 26.28548812866211, "margin_dpo/margin_std": 32.44743728637695, "step": 218 }, { "KL/chosen_KL_mean": -26.7744140625, "KL/mean": -38.57316970825195, "KL/rejected_KL_mean": -50.371925354003906, "KL/std": 31.48541259765625, "epoch": 0.32158590308370044, "fcm_dpo/beta": 0.019057394936680794, "fcm_dpo/delta": -0.00039842724800109863, "fcm_dpo/margin": 23.59750747680664, "fcm_dpo/q_t": 0.398790180683136, "grad_norm": 10.948594093322754, "learning_rate": 4.303689819449636e-07, "logits/chosen": -1.679978609085083, "logits/rejected": -1.2237684726715088, "logps/chosen": -127.78175354003906, "logps/ref_chosen": -101.00733947753906, "logps/ref_rejected": -92.46794128417969, "logps/rejected": -142.83987426757812, "loss": 1.0807, "margin_dpo/margin_mean": 23.597511291503906, "margin_dpo/margin_std": 33.49624252319336, "step": 219 }, { "KL/chosen_KL_mean": -27.571399688720703, "KL/mean": -38.54227828979492, "KL/rejected_KL_mean": -49.513153076171875, "KL/std": 26.086421966552734, "epoch": 0.32305433186490456, "fcm_dpo/beta": 0.019048605114221573, "fcm_dpo/delta": 0.03341676667332649, "fcm_dpo/margin": 21.941770553588867, "fcm_dpo/q_t": 0.4032576084136963, "grad_norm": 11.072999000549316, "learning_rate": 4.2947798076611047e-07, "logits/chosen": -1.5224353075027466, "logits/rejected": -1.1750617027282715, "logps/chosen": -123.10861206054688, "logps/ref_chosen": -95.53721618652344, "logps/ref_rejected": -94.30703735351562, "logps/rejected": -143.8201904296875, "loss": 1.0687, "margin_dpo/margin_mean": 21.9417724609375, "margin_dpo/margin_std": 25.74374008178711, "step": 220 }, { "KL/chosen_KL_mean": -26.08209228515625, "KL/mean": -42.82952880859375, "KL/rejected_KL_mean": -59.576961517333984, "KL/std": 32.706634521484375, "epoch": 0.3245227606461087, "fcm_dpo/beta": 0.018682332709431648, "fcm_dpo/delta": -0.1871957927942276, "fcm_dpo/margin": 33.494869232177734, "fcm_dpo/q_t": 0.357664555311203, "grad_norm": 10.636537551879883, "learning_rate": 4.285822501755485e-07, "logits/chosen": -1.2643651962280273, "logits/rejected": -1.1244919300079346, "logps/chosen": -108.92695617675781, "logps/ref_chosen": -82.84486389160156, "logps/ref_rejected": -110.81179809570312, "logps/rejected": -170.38876342773438, "loss": 0.9268, "margin_dpo/margin_mean": 33.494869232177734, "margin_dpo/margin_std": 28.816444396972656, "step": 221 }, { "KL/chosen_KL_mean": -33.369239807128906, "KL/mean": -47.110816955566406, "KL/rejected_KL_mean": -60.85240173339844, "KL/std": 31.279739379882812, "epoch": 0.32599118942731276, "fcm_dpo/beta": 0.01830419711768627, "fcm_dpo/delta": -0.056051403284072876, "fcm_dpo/margin": 27.483156204223633, "fcm_dpo/q_t": 0.38697564601898193, "grad_norm": 11.075545310974121, "learning_rate": 4.276818137766118e-07, "logits/chosen": -1.9451358318328857, "logits/rejected": -1.5141632556915283, "logps/chosen": -128.51121520996094, "logps/ref_chosen": -95.14198303222656, "logps/ref_rejected": -106.80441284179688, "logps/rejected": -167.6568145751953, "loss": 1.0509, "margin_dpo/margin_mean": 27.483158111572266, "margin_dpo/margin_std": 36.74195861816406, "step": 222 }, { "KL/chosen_KL_mean": -32.391475677490234, "KL/mean": -42.24231719970703, "KL/rejected_KL_mean": -52.09315872192383, "KL/std": 27.651885986328125, "epoch": 0.3274596182085169, "fcm_dpo/beta": 0.01839200034737587, "fcm_dpo/delta": 0.0910235345363617, "fcm_dpo/margin": 19.70168685913086, "fcm_dpo/q_t": 0.41741105914115906, "grad_norm": 10.734125137329102, "learning_rate": 4.2677669529663686e-07, "logits/chosen": -2.1838834285736084, "logits/rejected": -1.7929104566574097, "logps/chosen": -117.96658325195312, "logps/ref_chosen": -85.57511138916016, "logps/ref_rejected": -86.45237731933594, "logps/rejected": -138.5455322265625, "loss": 1.1292, "margin_dpo/margin_mean": 19.70168685913086, "margin_dpo/margin_std": 30.232542037963867, "step": 223 }, { "KL/chosen_KL_mean": -29.820423126220703, "KL/mean": -43.66676712036133, "KL/rejected_KL_mean": -57.51311111450195, "KL/std": 30.592058181762695, "epoch": 0.328928046989721, "fcm_dpo/beta": 0.018324650824069977, "fcm_dpo/delta": -0.06056816875934601, "fcm_dpo/margin": 27.692689895629883, "fcm_dpo/q_t": 0.38475412130355835, "grad_norm": 10.024768829345703, "learning_rate": 4.2586691858633747e-07, "logits/chosen": -1.8784904479980469, "logits/rejected": -1.4315197467803955, "logps/chosen": -112.54422760009766, "logps/ref_chosen": -82.72380065917969, "logps/ref_rejected": -82.59538269042969, "logps/rejected": -140.10848999023438, "loss": 1.0197, "margin_dpo/margin_mean": 27.692691802978516, "margin_dpo/margin_std": 31.912351608276367, "step": 224 }, { "KL/chosen_KL_mean": -33.354976654052734, "KL/mean": -47.91131591796875, "KL/rejected_KL_mean": -62.46765899658203, "KL/std": 32.20822525024414, "epoch": 0.3303964757709251, "fcm_dpo/beta": 0.018125828355550766, "fcm_dpo/delta": -0.08200725167989731, "fcm_dpo/margin": 29.112688064575195, "fcm_dpo/q_t": 0.38220465183258057, "grad_norm": 11.154870986938477, "learning_rate": 4.249525076191759e-07, "logits/chosen": -1.878772258758545, "logits/rejected": -1.3943595886230469, "logps/chosen": -129.03265380859375, "logps/ref_chosen": -95.67768859863281, "logps/ref_rejected": -105.09687805175781, "logps/rejected": -167.56454467773438, "loss": 1.0206, "margin_dpo/margin_mean": 29.112688064575195, "margin_dpo/margin_std": 35.91593933105469, "step": 225 }, { "KL/chosen_KL_mean": -27.574657440185547, "KL/mean": -41.40182876586914, "KL/rejected_KL_mean": -55.22899627685547, "KL/std": 31.36874008178711, "epoch": 0.33186490455212925, "fcm_dpo/beta": 0.017766930162906647, "fcm_dpo/delta": -0.04428989440202713, "fcm_dpo/margin": 27.654335021972656, "fcm_dpo/q_t": 0.3866173326969147, "grad_norm": 9.041952133178711, "learning_rate": 4.2403348649073167e-07, "logits/chosen": -1.857753038406372, "logits/rejected": -1.326967477798462, "logps/chosen": -121.03558349609375, "logps/ref_chosen": -93.46092224121094, "logps/ref_rejected": -86.7017593383789, "logps/rejected": -141.93075561523438, "loss": 1.0131, "margin_dpo/margin_mean": 27.654335021972656, "margin_dpo/margin_std": 27.987306594848633, "step": 226 }, { "KL/chosen_KL_mean": -37.83222198486328, "KL/mean": -52.051429748535156, "KL/rejected_KL_mean": -66.27063751220703, "KL/std": 30.28497314453125, "epoch": 0.3333333333333333, "fcm_dpo/beta": 0.017683152109384537, "fcm_dpo/delta": -0.055658984929323196, "fcm_dpo/margin": 28.438419342041016, "fcm_dpo/q_t": 0.3871355652809143, "grad_norm": 10.029187202453613, "learning_rate": 4.2310987941806615e-07, "logits/chosen": -1.5599040985107422, "logits/rejected": -1.2161908149719238, "logps/chosen": -130.646484375, "logps/ref_chosen": -92.81427001953125, "logps/ref_rejected": -104.73693084716797, "logps/rejected": -171.007568359375, "loss": 1.038, "margin_dpo/margin_mean": 28.438419342041016, "margin_dpo/margin_std": 35.90989685058594, "step": 227 }, { "KL/chosen_KL_mean": -35.73163604736328, "KL/mean": -46.58448791503906, "KL/rejected_KL_mean": -57.43733596801758, "KL/std": 28.182226181030273, "epoch": 0.33480176211453744, "fcm_dpo/beta": 0.017788201570510864, "fcm_dpo/delta": 0.06649549305438995, "fcm_dpo/margin": 21.705698013305664, "fcm_dpo/q_t": 0.4111056923866272, "grad_norm": 10.444371223449707, "learning_rate": 4.2218171073908463e-07, "logits/chosen": -1.8236147165298462, "logits/rejected": -1.3874504566192627, "logps/chosen": -129.76876831054688, "logps/ref_chosen": -94.03712463378906, "logps/ref_rejected": -96.02151489257812, "logps/rejected": -153.4588623046875, "loss": 1.1114, "margin_dpo/margin_mean": 21.705698013305664, "margin_dpo/margin_std": 31.48227310180664, "step": 228 }, { "KL/chosen_KL_mean": -31.76717758178711, "KL/mean": -49.680389404296875, "KL/rejected_KL_mean": -67.59359741210938, "KL/std": 34.649620056152344, "epoch": 0.33627019089574156, "fcm_dpo/beta": 0.017390307039022446, "fcm_dpo/delta": -0.18441686034202576, "fcm_dpo/margin": 35.82642364501953, "fcm_dpo/q_t": 0.361656129360199, "grad_norm": 9.87229061126709, "learning_rate": 4.212490049118951e-07, "logits/chosen": -1.7755630016326904, "logits/rejected": -1.1372489929199219, "logps/chosen": -127.3448486328125, "logps/ref_chosen": -95.57766723632812, "logps/ref_rejected": -89.17379760742188, "logps/rejected": -156.76739501953125, "loss": 0.9497, "margin_dpo/margin_mean": 35.82642364501953, "margin_dpo/margin_std": 37.666080474853516, "step": 229 }, { "KL/chosen_KL_mean": -33.3121452331543, "KL/mean": -51.62913513183594, "KL/rejected_KL_mean": -69.94612121582031, "KL/std": 34.056793212890625, "epoch": 0.3377386196769457, "fcm_dpo/beta": 0.016771184280514717, "fcm_dpo/delta": -0.17516855895519257, "fcm_dpo/margin": 36.63397979736328, "fcm_dpo/q_t": 0.3589358925819397, "grad_norm": 10.829142570495605, "learning_rate": 4.203117865141635e-07, "logits/chosen": -1.6221001148223877, "logits/rejected": -1.5562043190002441, "logps/chosen": -97.02577209472656, "logps/ref_chosen": -63.713626861572266, "logps/ref_rejected": -91.90870666503906, "logps/rejected": -161.85482788085938, "loss": 0.9346, "margin_dpo/margin_mean": 36.63397979736328, "margin_dpo/margin_std": 33.538787841796875, "step": 230 }, { "KL/chosen_KL_mean": -37.03425598144531, "KL/mean": -52.77891159057617, "KL/rejected_KL_mean": -68.5235595703125, "KL/std": 32.92323303222656, "epoch": 0.3392070484581498, "fcm_dpo/beta": 0.01636582799255848, "fcm_dpo/delta": -0.06978515535593033, "fcm_dpo/margin": 31.48931121826172, "fcm_dpo/q_t": 0.3800153434276581, "grad_norm": 8.929129600524902, "learning_rate": 4.1937008024246625e-07, "logits/chosen": -1.8607261180877686, "logits/rejected": -1.3150267601013184, "logps/chosen": -132.48992919921875, "logps/ref_chosen": -95.45567321777344, "logps/ref_rejected": -80.95568084716797, "logps/rejected": -149.479248046875, "loss": 0.9881, "margin_dpo/margin_mean": 31.48931312561035, "margin_dpo/margin_std": 28.539264678955078, "step": 231 }, { "KL/chosen_KL_mean": -38.97697448730469, "KL/mean": -52.253780364990234, "KL/rejected_KL_mean": -65.53057861328125, "KL/std": 30.39307403564453, "epoch": 0.3406754772393539, "fcm_dpo/beta": 0.016406547278165817, "fcm_dpo/delta": 0.014987261965870857, "fcm_dpo/margin": 26.553607940673828, "fcm_dpo/q_t": 0.39936891198158264, "grad_norm": 9.567919731140137, "learning_rate": 4.1842391091163933e-07, "logits/chosen": -1.363256812095642, "logits/rejected": -0.8709152936935425, "logps/chosen": -135.87423706054688, "logps/ref_chosen": -96.89726257324219, "logps/ref_rejected": -89.76461791992188, "logps/rejected": -155.29519653320312, "loss": 1.0591, "margin_dpo/margin_mean": 26.553607940673828, "margin_dpo/margin_std": 31.093616485595703, "step": 232 }, { "KL/chosen_KL_mean": -40.90872573852539, "KL/mean": -59.25398254394531, "KL/rejected_KL_mean": -77.59922790527344, "KL/std": 38.152381896972656, "epoch": 0.342143906020558, "fcm_dpo/beta": 0.016056008636951447, "fcm_dpo/delta": -0.1481083482503891, "fcm_dpo/margin": 36.69049835205078, "fcm_dpo/q_t": 0.36577850580215454, "grad_norm": 10.34557056427002, "learning_rate": 4.174733034541245e-07, "logits/chosen": -1.6242706775665283, "logits/rejected": -1.4135671854019165, "logps/chosen": -129.95904541015625, "logps/ref_chosen": -89.05032348632812, "logps/ref_rejected": -112.75917053222656, "logps/rejected": -190.3583984375, "loss": 0.9602, "margin_dpo/margin_mean": 36.69050216674805, "margin_dpo/margin_std": 36.75348663330078, "step": 233 }, { "KL/chosen_KL_mean": -39.851036071777344, "KL/mean": -59.33667755126953, "KL/rejected_KL_mean": -78.82231903076172, "KL/std": 36.616676330566406, "epoch": 0.3436123348017621, "fcm_dpo/beta": 0.01545102708041668, "fcm_dpo/delta": -0.1645185798406601, "fcm_dpo/margin": 38.971282958984375, "fcm_dpo/q_t": 0.3646003007888794, "grad_norm": 9.245274543762207, "learning_rate": 4.165182829193126e-07, "logits/chosen": -1.4410686492919922, "logits/rejected": -1.3687561750411987, "logps/chosen": -114.16940307617188, "logps/ref_chosen": -74.318359375, "logps/ref_rejected": -106.38758850097656, "logps/rejected": -185.20989990234375, "loss": 0.9603, "margin_dpo/margin_mean": 38.971282958984375, "margin_dpo/margin_std": 40.08073425292969, "step": 234 }, { "KL/chosen_KL_mean": -47.7650260925293, "KL/mean": -62.5955924987793, "KL/rejected_KL_mean": -77.4261474609375, "KL/std": 32.802406311035156, "epoch": 0.34508076358296624, "fcm_dpo/beta": 0.015446878969669342, "fcm_dpo/delta": -0.008598173037171364, "fcm_dpo/margin": 29.66112518310547, "fcm_dpo/q_t": 0.3935566544532776, "grad_norm": 9.225977897644043, "learning_rate": 4.1555887447288255e-07, "logits/chosen": -1.6084418296813965, "logits/rejected": -1.0536733865737915, "logps/chosen": -145.98207092285156, "logps/ref_chosen": -98.217041015625, "logps/ref_rejected": -97.24677276611328, "logps/rejected": -174.67291259765625, "loss": 1.0335, "margin_dpo/margin_mean": 29.66112518310547, "margin_dpo/margin_std": 31.05475425720215, "step": 235 }, { "KL/chosen_KL_mean": -40.224674224853516, "KL/mean": -59.228267669677734, "KL/rejected_KL_mean": -78.23187255859375, "KL/std": 41.153526306152344, "epoch": 0.3465491923641703, "fcm_dpo/beta": 0.015149770304560661, "fcm_dpo/delta": -0.13328908383846283, "fcm_dpo/margin": 38.00719451904297, "fcm_dpo/q_t": 0.37542450428009033, "grad_norm": 9.858134269714355, "learning_rate": 4.1459510339613946e-07, "logits/chosen": -1.3845889568328857, "logits/rejected": -1.3828411102294922, "logps/chosen": -119.06240844726562, "logps/ref_chosen": -78.83773040771484, "logps/ref_rejected": -109.06343078613281, "logps/rejected": -187.29530334472656, "loss": 0.9855, "margin_dpo/margin_mean": 38.00719451904297, "margin_dpo/margin_std": 44.75410461425781, "step": 236 }, { "KL/chosen_KL_mean": -47.65378952026367, "KL/mean": -61.71543884277344, "KL/rejected_KL_mean": -75.7770767211914, "KL/std": 36.32740020751953, "epoch": 0.34801762114537443, "fcm_dpo/beta": 0.015071479603648186, "fcm_dpo/delta": 0.02729911357164383, "fcm_dpo/margin": 28.123275756835938, "fcm_dpo/q_t": 0.4083618223667145, "grad_norm": 9.366373062133789, "learning_rate": 4.136269950853473e-07, "logits/chosen": -1.4898028373718262, "logits/rejected": -1.1301348209381104, "logps/chosen": -132.86508178710938, "logps/ref_chosen": -85.21128845214844, "logps/ref_rejected": -99.90999603271484, "logps/rejected": -175.68707275390625, "loss": 1.1194, "margin_dpo/margin_mean": 28.123275756835938, "margin_dpo/margin_std": 47.189090728759766, "step": 237 }, { "KL/chosen_KL_mean": -52.024654388427734, "KL/mean": -68.32868957519531, "KL/rejected_KL_mean": -84.63272094726562, "KL/std": 37.93627166748047, "epoch": 0.34948604992657856, "fcm_dpo/beta": 0.014987221919000149, "fcm_dpo/delta": -0.040753819048404694, "fcm_dpo/margin": 32.60807800292969, "fcm_dpo/q_t": 0.38997578620910645, "grad_norm": 8.857922554016113, "learning_rate": 4.126545750510605e-07, "logits/chosen": -1.2862224578857422, "logits/rejected": -1.1352065801620483, "logps/chosen": -130.75588989257812, "logps/ref_chosen": -78.73123168945312, "logps/ref_rejected": -95.41840362548828, "logps/rejected": -180.05111694335938, "loss": 1.0342, "margin_dpo/margin_mean": 32.60807800292969, "margin_dpo/margin_std": 39.337013244628906, "step": 238 }, { "KL/chosen_KL_mean": -47.21555709838867, "KL/mean": -62.49492645263672, "KL/rejected_KL_mean": -77.77430725097656, "KL/std": 33.71417236328125, "epoch": 0.3509544787077827, "fcm_dpo/beta": 0.014858734793961048, "fcm_dpo/delta": -0.00522758811712265, "fcm_dpo/margin": 30.55875015258789, "fcm_dpo/q_t": 0.3967739939689636, "grad_norm": 9.374117851257324, "learning_rate": 4.116778689174514e-07, "logits/chosen": -1.7733337879180908, "logits/rejected": -1.2934105396270752, "logps/chosen": -139.81649780273438, "logps/ref_chosen": -92.60093688964844, "logps/ref_rejected": -100.51769256591797, "logps/rejected": -178.2919921875, "loss": 1.0604, "margin_dpo/margin_mean": 30.55875015258789, "margin_dpo/margin_std": 37.68760299682617, "step": 239 }, { "KL/chosen_KL_mean": -49.013118743896484, "KL/mean": -62.57707977294922, "KL/rejected_KL_mean": -76.14105224609375, "KL/std": 33.897212982177734, "epoch": 0.3524229074889868, "fcm_dpo/beta": 0.015017041936516762, "fcm_dpo/delta": 0.04441224783658981, "fcm_dpo/margin": 27.1279296875, "fcm_dpo/q_t": 0.4065741300582886, "grad_norm": 10.550976753234863, "learning_rate": 4.106969024216348e-07, "logits/chosen": -2.0598533153533936, "logits/rejected": -1.7029976844787598, "logps/chosen": -135.17288208007812, "logps/ref_chosen": -86.15977478027344, "logps/ref_rejected": -80.45567321777344, "logps/rejected": -156.59671020507812, "loss": 1.0894, "margin_dpo/margin_mean": 27.127927780151367, "margin_dpo/margin_std": 36.040008544921875, "step": 240 }, { "KL/chosen_KL_mean": -44.52872085571289, "KL/mean": -68.604736328125, "KL/rejected_KL_mean": -92.68074035644531, "KL/std": 39.203582763671875, "epoch": 0.35389133627019087, "fcm_dpo/beta": 0.014577025547623634, "fcm_dpo/delta": -0.27007779479026794, "fcm_dpo/margin": 48.15202331542969, "fcm_dpo/q_t": 0.34311577677726746, "grad_norm": 9.134992599487305, "learning_rate": 4.097117014129903e-07, "logits/chosen": -1.63786780834198, "logits/rejected": -0.9519686698913574, "logps/chosen": -145.5746612548828, "logps/ref_chosen": -101.04594421386719, "logps/ref_rejected": -94.04934692382812, "logps/rejected": -186.73008728027344, "loss": 0.8934, "margin_dpo/margin_mean": 48.15202331542969, "margin_dpo/margin_std": 43.82901382446289, "step": 241 }, { "KL/chosen_KL_mean": -52.94053268432617, "KL/mean": -70.04161071777344, "KL/rejected_KL_mean": -87.1427001953125, "KL/std": 34.48725891113281, "epoch": 0.355359765051395, "fcm_dpo/beta": 0.014258917421102524, "fcm_dpo/delta": -0.039715249091386795, "fcm_dpo/margin": 34.20215606689453, "fcm_dpo/q_t": 0.3874158263206482, "grad_norm": 9.531928062438965, "learning_rate": 4.087222918524807e-07, "logits/chosen": -1.6932920217514038, "logits/rejected": -1.013551115989685, "logps/chosen": -148.61318969726562, "logps/ref_chosen": -95.67266845703125, "logps/ref_rejected": -90.65454864501953, "logps/rejected": -177.7972412109375, "loss": 1.027, "margin_dpo/margin_mean": 34.20215606689453, "margin_dpo/margin_std": 38.174041748046875, "step": 242 }, { "KL/chosen_KL_mean": -51.86396789550781, "KL/mean": -68.801513671875, "KL/rejected_KL_mean": -85.73905181884766, "KL/std": 36.9276123046875, "epoch": 0.3568281938325991, "fcm_dpo/beta": 0.014229689724743366, "fcm_dpo/delta": -0.034833505749702454, "fcm_dpo/margin": 33.875083923339844, "fcm_dpo/q_t": 0.39147675037384033, "grad_norm": 9.327872276306152, "learning_rate": 4.07728699811968e-07, "logits/chosen": -1.8189271688461304, "logits/rejected": -0.9876618385314941, "logps/chosen": -149.89537048339844, "logps/ref_chosen": -98.03140258789062, "logps/ref_rejected": -83.18806457519531, "logps/rejected": -168.9271240234375, "loss": 1.0562, "margin_dpo/margin_mean": 33.875083923339844, "margin_dpo/margin_std": 43.51741027832031, "step": 243 }, { "KL/chosen_KL_mean": -54.084632873535156, "KL/mean": -72.06623840332031, "KL/rejected_KL_mean": -90.0478515625, "KL/std": 40.88813018798828, "epoch": 0.35829662261380324, "fcm_dpo/beta": 0.013942182995378971, "fcm_dpo/delta": -0.05431248992681503, "fcm_dpo/margin": 35.963226318359375, "fcm_dpo/q_t": 0.3859729468822479, "grad_norm": 9.260563850402832, "learning_rate": 4.067309514735267e-07, "logits/chosen": -2.1343822479248047, "logits/rejected": -1.8000872135162354, "logps/chosen": -142.97854614257812, "logps/ref_chosen": -88.8939208984375, "logps/ref_rejected": -102.57278442382812, "logps/rejected": -192.62063598632812, "loss": 1.0208, "margin_dpo/margin_mean": 35.963226318359375, "margin_dpo/margin_std": 41.07027816772461, "step": 244 }, { "KL/chosen_KL_mean": -45.61378860473633, "KL/mean": -64.84752655029297, "KL/rejected_KL_mean": -84.08126068115234, "KL/std": 38.089500427246094, "epoch": 0.35976505139500736, "fcm_dpo/beta": 0.013776318170130253, "fcm_dpo/delta": -0.08433786034584045, "fcm_dpo/margin": 38.467464447021484, "fcm_dpo/q_t": 0.3781154155731201, "grad_norm": 8.69044303894043, "learning_rate": 4.057290731287531e-07, "logits/chosen": -2.269152879714966, "logits/rejected": -1.4394030570983887, "logps/chosen": -149.8077850341797, "logps/ref_chosen": -104.19400024414062, "logps/ref_rejected": -92.65645599365234, "logps/rejected": -176.7377166748047, "loss": 0.9899, "margin_dpo/margin_mean": 38.467464447021484, "margin_dpo/margin_std": 38.598819732666016, "step": 245 }, { "KL/chosen_KL_mean": -59.491981506347656, "KL/mean": -74.54719543457031, "KL/rejected_KL_mean": -89.60240173339844, "KL/std": 36.99351119995117, "epoch": 0.36123348017621143, "fcm_dpo/beta": 0.01372530311346054, "fcm_dpo/delta": 0.03816515952348709, "fcm_dpo/margin": 30.110424041748047, "fcm_dpo/q_t": 0.4057997465133667, "grad_norm": 8.911885261535645, "learning_rate": 4.047230911780736e-07, "logits/chosen": -1.8584641218185425, "logits/rejected": -1.3190970420837402, "logps/chosen": -162.71102905273438, "logps/ref_chosen": -103.21904754638672, "logps/ref_rejected": -90.99221801757812, "logps/rejected": -180.5946044921875, "loss": 1.0846, "margin_dpo/margin_mean": 30.11042594909668, "margin_dpo/margin_std": 39.586910247802734, "step": 246 }, { "KL/chosen_KL_mean": -53.546241760253906, "KL/mean": -77.9178695678711, "KL/rejected_KL_mean": -102.28948974609375, "KL/std": 41.55171203613281, "epoch": 0.36270190895741555, "fcm_dpo/beta": 0.013310082256793976, "fcm_dpo/delta": -0.21401762962341309, "fcm_dpo/margin": 48.74325942993164, "fcm_dpo/q_t": 0.354427695274353, "grad_norm": 8.816831588745117, "learning_rate": 4.0371303213004814e-07, "logits/chosen": -2.1500349044799805, "logits/rejected": -1.96571946144104, "logps/chosen": -140.54061889648438, "logps/ref_chosen": -86.99437713623047, "logps/ref_rejected": -111.33802795410156, "logps/rejected": -213.6275177001953, "loss": 0.9249, "margin_dpo/margin_mean": 48.743263244628906, "margin_dpo/margin_std": 46.65956115722656, "step": 247 }, { "KL/chosen_KL_mean": -61.57766342163086, "KL/mean": -77.98873901367188, "KL/rejected_KL_mean": -94.39981842041016, "KL/std": 36.767066955566406, "epoch": 0.3641703377386197, "fcm_dpo/beta": 0.013294167816638947, "fcm_dpo/delta": 0.014036701992154121, "fcm_dpo/margin": 32.822166442871094, "fcm_dpo/q_t": 0.39929088950157166, "grad_norm": 9.573286056518555, "learning_rate": 4.0269892260067197e-07, "logits/chosen": -1.8852083683013916, "logits/rejected": -1.853461742401123, "logps/chosen": -136.36318969726562, "logps/ref_chosen": -74.7855224609375, "logps/ref_rejected": -98.27689361572266, "logps/rejected": -192.6767120361328, "loss": 1.056, "margin_dpo/margin_mean": 32.82216262817383, "margin_dpo/margin_std": 37.72064208984375, "step": 248 }, { "KL/chosen_KL_mean": -67.73788452148438, "KL/mean": -82.79411315917969, "KL/rejected_KL_mean": -97.85035705566406, "KL/std": 39.32136535644531, "epoch": 0.3656387665198238, "fcm_dpo/beta": 0.013312667608261108, "fcm_dpo/delta": 0.0510699599981308, "fcm_dpo/margin": 30.112468719482422, "fcm_dpo/q_t": 0.40923529863357544, "grad_norm": 10.465920448303223, "learning_rate": 4.0168078931267426e-07, "logits/chosen": -2.1551313400268555, "logits/rejected": -1.6154589653015137, "logps/chosen": -163.44168090820312, "logps/ref_chosen": -95.70379638671875, "logps/ref_rejected": -87.14646911621094, "logps/rejected": -184.996826171875, "loss": 1.1095, "margin_dpo/margin_mean": 30.112470626831055, "margin_dpo/margin_std": 44.744789123535156, "step": 249 }, { "KL/chosen_KL_mean": -60.65462875366211, "KL/mean": -77.22984313964844, "KL/rejected_KL_mean": -93.80506896972656, "KL/std": 39.87385177612305, "epoch": 0.3671071953010279, "fcm_dpo/beta": 0.013342011719942093, "fcm_dpo/delta": 0.007517479360103607, "fcm_dpo/margin": 33.15043258666992, "fcm_dpo/q_t": 0.39819926023483276, "grad_norm": 10.026691436767578, "learning_rate": 4.006586590948141e-07, "logits/chosen": -1.9156348705291748, "logits/rejected": -0.9815178513526917, "logps/chosen": -174.70684814453125, "logps/ref_chosen": -114.05221557617188, "logps/ref_rejected": -81.08767700195312, "logps/rejected": -174.8927459716797, "loss": 1.0735, "margin_dpo/margin_mean": 33.15043258666992, "margin_dpo/margin_std": 42.58338165283203, "step": 250 }, { "KL/chosen_KL_mean": -60.36662292480469, "KL/mean": -75.32786560058594, "KL/rejected_KL_mean": -90.28909301757812, "KL/std": 37.09200668334961, "epoch": 0.368575624082232, "fcm_dpo/beta": 0.013437955640256405, "fcm_dpo/delta": 0.04947926104068756, "fcm_dpo/margin": 29.9224796295166, "fcm_dpo/q_t": 0.40989112854003906, "grad_norm": 10.422486305236816, "learning_rate": 3.9963255888117325e-07, "logits/chosen": -1.9565802812576294, "logits/rejected": -1.2686119079589844, "logps/chosen": -158.07791137695312, "logps/ref_chosen": -97.71128845214844, "logps/ref_rejected": -83.52742004394531, "logps/rejected": -173.81651306152344, "loss": 1.1108, "margin_dpo/margin_mean": 29.9224796295166, "margin_dpo/margin_std": 44.29001998901367, "step": 251 }, { "KL/chosen_KL_mean": -60.423797607421875, "KL/mean": -75.56443786621094, "KL/rejected_KL_mean": -90.70506286621094, "KL/std": 39.300804138183594, "epoch": 0.3700440528634361, "fcm_dpo/beta": 0.01364966668188572, "fcm_dpo/delta": 0.03814868628978729, "fcm_dpo/margin": 30.281265258789062, "fcm_dpo/q_t": 0.40516144037246704, "grad_norm": 10.538346290588379, "learning_rate": 3.9860251571044666e-07, "logits/chosen": -2.0909392833709717, "logits/rejected": -1.5616133213043213, "logps/chosen": -169.409912109375, "logps/ref_chosen": -108.9861068725586, "logps/ref_rejected": -91.56423950195312, "logps/rejected": -182.26931762695312, "loss": 1.093, "margin_dpo/margin_mean": 30.281265258789062, "margin_dpo/margin_std": 41.773738861083984, "step": 252 }, { "KL/chosen_KL_mean": -66.49739074707031, "KL/mean": -80.59434509277344, "KL/rejected_KL_mean": -94.69131469726562, "KL/std": 42.872676849365234, "epoch": 0.37151248164464024, "fcm_dpo/beta": 0.013871677219867706, "fcm_dpo/delta": 0.05996757373213768, "fcm_dpo/margin": 28.193923950195312, "fcm_dpo/q_t": 0.4091401696205139, "grad_norm": 10.558040618896484, "learning_rate": 3.9756855672522986e-07, "logits/chosen": -1.8401331901550293, "logits/rejected": -1.6228747367858887, "logps/chosen": -166.71368408203125, "logps/ref_chosen": -100.21630096435547, "logps/ref_rejected": -105.67670440673828, "logps/rejected": -200.36801147460938, "loss": 1.099, "margin_dpo/margin_mean": 28.193923950195312, "margin_dpo/margin_std": 37.02585220336914, "step": 253 }, { "KL/chosen_KL_mean": -64.74571228027344, "KL/mean": -79.40669250488281, "KL/rejected_KL_mean": -94.06766510009766, "KL/std": 40.28760528564453, "epoch": 0.37298091042584436, "fcm_dpo/beta": 0.013849547132849693, "fcm_dpo/delta": 0.04552397131919861, "fcm_dpo/margin": 29.32195472717285, "fcm_dpo/q_t": 0.40663406252861023, "grad_norm": 10.165385246276855, "learning_rate": 3.965307091713037e-07, "logits/chosen": -1.99363374710083, "logits/rejected": -1.4603936672210693, "logps/chosen": -163.48089599609375, "logps/ref_chosen": -98.73518371582031, "logps/ref_rejected": -93.73825073242188, "logps/rejected": -187.805908203125, "loss": 1.1056, "margin_dpo/margin_mean": 29.32195281982422, "margin_dpo/margin_std": 42.65867614746094, "step": 254 }, { "KL/chosen_KL_mean": -65.8162841796875, "KL/mean": -87.46654510498047, "KL/rejected_KL_mean": -109.11680603027344, "KL/std": 45.29289245605469, "epoch": 0.3744493392070485, "fcm_dpo/beta": 0.013654084876179695, "fcm_dpo/delta": -0.1499733328819275, "fcm_dpo/margin": 43.30052947998047, "fcm_dpo/q_t": 0.3662024140357971, "grad_norm": 9.934861183166504, "learning_rate": 3.954890003969163e-07, "logits/chosen": -1.779320478439331, "logits/rejected": -1.4321630001068115, "logps/chosen": -156.1988525390625, "logps/ref_chosen": -90.382568359375, "logps/ref_rejected": -97.07624816894531, "logps/rejected": -206.19305419921875, "loss": 0.9687, "margin_dpo/margin_mean": 43.30052947998047, "margin_dpo/margin_std": 45.886634826660156, "step": 255 }, { "KL/chosen_KL_mean": -63.502994537353516, "KL/mean": -83.19683837890625, "KL/rejected_KL_mean": -102.89068603515625, "KL/std": 46.93716049194336, "epoch": 0.37591776798825255, "fcm_dpo/beta": 0.013406183570623398, "fcm_dpo/delta": -0.08231563866138458, "fcm_dpo/margin": 39.3876953125, "fcm_dpo/q_t": 0.38315078616142273, "grad_norm": 10.551952362060547, "learning_rate": 3.944434578520628e-07, "logits/chosen": -2.149305820465088, "logits/rejected": -1.8200596570968628, "logps/chosen": -152.25579833984375, "logps/ref_chosen": -88.7528076171875, "logps/ref_rejected": -98.49383544921875, "logps/rejected": -201.384521484375, "loss": 1.046, "margin_dpo/margin_mean": 39.3876953125, "margin_dpo/margin_std": 54.235328674316406, "step": 256 }, { "KL/chosen_KL_mean": -65.27300262451172, "KL/mean": -85.2200927734375, "KL/rejected_KL_mean": -105.16717529296875, "KL/std": 48.23341751098633, "epoch": 0.37738619676945667, "fcm_dpo/beta": 0.01303618960082531, "fcm_dpo/delta": -0.07658988237380981, "fcm_dpo/margin": 39.894187927246094, "fcm_dpo/q_t": 0.3826878070831299, "grad_norm": 9.485233306884766, "learning_rate": 3.933941090877615e-07, "logits/chosen": -1.8701181411743164, "logits/rejected": -1.4704567193984985, "logps/chosen": -148.07652282714844, "logps/ref_chosen": -82.80352020263672, "logps/ref_rejected": -85.8677978515625, "logps/rejected": -191.03497314453125, "loss": 1.0246, "margin_dpo/margin_mean": 39.894187927246094, "margin_dpo/margin_std": 46.648895263671875, "step": 257 }, { "KL/chosen_KL_mean": -68.06910705566406, "KL/mean": -86.16755676269531, "KL/rejected_KL_mean": -104.2660140991211, "KL/std": 45.84845733642578, "epoch": 0.3788546255506608, "fcm_dpo/beta": 0.013144619762897491, "fcm_dpo/delta": -0.02780015394091606, "fcm_dpo/margin": 36.196903228759766, "fcm_dpo/q_t": 0.39086252450942993, "grad_norm": 10.46627426147461, "learning_rate": 3.923409817553284e-07, "logits/chosen": -2.121572494506836, "logits/rejected": -1.7644829750061035, "logps/chosen": -158.25685119628906, "logps/ref_chosen": -90.187744140625, "logps/ref_rejected": -103.47068786621094, "logps/rejected": -207.7366943359375, "loss": 1.0464, "margin_dpo/margin_mean": 36.196903228759766, "margin_dpo/margin_std": 43.453529357910156, "step": 258 }, { "KL/chosen_KL_mean": -74.21199798583984, "KL/mean": -89.64044952392578, "KL/rejected_KL_mean": -105.06889343261719, "KL/std": 45.95478820800781, "epoch": 0.3803230543318649, "fcm_dpo/beta": 0.013054771348834038, "fcm_dpo/delta": 0.04869488999247551, "fcm_dpo/margin": 30.85689926147461, "fcm_dpo/q_t": 0.408587783575058, "grad_norm": 10.180980682373047, "learning_rate": 3.9128410360564793e-07, "logits/chosen": -1.2585045099258423, "logits/rejected": -1.0040603876113892, "logps/chosen": -164.9845428466797, "logps/ref_chosen": -90.77254486083984, "logps/ref_rejected": -94.58816528320312, "logps/rejected": -199.6570587158203, "loss": 1.1307, "margin_dpo/margin_mean": 30.856897354125977, "margin_dpo/margin_std": 51.554019927978516, "step": 259 }, { "KL/chosen_KL_mean": -75.8703384399414, "KL/mean": -89.35120391845703, "KL/rejected_KL_mean": -102.83206176757812, "KL/std": 46.482208251953125, "epoch": 0.38179148311306904, "fcm_dpo/beta": 0.01333899050951004, "fcm_dpo/delta": 0.0938006117939949, "fcm_dpo/margin": 26.961729049682617, "fcm_dpo/q_t": 0.4186316728591919, "grad_norm": 12.10897159576416, "learning_rate": 3.9022350248844246e-07, "logits/chosen": -1.7895164489746094, "logits/rejected": -1.812833309173584, "logps/chosen": -151.46304321289062, "logps/ref_chosen": -75.59269714355469, "logps/ref_rejected": -100.84554290771484, "logps/rejected": -203.6776123046875, "loss": 1.1666, "margin_dpo/margin_mean": 26.961729049682617, "margin_dpo/margin_std": 50.884849548339844, "step": 260 }, { "KL/chosen_KL_mean": -59.24699783325195, "KL/mean": -82.04939270019531, "KL/rejected_KL_mean": -104.8517837524414, "KL/std": 46.922218322753906, "epoch": 0.3832599118942731, "fcm_dpo/beta": 0.013034466654062271, "fcm_dpo/delta": -0.15515339374542236, "fcm_dpo/margin": 45.60478973388672, "fcm_dpo/q_t": 0.3651568591594696, "grad_norm": 9.479791641235352, "learning_rate": 3.891592063515376e-07, "logits/chosen": -2.04919171333313, "logits/rejected": -1.6856176853179932, "logps/chosen": -153.5019073486328, "logps/ref_chosen": -94.25491333007812, "logps/ref_rejected": -93.65699005126953, "logps/rejected": -198.50877380371094, "loss": 0.9571, "margin_dpo/margin_mean": 45.60478973388672, "margin_dpo/margin_std": 45.41069412231445, "step": 261 }, { "KL/chosen_KL_mean": -63.915008544921875, "KL/mean": -85.09645080566406, "KL/rejected_KL_mean": -106.27789306640625, "KL/std": 49.078636169433594, "epoch": 0.38472834067547723, "fcm_dpo/beta": 0.012775479815900326, "fcm_dpo/delta": -0.09700441360473633, "fcm_dpo/margin": 42.362884521484375, "fcm_dpo/q_t": 0.3776552975177765, "grad_norm": 9.474481582641602, "learning_rate": 3.880912432401264e-07, "logits/chosen": -1.8166594505310059, "logits/rejected": -1.2828445434570312, "logps/chosen": -149.18231201171875, "logps/ref_chosen": -85.26730346679688, "logps/ref_rejected": -90.82609558105469, "logps/rejected": -197.10397338867188, "loss": 1.0037, "margin_dpo/margin_mean": 42.362884521484375, "margin_dpo/margin_std": 48.5179443359375, "step": 262 }, { "KL/chosen_KL_mean": -51.72381591796875, "KL/mean": -80.45867919921875, "KL/rejected_KL_mean": -109.19354248046875, "KL/std": 53.76057815551758, "epoch": 0.38619676945668135, "fcm_dpo/beta": 0.012269420549273491, "fcm_dpo/delta": -0.27408623695373535, "fcm_dpo/margin": 57.4697265625, "fcm_dpo/q_t": 0.34254956245422363, "grad_norm": 9.069549560546875, "learning_rate": 3.870196412960302e-07, "logits/chosen": -2.10990047454834, "logits/rejected": -1.5606751441955566, "logps/chosen": -147.48171997070312, "logps/ref_chosen": -95.75790405273438, "logps/ref_rejected": -101.83377075195312, "logps/rejected": -211.02731323242188, "loss": 0.9073, "margin_dpo/margin_mean": 57.4697265625, "margin_dpo/margin_std": 56.68013000488281, "step": 263 }, { "KL/chosen_KL_mean": -60.24684143066406, "KL/mean": -84.13995361328125, "KL/rejected_KL_mean": -108.03305053710938, "KL/std": 50.6898307800293, "epoch": 0.3876651982378855, "fcm_dpo/beta": 0.011833982542157173, "fcm_dpo/delta": -0.12274721264839172, "fcm_dpo/margin": 47.786216735839844, "fcm_dpo/q_t": 0.3716976046562195, "grad_norm": 8.860624313354492, "learning_rate": 3.8594442875695665e-07, "logits/chosen": -2.292080879211426, "logits/rejected": -2.016915798187256, "logps/chosen": -150.8695068359375, "logps/ref_chosen": -90.6226577758789, "logps/ref_rejected": -100.32554626464844, "logps/rejected": -208.3585968017578, "loss": 0.9837, "margin_dpo/margin_mean": 47.786216735839844, "margin_dpo/margin_std": 51.46210479736328, "step": 264 }, { "KL/chosen_KL_mean": -62.92628479003906, "KL/mean": -84.69291687011719, "KL/rejected_KL_mean": -106.45956420898438, "KL/std": 51.68177032470703, "epoch": 0.3891336270190896, "fcm_dpo/beta": 0.011622949503362179, "fcm_dpo/delta": -0.059719592332839966, "fcm_dpo/margin": 43.53327941894531, "fcm_dpo/q_t": 0.3872433602809906, "grad_norm": 9.03610897064209, "learning_rate": 3.848656339557562e-07, "logits/chosen": -2.082602024078369, "logits/rejected": -1.6820037364959717, "logps/chosen": -155.29861450195312, "logps/ref_chosen": -92.37232971191406, "logps/ref_rejected": -94.62757873535156, "logps/rejected": -201.08714294433594, "loss": 1.0401, "margin_dpo/margin_mean": 43.53327941894531, "margin_dpo/margin_std": 55.71157455444336, "step": 265 }, { "KL/chosen_KL_mean": -73.42080688476562, "KL/mean": -93.02494049072266, "KL/rejected_KL_mean": -112.62907409667969, "KL/std": 49.6181526184082, "epoch": 0.39060205580029367, "fcm_dpo/beta": 0.01160573773086071, "fcm_dpo/delta": -0.005409969016909599, "fcm_dpo/margin": 39.20826721191406, "fcm_dpo/q_t": 0.397316575050354, "grad_norm": 10.500243186950684, "learning_rate": 3.8378328531967507e-07, "logits/chosen": -2.4886555671691895, "logits/rejected": -1.707979679107666, "logps/chosen": -175.62083435058594, "logps/ref_chosen": -102.20002746582031, "logps/ref_rejected": -74.36642456054688, "logps/rejected": -186.99549865722656, "loss": 1.0681, "margin_dpo/margin_mean": 39.20826721191406, "margin_dpo/margin_std": 51.86351776123047, "step": 266 }, { "KL/chosen_KL_mean": -63.99808120727539, "KL/mean": -86.74386596679688, "KL/rejected_KL_mean": -109.4896469116211, "KL/std": 50.218692779541016, "epoch": 0.3920704845814978, "fcm_dpo/beta": 0.01145089603960514, "fcm_dpo/delta": -0.07552241533994675, "fcm_dpo/margin": 45.4915771484375, "fcm_dpo/q_t": 0.3806152939796448, "grad_norm": 9.601058959960938, "learning_rate": 3.8269741136960646e-07, "logits/chosen": -1.6615180969238281, "logits/rejected": -1.0291712284088135, "logps/chosen": -168.28408813476562, "logps/ref_chosen": -104.28599548339844, "logps/ref_rejected": -95.98719024658203, "logps/rejected": -205.47683715820312, "loss": 1.0, "margin_dpo/margin_mean": 45.4915771484375, "margin_dpo/margin_std": 47.074493408203125, "step": 267 }, { "KL/chosen_KL_mean": -63.305763244628906, "KL/mean": -89.70006561279297, "KL/rejected_KL_mean": -116.09436798095703, "KL/std": 50.55253601074219, "epoch": 0.3935389133627019, "fcm_dpo/beta": 0.0111553855240345, "fcm_dpo/delta": -0.14865517616271973, "fcm_dpo/margin": 52.788612365722656, "fcm_dpo/q_t": 0.36453887820243835, "grad_norm": 9.604703903198242, "learning_rate": 3.8160804071933894e-07, "logits/chosen": -1.68306565284729, "logits/rejected": -1.524566888809204, "logps/chosen": -150.00198364257812, "logps/ref_chosen": -86.69622039794922, "logps/ref_rejected": -109.19184112548828, "logps/rejected": -225.2862091064453, "loss": 0.9527, "margin_dpo/margin_mean": 52.788612365722656, "margin_dpo/margin_std": 49.61883544921875, "step": 268 }, { "KL/chosen_KL_mean": -67.03485107421875, "KL/mean": -92.83975219726562, "KL/rejected_KL_mean": -118.64463806152344, "KL/std": 54.211143493652344, "epoch": 0.39500734214390604, "fcm_dpo/beta": 0.010949023067951202, "fcm_dpo/delta": -0.12179264426231384, "fcm_dpo/margin": 51.60980987548828, "fcm_dpo/q_t": 0.37075239419937134, "grad_norm": 8.980087280273438, "learning_rate": 3.8051520207480204e-07, "logits/chosen": -1.77012300491333, "logits/rejected": -1.3072271347045898, "logps/chosen": -172.00665283203125, "logps/ref_chosen": -104.97181701660156, "logps/ref_rejected": -112.47647094726562, "logps/rejected": -231.12110900878906, "loss": 0.9722, "margin_dpo/margin_mean": 51.60980987548828, "margin_dpo/margin_std": 51.5111083984375, "step": 269 }, { "KL/chosen_KL_mean": -74.47264099121094, "KL/mean": -96.40638732910156, "KL/rejected_KL_mean": -118.34014892578125, "KL/std": 53.79513168334961, "epoch": 0.3964757709251101, "fcm_dpo/beta": 0.010844661854207516, "fcm_dpo/delta": -0.027146557345986366, "fcm_dpo/margin": 43.86751174926758, "fcm_dpo/q_t": 0.39237266778945923, "grad_norm": 9.225295066833496, "learning_rate": 3.794189242333106e-07, "logits/chosen": -2.355654001235962, "logits/rejected": -1.9361772537231445, "logps/chosen": -175.54647827148438, "logps/ref_chosen": -101.07383728027344, "logps/ref_rejected": -117.75289154052734, "logps/rejected": -236.09304809570312, "loss": 1.0678, "margin_dpo/margin_mean": 43.86750793457031, "margin_dpo/margin_std": 60.640625, "step": 270 }, { "KL/chosen_KL_mean": -60.621280670166016, "KL/mean": -88.11228942871094, "KL/rejected_KL_mean": -115.6032943725586, "KL/std": 50.711997985839844, "epoch": 0.39794419970631423, "fcm_dpo/beta": 0.01055808737874031, "fcm_dpo/delta": -0.1387995183467865, "fcm_dpo/margin": 54.981998443603516, "fcm_dpo/q_t": 0.3661203384399414, "grad_norm": 9.050552368164062, "learning_rate": 3.7831923608280514e-07, "logits/chosen": -1.4702354669570923, "logits/rejected": -1.0143587589263916, "logps/chosen": -157.34588623046875, "logps/ref_chosen": -96.72459411621094, "logps/ref_rejected": -98.5244140625, "logps/rejected": -214.12771606445312, "loss": 0.9747, "margin_dpo/margin_mean": 54.981998443603516, "margin_dpo/margin_std": 57.74799728393555, "step": 271 }, { "KL/chosen_KL_mean": -52.731178283691406, "KL/mean": -78.52455139160156, "KL/rejected_KL_mean": -104.31791687011719, "KL/std": 55.571563720703125, "epoch": 0.39941262848751835, "fcm_dpo/beta": 0.01035686582326889, "fcm_dpo/delta": -0.08906619250774384, "fcm_dpo/margin": 51.58673858642578, "fcm_dpo/q_t": 0.37962785363197327, "grad_norm": 9.368598937988281, "learning_rate": 3.772161666010912e-07, "logits/chosen": -1.3417601585388184, "logits/rejected": -1.2235678434371948, "logps/chosen": -133.70840454101562, "logps/ref_chosen": -80.97721862792969, "logps/ref_rejected": -108.55535888671875, "logps/rejected": -212.87327575683594, "loss": 1.0108, "margin_dpo/margin_mean": 51.58673858642578, "margin_dpo/margin_std": 60.24427032470703, "step": 272 }, { "KL/chosen_KL_mean": -72.3418960571289, "KL/mean": -98.98159790039062, "KL/rejected_KL_mean": -125.62128448486328, "KL/std": 57.446327209472656, "epoch": 0.4008810572687225, "fcm_dpo/beta": 0.010085253976285458, "fcm_dpo/delta": -0.09449248015880585, "fcm_dpo/margin": 53.27939224243164, "fcm_dpo/q_t": 0.378003865480423, "grad_norm": 8.671329498291016, "learning_rate": 3.761097448550755e-07, "logits/chosen": -1.827613353729248, "logits/rejected": -1.4322266578674316, "logps/chosen": -164.56649780273438, "logps/ref_chosen": -92.22460174560547, "logps/ref_rejected": -97.3630599975586, "logps/rejected": -222.98434448242188, "loss": 1.0218, "margin_dpo/margin_mean": 53.279388427734375, "margin_dpo/margin_std": 64.60359954833984, "step": 273 }, { "KL/chosen_KL_mean": -74.13670349121094, "KL/mean": -94.80648040771484, "KL/rejected_KL_mean": -115.47625732421875, "KL/std": 55.86241912841797, "epoch": 0.4023494860499266, "fcm_dpo/beta": 0.010142168030142784, "fcm_dpo/delta": 0.032018598169088364, "fcm_dpo/margin": 41.33954620361328, "fcm_dpo/q_t": 0.40365350246429443, "grad_norm": 13.00503921508789, "learning_rate": 3.75e-07, "logits/chosen": -2.043581008911133, "logits/rejected": -1.6510529518127441, "logps/chosen": -156.39759826660156, "logps/ref_chosen": -82.26089477539062, "logps/ref_rejected": -83.87699890136719, "logps/rejected": -199.35324096679688, "loss": 1.0741, "margin_dpo/margin_mean": 41.33954620361328, "margin_dpo/margin_std": 51.26081085205078, "step": 274 }, { "KL/chosen_KL_mean": -63.02811813354492, "KL/mean": -86.29637145996094, "KL/rejected_KL_mean": -109.56463623046875, "KL/std": 55.224388122558594, "epoch": 0.40381791483113066, "fcm_dpo/beta": 0.01020372286438942, "fcm_dpo/delta": -0.026614777743816376, "fcm_dpo/margin": 46.536502838134766, "fcm_dpo/q_t": 0.3922935128211975, "grad_norm": 9.820210456848145, "learning_rate": 3.738869612786737e-07, "logits/chosen": -2.1946425437927246, "logits/rejected": -1.8749346733093262, "logps/chosen": -142.715087890625, "logps/ref_chosen": -79.68696594238281, "logps/ref_rejected": -98.7509765625, "logps/rejected": -208.31561279296875, "loss": 1.0727, "margin_dpo/margin_mean": 46.536502838134766, "margin_dpo/margin_std": 64.95244598388672, "step": 275 }, { "KL/chosen_KL_mean": -75.37953186035156, "KL/mean": -100.19548034667969, "KL/rejected_KL_mean": -125.01143646240234, "KL/std": 55.71049499511719, "epoch": 0.4052863436123348, "fcm_dpo/beta": 0.00998476892709732, "fcm_dpo/delta": -0.04935610294342041, "fcm_dpo/margin": 49.63191604614258, "fcm_dpo/q_t": 0.3882775902748108, "grad_norm": 10.756178855895996, "learning_rate": 3.7277065802070204e-07, "logits/chosen": -2.2146334648132324, "logits/rejected": -1.6574091911315918, "logps/chosen": -161.9192352294922, "logps/ref_chosen": -86.53970336914062, "logps/ref_rejected": -77.85394287109375, "logps/rejected": -202.86538696289062, "loss": 1.0418, "margin_dpo/margin_mean": 49.631919860839844, "margin_dpo/margin_std": 61.964561462402344, "step": 276 }, { "KL/chosen_KL_mean": -81.672607421875, "KL/mean": -100.11717224121094, "KL/rejected_KL_mean": -118.5617446899414, "KL/std": 53.90569305419922, "epoch": 0.4067547723935389, "fcm_dpo/beta": 0.010115118697285652, "fcm_dpo/delta": 0.07986337691545486, "fcm_dpo/margin": 36.889129638671875, "fcm_dpo/q_t": 0.4145438075065613, "grad_norm": 10.247102737426758, "learning_rate": 3.71651119641714e-07, "logits/chosen": -1.983532190322876, "logits/rejected": -1.5623936653137207, "logps/chosen": -165.91671752929688, "logps/ref_chosen": -84.24411010742188, "logps/ref_rejected": -98.83421325683594, "logps/rejected": -217.39596557617188, "loss": 1.1201, "margin_dpo/margin_mean": 36.889129638671875, "margin_dpo/margin_std": 54.23590087890625, "step": 277 }, { "KL/chosen_KL_mean": -78.61597442626953, "KL/mean": -101.62483215332031, "KL/rejected_KL_mean": -124.63369750976562, "KL/std": 57.92980194091797, "epoch": 0.40822320117474303, "fcm_dpo/beta": 0.01014047022908926, "fcm_dpo/delta": -0.017631515860557556, "fcm_dpo/margin": 46.01770782470703, "fcm_dpo/q_t": 0.39241647720336914, "grad_norm": 11.945178985595703, "learning_rate": 3.705283756425872e-07, "logits/chosen": -2.090693950653076, "logits/rejected": -1.8765650987625122, "logps/chosen": -161.0478515625, "logps/ref_chosen": -82.431884765625, "logps/ref_rejected": -97.85691833496094, "logps/rejected": -222.4906005859375, "loss": 1.0552, "margin_dpo/margin_mean": 46.01770782470703, "margin_dpo/margin_std": 57.620609283447266, "step": 278 }, { "KL/chosen_KL_mean": -81.6851577758789, "KL/mean": -110.146484375, "KL/rejected_KL_mean": -138.60781860351562, "KL/std": 61.24153137207031, "epoch": 0.40969162995594716, "fcm_dpo/beta": 0.009937990456819534, "fcm_dpo/delta": -0.12331337481737137, "fcm_dpo/margin": 56.922672271728516, "fcm_dpo/q_t": 0.37146955728530884, "grad_norm": 10.838581085205078, "learning_rate": 3.6940245560867e-07, "logits/chosen": -1.9057362079620361, "logits/rejected": -1.6948736906051636, "logps/chosen": -166.8531494140625, "logps/ref_chosen": -85.16799926757812, "logps/ref_rejected": -94.12664794921875, "logps/rejected": -232.73446655273438, "loss": 0.9809, "margin_dpo/margin_mean": 56.922672271728516, "margin_dpo/margin_std": 60.66328430175781, "step": 279 }, { "KL/chosen_KL_mean": -84.80870056152344, "KL/mean": -113.30152893066406, "KL/rejected_KL_mean": -141.79437255859375, "KL/std": 56.13744354248047, "epoch": 0.4111600587371512, "fcm_dpo/beta": 0.009764859452843666, "fcm_dpo/delta": -0.1125873476266861, "fcm_dpo/margin": 56.985660552978516, "fcm_dpo/q_t": 0.3725709617137909, "grad_norm": 9.573436737060547, "learning_rate": 3.6827338920900253e-07, "logits/chosen": -1.7208222150802612, "logits/rejected": -1.5179696083068848, "logps/chosen": -170.6651153564453, "logps/ref_chosen": -85.85641479492188, "logps/ref_rejected": -104.11859130859375, "logps/rejected": -245.9129638671875, "loss": 0.9809, "margin_dpo/margin_mean": 56.985660552978516, "margin_dpo/margin_std": 58.380191802978516, "step": 280 }, { "KL/chosen_KL_mean": -90.166748046875, "KL/mean": -118.92213439941406, "KL/rejected_KL_mean": -147.67752075195312, "KL/std": 65.98188781738281, "epoch": 0.41262848751835535, "fcm_dpo/beta": 0.009521182626485825, "fcm_dpo/delta": -0.1035277247428894, "fcm_dpo/margin": 57.510780334472656, "fcm_dpo/q_t": 0.3771226406097412, "grad_norm": 9.767707824707031, "learning_rate": 3.6714120619553435e-07, "logits/chosen": -1.9440927505493164, "logits/rejected": -1.1883043050765991, "logps/chosen": -176.7175750732422, "logps/ref_chosen": -86.55082702636719, "logps/ref_rejected": -88.62866973876953, "logps/rejected": -236.3061981201172, "loss": 1.0028, "margin_dpo/margin_mean": 57.510780334472656, "margin_dpo/margin_std": 66.96490478515625, "step": 281 }, { "KL/chosen_KL_mean": -88.52243041992188, "KL/mean": -123.98078918457031, "KL/rejected_KL_mean": -159.43914794921875, "KL/std": 63.78892517089844, "epoch": 0.41409691629955947, "fcm_dpo/beta": 0.00918116606771946, "fcm_dpo/delta": -0.2153972089290619, "fcm_dpo/margin": 70.91671752929688, "fcm_dpo/q_t": 0.3512480854988098, "grad_norm": 8.765600204467773, "learning_rate": 3.660059364023408e-07, "logits/chosen": -2.027914047241211, "logits/rejected": -1.6634950637817383, "logps/chosen": -193.62753295898438, "logps/ref_chosen": -105.10511016845703, "logps/ref_rejected": -102.85336303710938, "logps/rejected": -262.29254150390625, "loss": 0.9117, "margin_dpo/margin_mean": 70.91671752929688, "margin_dpo/margin_std": 63.18724822998047, "step": 282 }, { "KL/chosen_KL_mean": -102.74705505371094, "KL/mean": -128.06011962890625, "KL/rejected_KL_mean": -153.37319946289062, "KL/std": 63.34912872314453, "epoch": 0.4155653450807636, "fcm_dpo/beta": 0.009073339402675629, "fcm_dpo/delta": -0.009805109351873398, "fcm_dpo/margin": 50.626121520996094, "fcm_dpo/q_t": 0.39625316858291626, "grad_norm": 10.84497356414795, "learning_rate": 3.6486760974483685e-07, "logits/chosen": -2.187713623046875, "logits/rejected": -1.84690260887146, "logps/chosen": -197.79965209960938, "logps/ref_chosen": -95.05259704589844, "logps/ref_rejected": -103.54454803466797, "logps/rejected": -256.917724609375, "loss": 1.0705, "margin_dpo/margin_mean": 50.62611770629883, "margin_dpo/margin_std": 69.36991882324219, "step": 283 }, { "KL/chosen_KL_mean": -93.44561767578125, "KL/mean": -125.770263671875, "KL/rejected_KL_mean": -158.0948944091797, "KL/std": 62.93775939941406, "epoch": 0.4170337738619677, "fcm_dpo/beta": 0.008855831809341908, "fcm_dpo/delta": -0.13103410601615906, "fcm_dpo/margin": 64.64927673339844, "fcm_dpo/q_t": 0.36667758226394653, "grad_norm": 10.206171035766602, "learning_rate": 3.6372625621898863e-07, "logits/chosen": -1.7394795417785645, "logits/rejected": -1.4053385257720947, "logps/chosen": -181.11209106445312, "logps/ref_chosen": -87.66647338867188, "logps/ref_rejected": -98.75103759765625, "logps/rejected": -256.845947265625, "loss": 0.9497, "margin_dpo/margin_mean": 64.64927673339844, "margin_dpo/margin_std": 55.73645782470703, "step": 284 }, { "KL/chosen_KL_mean": -104.77790069580078, "KL/mean": -127.95679473876953, "KL/rejected_KL_mean": -151.13571166992188, "KL/std": 56.91664123535156, "epoch": 0.4185022026431718, "fcm_dpo/beta": 0.00891127996146679, "fcm_dpo/delta": 0.03824569284915924, "fcm_dpo/margin": 46.357810974121094, "fcm_dpo/q_t": 0.4031790494918823, "grad_norm": 10.669732093811035, "learning_rate": 3.625819059005228e-07, "logits/chosen": -2.5109004974365234, "logits/rejected": -2.2383971214294434, "logps/chosen": -199.2109375, "logps/ref_chosen": -94.43303680419922, "logps/ref_rejected": -104.07194519042969, "logps/rejected": -255.2076416015625, "loss": 1.0699, "margin_dpo/margin_mean": 46.35780334472656, "margin_dpo/margin_std": 53.50532531738281, "step": 285 }, { "KL/chosen_KL_mean": -108.8774185180664, "KL/mean": -139.5762939453125, "KL/rejected_KL_mean": -170.275146484375, "KL/std": 61.411163330078125, "epoch": 0.4199706314243759, "fcm_dpo/beta": 0.008768124505877495, "fcm_dpo/delta": -0.09358012676239014, "fcm_dpo/margin": 61.39774703979492, "fcm_dpo/q_t": 0.37601304054260254, "grad_norm": 8.792093276977539, "learning_rate": 3.614345889441346e-07, "logits/chosen": -1.7636845111846924, "logits/rejected": -1.285810947418213, "logps/chosen": -212.59780883789062, "logps/ref_chosen": -103.72039794921875, "logps/ref_rejected": -96.25775909423828, "logps/rejected": -266.53289794921875, "loss": 0.9919, "margin_dpo/margin_mean": 61.39775085449219, "margin_dpo/margin_std": 63.375953674316406, "step": 286 }, { "KL/chosen_KL_mean": -125.4956283569336, "KL/mean": -143.25682067871094, "KL/rejected_KL_mean": -161.01800537109375, "KL/std": 56.276981353759766, "epoch": 0.42143906020558003, "fcm_dpo/beta": 0.00896537210792303, "fcm_dpo/delta": 0.13455955684185028, "fcm_dpo/margin": 35.52237319946289, "fcm_dpo/q_t": 0.4286572337150574, "grad_norm": 8.975525856018066, "learning_rate": 3.6028433558269275e-07, "logits/chosen": -2.0645358562469482, "logits/rejected": -1.4521374702453613, "logps/chosen": -219.38551330566406, "logps/ref_chosen": -93.88988494873047, "logps/ref_rejected": -83.33365631103516, "logps/rejected": -244.35165405273438, "loss": 1.1753, "margin_dpo/margin_mean": 35.52237319946289, "margin_dpo/margin_std": 64.4245376586914, "step": 287 }, { "KL/chosen_KL_mean": -119.27923583984375, "KL/mean": -151.07376098632812, "KL/rejected_KL_mean": -182.86830139160156, "KL/std": 63.64386749267578, "epoch": 0.42290748898678415, "fcm_dpo/beta": 0.00880710780620575, "fcm_dpo/delta": -0.11687169224023819, "fcm_dpo/margin": 63.58906173706055, "fcm_dpo/q_t": 0.3747601807117462, "grad_norm": 8.365121841430664, "learning_rate": 3.5913117612644327e-07, "logits/chosen": -1.878031611442566, "logits/rejected": -1.4685653448104858, "logps/chosen": -207.43524169921875, "logps/ref_chosen": -88.15602111816406, "logps/ref_rejected": -93.28195190429688, "logps/rejected": -276.1502685546875, "loss": 0.989, "margin_dpo/margin_mean": 63.58906555175781, "margin_dpo/margin_std": 70.78921508789062, "step": 288 }, { "KL/chosen_KL_mean": -116.05216217041016, "KL/mean": -148.35260009765625, "KL/rejected_KL_mean": -180.6530303955078, "KL/std": 66.90855407714844, "epoch": 0.4243759177679883, "fcm_dpo/beta": 0.008643142879009247, "fcm_dpo/delta": -0.11460825055837631, "fcm_dpo/margin": 64.60086059570312, "fcm_dpo/q_t": 0.3773513436317444, "grad_norm": 7.985620021820068, "learning_rate": 3.5797514096221024e-07, "logits/chosen": -2.0833332538604736, "logits/rejected": -1.8857743740081787, "logps/chosen": -191.44509887695312, "logps/ref_chosen": -75.39292907714844, "logps/ref_rejected": -93.15428161621094, "logps/rejected": -273.80731201171875, "loss": 1.0012, "margin_dpo/margin_mean": 64.60086059570312, "margin_dpo/margin_std": 76.6702880859375, "step": 289 }, { "KL/chosen_KL_mean": -125.43167877197266, "KL/mean": -159.2182159423828, "KL/rejected_KL_mean": -193.0047607421875, "KL/std": 70.69041442871094, "epoch": 0.42584434654919234, "fcm_dpo/beta": 0.008467867970466614, "fcm_dpo/delta": -0.12969830632209778, "fcm_dpo/margin": 67.57308959960938, "fcm_dpo/q_t": 0.3721858859062195, "grad_norm": 8.800580024719238, "learning_rate": 3.568162605525952e-07, "logits/chosen": -1.1682285070419312, "logits/rejected": -1.2103686332702637, "logps/chosen": -213.47366333007812, "logps/ref_chosen": -88.04197692871094, "logps/ref_rejected": -123.212158203125, "logps/rejected": -316.2169189453125, "loss": 0.9826, "margin_dpo/margin_mean": 67.57308959960938, "margin_dpo/margin_std": 73.43978881835938, "step": 290 }, { "KL/chosen_KL_mean": -117.7747802734375, "KL/mean": -146.55003356933594, "KL/rejected_KL_mean": -175.32530212402344, "KL/std": 67.0921630859375, "epoch": 0.42731277533039647, "fcm_dpo/beta": 0.008327526971697807, "fcm_dpo/delta": -0.030725864693522453, "fcm_dpo/margin": 57.550506591796875, "fcm_dpo/q_t": 0.3891611695289612, "grad_norm": 9.72525405883789, "learning_rate": 3.5565456543517485e-07, "logits/chosen": -2.4693450927734375, "logits/rejected": -2.0153958797454834, "logps/chosen": -211.87002563476562, "logps/ref_chosen": -94.09524536132812, "logps/ref_rejected": -96.05006408691406, "logps/rejected": -271.3753662109375, "loss": 1.0355, "margin_dpo/margin_mean": 57.550506591796875, "margin_dpo/margin_std": 66.60820770263672, "step": 291 }, { "KL/chosen_KL_mean": -121.39501190185547, "KL/mean": -151.31764221191406, "KL/rejected_KL_mean": -181.24029541015625, "KL/std": 64.85142517089844, "epoch": 0.4287812041116006, "fcm_dpo/beta": 0.008243793621659279, "fcm_dpo/delta": -0.04561452940106392, "fcm_dpo/margin": 59.84527587890625, "fcm_dpo/q_t": 0.3857780992984772, "grad_norm": 9.154579162597656, "learning_rate": 3.5449008622169583e-07, "logits/chosen": -2.019760847091675, "logits/rejected": -1.5835098028182983, "logps/chosen": -209.64541625976562, "logps/ref_chosen": -88.25041198730469, "logps/ref_rejected": -96.41764068603516, "logps/rejected": -277.6579284667969, "loss": 1.0089, "margin_dpo/margin_mean": 59.84527587890625, "margin_dpo/margin_std": 60.23894500732422, "step": 292 }, { "KL/chosen_KL_mean": -120.842529296875, "KL/mean": -144.92396545410156, "KL/rejected_KL_mean": -169.00540161132812, "KL/std": 69.42327117919922, "epoch": 0.4302496328928047, "fcm_dpo/beta": 0.008326580747961998, "fcm_dpo/delta": 0.05035467445850372, "fcm_dpo/margin": 48.162879943847656, "fcm_dpo/q_t": 0.41038620471954346, "grad_norm": 8.575159072875977, "learning_rate": 3.5332285359726846e-07, "logits/chosen": -2.351748466491699, "logits/rejected": -1.7826613187789917, "logps/chosen": -208.2190704345703, "logps/ref_chosen": -87.37654113769531, "logps/ref_rejected": -85.75579833984375, "logps/rejected": -254.76121520996094, "loss": 1.1087, "margin_dpo/margin_mean": 48.162879943847656, "margin_dpo/margin_std": 71.87301635742188, "step": 293 }, { "KL/chosen_KL_mean": -124.97944641113281, "KL/mean": -144.67184448242188, "KL/rejected_KL_mean": -164.36422729492188, "KL/std": 65.12646484375, "epoch": 0.43171806167400884, "fcm_dpo/beta": 0.00848595891147852, "fcm_dpo/delta": 0.11919843405485153, "fcm_dpo/margin": 39.3847770690918, "fcm_dpo/q_t": 0.4253361225128174, "grad_norm": 9.369819641113281, "learning_rate": 3.5215289831955786e-07, "logits/chosen": -1.8082489967346191, "logits/rejected": -1.6395163536071777, "logps/chosen": -198.48739624023438, "logps/ref_chosen": -73.50794982910156, "logps/ref_rejected": -88.08877563476562, "logps/rejected": -252.4530029296875, "loss": 1.1789, "margin_dpo/margin_mean": 39.38477325439453, "margin_dpo/margin_std": 75.7198486328125, "step": 294 }, { "KL/chosen_KL_mean": -118.93072509765625, "KL/mean": -149.1732177734375, "KL/rejected_KL_mean": -179.41571044921875, "KL/std": 66.96073150634766, "epoch": 0.4331864904552129, "fcm_dpo/beta": 0.008392706513404846, "fcm_dpo/delta": -0.061477065086364746, "fcm_dpo/margin": 60.485008239746094, "fcm_dpo/q_t": 0.3851470351219177, "grad_norm": 8.884387969970703, "learning_rate": 3.509802512179737e-07, "logits/chosen": -1.5656354427337646, "logits/rejected": -1.409606695175171, "logps/chosen": -196.69619750976562, "logps/ref_chosen": -77.76548767089844, "logps/ref_rejected": -94.24726867675781, "logps/rejected": -273.6629638671875, "loss": 1.0254, "margin_dpo/margin_mean": 60.485008239746094, "margin_dpo/margin_std": 71.83735656738281, "step": 295 }, { "KL/chosen_KL_mean": -127.09491729736328, "KL/mean": -159.75466918945312, "KL/rejected_KL_mean": -192.41441345214844, "KL/std": 67.45562744140625, "epoch": 0.434654919236417, "fcm_dpo/beta": 0.008258884772658348, "fcm_dpo/delta": -0.0951654464006424, "fcm_dpo/margin": 65.31949615478516, "fcm_dpo/q_t": 0.3758048713207245, "grad_norm": 11.877053260803223, "learning_rate": 3.498049431928577e-07, "logits/chosen": -2.6076111793518066, "logits/rejected": -2.1342110633850098, "logps/chosen": -224.95132446289062, "logps/ref_chosen": -97.85641479492188, "logps/ref_rejected": -100.81631469726562, "logps/rejected": -293.230712890625, "loss": 0.9905, "margin_dpo/margin_mean": 65.31949615478516, "margin_dpo/margin_std": 67.6308364868164, "step": 296 }, { "KL/chosen_KL_mean": -129.87213134765625, "KL/mean": -156.85235595703125, "KL/rejected_KL_mean": -183.8325653076172, "KL/std": 66.62882232666016, "epoch": 0.43612334801762115, "fcm_dpo/beta": 0.008269837126135826, "fcm_dpo/delta": 0.003833906725049019, "fcm_dpo/margin": 53.96044158935547, "fcm_dpo/q_t": 0.3997609615325928, "grad_norm": 12.508593559265137, "learning_rate": 3.486270052146694e-07, "logits/chosen": -1.3853625059127808, "logits/rejected": -1.080672025680542, "logps/chosen": -218.43797302246094, "logps/ref_chosen": -88.56583404541016, "logps/ref_rejected": -101.55656433105469, "logps/rejected": -285.3891296386719, "loss": 1.068, "margin_dpo/margin_mean": 53.96044158935547, "margin_dpo/margin_std": 71.1764907836914, "step": 297 }, { "KL/chosen_KL_mean": -109.5050048828125, "KL/mean": -144.00445556640625, "KL/rejected_KL_mean": -178.50390625, "KL/std": 73.2869873046875, "epoch": 0.43759177679882527, "fcm_dpo/beta": 0.0081322081387043, "fcm_dpo/delta": -0.11755289137363434, "fcm_dpo/margin": 68.9989013671875, "fcm_dpo/q_t": 0.3743922710418701, "grad_norm": 12.788277626037598, "learning_rate": 3.474464683231698e-07, "logits/chosen": -2.0523953437805176, "logits/rejected": -2.0216622352600098, "logps/chosen": -204.38543701171875, "logps/ref_chosen": -94.88043975830078, "logps/ref_rejected": -122.31101989746094, "logps/rejected": -300.81494140625, "loss": 0.9822, "margin_dpo/margin_mean": 68.9989013671875, "margin_dpo/margin_std": 74.03498840332031, "step": 298 }, { "KL/chosen_KL_mean": -112.6458740234375, "KL/mean": -147.16741943359375, "KL/rejected_KL_mean": -181.68899536132812, "KL/std": 70.91508483886719, "epoch": 0.4390602055800294, "fcm_dpo/beta": 0.007931895554065704, "fcm_dpo/delta": -0.10341779887676239, "fcm_dpo/margin": 69.0431137084961, "fcm_dpo/q_t": 0.3754294514656067, "grad_norm": 10.664654731750488, "learning_rate": 3.462633636266041e-07, "logits/chosen": -0.9453160762786865, "logits/rejected": -0.7165822982788086, "logps/chosen": -193.05422973632812, "logps/ref_chosen": -80.40835571289062, "logps/ref_rejected": -89.53717041015625, "logps/rejected": -271.22613525390625, "loss": 0.9897, "margin_dpo/margin_mean": 69.0431137084961, "margin_dpo/margin_std": 73.78032684326172, "step": 299 }, { "KL/chosen_KL_mean": -118.73129272460938, "KL/mean": -147.56182861328125, "KL/rejected_KL_mean": -176.392333984375, "KL/std": 74.04791259765625, "epoch": 0.44052863436123346, "fcm_dpo/beta": 0.00785842351615429, "fcm_dpo/delta": -0.0036856867372989655, "fcm_dpo/margin": 57.66106414794922, "fcm_dpo/q_t": 0.39694181084632874, "grad_norm": 9.227557182312012, "learning_rate": 3.4507772230088147e-07, "logits/chosen": -1.436816930770874, "logits/rejected": -1.2209242582321167, "logps/chosen": -206.89019775390625, "logps/ref_chosen": -88.15890502929688, "logps/ref_rejected": -100.93919372558594, "logps/rejected": -277.33154296875, "loss": 1.0727, "margin_dpo/margin_mean": 57.66106414794922, "margin_dpo/margin_std": 77.9786148071289, "step": 300 }, { "KL/chosen_KL_mean": -109.10418701171875, "KL/mean": -145.15988159179688, "KL/rejected_KL_mean": -181.21554565429688, "KL/std": 72.45738983154297, "epoch": 0.4419970631424376, "fcm_dpo/beta": 0.007712747436016798, "fcm_dpo/delta": -0.1138525903224945, "fcm_dpo/margin": 72.11134338378906, "fcm_dpo/q_t": 0.3746373653411865, "grad_norm": 10.497952461242676, "learning_rate": 3.4388957558875316e-07, "logits/chosen": -1.5942405462265015, "logits/rejected": -1.1501681804656982, "logps/chosen": -194.60910034179688, "logps/ref_chosen": -85.50491333007812, "logps/ref_rejected": -98.91683959960938, "logps/rejected": -280.13238525390625, "loss": 0.9951, "margin_dpo/margin_mean": 72.11134338378906, "margin_dpo/margin_std": 80.96015167236328, "step": 301 }, { "KL/chosen_KL_mean": -105.61663818359375, "KL/mean": -136.363037109375, "KL/rejected_KL_mean": -167.1094512939453, "KL/std": 75.00934600830078, "epoch": 0.4434654919236417, "fcm_dpo/beta": 0.00767013244330883, "fcm_dpo/delta": -0.0228885430842638, "fcm_dpo/margin": 61.49281692504883, "fcm_dpo/q_t": 0.39133208990097046, "grad_norm": 9.270161628723145, "learning_rate": 3.426989547989902e-07, "logits/chosen": -1.4711213111877441, "logits/rejected": -1.2963201999664307, "logps/chosen": -185.06704711914062, "logps/ref_chosen": -79.45040893554688, "logps/ref_rejected": -103.0591049194336, "logps/rejected": -270.1685485839844, "loss": 1.0426, "margin_dpo/margin_mean": 61.49281311035156, "margin_dpo/margin_std": 73.03895568847656, "step": 302 }, { "KL/chosen_KL_mean": -98.44601440429688, "KL/mean": -132.39842224121094, "KL/rejected_KL_mean": -166.350830078125, "KL/std": 69.38361358642578, "epoch": 0.44493392070484583, "fcm_dpo/beta": 0.007617846131324768, "fcm_dpo/delta": -0.07091275602579117, "fcm_dpo/margin": 67.90481567382812, "fcm_dpo/q_t": 0.38092440366744995, "grad_norm": 8.717459678649902, "learning_rate": 3.4150589130555773e-07, "logits/chosen": -1.8716776371002197, "logits/rejected": -1.4163432121276855, "logps/chosen": -194.0443115234375, "logps/ref_chosen": -95.59829711914062, "logps/ref_rejected": -92.0645751953125, "logps/rejected": -258.4154052734375, "loss": 1.0075, "margin_dpo/margin_mean": 67.90481567382812, "margin_dpo/margin_std": 72.90766906738281, "step": 303 }, { "KL/chosen_KL_mean": -98.38722229003906, "KL/mean": -133.69200134277344, "KL/rejected_KL_mean": -168.99676513671875, "KL/std": 72.43708801269531, "epoch": 0.44640234948604995, "fcm_dpo/beta": 0.0074823698960244656, "fcm_dpo/delta": -0.08265557885169983, "fcm_dpo/margin": 70.60956573486328, "fcm_dpo/q_t": 0.3783053755760193, "grad_norm": 9.571141242980957, "learning_rate": 3.403104165467883e-07, "logits/chosen": -2.0542800426483154, "logits/rejected": -1.6292614936828613, "logps/chosen": -197.84036254882812, "logps/ref_chosen": -99.4531478881836, "logps/ref_rejected": -98.08226013183594, "logps/rejected": -267.07904052734375, "loss": 0.9976, "margin_dpo/margin_mean": 70.60955810546875, "margin_dpo/margin_std": 74.43731689453125, "step": 304 }, { "KL/chosen_KL_mean": -106.96412658691406, "KL/mean": -140.706787109375, "KL/rejected_KL_mean": -174.44943237304688, "KL/std": 75.149169921875, "epoch": 0.447870778267254, "fcm_dpo/beta": 0.007345802150666714, "fcm_dpo/delta": -0.04923749715089798, "fcm_dpo/margin": 67.48530578613281, "fcm_dpo/q_t": 0.3862135410308838, "grad_norm": 8.295806884765625, "learning_rate": 3.391125620245535e-07, "logits/chosen": -2.002380132675171, "logits/rejected": -1.506052851676941, "logps/chosen": -210.22752380371094, "logps/ref_chosen": -103.26339721679688, "logps/ref_rejected": -97.82897186279297, "logps/rejected": -272.2784118652344, "loss": 1.0347, "margin_dpo/margin_mean": 67.48530578613281, "margin_dpo/margin_std": 79.51161193847656, "step": 305 }, { "KL/chosen_KL_mean": -95.57650756835938, "KL/mean": -135.11154174804688, "KL/rejected_KL_mean": -174.64654541015625, "KL/std": 71.50088500976562, "epoch": 0.44933920704845814, "fcm_dpo/beta": 0.007213468197733164, "fcm_dpo/delta": -0.12819325923919678, "fcm_dpo/margin": 79.07003784179688, "fcm_dpo/q_t": 0.36620384454727173, "grad_norm": 10.3646240234375, "learning_rate": 3.3791235930343417e-07, "logits/chosen": -2.2244176864624023, "logits/rejected": -1.6680846214294434, "logps/chosen": -195.88034057617188, "logps/ref_chosen": -100.3038330078125, "logps/ref_rejected": -92.23086547851562, "logps/rejected": -266.87744140625, "loss": 0.9479, "margin_dpo/margin_mean": 79.07003784179688, "margin_dpo/margin_std": 65.43547058105469, "step": 306 }, { "KL/chosen_KL_mean": -101.2436294555664, "KL/mean": -130.86199951171875, "KL/rejected_KL_mean": -160.48036193847656, "KL/std": 72.14974975585938, "epoch": 0.45080763582966227, "fcm_dpo/beta": 0.007188752293586731, "fcm_dpo/delta": 0.02507771924138069, "fcm_dpo/margin": 59.23672866821289, "fcm_dpo/q_t": 0.4036938548088074, "grad_norm": 9.431406021118164, "learning_rate": 3.367098400098881e-07, "logits/chosen": -1.878330945968628, "logits/rejected": -1.5287479162216187, "logps/chosen": -197.89340209960938, "logps/ref_chosen": -96.64976501464844, "logps/ref_rejected": -93.34860229492188, "logps/rejected": -253.82896423339844, "loss": 1.0925, "margin_dpo/margin_mean": 59.23672866821289, "margin_dpo/margin_std": 84.82234954833984, "step": 307 }, { "KL/chosen_KL_mean": -105.28068542480469, "KL/mean": -134.8682403564453, "KL/rejected_KL_mean": -164.455810546875, "KL/std": 73.43102264404297, "epoch": 0.4522760646108664, "fcm_dpo/beta": 0.007201895117759705, "fcm_dpo/delta": 0.02417123317718506, "fcm_dpo/margin": 59.17512893676758, "fcm_dpo/q_t": 0.40439844131469727, "grad_norm": 10.738750457763672, "learning_rate": 3.355050358314172e-07, "logits/chosen": -1.7507621049880981, "logits/rejected": -1.4741660356521606, "logps/chosen": -191.00535583496094, "logps/ref_chosen": -85.72467041015625, "logps/ref_rejected": -86.42317962646484, "logps/rejected": -250.87899780273438, "loss": 1.0971, "margin_dpo/margin_mean": 59.17512893676758, "margin_dpo/margin_std": 85.20042419433594, "step": 308 }, { "KL/chosen_KL_mean": -118.82035827636719, "KL/mean": -145.4488067626953, "KL/rejected_KL_mean": -172.07725524902344, "KL/std": 77.83595275878906, "epoch": 0.45374449339207046, "fcm_dpo/beta": 0.0073315356858074665, "fcm_dpo/delta": 0.06189519166946411, "fcm_dpo/margin": 53.25689697265625, "fcm_dpo/q_t": 0.4107746481895447, "grad_norm": 13.122225761413574, "learning_rate": 3.3429797851573183e-07, "logits/chosen": -1.624879240989685, "logits/rejected": -1.1875228881835938, "logps/chosen": -201.97181701660156, "logps/ref_chosen": -83.15145874023438, "logps/ref_rejected": -86.58602142333984, "logps/rejected": -258.66326904296875, "loss": 1.1114, "margin_dpo/margin_mean": 53.25689697265625, "margin_dpo/margin_std": 76.73574829101562, "step": 309 }, { "KL/chosen_KL_mean": -106.3636474609375, "KL/mean": -136.62596130371094, "KL/rejected_KL_mean": -166.8883056640625, "KL/std": 71.94114685058594, "epoch": 0.4552129221732746, "fcm_dpo/beta": 0.00734830042347312, "fcm_dpo/delta": 0.005481800530105829, "fcm_dpo/margin": 60.52464294433594, "fcm_dpo/q_t": 0.3978798985481262, "grad_norm": 9.456814765930176, "learning_rate": 3.3308869986991487e-07, "logits/chosen": -2.310269832611084, "logits/rejected": -1.901537537574768, "logps/chosen": -196.1016387939453, "logps/ref_chosen": -89.73799133300781, "logps/ref_rejected": -87.07083129882812, "logps/rejected": -253.95912170410156, "loss": 1.0607, "margin_dpo/margin_mean": 60.52464294433594, "margin_dpo/margin_std": 74.69645690917969, "step": 310 }, { "KL/chosen_KL_mean": -126.24038696289062, "KL/mean": -164.61024475097656, "KL/rejected_KL_mean": -202.98008728027344, "KL/std": 73.11846160888672, "epoch": 0.4566813509544787, "fcm_dpo/beta": 0.007237396668642759, "fcm_dpo/delta": -0.11156634986400604, "fcm_dpo/margin": 76.73970794677734, "fcm_dpo/q_t": 0.3726646304130554, "grad_norm": 9.374836921691895, "learning_rate": 3.3187723175958346e-07, "logits/chosen": -1.5658994913101196, "logits/rejected": -0.8260637521743774, "logps/chosen": -214.76681518554688, "logps/ref_chosen": -88.52644348144531, "logps/ref_rejected": -82.26608276367188, "logps/rejected": -285.24615478515625, "loss": 0.9696, "margin_dpo/margin_mean": 76.73970031738281, "margin_dpo/margin_std": 73.58587646484375, "step": 311 }, { "KL/chosen_KL_mean": -122.65965270996094, "KL/mean": -154.30389404296875, "KL/rejected_KL_mean": -185.9481658935547, "KL/std": 80.49473571777344, "epoch": 0.4581497797356828, "fcm_dpo/beta": 0.007210931275039911, "fcm_dpo/delta": -0.0068823304027318954, "fcm_dpo/margin": 63.28850555419922, "fcm_dpo/q_t": 0.3952620029449463, "grad_norm": 9.447283744812012, "learning_rate": 3.306636061080487e-07, "logits/chosen": -1.6048760414123535, "logits/rejected": -1.0793572664260864, "logps/chosen": -203.0603485107422, "logps/ref_chosen": -80.40069580078125, "logps/ref_rejected": -82.4471435546875, "logps/rejected": -268.39532470703125, "loss": 1.0544, "margin_dpo/margin_mean": 63.28850555419922, "margin_dpo/margin_std": 77.36481475830078, "step": 312 }, { "KL/chosen_KL_mean": -115.8592529296875, "KL/mean": -147.1273193359375, "KL/rejected_KL_mean": -178.39537048339844, "KL/std": 78.93463134765625, "epoch": 0.45961820851688695, "fcm_dpo/beta": 0.0072022429667413235, "fcm_dpo/delta": -0.0005557965487241745, "fcm_dpo/margin": 62.53610610961914, "fcm_dpo/q_t": 0.3974963128566742, "grad_norm": 10.454935073852539, "learning_rate": 3.2944785489547537e-07, "logits/chosen": -2.4720208644866943, "logits/rejected": -2.070781946182251, "logps/chosen": -200.00115966796875, "logps/ref_chosen": -84.14190673828125, "logps/ref_rejected": -93.42684173583984, "logps/rejected": -271.82220458984375, "loss": 1.0722, "margin_dpo/margin_mean": 62.53610610961914, "margin_dpo/margin_std": 84.5926513671875, "step": 313 }, { "KL/chosen_KL_mean": -118.06554412841797, "KL/mean": -148.58663940429688, "KL/rejected_KL_mean": -179.10772705078125, "KL/std": 78.61907958984375, "epoch": 0.461086637298091, "fcm_dpo/beta": 0.007175165228545666, "fcm_dpo/delta": 0.012416481971740723, "fcm_dpo/margin": 61.042179107666016, "fcm_dpo/q_t": 0.4005988538265228, "grad_norm": 10.791749000549316, "learning_rate": 3.2823001015803857e-07, "logits/chosen": -1.8073995113372803, "logits/rejected": -1.5847784280776978, "logps/chosen": -206.87527465820312, "logps/ref_chosen": -88.80972290039062, "logps/ref_rejected": -104.15892028808594, "logps/rejected": -283.26666259765625, "loss": 1.069, "margin_dpo/margin_mean": 61.042179107666016, "margin_dpo/margin_std": 78.028564453125, "step": 314 }, { "KL/chosen_KL_mean": -115.33090209960938, "KL/mean": -146.99432373046875, "KL/rejected_KL_mean": -178.65774536132812, "KL/std": 83.00553131103516, "epoch": 0.46255506607929514, "fcm_dpo/beta": 0.007191159762442112, "fcm_dpo/delta": -0.005660181399434805, "fcm_dpo/margin": 63.32685089111328, "fcm_dpo/q_t": 0.3985491394996643, "grad_norm": 9.554756164550781, "learning_rate": 3.270101039870797e-07, "logits/chosen": -1.351799726486206, "logits/rejected": -1.1717203855514526, "logps/chosen": -196.12460327148438, "logps/ref_chosen": -80.793701171875, "logps/ref_rejected": -92.13711547851562, "logps/rejected": -270.79486083984375, "loss": 1.0895, "margin_dpo/margin_mean": 63.32685089111328, "margin_dpo/margin_std": 95.18093872070312, "step": 315 }, { "KL/chosen_KL_mean": -118.03253173828125, "KL/mean": -151.88902282714844, "KL/rejected_KL_mean": -185.74551391601562, "KL/std": 81.04499816894531, "epoch": 0.46402349486049926, "fcm_dpo/beta": 0.0071615735068917274, "fcm_dpo/delta": -0.03669600933790207, "fcm_dpo/margin": 67.71298217773438, "fcm_dpo/q_t": 0.39057457447052, "grad_norm": 9.565749168395996, "learning_rate": 3.2578816852826086e-07, "logits/chosen": -1.9321346282958984, "logits/rejected": -1.9324355125427246, "logps/chosen": -202.216796875, "logps/ref_chosen": -84.18424987792969, "logps/ref_rejected": -105.6908187866211, "logps/rejected": -291.43634033203125, "loss": 1.0399, "margin_dpo/margin_mean": 67.71298217773438, "margin_dpo/margin_std": 83.6407241821289, "step": 316 }, { "KL/chosen_KL_mean": -123.20557403564453, "KL/mean": -165.17007446289062, "KL/rejected_KL_mean": -207.13458251953125, "KL/std": 90.36332702636719, "epoch": 0.4654919236417034, "fcm_dpo/beta": 0.007012120448052883, "fcm_dpo/delta": -0.14692042768001556, "fcm_dpo/margin": 83.92901611328125, "fcm_dpo/q_t": 0.36895453929901123, "grad_norm": 10.822399139404297, "learning_rate": 3.2456423598071783e-07, "logits/chosen": -1.8852722644805908, "logits/rejected": -1.4041595458984375, "logps/chosen": -211.07907104492188, "logps/ref_chosen": -87.87348937988281, "logps/ref_rejected": -108.26519775390625, "logps/rejected": -315.3997802734375, "loss": 0.9856, "margin_dpo/margin_mean": 83.92901611328125, "margin_dpo/margin_std": 95.92436218261719, "step": 317 }, { "KL/chosen_KL_mean": -119.85832214355469, "KL/mean": -157.14173889160156, "KL/rejected_KL_mean": -194.42515563964844, "KL/std": 75.50823974609375, "epoch": 0.4669603524229075, "fcm_dpo/beta": 0.006859579123556614, "fcm_dpo/delta": -0.06490808725357056, "fcm_dpo/margin": 74.56683349609375, "fcm_dpo/q_t": 0.3828851282596588, "grad_norm": 11.389307022094727, "learning_rate": 3.233383385962115e-07, "logits/chosen": -2.099835157394409, "logits/rejected": -1.378746747970581, "logps/chosen": -218.06385803222656, "logps/ref_chosen": -98.20553588867188, "logps/ref_rejected": -88.13629150390625, "logps/rejected": -282.56146240234375, "loss": 1.0126, "margin_dpo/margin_mean": 74.56683349609375, "margin_dpo/margin_std": 82.71302795410156, "step": 318 }, { "KL/chosen_KL_mean": -104.32672119140625, "KL/mean": -144.81991577148438, "KL/rejected_KL_mean": -185.31312561035156, "KL/std": 78.60574340820312, "epoch": 0.4684287812041116, "fcm_dpo/beta": 0.00672875763848424, "fcm_dpo/delta": -0.100704126060009, "fcm_dpo/margin": 80.98640441894531, "fcm_dpo/q_t": 0.37574416399002075, "grad_norm": 10.522967338562012, "learning_rate": 3.2211050867827805e-07, "logits/chosen": -1.8933600187301636, "logits/rejected": -1.820589303970337, "logps/chosen": -186.83010864257812, "logps/ref_chosen": -82.50337219238281, "logps/ref_rejected": -118.65068054199219, "logps/rejected": -303.96380615234375, "loss": 0.9941, "margin_dpo/margin_mean": 80.98640441894531, "margin_dpo/margin_std": 88.18949890136719, "step": 319 }, { "KL/chosen_KL_mean": -107.90133666992188, "KL/mean": -141.27035522460938, "KL/rejected_KL_mean": -174.63937377929688, "KL/std": 80.5972900390625, "epoch": 0.4698972099853157, "fcm_dpo/beta": 0.006698761135339737, "fcm_dpo/delta": 0.0030147507786750793, "fcm_dpo/margin": 66.73802185058594, "fcm_dpo/q_t": 0.3972882628440857, "grad_norm": 8.973281860351562, "learning_rate": 3.208807785813777e-07, "logits/chosen": -2.0102860927581787, "logits/rejected": -1.8216140270233154, "logps/chosen": -195.03857421875, "logps/ref_chosen": -87.13722229003906, "logps/ref_rejected": -103.22412109375, "logps/rejected": -277.86346435546875, "loss": 1.0507, "margin_dpo/margin_mean": 66.73802185058594, "margin_dpo/margin_std": 77.56290435791016, "step": 320 }, { "KL/chosen_KL_mean": -119.27159881591797, "KL/mean": -158.0006103515625, "KL/rejected_KL_mean": -196.72964477539062, "KL/std": 82.33745574951172, "epoch": 0.4713656387665198, "fcm_dpo/beta": 0.00661929976195097, "fcm_dpo/delta": -0.06669492274522781, "fcm_dpo/margin": 77.45805358886719, "fcm_dpo/q_t": 0.3806576132774353, "grad_norm": 8.919323921203613, "learning_rate": 3.1964918071004217e-07, "logits/chosen": -1.880910873413086, "logits/rejected": -1.4864349365234375, "logps/chosen": -212.3762664794922, "logps/ref_chosen": -93.10466766357422, "logps/ref_rejected": -97.4681396484375, "logps/rejected": -294.1977844238281, "loss": 1.0063, "margin_dpo/margin_mean": 77.45805358886719, "margin_dpo/margin_std": 81.26376342773438, "step": 321 }, { "KL/chosen_KL_mean": -111.59713745117188, "KL/mean": -153.31509399414062, "KL/rejected_KL_mean": -195.0330352783203, "KL/std": 82.57926940917969, "epoch": 0.47283406754772395, "fcm_dpo/beta": 0.006480282172560692, "fcm_dpo/delta": -0.09766355156898499, "fcm_dpo/margin": 83.43589782714844, "fcm_dpo/q_t": 0.37637168169021606, "grad_norm": 13.370867729187012, "learning_rate": 3.184157475180207e-07, "logits/chosen": -1.5116252899169922, "logits/rejected": -1.1432104110717773, "logps/chosen": -203.79005432128906, "logps/ref_chosen": -92.19291687011719, "logps/ref_rejected": -100.89321899414062, "logps/rejected": -295.92626953125, "loss": 0.9981, "margin_dpo/margin_mean": 83.43589782714844, "margin_dpo/margin_std": 89.31085205078125, "step": 322 }, { "KL/chosen_KL_mean": -128.08004760742188, "KL/mean": -163.47006225585938, "KL/rejected_KL_mean": -198.8600311279297, "KL/std": 82.79885864257812, "epoch": 0.47430249632892807, "fcm_dpo/beta": 0.00647247489541769, "fcm_dpo/delta": -0.008625701069831848, "fcm_dpo/margin": 70.77998352050781, "fcm_dpo/q_t": 0.3960116505622864, "grad_norm": 10.688695907592773, "learning_rate": 3.171805115074251e-07, "logits/chosen": -2.0449767112731934, "logits/rejected": -1.6692168712615967, "logps/chosen": -208.35215759277344, "logps/ref_chosen": -80.27210235595703, "logps/ref_rejected": -80.3033447265625, "logps/rejected": -279.16339111328125, "loss": 1.0508, "margin_dpo/margin_mean": 70.77998352050781, "margin_dpo/margin_std": 85.38031005859375, "step": 323 }, { "KL/chosen_KL_mean": -127.90582275390625, "KL/mean": -164.725341796875, "KL/rejected_KL_mean": -201.54489135742188, "KL/std": 82.27083587646484, "epoch": 0.47577092511013214, "fcm_dpo/beta": 0.006465715356171131, "fcm_dpo/delta": -0.02743496000766754, "fcm_dpo/margin": 73.63906860351562, "fcm_dpo/q_t": 0.38912340998649597, "grad_norm": 9.375765800476074, "learning_rate": 3.1594350522787295e-07, "logits/chosen": -1.6269385814666748, "logits/rejected": -0.9706502556800842, "logps/chosen": -223.6351776123047, "logps/ref_chosen": -95.72935485839844, "logps/ref_rejected": -94.16924285888672, "logps/rejected": -295.714111328125, "loss": 1.0244, "margin_dpo/margin_mean": 73.63906860351562, "margin_dpo/margin_std": 77.25653076171875, "step": 324 }, { "KL/chosen_KL_mean": -117.1956787109375, "KL/mean": -156.62721252441406, "KL/rejected_KL_mean": -196.05874633789062, "KL/std": 78.86705017089844, "epoch": 0.47723935389133626, "fcm_dpo/beta": 0.00639908853918314, "fcm_dpo/delta": -0.05752098932862282, "fcm_dpo/margin": 78.86307525634766, "fcm_dpo/q_t": 0.3828526735305786, "grad_norm": 8.189767837524414, "learning_rate": 3.147047612756302e-07, "logits/chosen": -1.8238208293914795, "logits/rejected": -1.0989253520965576, "logps/chosen": -215.9025421142578, "logps/ref_chosen": -98.70686340332031, "logps/ref_rejected": -76.40809631347656, "logps/rejected": -272.46685791015625, "loss": 1.0077, "margin_dpo/margin_mean": 78.86307525634766, "margin_dpo/margin_std": 81.07893371582031, "step": 325 }, { "KL/chosen_KL_mean": -130.5388641357422, "KL/mean": -166.12429809570312, "KL/rejected_KL_mean": -201.709716796875, "KL/std": 78.67150115966797, "epoch": 0.4787077826725404, "fcm_dpo/beta": 0.00636826828122139, "fcm_dpo/delta": -0.0033944009337574244, "fcm_dpo/margin": 71.17085266113281, "fcm_dpo/q_t": 0.39494964480400085, "grad_norm": 8.22679615020752, "learning_rate": 3.134643122927519e-07, "logits/chosen": -2.344250440597534, "logits/rejected": -1.6645500659942627, "logps/chosen": -235.20016479492188, "logps/ref_chosen": -104.66130828857422, "logps/ref_rejected": -86.82537841796875, "logps/rejected": -288.53509521484375, "loss": 1.0511, "margin_dpo/margin_mean": 71.17085266113281, "margin_dpo/margin_std": 84.024658203125, "step": 326 }, { "KL/chosen_KL_mean": -126.83247375488281, "KL/mean": -161.19143676757812, "KL/rejected_KL_mean": -195.55038452148438, "KL/std": 72.33634948730469, "epoch": 0.4801762114537445, "fcm_dpo/beta": 0.006365356035530567, "fcm_dpo/delta": 0.013049425557255745, "fcm_dpo/margin": 68.71792602539062, "fcm_dpo/q_t": 0.4000307321548462, "grad_norm": 9.14709758758545, "learning_rate": 3.1222219096622264e-07, "logits/chosen": -1.6792106628417969, "logits/rejected": -1.0810998678207397, "logps/chosen": -234.272705078125, "logps/ref_chosen": -107.44023895263672, "logps/ref_rejected": -117.05323028564453, "logps/rejected": -312.6036376953125, "loss": 1.0584, "margin_dpo/margin_mean": 68.71792602539062, "margin_dpo/margin_std": 81.67416381835938, "step": 327 }, { "KL/chosen_KL_mean": -123.31852722167969, "KL/mean": -168.656005859375, "KL/rejected_KL_mean": -213.99349975585938, "KL/std": 86.12576293945312, "epoch": 0.48164464023494863, "fcm_dpo/beta": 0.006271493621170521, "fcm_dpo/delta": -0.12576939165592194, "fcm_dpo/margin": 90.67498779296875, "fcm_dpo/q_t": 0.36831027269363403, "grad_norm": 10.82209300994873, "learning_rate": 3.1097843002709427e-07, "logits/chosen": -1.7117938995361328, "logits/rejected": -1.4164050817489624, "logps/chosen": -209.92922973632812, "logps/ref_chosen": -86.6106948852539, "logps/ref_rejected": -97.40188598632812, "logps/rejected": -311.3953857421875, "loss": 0.9572, "margin_dpo/margin_mean": 90.67498779296875, "margin_dpo/margin_std": 82.11698913574219, "step": 328 }, { "KL/chosen_KL_mean": -128.87521362304688, "KL/mean": -164.40403747558594, "KL/rejected_KL_mean": -199.93287658691406, "KL/std": 81.97160339355469, "epoch": 0.4831130690161527, "fcm_dpo/beta": 0.006211411207914352, "fcm_dpo/delta": 0.008740127086639404, "fcm_dpo/margin": 71.05769348144531, "fcm_dpo/q_t": 0.3989870548248291, "grad_norm": 10.588635444641113, "learning_rate": 3.0973306224962437e-07, "logits/chosen": -1.552767276763916, "logits/rejected": -1.054271936416626, "logps/chosen": -229.48828125, "logps/ref_chosen": -100.61308288574219, "logps/ref_rejected": -104.95370483398438, "logps/rejected": -304.8865966796875, "loss": 1.0613, "margin_dpo/margin_mean": 71.05769348144531, "margin_dpo/margin_std": 86.44296264648438, "step": 329 }, { "KL/chosen_KL_mean": -110.63896942138672, "KL/mean": -148.5820770263672, "KL/rejected_KL_mean": -186.5251922607422, "KL/std": 80.47824096679688, "epoch": 0.4845814977973568, "fcm_dpo/beta": 0.006208469159901142, "fcm_dpo/delta": -0.022268781438469887, "fcm_dpo/margin": 75.88623046875, "fcm_dpo/q_t": 0.38848236203193665, "grad_norm": 8.516870498657227, "learning_rate": 3.084861204504122e-07, "logits/chosen": -1.7653732299804688, "logits/rejected": -1.6858103275299072, "logps/chosen": -187.597412109375, "logps/ref_chosen": -76.95843505859375, "logps/ref_rejected": -92.81494140625, "logps/rejected": -279.34014892578125, "loss": 1.0166, "margin_dpo/margin_mean": 75.88623046875, "margin_dpo/margin_std": 70.37911987304688, "step": 330 }, { "KL/chosen_KL_mean": -119.34526824951172, "KL/mean": -161.75233459472656, "KL/rejected_KL_mean": -204.15940856933594, "KL/std": 78.9277572631836, "epoch": 0.48604992657856094, "fcm_dpo/beta": 0.0061643123626708984, "fcm_dpo/delta": -0.07684408873319626, "fcm_dpo/margin": 84.81415557861328, "fcm_dpo/q_t": 0.3781358599662781, "grad_norm": 10.151162147521973, "learning_rate": 3.072376374875335e-07, "logits/chosen": -1.8666870594024658, "logits/rejected": -1.562316656112671, "logps/chosen": -202.40469360351562, "logps/ref_chosen": -83.05941772460938, "logps/ref_rejected": -94.13113403320312, "logps/rejected": -298.29052734375, "loss": 0.984, "margin_dpo/margin_mean": 84.81414794921875, "margin_dpo/margin_std": 76.9617919921875, "step": 331 }, { "KL/chosen_KL_mean": -125.17555236816406, "KL/mean": -154.91082763671875, "KL/rejected_KL_mean": -184.6461181640625, "KL/std": 87.22966003417969, "epoch": 0.48751835535976507, "fcm_dpo/beta": 0.006201116368174553, "fcm_dpo/delta": 0.08419719338417053, "fcm_dpo/margin": 59.470550537109375, "fcm_dpo/q_t": 0.41479602456092834, "grad_norm": 8.157636642456055, "learning_rate": 3.059876462596758e-07, "logits/chosen": -2.1781392097473145, "logits/rejected": -1.7544598579406738, "logps/chosen": -207.1064453125, "logps/ref_chosen": -81.93089294433594, "logps/ref_rejected": -82.84590148925781, "logps/rejected": -267.49200439453125, "loss": 1.1158, "margin_dpo/margin_mean": 59.470558166503906, "margin_dpo/margin_std": 84.43423461914062, "step": 332 }, { "KL/chosen_KL_mean": -145.17108154296875, "KL/mean": -186.31817626953125, "KL/rejected_KL_mean": -227.46524047851562, "KL/std": 90.61051177978516, "epoch": 0.4889867841409692, "fcm_dpo/beta": 0.006128801964223385, "fcm_dpo/delta": -0.058387283235788345, "fcm_dpo/margin": 82.2941665649414, "fcm_dpo/q_t": 0.3855236768722534, "grad_norm": 8.680399894714355, "learning_rate": 3.0473617970527015e-07, "logits/chosen": -1.5334293842315674, "logits/rejected": -1.1944446563720703, "logps/chosen": -233.70367431640625, "logps/ref_chosen": -88.53257751464844, "logps/ref_rejected": -102.46601867675781, "logps/rejected": -329.9312744140625, "loss": 1.0292, "margin_dpo/margin_mean": 82.29415893554688, "margin_dpo/margin_std": 97.24163055419922, "step": 333 }, { "KL/chosen_KL_mean": -131.6866912841797, "KL/mean": -168.419189453125, "KL/rejected_KL_mean": -205.1516876220703, "KL/std": 85.39486694335938, "epoch": 0.49045521292217326, "fcm_dpo/beta": 0.00611284002661705, "fcm_dpo/delta": 0.0002731010317802429, "fcm_dpo/margin": 73.46499633789062, "fcm_dpo/q_t": 0.39615800976753235, "grad_norm": 9.491837501525879, "learning_rate": 3.034832708016243e-07, "logits/chosen": -1.3197132349014282, "logits/rejected": -1.0197309255599976, "logps/chosen": -229.44447326660156, "logps/ref_chosen": -97.75778198242188, "logps/ref_rejected": -100.0058364868164, "logps/rejected": -305.15753173828125, "loss": 1.0696, "margin_dpo/margin_mean": 73.46499633789062, "margin_dpo/margin_std": 95.07492065429688, "step": 334 }, { "KL/chosen_KL_mean": -131.00123596191406, "KL/mean": -163.1631317138672, "KL/rejected_KL_mean": -195.32501220703125, "KL/std": 85.37344360351562, "epoch": 0.4919236417033774, "fcm_dpo/beta": 0.0061988988891243935, "fcm_dpo/delta": 0.053380027413368225, "fcm_dpo/margin": 64.32377624511719, "fcm_dpo/q_t": 0.40897828340530396, "grad_norm": 11.817728042602539, "learning_rate": 3.022289525640531e-07, "logits/chosen": -2.154109477996826, "logits/rejected": -1.7217153310775757, "logps/chosen": -227.3772735595703, "logps/ref_chosen": -96.37603759765625, "logps/ref_rejected": -94.92401123046875, "logps/rejected": -290.2490234375, "loss": 1.1049, "margin_dpo/margin_mean": 64.32377624511719, "margin_dpo/margin_std": 93.01960754394531, "step": 335 }, { "KL/chosen_KL_mean": -133.98861694335938, "KL/mean": -174.62728881835938, "KL/rejected_KL_mean": -215.26593017578125, "KL/std": 87.84451293945312, "epoch": 0.4933920704845815, "fcm_dpo/beta": 0.006177923642098904, "fcm_dpo/delta": -0.054861292243003845, "fcm_dpo/margin": 81.27732849121094, "fcm_dpo/q_t": 0.38504862785339355, "grad_norm": 11.14604377746582, "learning_rate": 3.009732580450086e-07, "logits/chosen": -1.5421233177185059, "logits/rejected": -1.3872023820877075, "logps/chosen": -222.5181427001953, "logps/ref_chosen": -88.52952575683594, "logps/ref_rejected": -109.5557861328125, "logps/rejected": -324.82171630859375, "loss": 1.019, "margin_dpo/margin_mean": 81.27732849121094, "margin_dpo/margin_std": 92.28482055664062, "step": 336 }, { "KL/chosen_KL_mean": -132.160888671875, "KL/mean": -170.9864501953125, "KL/rejected_KL_mean": -209.81201171875, "KL/std": 88.68861389160156, "epoch": 0.4948604992657856, "fcm_dpo/beta": 0.006131826899945736, "fcm_dpo/delta": -0.02746148779988289, "fcm_dpo/margin": 77.65113067626953, "fcm_dpo/q_t": 0.39256367087364197, "grad_norm": 9.663256645202637, "learning_rate": 2.9971622033320914e-07, "logits/chosen": -2.5155861377716064, "logits/rejected": -2.2384302616119385, "logps/chosen": -229.4732666015625, "logps/ref_chosen": -97.3123779296875, "logps/ref_rejected": -110.13134765625, "logps/rejected": -319.943359375, "loss": 1.0572, "margin_dpo/margin_mean": 77.65113067626953, "margin_dpo/margin_std": 102.71278381347656, "step": 337 }, { "KL/chosen_KL_mean": -136.95333862304688, "KL/mean": -172.71170043945312, "KL/rejected_KL_mean": -208.47006225585938, "KL/std": 89.64544677734375, "epoch": 0.49632892804698975, "fcm_dpo/beta": 0.006117684300988913, "fcm_dpo/delta": 0.01304161548614502, "fcm_dpo/margin": 71.51670837402344, "fcm_dpo/q_t": 0.39828041195869446, "grad_norm": 9.636395454406738, "learning_rate": 2.984578725527675e-07, "logits/chosen": -1.8111103773117065, "logits/rejected": -1.6560990810394287, "logps/chosen": -220.58346557617188, "logps/ref_chosen": -83.63011169433594, "logps/ref_rejected": -93.69235229492188, "logps/rejected": -302.16241455078125, "loss": 1.0626, "margin_dpo/margin_mean": 71.51671600341797, "margin_dpo/margin_std": 86.73267364501953, "step": 338 }, { "KL/chosen_KL_mean": -145.28753662109375, "KL/mean": -183.42254638671875, "KL/rejected_KL_mean": -221.55752563476562, "KL/std": 88.99887084960938, "epoch": 0.4977973568281938, "fcm_dpo/beta": 0.006152212619781494, "fcm_dpo/delta": -0.02100226655602455, "fcm_dpo/margin": 76.27001953125, "fcm_dpo/q_t": 0.3927009403705597, "grad_norm": 8.57188606262207, "learning_rate": 2.9719824786231796e-07, "logits/chosen": -2.97969913482666, "logits/rejected": -2.559129238128662, "logps/chosen": -245.1393585205078, "logps/ref_chosen": -99.85182189941406, "logps/ref_rejected": -109.0743637084961, "logps/rejected": -330.63189697265625, "loss": 1.0591, "margin_dpo/margin_mean": 76.27002716064453, "margin_dpo/margin_std": 97.40506744384766, "step": 339 }, { "KL/chosen_KL_mean": -155.9953155517578, "KL/mean": -189.56283569335938, "KL/rejected_KL_mean": -223.13035583496094, "KL/std": 81.51522827148438, "epoch": 0.49926578560939794, "fcm_dpo/beta": 0.00614144466817379, "fcm_dpo/delta": 0.03929678723216057, "fcm_dpo/margin": 67.13504028320312, "fcm_dpo/q_t": 0.40457192063331604, "grad_norm": 10.448555946350098, "learning_rate": 2.959373794541426e-07, "logits/chosen": -1.5166605710983276, "logits/rejected": -1.0412172079086304, "logps/chosen": -240.96926879882812, "logps/ref_chosen": -84.97396850585938, "logps/ref_rejected": -78.80033874511719, "logps/rejected": -301.9306945800781, "loss": 1.0843, "margin_dpo/margin_mean": 67.13504028320312, "margin_dpo/margin_std": 88.80570983886719, "step": 340 }, { "KL/chosen_KL_mean": -163.71954345703125, "KL/mean": -196.01004028320312, "KL/rejected_KL_mean": -228.30056762695312, "KL/std": 85.64817810058594, "epoch": 0.5007342143906021, "fcm_dpo/beta": 0.0061846161261200905, "fcm_dpo/delta": 0.05270107463002205, "fcm_dpo/margin": 64.58100128173828, "fcm_dpo/q_t": 0.40958231687545776, "grad_norm": 9.756683349609375, "learning_rate": 2.946753005532965e-07, "logits/chosen": -1.4001996517181396, "logits/rejected": -1.1443207263946533, "logps/chosen": -248.98353576660156, "logps/ref_chosen": -85.26399230957031, "logps/ref_rejected": -106.08595275878906, "logps/rejected": -334.38653564453125, "loss": 1.1137, "margin_dpo/margin_mean": 64.58100128173828, "margin_dpo/margin_std": 97.7337646484375, "step": 341 }, { "KL/chosen_KL_mean": -130.64772033691406, "KL/mean": -177.71002197265625, "KL/rejected_KL_mean": -224.77235412597656, "KL/std": 86.38710021972656, "epoch": 0.5022026431718062, "fcm_dpo/beta": 0.006058323662728071, "fcm_dpo/delta": -0.12925466895103455, "fcm_dpo/margin": 94.12464904785156, "fcm_dpo/q_t": 0.36713290214538574, "grad_norm": 8.498157501220703, "learning_rate": 2.934120444167326e-07, "logits/chosen": -1.437894582748413, "logits/rejected": -0.6820884943008423, "logps/chosen": -231.6863250732422, "logps/ref_chosen": -101.03860473632812, "logps/ref_rejected": -98.89851379394531, "logps/rejected": -323.6708679199219, "loss": 0.9732, "margin_dpo/margin_mean": 94.12464904785156, "margin_dpo/margin_std": 91.66548156738281, "step": 342 }, { "KL/chosen_KL_mean": -142.7459259033203, "KL/mean": -181.76763916015625, "KL/rejected_KL_mean": -220.78933715820312, "KL/std": 89.77273559570312, "epoch": 0.5036710719530103, "fcm_dpo/beta": 0.0060761794447898865, "fcm_dpo/delta": -0.026198003441095352, "fcm_dpo/margin": 78.04341125488281, "fcm_dpo/q_t": 0.39156705141067505, "grad_norm": 11.090639114379883, "learning_rate": 2.9214764433242476e-07, "logits/chosen": -2.0876166820526123, "logits/rejected": -1.9741833209991455, "logps/chosen": -226.4932861328125, "logps/ref_chosen": -83.74736022949219, "logps/ref_rejected": -116.57908630371094, "logps/rejected": -337.368408203125, "loss": 1.0463, "margin_dpo/margin_mean": 78.04341125488281, "margin_dpo/margin_std": 92.98764038085938, "step": 343 }, { "KL/chosen_KL_mean": -142.91702270507812, "KL/mean": -183.55194091796875, "KL/rejected_KL_mean": -224.18685913085938, "KL/std": 95.46722412109375, "epoch": 0.5051395007342144, "fcm_dpo/beta": 0.006021049339324236, "fcm_dpo/delta": -0.04170961305499077, "fcm_dpo/margin": 81.26984405517578, "fcm_dpo/q_t": 0.38922470808029175, "grad_norm": 10.0555419921875, "learning_rate": 2.9088213361849126e-07, "logits/chosen": -1.6818866729736328, "logits/rejected": -1.4340788125991821, "logps/chosen": -221.17098999023438, "logps/ref_chosen": -78.25396728515625, "logps/ref_rejected": -95.42870330810547, "logps/rejected": -319.6155700683594, "loss": 1.0427, "margin_dpo/margin_mean": 81.26985168457031, "margin_dpo/margin_std": 100.70010375976562, "step": 344 }, { "KL/chosen_KL_mean": -147.00729370117188, "KL/mean": -190.7525634765625, "KL/rejected_KL_mean": -234.49781799316406, "KL/std": 95.71751403808594, "epoch": 0.5066079295154186, "fcm_dpo/beta": 0.0059035178273916245, "fcm_dpo/delta": -0.07031524181365967, "fcm_dpo/margin": 87.49053192138672, "fcm_dpo/q_t": 0.38228651881217957, "grad_norm": 9.630701065063477, "learning_rate": 2.896155456223163e-07, "logits/chosen": -1.591372013092041, "logits/rejected": -1.2972593307495117, "logps/chosen": -245.0867919921875, "logps/ref_chosen": -98.0794906616211, "logps/ref_rejected": -104.03477478027344, "logps/rejected": -338.5325927734375, "loss": 1.012, "margin_dpo/margin_mean": 87.49052429199219, "margin_dpo/margin_std": 98.24116516113281, "step": 345 }, { "KL/chosen_KL_mean": -163.81016540527344, "KL/mean": -197.99606323242188, "KL/rejected_KL_mean": -232.18194580078125, "KL/std": 88.83554077148438, "epoch": 0.5080763582966226, "fcm_dpo/beta": 0.005932542495429516, "fcm_dpo/delta": 0.046264972537755966, "fcm_dpo/margin": 68.37179565429688, "fcm_dpo/q_t": 0.40812280774116516, "grad_norm": 8.503084182739258, "learning_rate": 2.883479137196714e-07, "logits/chosen": -1.808495044708252, "logits/rejected": -1.494280219078064, "logps/chosen": -244.49365234375, "logps/ref_chosen": -80.68348693847656, "logps/ref_rejected": -85.89260864257812, "logps/rejected": -318.0745544433594, "loss": 1.104, "margin_dpo/margin_mean": 68.37179565429688, "margin_dpo/margin_std": 100.7035140991211, "step": 346 }, { "KL/chosen_KL_mean": -153.70419311523438, "KL/mean": -186.50839233398438, "KL/rejected_KL_mean": -219.3125762939453, "KL/std": 87.70765686035156, "epoch": 0.5095447870778267, "fcm_dpo/beta": 0.006018254440277815, "fcm_dpo/delta": 0.05700414627790451, "fcm_dpo/margin": 65.60836791992188, "fcm_dpo/q_t": 0.4102572798728943, "grad_norm": 8.964280128479004, "learning_rate": 2.8707927131383614e-07, "logits/chosen": -1.884059190750122, "logits/rejected": -1.5605113506317139, "logps/chosen": -247.38357543945312, "logps/ref_chosen": -93.67938232421875, "logps/ref_rejected": -97.54251098632812, "logps/rejected": -316.8551025390625, "loss": 1.1063, "margin_dpo/margin_mean": 65.60836791992188, "margin_dpo/margin_std": 94.55398559570312, "step": 347 }, { "KL/chosen_KL_mean": -133.36244201660156, "KL/mean": -170.93353271484375, "KL/rejected_KL_mean": -208.50462341308594, "KL/std": 83.08096313476562, "epoch": 0.5110132158590308, "fcm_dpo/beta": 0.005962018854916096, "fcm_dpo/delta": 0.0004133060574531555, "fcm_dpo/margin": 75.14217376708984, "fcm_dpo/q_t": 0.3959684669971466, "grad_norm": 8.280686378479004, "learning_rate": 2.858096518347179e-07, "logits/chosen": -1.851686954498291, "logits/rejected": -1.5706337690353394, "logps/chosen": -223.61468505859375, "logps/ref_chosen": -90.25224304199219, "logps/ref_rejected": -95.77027893066406, "logps/rejected": -304.27490234375, "loss": 1.0563, "margin_dpo/margin_mean": 75.14216613769531, "margin_dpo/margin_std": 84.9925537109375, "step": 348 }, { "KL/chosen_KL_mean": -143.33042907714844, "KL/mean": -178.51885986328125, "KL/rejected_KL_mean": -213.707275390625, "KL/std": 91.03848266601562, "epoch": 0.5124816446402349, "fcm_dpo/beta": 0.006063205190002918, "fcm_dpo/delta": 0.02386452443897724, "fcm_dpo/margin": 70.37683868408203, "fcm_dpo/q_t": 0.4028947353363037, "grad_norm": 10.386116027832031, "learning_rate": 2.845390887379706e-07, "logits/chosen": -2.064490556716919, "logits/rejected": -1.8402228355407715, "logps/chosen": -235.38851928710938, "logps/ref_chosen": -92.05809020996094, "logps/ref_rejected": -103.96527099609375, "logps/rejected": -317.67254638671875, "loss": 1.0795, "margin_dpo/margin_mean": 70.37683868408203, "margin_dpo/margin_std": 92.30020904541016, "step": 349 }, { "KL/chosen_KL_mean": -152.83030700683594, "KL/mean": -181.812744140625, "KL/rejected_KL_mean": -210.79519653320312, "KL/std": 83.527099609375, "epoch": 0.5139500734214391, "fcm_dpo/beta": 0.006126874126493931, "fcm_dpo/delta": 0.09844039380550385, "fcm_dpo/margin": 57.96487808227539, "fcm_dpo/q_t": 0.41880983114242554, "grad_norm": 11.032787322998047, "learning_rate": 2.8326761550411346e-07, "logits/chosen": -2.0019736289978027, "logits/rejected": -1.7813717126846313, "logps/chosen": -239.88450622558594, "logps/ref_chosen": -87.05419921875, "logps/ref_rejected": -95.72193145751953, "logps/rejected": -306.51715087890625, "loss": 1.1341, "margin_dpo/margin_mean": 57.964874267578125, "margin_dpo/margin_std": 89.42626190185547, "step": 350 }, { "KL/chosen_KL_mean": -151.509033203125, "KL/mean": -193.4451141357422, "KL/rejected_KL_mean": -235.38119506835938, "KL/std": 93.31094360351562, "epoch": 0.5154185022026432, "fcm_dpo/beta": 0.006119257770478725, "fcm_dpo/delta": -0.06661495566368103, "fcm_dpo/margin": 83.87215423583984, "fcm_dpo/q_t": 0.38258159160614014, "grad_norm": 9.132224082946777, "learning_rate": 2.819952656376487e-07, "logits/chosen": -1.707326054573059, "logits/rejected": -1.4008855819702148, "logps/chosen": -240.67807006835938, "logps/ref_chosen": -89.16903686523438, "logps/ref_rejected": -108.29801940917969, "logps/rejected": -343.67919921875, "loss": 1.0223, "margin_dpo/margin_mean": 83.87214660644531, "margin_dpo/margin_std": 98.51606750488281, "step": 351 }, { "KL/chosen_KL_mean": -138.6739501953125, "KL/mean": -178.33370971679688, "KL/rejected_KL_mean": -217.99343872070312, "KL/std": 92.34928894042969, "epoch": 0.5168869309838473, "fcm_dpo/beta": 0.006055803969502449, "fcm_dpo/delta": -0.03186991065740585, "fcm_dpo/margin": 79.31950378417969, "fcm_dpo/q_t": 0.39239203929901123, "grad_norm": 9.162763595581055, "learning_rate": 2.8072207266617854e-07, "logits/chosen": -1.8129549026489258, "logits/rejected": -1.2578930854797363, "logps/chosen": -235.48544311523438, "logps/ref_chosen": -96.81147766113281, "logps/ref_rejected": -84.40483093261719, "logps/rejected": -302.39825439453125, "loss": 1.0462, "margin_dpo/margin_mean": 79.31949615478516, "margin_dpo/margin_std": 98.95259857177734, "step": 352 }, { "KL/chosen_KL_mean": -159.00277709960938, "KL/mean": -184.40956115722656, "KL/rejected_KL_mean": -209.81634521484375, "KL/std": 82.14775848388672, "epoch": 0.5183553597650514, "fcm_dpo/beta": 0.006155360955744982, "fcm_dpo/delta": 0.14189966022968292, "fcm_dpo/margin": 50.813568115234375, "fcm_dpo/q_t": 0.4280507564544678, "grad_norm": 10.709700584411621, "learning_rate": 2.794480701395219e-07, "logits/chosen": -2.196394681930542, "logits/rejected": -1.8158748149871826, "logps/chosen": -244.36526489257812, "logps/ref_chosen": -85.36248779296875, "logps/ref_rejected": -87.36732482910156, "logps/rejected": -297.18365478515625, "loss": 1.1751, "margin_dpo/margin_mean": 50.813568115234375, "margin_dpo/margin_std": 91.15338134765625, "step": 353 }, { "KL/chosen_KL_mean": -139.20684814453125, "KL/mean": -178.58595275878906, "KL/rejected_KL_mean": -217.9650421142578, "KL/std": 83.68165588378906, "epoch": 0.5198237885462555, "fcm_dpo/beta": 0.006197898183017969, "fcm_dpo/delta": -0.04012412577867508, "fcm_dpo/margin": 78.75819396972656, "fcm_dpo/q_t": 0.3867066502571106, "grad_norm": 9.191285133361816, "learning_rate": 2.781732916288303e-07, "logits/chosen": -1.6556808948516846, "logits/rejected": -1.168824553489685, "logps/chosen": -235.00802612304688, "logps/ref_chosen": -95.8011703491211, "logps/ref_rejected": -94.86614227294922, "logps/rejected": -312.8311767578125, "loss": 1.0088, "margin_dpo/margin_mean": 78.75819396972656, "margin_dpo/margin_std": 76.52041625976562, "step": 354 }, { "KL/chosen_KL_mean": -137.57781982421875, "KL/mean": -174.31130981445312, "KL/rejected_KL_mean": -211.0447998046875, "KL/std": 85.07630920410156, "epoch": 0.5212922173274597, "fcm_dpo/beta": 0.0061437515541911125, "fcm_dpo/delta": -0.001744445413351059, "fcm_dpo/margin": 73.46697235107422, "fcm_dpo/q_t": 0.39597606658935547, "grad_norm": 10.174688339233398, "learning_rate": 2.7689777072570284e-07, "logits/chosen": -2.5813350677490234, "logits/rejected": -2.1107242107391357, "logps/chosen": -223.7633819580078, "logps/ref_chosen": -86.185546875, "logps/ref_rejected": -88.4615707397461, "logps/rejected": -299.50634765625, "loss": 1.0601, "margin_dpo/margin_mean": 73.46696472167969, "margin_dpo/margin_std": 89.4022216796875, "step": 355 }, { "KL/chosen_KL_mean": -161.46664428710938, "KL/mean": -189.98309326171875, "KL/rejected_KL_mean": -218.49954223632812, "KL/std": 83.07925415039062, "epoch": 0.5227606461086637, "fcm_dpo/beta": 0.006275987718254328, "fcm_dpo/delta": 0.09493474662303925, "fcm_dpo/margin": 57.03289031982422, "fcm_dpo/q_t": 0.41823703050613403, "grad_norm": 10.077198028564453, "learning_rate": 2.7562154104130176e-07, "logits/chosen": -1.6768903732299805, "logits/rejected": -1.1983524560928345, "logps/chosen": -252.28475952148438, "logps/ref_chosen": -90.818115234375, "logps/ref_rejected": -82.1273193359375, "logps/rejected": -300.6268615722656, "loss": 1.1432, "margin_dpo/margin_mean": 57.032894134521484, "margin_dpo/margin_std": 94.3851547241211, "step": 356 }, { "KL/chosen_KL_mean": -134.25582885742188, "KL/mean": -180.50140380859375, "KL/rejected_KL_mean": -226.74700927734375, "KL/std": 91.9460678100586, "epoch": 0.5242290748898678, "fcm_dpo/beta": 0.006120544858276844, "fcm_dpo/delta": -0.12536004185676575, "fcm_dpo/margin": 92.49116516113281, "fcm_dpo/q_t": 0.36878687143325806, "grad_norm": 8.692706108093262, "learning_rate": 2.7434463620546594e-07, "logits/chosen": -2.0856592655181885, "logits/rejected": -1.7231680154800415, "logps/chosen": -221.76058959960938, "logps/ref_chosen": -87.5047607421875, "logps/ref_rejected": -92.24937438964844, "logps/rejected": -318.9963684082031, "loss": 0.9641, "margin_dpo/margin_mean": 92.49116516113281, "margin_dpo/margin_std": 83.96719360351562, "step": 357 }, { "KL/chosen_KL_mean": -146.8455047607422, "KL/mean": -180.4937744140625, "KL/rejected_KL_mean": -214.1420135498047, "KL/std": 85.15274047851562, "epoch": 0.5256975036710719, "fcm_dpo/beta": 0.0061625586822628975, "fcm_dpo/delta": 0.036812763661146164, "fcm_dpo/margin": 67.29647827148438, "fcm_dpo/q_t": 0.4069470167160034, "grad_norm": 9.665899276733398, "learning_rate": 2.730670898658255e-07, "logits/chosen": -1.705322504043579, "logits/rejected": -1.237384557723999, "logps/chosen": -241.90097045898438, "logps/ref_chosen": -95.05546569824219, "logps/ref_rejected": -96.47611999511719, "logps/rejected": -310.6181335449219, "loss": 1.0868, "margin_dpo/margin_mean": 67.29647827148438, "margin_dpo/margin_std": 91.71583557128906, "step": 358 }, { "KL/chosen_KL_mean": -163.5113525390625, "KL/mean": -194.93548583984375, "KL/rejected_KL_mean": -226.359619140625, "KL/std": 80.23503112792969, "epoch": 0.527165932452276, "fcm_dpo/beta": 0.006196748465299606, "fcm_dpo/delta": 0.06266428530216217, "fcm_dpo/margin": 62.8482666015625, "fcm_dpo/q_t": 0.4122501611709595, "grad_norm": 9.812956809997559, "learning_rate": 2.717889356869146e-07, "logits/chosen": -1.5021560192108154, "logits/rejected": -0.928576648235321, "logps/chosen": -250.8846893310547, "logps/ref_chosen": -87.37332916259766, "logps/ref_rejected": -88.77188110351562, "logps/rejected": -315.1315002441406, "loss": 1.1143, "margin_dpo/margin_mean": 62.8482666015625, "margin_dpo/margin_std": 92.45555114746094, "step": 359 }, { "KL/chosen_KL_mean": -150.11993408203125, "KL/mean": -190.4580078125, "KL/rejected_KL_mean": -230.7960968017578, "KL/std": 81.05520629882812, "epoch": 0.5286343612334802, "fcm_dpo/beta": 0.006197527050971985, "fcm_dpo/delta": -0.05270250886678696, "fcm_dpo/margin": 80.67617797851562, "fcm_dpo/q_t": 0.3835641145706177, "grad_norm": 12.480203628540039, "learning_rate": 2.7051020734928443e-07, "logits/chosen": -1.594376564025879, "logits/rejected": -1.2482199668884277, "logps/chosen": -225.3433837890625, "logps/ref_chosen": -75.22344970703125, "logps/ref_rejected": -76.09400177001953, "logps/rejected": -306.89007568359375, "loss": 1.0112, "margin_dpo/margin_mean": 80.67617797851562, "margin_dpo/margin_std": 84.2534408569336, "step": 360 }, { "KL/chosen_KL_mean": -153.8967742919922, "KL/mean": -188.97772216796875, "KL/rejected_KL_mean": -224.05865478515625, "KL/std": 91.28170776367188, "epoch": 0.5301027900146843, "fcm_dpo/beta": 0.006206847261637449, "fcm_dpo/delta": 0.015146945603191853, "fcm_dpo/margin": 70.16188049316406, "fcm_dpo/q_t": 0.4020695984363556, "grad_norm": 11.036417007446289, "learning_rate": 2.6923093854861593e-07, "logits/chosen": -1.7453486919403076, "logits/rejected": -1.5000078678131104, "logps/chosen": -235.85011291503906, "logps/ref_chosen": -81.95333099365234, "logps/ref_rejected": -96.67919921875, "logps/rejected": -320.73785400390625, "loss": 1.0814, "margin_dpo/margin_mean": 70.16187286376953, "margin_dpo/margin_std": 97.12460327148438, "step": 361 }, { "KL/chosen_KL_mean": -158.99661254882812, "KL/mean": -195.830078125, "KL/rejected_KL_mean": -232.66351318359375, "KL/std": 85.33515930175781, "epoch": 0.5315712187958884, "fcm_dpo/beta": 0.006188073195517063, "fcm_dpo/delta": -0.006228707730770111, "fcm_dpo/margin": 73.66690063476562, "fcm_dpo/q_t": 0.39482009410858154, "grad_norm": 10.10085678100586, "learning_rate": 2.679511629948319e-07, "logits/chosen": -1.9562934637069702, "logits/rejected": -1.7549805641174316, "logps/chosen": -248.33633422851562, "logps/ref_chosen": -89.33971405029297, "logps/ref_rejected": -110.50196838378906, "logps/rejected": -343.16546630859375, "loss": 1.055, "margin_dpo/margin_mean": 73.66690063476562, "margin_dpo/margin_std": 89.30767822265625, "step": 362 }, { "KL/chosen_KL_mean": -149.33816528320312, "KL/mean": -196.967529296875, "KL/rejected_KL_mean": -244.59689331054688, "KL/std": 90.84678649902344, "epoch": 0.5330396475770925, "fcm_dpo/beta": 0.006099109537899494, "fcm_dpo/delta": -0.13886189460754395, "fcm_dpo/margin": 95.25874328613281, "fcm_dpo/q_t": 0.36868971586227417, "grad_norm": 11.294530868530273, "learning_rate": 2.6667091441120816e-07, "logits/chosen": -1.7525750398635864, "logits/rejected": -1.3133634328842163, "logps/chosen": -230.1131591796875, "logps/ref_chosen": -80.77499389648438, "logps/ref_rejected": -79.43267822265625, "logps/rejected": -324.0295715332031, "loss": 0.9702, "margin_dpo/margin_mean": 95.25874328613281, "margin_dpo/margin_std": 98.54278564453125, "step": 363 }, { "KL/chosen_KL_mean": -147.554931640625, "KL/mean": -186.21170043945312, "KL/rejected_KL_mean": -224.86843872070312, "KL/std": 84.30770874023438, "epoch": 0.5345080763582967, "fcm_dpo/beta": 0.006051627919077873, "fcm_dpo/delta": -0.01953640580177307, "fcm_dpo/margin": 77.31351470947266, "fcm_dpo/q_t": 0.39090150594711304, "grad_norm": 12.098045349121094, "learning_rate": 2.6539022653348575e-07, "logits/chosen": -2.1194944381713867, "logits/rejected": -1.9742317199707031, "logps/chosen": -224.08453369140625, "logps/ref_chosen": -76.52960205078125, "logps/ref_rejected": -97.56173706054688, "logps/rejected": -322.43017578125, "loss": 1.0315, "margin_dpo/margin_mean": 77.31351470947266, "margin_dpo/margin_std": 79.7694091796875, "step": 364 }, { "KL/chosen_KL_mean": -143.73880004882812, "KL/mean": -185.5823974609375, "KL/rejected_KL_mean": -227.4259796142578, "KL/std": 90.41179656982422, "epoch": 0.5359765051395007, "fcm_dpo/beta": 0.005980190820991993, "fcm_dpo/delta": -0.053211845457553864, "fcm_dpo/margin": 83.68717956542969, "fcm_dpo/q_t": 0.38757264614105225, "grad_norm": 9.37916088104248, "learning_rate": 2.641091331089811e-07, "logits/chosen": -1.7654857635498047, "logits/rejected": -1.6367874145507812, "logps/chosen": -224.28369140625, "logps/ref_chosen": -80.54489135742188, "logps/ref_rejected": -99.32316589355469, "logps/rejected": -326.7491455078125, "loss": 1.0336, "margin_dpo/margin_mean": 83.68717956542969, "margin_dpo/margin_std": 102.36875915527344, "step": 365 }, { "KL/chosen_KL_mean": -145.4033203125, "KL/mean": -180.5133056640625, "KL/rejected_KL_mean": -215.62332153320312, "KL/std": 94.08183288574219, "epoch": 0.5374449339207048, "fcm_dpo/beta": 0.00597140658646822, "fcm_dpo/delta": 0.03202168643474579, "fcm_dpo/margin": 70.22000122070312, "fcm_dpo/q_t": 0.4060773253440857, "grad_norm": 9.141742706298828, "learning_rate": 2.6282766789569736e-07, "logits/chosen": -1.7277851104736328, "logits/rejected": -1.7594456672668457, "logps/chosen": -217.64138793945312, "logps/ref_chosen": -72.23806762695312, "logps/ref_rejected": -91.67182922363281, "logps/rejected": -307.295166015625, "loss": 1.0767, "margin_dpo/margin_mean": 70.22000885009766, "margin_dpo/margin_std": 91.1526870727539, "step": 366 }, { "KL/chosen_KL_mean": -132.71734619140625, "KL/mean": -174.53428649902344, "KL/rejected_KL_mean": -216.35122680664062, "KL/std": 83.21417999267578, "epoch": 0.5389133627019089, "fcm_dpo/beta": 0.005939566530287266, "fcm_dpo/delta": -0.04917112737894058, "fcm_dpo/margin": 83.63388061523438, "fcm_dpo/q_t": 0.38521942496299744, "grad_norm": 8.794506072998047, "learning_rate": 2.615458646614349e-07, "logits/chosen": -1.76117742061615, "logits/rejected": -1.4162061214447021, "logps/chosen": -218.13934326171875, "logps/ref_chosen": -85.42201232910156, "logps/ref_rejected": -83.51779174804688, "logps/rejected": -299.8690185546875, "loss": 1.0102, "margin_dpo/margin_mean": 83.63388061523438, "margin_dpo/margin_std": 86.39166259765625, "step": 367 }, { "KL/chosen_KL_mean": -124.18267059326172, "KL/mean": -170.06015014648438, "KL/rejected_KL_mean": -215.93765258789062, "KL/std": 91.82571411132812, "epoch": 0.540381791483113, "fcm_dpo/beta": 0.005866661202162504, "fcm_dpo/delta": -0.09332215785980225, "fcm_dpo/margin": 91.75496673583984, "fcm_dpo/q_t": 0.3754565119743347, "grad_norm": 9.531155586242676, "learning_rate": 2.6026375718290083e-07, "logits/chosen": -1.468907356262207, "logits/rejected": -1.4333953857421875, "logps/chosen": -198.0238037109375, "logps/ref_chosen": -73.84112548828125, "logps/ref_rejected": -104.69528198242188, "logps/rejected": -320.6329345703125, "loss": 0.992, "margin_dpo/margin_mean": 91.75496673583984, "margin_dpo/margin_std": 93.23069763183594, "step": 368 }, { "KL/chosen_KL_mean": -137.4363250732422, "KL/mean": -181.03756713867188, "KL/rejected_KL_mean": -224.63880920410156, "KL/std": 84.01296997070312, "epoch": 0.5418502202643172, "fcm_dpo/beta": 0.005774751305580139, "fcm_dpo/delta": -0.05638273060321808, "fcm_dpo/margin": 87.20248413085938, "fcm_dpo/q_t": 0.38302382826805115, "grad_norm": 9.568777084350586, "learning_rate": 2.589813792448196e-07, "logits/chosen": -1.5823924541473389, "logits/rejected": -1.2088656425476074, "logps/chosen": -235.89828491210938, "logps/ref_chosen": -98.46195983886719, "logps/ref_rejected": -99.15672302246094, "logps/rejected": -323.7955322265625, "loss": 0.9992, "margin_dpo/margin_mean": 87.20248413085938, "margin_dpo/margin_std": 83.74822998046875, "step": 369 }, { "KL/chosen_KL_mean": -152.99267578125, "KL/mean": -182.20016479492188, "KL/rejected_KL_mean": -211.40765380859375, "KL/std": 81.64815521240234, "epoch": 0.5433186490455213, "fcm_dpo/beta": 0.005848081316798925, "fcm_dpo/delta": 0.11211700737476349, "fcm_dpo/margin": 58.41497802734375, "fcm_dpo/q_t": 0.4209824204444885, "grad_norm": 9.319167137145996, "learning_rate": 2.5769876463904263e-07, "logits/chosen": -1.6919502019882202, "logits/rejected": -1.4850552082061768, "logps/chosen": -249.76840209960938, "logps/ref_chosen": -96.77574157714844, "logps/ref_rejected": -102.74411010742188, "logps/rejected": -314.1517639160156, "loss": 1.1455, "margin_dpo/margin_mean": 58.41497802734375, "margin_dpo/margin_std": 93.90602111816406, "step": 370 }, { "KL/chosen_KL_mean": -145.09376525878906, "KL/mean": -177.24945068359375, "KL/rejected_KL_mean": -209.40518188476562, "KL/std": 82.22689819335938, "epoch": 0.5447870778267254, "fcm_dpo/beta": 0.005954510532319546, "fcm_dpo/delta": 0.0693742036819458, "fcm_dpo/margin": 64.31141662597656, "fcm_dpo/q_t": 0.4113882780075073, "grad_norm": 11.732169151306152, "learning_rate": 2.5641594716365744e-07, "logits/chosen": -1.8820791244506836, "logits/rejected": -1.5166668891906738, "logps/chosen": -248.08396911621094, "logps/ref_chosen": -102.99020385742188, "logps/ref_rejected": -105.35062408447266, "logps/rejected": -314.75579833984375, "loss": 1.1068, "margin_dpo/margin_mean": 64.31141662597656, "margin_dpo/margin_std": 88.78504943847656, "step": 371 }, { "KL/chosen_KL_mean": -124.0773696899414, "KL/mean": -172.28167724609375, "KL/rejected_KL_mean": -220.4859619140625, "KL/std": 95.52281951904297, "epoch": 0.5462555066079295, "fcm_dpo/beta": 0.00585830956697464, "fcm_dpo/delta": -0.12156160175800323, "fcm_dpo/margin": 96.40857696533203, "fcm_dpo/q_t": 0.3712029457092285, "grad_norm": 8.286251068115234, "learning_rate": 2.551329606220976e-07, "logits/chosen": -2.0169198513031006, "logits/rejected": -1.3986923694610596, "logps/chosen": -215.10360717773438, "logps/ref_chosen": -91.02623748779297, "logps/ref_rejected": -85.6102294921875, "logps/rejected": -306.09619140625, "loss": 0.9733, "margin_dpo/margin_mean": 96.40858459472656, "margin_dpo/margin_std": 97.09080505371094, "step": 372 }, { "KL/chosen_KL_mean": -141.81610107421875, "KL/mean": -177.65072631835938, "KL/rejected_KL_mean": -213.48536682128906, "KL/std": 90.250732421875, "epoch": 0.5477239353891337, "fcm_dpo/beta": 0.005838732700794935, "fcm_dpo/delta": 0.032932497560977936, "fcm_dpo/margin": 71.66928100585938, "fcm_dpo/q_t": 0.4041418731212616, "grad_norm": 10.926004409790039, "learning_rate": 2.538498388222517e-07, "logits/chosen": -2.1831014156341553, "logits/rejected": -1.6671464443206787, "logps/chosen": -241.88919067382812, "logps/ref_chosen": -100.07307434082031, "logps/ref_rejected": -91.11634826660156, "logps/rejected": -304.6017150878906, "loss": 1.0851, "margin_dpo/margin_mean": 71.66926574707031, "margin_dpo/margin_std": 96.1834945678711, "step": 373 }, { "KL/chosen_KL_mean": -124.3795166015625, "KL/mean": -163.28689575195312, "KL/rejected_KL_mean": -202.1942596435547, "KL/std": 85.82447814941406, "epoch": 0.5491923641703378, "fcm_dpo/beta": 0.005836909636855125, "fcm_dpo/delta": -0.004515616223216057, "fcm_dpo/margin": 77.81475830078125, "fcm_dpo/q_t": 0.3948688507080078, "grad_norm": 8.858715057373047, "learning_rate": 2.525666155755725e-07, "logits/chosen": -2.284390449523926, "logits/rejected": -1.8986577987670898, "logps/chosen": -222.39186096191406, "logps/ref_chosen": -98.01234436035156, "logps/ref_rejected": -100.75224304199219, "logps/rejected": -302.9465026855469, "loss": 1.0365, "margin_dpo/margin_mean": 77.81475830078125, "margin_dpo/margin_std": 82.1981201171875, "step": 374 }, { "KL/chosen_KL_mean": -126.76860809326172, "KL/mean": -171.35986328125, "KL/rejected_KL_mean": -215.95111083984375, "KL/std": 83.42938232421875, "epoch": 0.5506607929515418, "fcm_dpo/beta": 0.005751289427280426, "fcm_dpo/delta": -0.06771738827228546, "fcm_dpo/margin": 89.18248748779297, "fcm_dpo/q_t": 0.3805190324783325, "grad_norm": 9.776904106140137, "learning_rate": 2.512833246961859e-07, "logits/chosen": -1.632298231124878, "logits/rejected": -1.4000362157821655, "logps/chosen": -211.16488647460938, "logps/ref_chosen": -84.39627838134766, "logps/ref_rejected": -95.7188491821289, "logps/rejected": -311.6699523925781, "loss": 1.0013, "margin_dpo/margin_mean": 89.18248748779297, "margin_dpo/margin_std": 88.08191680908203, "step": 375 }, { "KL/chosen_KL_mean": -121.34154510498047, "KL/mean": -170.11871337890625, "KL/rejected_KL_mean": -218.89590454101562, "KL/std": 94.6286392211914, "epoch": 0.5521292217327459, "fcm_dpo/beta": 0.005672710482031107, "fcm_dpo/delta": -0.10955370217561722, "fcm_dpo/margin": 97.55435180664062, "fcm_dpo/q_t": 0.37314870953559875, "grad_norm": 10.597599029541016, "learning_rate": 2.5e-07, "logits/chosen": -1.7344553470611572, "logits/rejected": -1.422084093093872, "logps/chosen": -217.07925415039062, "logps/ref_chosen": -95.73770141601562, "logps/ref_rejected": -111.97331237792969, "logps/rejected": -330.86920166015625, "loss": 0.9814, "margin_dpo/margin_mean": 97.55435180664062, "margin_dpo/margin_std": 100.21503448486328, "step": 376 }, { "KL/chosen_KL_mean": -129.27520751953125, "KL/mean": -180.42337036132812, "KL/rejected_KL_mean": -231.571533203125, "KL/std": 96.63461303710938, "epoch": 0.55359765051395, "fcm_dpo/beta": 0.005520098842680454, "fcm_dpo/delta": -0.12243510782718658, "fcm_dpo/margin": 102.29632568359375, "fcm_dpo/q_t": 0.36939406394958496, "grad_norm": 8.624650001525879, "learning_rate": 2.487166753038141e-07, "logits/chosen": -1.474902629852295, "logits/rejected": -1.3547477722167969, "logps/chosen": -208.20216369628906, "logps/ref_chosen": -78.92694854736328, "logps/ref_rejected": -105.89892578125, "logps/rejected": -337.470458984375, "loss": 0.9649, "margin_dpo/margin_mean": 102.29632568359375, "margin_dpo/margin_std": 96.22500610351562, "step": 377 }, { "KL/chosen_KL_mean": -126.4132080078125, "KL/mean": -167.10244750976562, "KL/rejected_KL_mean": -207.79168701171875, "KL/std": 91.60065460205078, "epoch": 0.5550660792951542, "fcm_dpo/beta": 0.005534189287573099, "fcm_dpo/delta": -0.0008363649249076843, "fcm_dpo/margin": 81.37847900390625, "fcm_dpo/q_t": 0.39472198486328125, "grad_norm": 10.012763977050781, "learning_rate": 2.4743338442442754e-07, "logits/chosen": -2.034902334213257, "logits/rejected": -1.7558884620666504, "logps/chosen": -201.78561401367188, "logps/ref_chosen": -75.37240600585938, "logps/ref_rejected": -93.70409393310547, "logps/rejected": -301.49578857421875, "loss": 1.0517, "margin_dpo/margin_mean": 81.37847900390625, "margin_dpo/margin_std": 93.79391479492188, "step": 378 }, { "KL/chosen_KL_mean": -147.85743713378906, "KL/mean": -183.8848876953125, "KL/rejected_KL_mean": -219.91232299804688, "KL/std": 94.57432556152344, "epoch": 0.5565345080763583, "fcm_dpo/beta": 0.005548754706978798, "fcm_dpo/delta": 0.05229198560118675, "fcm_dpo/margin": 72.05487823486328, "fcm_dpo/q_t": 0.41010457277297974, "grad_norm": 9.807597160339355, "learning_rate": 2.461501611777483e-07, "logits/chosen": -1.740838646888733, "logits/rejected": -1.659019112586975, "logps/chosen": -236.71311950683594, "logps/ref_chosen": -88.85568237304688, "logps/ref_rejected": -121.38114166259766, "logps/rejected": -341.29345703125, "loss": 1.11, "margin_dpo/margin_mean": 72.05488586425781, "margin_dpo/margin_std": 108.07455444335938, "step": 379 }, { "KL/chosen_KL_mean": -162.68089294433594, "KL/mean": -206.72476196289062, "KL/rejected_KL_mean": -250.76861572265625, "KL/std": 97.05766296386719, "epoch": 0.5580029368575624, "fcm_dpo/beta": 0.0055307662114501, "fcm_dpo/delta": -0.03912016749382019, "fcm_dpo/margin": 88.08770751953125, "fcm_dpo/q_t": 0.3904613256454468, "grad_norm": 13.091938018798828, "learning_rate": 2.4486703937790243e-07, "logits/chosen": -1.376621961593628, "logits/rejected": -1.3481290340423584, "logps/chosen": -242.51409912109375, "logps/ref_chosen": -79.83321380615234, "logps/ref_rejected": -110.7447509765625, "logps/rejected": -361.51336669921875, "loss": 1.0387, "margin_dpo/margin_mean": 88.08770751953125, "margin_dpo/margin_std": 109.23361206054688, "step": 380 }, { "KL/chosen_KL_mean": -142.77163696289062, "KL/mean": -181.06393432617188, "KL/rejected_KL_mean": -219.35623168945312, "KL/std": 92.96479797363281, "epoch": 0.5594713656387665, "fcm_dpo/beta": 0.005546100903302431, "fcm_dpo/delta": 0.026361431926488876, "fcm_dpo/margin": 76.58457946777344, "fcm_dpo/q_t": 0.40147024393081665, "grad_norm": 10.48298454284668, "learning_rate": 2.435840528363426e-07, "logits/chosen": -1.6954889297485352, "logits/rejected": -1.1373002529144287, "logps/chosen": -229.33847045898438, "logps/ref_chosen": -86.56683349609375, "logps/ref_rejected": -84.416748046875, "logps/rejected": -303.7729797363281, "loss": 1.0714, "margin_dpo/margin_mean": 76.5845947265625, "margin_dpo/margin_std": 94.87086486816406, "step": 381 }, { "KL/chosen_KL_mean": -140.79415893554688, "KL/mean": -184.33340454101562, "KL/rejected_KL_mean": -227.87265014648438, "KL/std": 91.3126220703125, "epoch": 0.5609397944199707, "fcm_dpo/beta": 0.005491263698786497, "fcm_dpo/delta": -0.030609525740146637, "fcm_dpo/margin": 87.07847595214844, "fcm_dpo/q_t": 0.3894699811935425, "grad_norm": 9.28026008605957, "learning_rate": 2.4230123536095745e-07, "logits/chosen": -1.9678525924682617, "logits/rejected": -1.7035901546478271, "logps/chosen": -245.25596618652344, "logps/ref_chosen": -104.46180725097656, "logps/ref_rejected": -114.60223388671875, "logps/rejected": -342.4748840332031, "loss": 1.0282, "margin_dpo/margin_mean": 87.07847595214844, "margin_dpo/margin_std": 92.77288818359375, "step": 382 }, { "KL/chosen_KL_mean": -152.9261932373047, "KL/mean": -189.8676300048828, "KL/rejected_KL_mean": -226.80906677246094, "KL/std": 94.07464599609375, "epoch": 0.5624082232011748, "fcm_dpo/beta": 0.005541680380702019, "fcm_dpo/delta": 0.04230116680264473, "fcm_dpo/margin": 73.88288879394531, "fcm_dpo/q_t": 0.4054419994354248, "grad_norm": 10.031593322753906, "learning_rate": 2.4101862075518037e-07, "logits/chosen": -1.6223798990249634, "logits/rejected": -1.500431776046753, "logps/chosen": -235.04226684570312, "logps/ref_chosen": -82.11607360839844, "logps/ref_rejected": -98.77716064453125, "logps/rejected": -325.58624267578125, "loss": 1.0805, "margin_dpo/margin_mean": 73.88288879394531, "margin_dpo/margin_std": 91.41796875, "step": 383 }, { "KL/chosen_KL_mean": -163.90792846679688, "KL/mean": -200.69241333007812, "KL/rejected_KL_mean": -237.4769287109375, "KL/std": 97.68135070800781, "epoch": 0.5638766519823789, "fcm_dpo/beta": 0.005634545348584652, "fcm_dpo/delta": 0.035694241523742676, "fcm_dpo/margin": 73.56901550292969, "fcm_dpo/q_t": 0.4049611985683441, "grad_norm": 11.313835144042969, "learning_rate": 2.397362428170992e-07, "logits/chosen": -1.7594760656356812, "logits/rejected": -1.5330588817596436, "logps/chosen": -244.4931640625, "logps/ref_chosen": -80.58525085449219, "logps/ref_rejected": -91.44789123535156, "logps/rejected": -328.9248046875, "loss": 1.1073, "margin_dpo/margin_mean": 73.56900787353516, "margin_dpo/margin_std": 107.26543426513672, "step": 384 }, { "KL/chosen_KL_mean": -136.58233642578125, "KL/mean": -183.89329528808594, "KL/rejected_KL_mean": -231.2042236328125, "KL/std": 88.43669891357422, "epoch": 0.5653450807635829, "fcm_dpo/beta": 0.005509526468813419, "fcm_dpo/delta": -0.07635048031806946, "fcm_dpo/margin": 94.62188720703125, "fcm_dpo/q_t": 0.38109469413757324, "grad_norm": 10.951947212219238, "learning_rate": 2.3845413533856514e-07, "logits/chosen": -2.3123326301574707, "logits/rejected": -1.7141015529632568, "logps/chosen": -229.34251403808594, "logps/ref_chosen": -92.76016235351562, "logps/ref_rejected": -84.45997619628906, "logps/rejected": -315.6641845703125, "loss": 1.0071, "margin_dpo/margin_mean": 94.62188720703125, "margin_dpo/margin_std": 102.41546630859375, "step": 385 }, { "KL/chosen_KL_mean": -145.4132080078125, "KL/mean": -190.88641357421875, "KL/rejected_KL_mean": -236.35960388183594, "KL/std": 90.42179870605469, "epoch": 0.566813509544787, "fcm_dpo/beta": 0.0054764775559306145, "fcm_dpo/delta": -0.050601229071617126, "fcm_dpo/margin": 90.94638061523438, "fcm_dpo/q_t": 0.38333213329315186, "grad_norm": 8.21135425567627, "learning_rate": 2.3717233210430254e-07, "logits/chosen": -1.616806983947754, "logits/rejected": -1.330208420753479, "logps/chosen": -230.7721710205078, "logps/ref_chosen": -85.35896301269531, "logps/ref_rejected": -96.99065399169922, "logps/rejected": -333.3502502441406, "loss": 1.0171, "margin_dpo/margin_mean": 90.94638061523438, "margin_dpo/margin_std": 97.93557739257812, "step": 386 }, { "KL/chosen_KL_mean": -168.45127868652344, "KL/mean": -209.4113006591797, "KL/rejected_KL_mean": -250.371337890625, "KL/std": 87.11465454101562, "epoch": 0.5682819383259912, "fcm_dpo/beta": 0.0054756514728069305, "fcm_dpo/delta": 0.0014210660010576248, "fcm_dpo/margin": 81.92005920410156, "fcm_dpo/q_t": 0.39575254917144775, "grad_norm": 11.177667617797852, "learning_rate": 2.3589086689101889e-07, "logits/chosen": -2.4550271034240723, "logits/rejected": -1.772486925125122, "logps/chosen": -271.139404296875, "logps/ref_chosen": -102.68814086914062, "logps/ref_rejected": -99.42858123779297, "logps/rejected": -349.7999267578125, "loss": 1.0443, "margin_dpo/margin_mean": 81.92005920410156, "margin_dpo/margin_std": 89.9079818725586, "step": 387 }, { "KL/chosen_KL_mean": -170.07723999023438, "KL/mean": -215.07505798339844, "KL/rejected_KL_mean": -260.0728759765625, "KL/std": 97.01229858398438, "epoch": 0.5697503671071953, "fcm_dpo/beta": 0.005427849944680929, "fcm_dpo/delta": -0.04050945118069649, "fcm_dpo/margin": 89.99563598632812, "fcm_dpo/q_t": 0.3870543837547302, "grad_norm": 9.495423316955566, "learning_rate": 2.3460977346651428e-07, "logits/chosen": -1.6890193223953247, "logits/rejected": -1.5671308040618896, "logps/chosen": -253.6624755859375, "logps/ref_chosen": -83.58523559570312, "logps/ref_rejected": -110.10037231445312, "logps/rejected": -370.1732482910156, "loss": 1.0242, "margin_dpo/margin_mean": 89.99563598632812, "margin_dpo/margin_std": 99.14422607421875, "step": 388 }, { "KL/chosen_KL_mean": -173.98683166503906, "KL/mean": -213.3004608154297, "KL/rejected_KL_mean": -252.61410522460938, "KL/std": 99.34040832519531, "epoch": 0.5712187958883994, "fcm_dpo/beta": 0.005437172017991543, "fcm_dpo/delta": 0.023501064628362656, "fcm_dpo/margin": 78.62728881835938, "fcm_dpo/q_t": 0.40241333842277527, "grad_norm": 11.630664825439453, "learning_rate": 2.3332908558879177e-07, "logits/chosen": -2.2833778858184814, "logits/rejected": -1.8254132270812988, "logps/chosen": -263.80908203125, "logps/ref_chosen": -89.822265625, "logps/ref_rejected": -84.85376739501953, "logps/rejected": -337.4678649902344, "loss": 1.0825, "margin_dpo/margin_mean": 78.62728881835938, "margin_dpo/margin_std": 107.08319091796875, "step": 389 }, { "KL/chosen_KL_mean": -175.72840881347656, "KL/mean": -212.7960205078125, "KL/rejected_KL_mean": -249.86367797851562, "KL/std": 96.76557159423828, "epoch": 0.5726872246696035, "fcm_dpo/beta": 0.005455967970192432, "fcm_dpo/delta": 0.04710370674729347, "fcm_dpo/margin": 74.13525390625, "fcm_dpo/q_t": 0.41013091802597046, "grad_norm": 9.654455184936523, "learning_rate": 2.320488370051681e-07, "logits/chosen": -1.3380204439163208, "logits/rejected": -1.121154546737671, "logps/chosen": -253.26084899902344, "logps/ref_chosen": -77.53244018554688, "logps/ref_rejected": -92.58392333984375, "logps/rejected": -342.44757080078125, "loss": 1.1081, "margin_dpo/margin_mean": 74.13525390625, "margin_dpo/margin_std": 110.93473815917969, "step": 390 }, { "KL/chosen_KL_mean": -158.9708709716797, "KL/mean": -199.36349487304688, "KL/rejected_KL_mean": -239.75613403320312, "KL/std": 92.69353485107422, "epoch": 0.5741556534508077, "fcm_dpo/beta": 0.005487093701958656, "fcm_dpo/delta": 0.006862584501504898, "fcm_dpo/margin": 80.78526306152344, "fcm_dpo/q_t": 0.39808547496795654, "grad_norm": 9.405406951904297, "learning_rate": 2.3076906145138405e-07, "logits/chosen": -2.024125576019287, "logits/rejected": -1.5786292552947998, "logps/chosen": -247.20379638671875, "logps/ref_chosen": -88.2329330444336, "logps/ref_rejected": -93.00756072998047, "logps/rejected": -332.7637023925781, "loss": 1.0608, "margin_dpo/margin_mean": 80.78526306152344, "margin_dpo/margin_std": 98.21650695800781, "step": 391 }, { "KL/chosen_KL_mean": -161.13150024414062, "KL/mean": -209.9306640625, "KL/rejected_KL_mean": -258.7298583984375, "KL/std": 103.97460174560547, "epoch": 0.5756240822320118, "fcm_dpo/beta": 0.005453658290207386, "fcm_dpo/delta": -0.08683225512504578, "fcm_dpo/margin": 97.59834289550781, "fcm_dpo/q_t": 0.37842029333114624, "grad_norm": 10.277584075927734, "learning_rate": 2.294897926507156e-07, "logits/chosen": -1.775519609451294, "logits/rejected": -1.6575126647949219, "logps/chosen": -248.87591552734375, "logps/ref_chosen": -87.74442291259766, "logps/ref_rejected": -108.57071685791016, "logps/rejected": -367.300537109375, "loss": 1.0096, "margin_dpo/margin_mean": 97.59835052490234, "margin_dpo/margin_std": 111.62520599365234, "step": 392 }, { "KL/chosen_KL_mean": -143.23472595214844, "KL/mean": -187.03500366210938, "KL/rejected_KL_mean": -230.83529663085938, "KL/std": 98.00140380859375, "epoch": 0.5770925110132159, "fcm_dpo/beta": 0.005389312282204628, "fcm_dpo/delta": -0.02322268858551979, "fcm_dpo/margin": 87.60055541992188, "fcm_dpo/q_t": 0.39091381430625916, "grad_norm": 8.891741752624512, "learning_rate": 2.2821106431308543e-07, "logits/chosen": -1.800893783569336, "logits/rejected": -1.484067678451538, "logps/chosen": -219.7985382080078, "logps/ref_chosen": -76.56381225585938, "logps/ref_rejected": -79.72618103027344, "logps/rejected": -310.56146240234375, "loss": 1.0313, "margin_dpo/margin_mean": 87.60055541992188, "margin_dpo/margin_std": 96.00604248046875, "step": 393 }, { "KL/chosen_KL_mean": -151.06854248046875, "KL/mean": -199.05972290039062, "KL/rejected_KL_mean": -247.05091857910156, "KL/std": 93.27932739257812, "epoch": 0.57856093979442, "fcm_dpo/beta": 0.005334894172847271, "fcm_dpo/delta": -0.06539418548345566, "fcm_dpo/margin": 95.98237609863281, "fcm_dpo/q_t": 0.380257248878479, "grad_norm": 9.333141326904297, "learning_rate": 2.2693291013417452e-07, "logits/chosen": -1.4963953495025635, "logits/rejected": -1.2618308067321777, "logps/chosen": -235.7672119140625, "logps/ref_chosen": -84.69866180419922, "logps/ref_rejected": -97.457763671875, "logps/rejected": -344.5086669921875, "loss": 0.9934, "margin_dpo/margin_mean": 95.98236846923828, "margin_dpo/margin_std": 90.62801361083984, "step": 394 }, { "KL/chosen_KL_mean": -156.96026611328125, "KL/mean": -193.9337158203125, "KL/rejected_KL_mean": -230.90713500976562, "KL/std": 98.38123321533203, "epoch": 0.580029368575624, "fcm_dpo/beta": 0.005336286500096321, "fcm_dpo/delta": 0.057480037212371826, "fcm_dpo/margin": 73.94686889648438, "fcm_dpo/q_t": 0.4133360981941223, "grad_norm": 9.802746772766113, "learning_rate": 2.2565536379453404e-07, "logits/chosen": -2.467777729034424, "logits/rejected": -2.0613045692443848, "logps/chosen": -245.84791564941406, "logps/ref_chosen": -88.88763427734375, "logps/ref_rejected": -91.0455322265625, "logps/rejected": -321.9526672363281, "loss": 1.148, "margin_dpo/margin_mean": 73.94686889648438, "margin_dpo/margin_std": 132.66213989257812, "step": 395 }, { "KL/chosen_KL_mean": -166.8684844970703, "KL/mean": -199.56668090820312, "KL/rejected_KL_mean": -232.26490783691406, "KL/std": 86.83419036865234, "epoch": 0.5814977973568282, "fcm_dpo/beta": 0.00545476283878088, "fcm_dpo/delta": 0.09674517810344696, "fcm_dpo/margin": 65.39639282226562, "fcm_dpo/q_t": 0.4183182418346405, "grad_norm": 12.155906677246094, "learning_rate": 2.2437845895869825e-07, "logits/chosen": -1.90079665184021, "logits/rejected": -1.408474326133728, "logps/chosen": -269.3398132324219, "logps/ref_chosen": -102.47132873535156, "logps/ref_rejected": -93.23478698730469, "logps/rejected": -325.49969482421875, "loss": 1.1341, "margin_dpo/margin_mean": 65.39639282226562, "margin_dpo/margin_std": 101.74411010742188, "step": 396 }, { "KL/chosen_KL_mean": -149.63766479492188, "KL/mean": -208.14910888671875, "KL/rejected_KL_mean": -266.6605529785156, "KL/std": 100.67845153808594, "epoch": 0.5829662261380323, "fcm_dpo/beta": 0.00532365869730711, "fcm_dpo/delta": -0.18493586778640747, "fcm_dpo/margin": 117.02285766601562, "fcm_dpo/q_t": 0.3586190342903137, "grad_norm": 9.541089057922363, "learning_rate": 2.2310222927429716e-07, "logits/chosen": -2.185824394226074, "logits/rejected": -1.5791468620300293, "logps/chosen": -250.68209838867188, "logps/ref_chosen": -101.04441833496094, "logps/ref_rejected": -108.8511734008789, "logps/rejected": -375.51171875, "loss": 0.9507, "margin_dpo/margin_mean": 117.02285766601562, "margin_dpo/margin_std": 120.26101684570312, "step": 397 }, { "KL/chosen_KL_mean": -158.05035400390625, "KL/mean": -200.77044677734375, "KL/rejected_KL_mean": -243.4905548095703, "KL/std": 98.89729309082031, "epoch": 0.5844346549192364, "fcm_dpo/beta": 0.00527543667703867, "fcm_dpo/delta": -0.0008107442408800125, "fcm_dpo/margin": 85.44017028808594, "fcm_dpo/q_t": 0.39793872833251953, "grad_norm": 8.96392822265625, "learning_rate": 2.2182670837116972e-07, "logits/chosen": -2.0579802989959717, "logits/rejected": -1.7727158069610596, "logps/chosen": -253.09957885742188, "logps/ref_chosen": -95.04922485351562, "logps/ref_rejected": -114.00563049316406, "logps/rejected": -357.4961853027344, "loss": 1.0733, "margin_dpo/margin_mean": 85.44017791748047, "margin_dpo/margin_std": 117.18036651611328, "step": 398 }, { "KL/chosen_KL_mean": -138.51805114746094, "KL/mean": -192.81028747558594, "KL/rejected_KL_mean": -247.10250854492188, "KL/std": 89.64627838134766, "epoch": 0.5859030837004405, "fcm_dpo/beta": 0.005199803970754147, "fcm_dpo/delta": -0.12131005525588989, "fcm_dpo/margin": 108.58445739746094, "fcm_dpo/q_t": 0.36873090267181396, "grad_norm": 9.179891586303711, "learning_rate": 2.2055192986047804e-07, "logits/chosen": -2.100039005279541, "logits/rejected": -1.2997509241104126, "logps/chosen": -235.15660095214844, "logps/ref_chosen": -96.63854217529297, "logps/ref_rejected": -83.53645324707031, "logps/rejected": -330.63897705078125, "loss": 0.9565, "margin_dpo/margin_mean": 108.5844497680664, "margin_dpo/margin_std": 95.44795989990234, "step": 399 }, { "KL/chosen_KL_mean": -144.21548461914062, "KL/mean": -199.53765869140625, "KL/rejected_KL_mean": -254.85983276367188, "KL/std": 99.18544006347656, "epoch": 0.5873715124816447, "fcm_dpo/beta": 0.005064649507403374, "fcm_dpo/delta": -0.11699317395687103, "fcm_dpo/margin": 110.64436340332031, "fcm_dpo/q_t": 0.37149012088775635, "grad_norm": 13.007590293884277, "learning_rate": 2.192779273338215e-07, "logits/chosen": -2.2159924507141113, "logits/rejected": -1.7322194576263428, "logps/chosen": -249.5186309814453, "logps/ref_chosen": -105.30314636230469, "logps/ref_rejected": -108.6555404663086, "logps/rejected": -363.515380859375, "loss": 0.98, "margin_dpo/margin_mean": 110.64436340332031, "margin_dpo/margin_std": 113.15621948242188, "step": 400 }, { "KL/chosen_KL_mean": -159.62994384765625, "KL/mean": -194.17129516601562, "KL/rejected_KL_mean": -228.71261596679688, "KL/std": 99.43193817138672, "epoch": 0.5888399412628488, "fcm_dpo/beta": 0.005099102854728699, "fcm_dpo/delta": 0.10143238306045532, "fcm_dpo/margin": 69.08269500732422, "fcm_dpo/q_t": 0.42251038551330566, "grad_norm": 11.749171257019043, "learning_rate": 2.1800473436235136e-07, "logits/chosen": -1.5369623899459839, "logits/rejected": -1.2931877374649048, "logps/chosen": -243.9191131591797, "logps/ref_chosen": -84.28916931152344, "logps/ref_rejected": -90.943115234375, "logps/rejected": -319.6557312011719, "loss": 1.1523, "margin_dpo/margin_mean": 69.08269500732422, "margin_dpo/margin_std": 121.23318481445312, "step": 401 }, { "KL/chosen_KL_mean": -143.09613037109375, "KL/mean": -207.28289794921875, "KL/rejected_KL_mean": -271.46966552734375, "KL/std": 106.4303207397461, "epoch": 0.5903083700440529, "fcm_dpo/beta": 0.004995099734514952, "fcm_dpo/delta": -0.20406782627105713, "fcm_dpo/margin": 128.37356567382812, "fcm_dpo/q_t": 0.3507021963596344, "grad_norm": 9.149144172668457, "learning_rate": 2.1673238449588665e-07, "logits/chosen": -1.9318034648895264, "logits/rejected": -1.465431809425354, "logps/chosen": -226.6892547607422, "logps/ref_chosen": -83.59312438964844, "logps/ref_rejected": -87.81027221679688, "logps/rejected": -359.27996826171875, "loss": 0.9069, "margin_dpo/margin_mean": 128.37356567382812, "margin_dpo/margin_std": 103.30931854248047, "step": 402 }, { "KL/chosen_KL_mean": -145.625244140625, "KL/mean": -192.91964721679688, "KL/rejected_KL_mean": -240.21401977539062, "KL/std": 102.69251251220703, "epoch": 0.591776798825257, "fcm_dpo/beta": 0.004911697469651699, "fcm_dpo/delta": -0.015448980033397675, "fcm_dpo/margin": 94.58877563476562, "fcm_dpo/q_t": 0.39277374744415283, "grad_norm": 9.459317207336426, "learning_rate": 2.154609112620295e-07, "logits/chosen": -2.316864013671875, "logits/rejected": -1.924619197845459, "logps/chosen": -219.37832641601562, "logps/ref_chosen": -73.75308227539062, "logps/ref_rejected": -83.92012786865234, "logps/rejected": -324.1341552734375, "loss": 1.0434, "margin_dpo/margin_mean": 94.58877563476562, "margin_dpo/margin_std": 109.79576110839844, "step": 403 }, { "KL/chosen_KL_mean": -174.01333618164062, "KL/mean": -214.6016387939453, "KL/rejected_KL_mean": -255.18992614746094, "KL/std": 98.91621398925781, "epoch": 0.593245227606461, "fcm_dpo/beta": 0.0049449363723397255, "fcm_dpo/delta": 0.05062542110681534, "fcm_dpo/margin": 81.17658233642578, "fcm_dpo/q_t": 0.40871748328208923, "grad_norm": 13.269368171691895, "learning_rate": 2.1419034816528218e-07, "logits/chosen": -2.0389392375946045, "logits/rejected": -1.7034201622009277, "logps/chosen": -253.68951416015625, "logps/ref_chosen": -79.67617797851562, "logps/ref_rejected": -84.280517578125, "logps/rejected": -339.470458984375, "loss": 1.1085, "margin_dpo/margin_mean": 81.17658996582031, "margin_dpo/margin_std": 121.17733764648438, "step": 404 }, { "KL/chosen_KL_mean": -176.22738647460938, "KL/mean": -217.8411865234375, "KL/rejected_KL_mean": -259.45501708984375, "KL/std": 93.14665222167969, "epoch": 0.5947136563876652, "fcm_dpo/beta": 0.00502049271017313, "fcm_dpo/delta": 0.032795634120702744, "fcm_dpo/margin": 83.22763061523438, "fcm_dpo/q_t": 0.4037052392959595, "grad_norm": 8.550498008728027, "learning_rate": 2.129207286861638e-07, "logits/chosen": -1.6770999431610107, "logits/rejected": -1.209843397140503, "logps/chosen": -272.6893310546875, "logps/ref_chosen": -96.46195220947266, "logps/ref_rejected": -92.87071228027344, "logps/rejected": -352.32574462890625, "loss": 1.0703, "margin_dpo/margin_mean": 83.22763061523438, "margin_dpo/margin_std": 98.129150390625, "step": 405 }, { "KL/chosen_KL_mean": -166.81361389160156, "KL/mean": -204.5878143310547, "KL/rejected_KL_mean": -242.36203002929688, "KL/std": 99.61241149902344, "epoch": 0.5961820851688693, "fcm_dpo/beta": 0.005056130699813366, "fcm_dpo/delta": 0.07070192694664001, "fcm_dpo/margin": 75.54840850830078, "fcm_dpo/q_t": 0.41290178894996643, "grad_norm": 16.26015281677246, "learning_rate": 2.1165208628032861e-07, "logits/chosen": -1.897040843963623, "logits/rejected": -1.7272096872329712, "logps/chosen": -244.94757080078125, "logps/ref_chosen": -78.13396453857422, "logps/ref_rejected": -98.28359985351562, "logps/rejected": -340.6456298828125, "loss": 1.1218, "margin_dpo/margin_mean": 75.54840850830078, "margin_dpo/margin_std": 116.21882629394531, "step": 406 }, { "KL/chosen_KL_mean": -174.8743896484375, "KL/mean": -208.07913208007812, "KL/rejected_KL_mean": -241.28387451171875, "KL/std": 97.82916259765625, "epoch": 0.5976505139500734, "fcm_dpo/beta": 0.005158255808055401, "fcm_dpo/delta": 0.11125224828720093, "fcm_dpo/margin": 66.40948486328125, "fcm_dpo/q_t": 0.4212542176246643, "grad_norm": 10.842120170593262, "learning_rate": 2.1038445437768375e-07, "logits/chosen": -1.99392569065094, "logits/rejected": -1.4557108879089355, "logps/chosen": -258.88720703125, "logps/ref_chosen": -84.01283264160156, "logps/ref_rejected": -82.78103637695312, "logps/rejected": -324.0649108886719, "loss": 1.1406, "margin_dpo/margin_mean": 66.40948486328125, "margin_dpo/margin_std": 104.44827270507812, "step": 407 }, { "KL/chosen_KL_mean": -166.17868041992188, "KL/mean": -199.85830688476562, "KL/rejected_KL_mean": -233.53793334960938, "KL/std": 92.88591003417969, "epoch": 0.5991189427312775, "fcm_dpo/beta": 0.0052575608715415, "fcm_dpo/delta": 0.099473737180233, "fcm_dpo/margin": 67.3592529296875, "fcm_dpo/q_t": 0.4203840494155884, "grad_norm": 10.597918510437012, "learning_rate": 2.0911786638150872e-07, "logits/chosen": -2.1638379096984863, "logits/rejected": -1.534118890762329, "logps/chosen": -270.64044189453125, "logps/ref_chosen": -104.46174621582031, "logps/ref_rejected": -96.37217712402344, "logps/rejected": -329.91009521484375, "loss": 1.149, "margin_dpo/margin_mean": 67.3592529296875, "margin_dpo/margin_std": 115.18268585205078, "step": 408 }, { "KL/chosen_KL_mean": -177.18463134765625, "KL/mean": -211.97927856445312, "KL/rejected_KL_mean": -246.77394104003906, "KL/std": 96.11148071289062, "epoch": 0.6005873715124816, "fcm_dpo/beta": 0.0053542363457381725, "fcm_dpo/delta": 0.080415278673172, "fcm_dpo/margin": 69.58931732177734, "fcm_dpo/q_t": 0.4129691421985626, "grad_norm": 9.510979652404785, "learning_rate": 2.0785235566757517e-07, "logits/chosen": -1.4348442554473877, "logits/rejected": -0.9889636635780334, "logps/chosen": -274.8529357910156, "logps/ref_chosen": -97.66830444335938, "logps/ref_rejected": -90.04583740234375, "logps/rejected": -336.81976318359375, "loss": 1.1267, "margin_dpo/margin_mean": 69.58930969238281, "margin_dpo/margin_std": 105.79730224609375, "step": 409 }, { "KL/chosen_KL_mean": -172.36636352539062, "KL/mean": -214.38494873046875, "KL/rejected_KL_mean": -256.40350341796875, "KL/std": 101.18763732910156, "epoch": 0.6020558002936858, "fcm_dpo/beta": 0.005375073757022619, "fcm_dpo/delta": -0.0017874417826533318, "fcm_dpo/margin": 84.03714752197266, "fcm_dpo/q_t": 0.3948698043823242, "grad_norm": 9.325094223022461, "learning_rate": 2.065879555832674e-07, "logits/chosen": -1.847787857055664, "logits/rejected": -1.7182414531707764, "logps/chosen": -248.83560180664062, "logps/ref_chosen": -76.46923828125, "logps/ref_rejected": -88.64064025878906, "logps/rejected": -345.0441589355469, "loss": 1.0554, "margin_dpo/margin_mean": 84.03715515136719, "margin_dpo/margin_std": 102.20069885253906, "step": 410 }, { "KL/chosen_KL_mean": -183.931640625, "KL/mean": -223.20455932617188, "KL/rejected_KL_mean": -262.4775390625, "KL/std": 93.54214477539062, "epoch": 0.6035242290748899, "fcm_dpo/beta": 0.005380285903811455, "fcm_dpo/delta": 0.02847486548125744, "fcm_dpo/margin": 78.54588317871094, "fcm_dpo/q_t": 0.40412479639053345, "grad_norm": 9.493480682373047, "learning_rate": 2.0532469944670343e-07, "logits/chosen": -2.276109218597412, "logits/rejected": -1.8670759201049805, "logps/chosen": -271.0979309082031, "logps/ref_chosen": -87.16630554199219, "logps/ref_rejected": -87.09603118896484, "logps/rejected": -349.57354736328125, "loss": 1.077, "margin_dpo/margin_mean": 78.54588317871094, "margin_dpo/margin_std": 100.51069641113281, "step": 411 }, { "KL/chosen_KL_mean": -174.45590209960938, "KL/mean": -213.2197723388672, "KL/rejected_KL_mean": -251.983642578125, "KL/std": 94.68341064453125, "epoch": 0.604992657856094, "fcm_dpo/beta": 0.005417516455054283, "fcm_dpo/delta": 0.03125259652733803, "fcm_dpo/margin": 77.52774047851562, "fcm_dpo/q_t": 0.40670841932296753, "grad_norm": 11.001174926757812, "learning_rate": 2.0406262054585738e-07, "logits/chosen": -2.3869190216064453, "logits/rejected": -2.1678786277770996, "logps/chosen": -253.40322875976562, "logps/ref_chosen": -78.94734191894531, "logps/ref_rejected": -106.10554504394531, "logps/rejected": -358.08917236328125, "loss": 1.1111, "margin_dpo/margin_mean": 77.52774047851562, "margin_dpo/margin_std": 122.05264282226562, "step": 412 }, { "KL/chosen_KL_mean": -187.71133422851562, "KL/mean": -230.51333618164062, "KL/rejected_KL_mean": -273.3153076171875, "KL/std": 99.9842529296875, "epoch": 0.6064610866372981, "fcm_dpo/beta": 0.005409596022218466, "fcm_dpo/delta": -0.014023840427398682, "fcm_dpo/margin": 85.6039810180664, "fcm_dpo/q_t": 0.3939126431941986, "grad_norm": 10.571904182434082, "learning_rate": 2.0280175213768205e-07, "logits/chosen": -1.0728927850723267, "logits/rejected": -0.8086060285568237, "logps/chosen": -283.40606689453125, "logps/ref_chosen": -95.69470977783203, "logps/ref_rejected": -107.96085357666016, "logps/rejected": -381.27618408203125, "loss": 1.069, "margin_dpo/margin_mean": 85.60398864746094, "margin_dpo/margin_std": 117.49549102783203, "step": 413 }, { "KL/chosen_KL_mean": -172.75274658203125, "KL/mean": -212.79336547851562, "KL/rejected_KL_mean": -252.833984375, "KL/std": 96.98312377929688, "epoch": 0.6079295154185022, "fcm_dpo/beta": 0.005443980917334557, "fcm_dpo/delta": 0.014622347429394722, "fcm_dpo/margin": 80.08126831054688, "fcm_dpo/q_t": 0.40238139033317566, "grad_norm": 14.67659854888916, "learning_rate": 2.0154212744723247e-07, "logits/chosen": -1.318266749382019, "logits/rejected": -0.9448731541633606, "logps/chosen": -261.0294189453125, "logps/ref_chosen": -88.27667236328125, "logps/ref_rejected": -92.87004089355469, "logps/rejected": -345.70404052734375, "loss": 1.1017, "margin_dpo/margin_mean": 80.08126831054688, "margin_dpo/margin_std": 123.9937973022461, "step": 414 }, { "KL/chosen_KL_mean": -187.19625854492188, "KL/mean": -220.89715576171875, "KL/rejected_KL_mean": -254.5980682373047, "KL/std": 87.36563873291016, "epoch": 0.6093979441997063, "fcm_dpo/beta": 0.005514100193977356, "fcm_dpo/delta": 0.08122564852237701, "fcm_dpo/margin": 67.40179443359375, "fcm_dpo/q_t": 0.41569462418556213, "grad_norm": 12.043018341064453, "learning_rate": 2.002837796667909e-07, "logits/chosen": -1.5819894075393677, "logits/rejected": -1.2009024620056152, "logps/chosen": -296.1121826171875, "logps/ref_chosen": -108.91590118408203, "logps/ref_rejected": -107.47135925292969, "logps/rejected": -362.0694274902344, "loss": 1.1328, "margin_dpo/margin_mean": 67.40178680419922, "margin_dpo/margin_std": 107.94573974609375, "step": 415 }, { "KL/chosen_KL_mean": -185.39361572265625, "KL/mean": -232.14376831054688, "KL/rejected_KL_mean": -278.8938903808594, "KL/std": 102.91915893554688, "epoch": 0.6108663729809104, "fcm_dpo/beta": 0.005518518388271332, "fcm_dpo/delta": -0.07041682302951813, "fcm_dpo/margin": 93.50028228759766, "fcm_dpo/q_t": 0.38325217366218567, "grad_norm": 9.76031494140625, "learning_rate": 1.990267419549914e-07, "logits/chosen": -1.8252341747283936, "logits/rejected": -1.321199655532837, "logps/chosen": -278.79248046875, "logps/ref_chosen": -93.39888000488281, "logps/ref_rejected": -97.6729736328125, "logps/rejected": -376.56689453125, "loss": 1.025, "margin_dpo/margin_mean": 93.50028228759766, "margin_dpo/margin_std": 108.26390075683594, "step": 416 }, { "KL/chosen_KL_mean": -172.47686767578125, "KL/mean": -216.36038208007812, "KL/rejected_KL_mean": -260.24383544921875, "KL/std": 95.44965362548828, "epoch": 0.6123348017621145, "fcm_dpo/beta": 0.005436611361801624, "fcm_dpo/delta": -0.028513753786683083, "fcm_dpo/margin": 87.76699829101562, "fcm_dpo/q_t": 0.38996344804763794, "grad_norm": 9.729513168334961, "learning_rate": 1.9777104743594686e-07, "logits/chosen": -2.2194457054138184, "logits/rejected": -1.305099606513977, "logps/chosen": -256.01220703125, "logps/ref_chosen": -83.53533172607422, "logps/ref_rejected": -74.44184875488281, "logps/rejected": -334.6856994628906, "loss": 1.0323, "margin_dpo/margin_mean": 87.76699829101562, "margin_dpo/margin_std": 98.33658599853516, "step": 417 }, { "KL/chosen_KL_mean": -165.25379943847656, "KL/mean": -219.8912353515625, "KL/rejected_KL_mean": -274.5286560058594, "KL/std": 102.3167724609375, "epoch": 0.6138032305433186, "fcm_dpo/beta": 0.005361407995223999, "fcm_dpo/delta": -0.14499600231647491, "fcm_dpo/margin": 109.27485656738281, "fcm_dpo/q_t": 0.3664282560348511, "grad_norm": 11.458939552307129, "learning_rate": 1.965167291983757e-07, "logits/chosen": -2.1761512756347656, "logits/rejected": -1.5184229612350464, "logps/chosen": -273.475341796875, "logps/ref_chosen": -108.22152709960938, "logps/ref_rejected": -111.8646469116211, "logps/rejected": -386.393310546875, "loss": 0.9592, "margin_dpo/margin_mean": 109.27485656738281, "margin_dpo/margin_std": 99.99575805664062, "step": 418 }, { "KL/chosen_KL_mean": -171.79110717773438, "KL/mean": -219.5113525390625, "KL/rejected_KL_mean": -267.2315979003906, "KL/std": 91.43870544433594, "epoch": 0.6152716593245228, "fcm_dpo/beta": 0.005250250920653343, "fcm_dpo/delta": -0.054010555148124695, "fcm_dpo/margin": 95.44049072265625, "fcm_dpo/q_t": 0.38451141119003296, "grad_norm": 11.30584716796875, "learning_rate": 1.9526382029472988e-07, "logits/chosen": -1.195124626159668, "logits/rejected": -0.7267280220985413, "logps/chosen": -268.9743957519531, "logps/ref_chosen": -97.18328094482422, "logps/ref_rejected": -98.18531799316406, "logps/rejected": -365.41693115234375, "loss": 1.0139, "margin_dpo/margin_mean": 95.44049072265625, "margin_dpo/margin_std": 100.7812271118164, "step": 419 }, { "KL/chosen_KL_mean": -176.08578491210938, "KL/mean": -218.35128784179688, "KL/rejected_KL_mean": -260.61676025390625, "KL/std": 96.43272399902344, "epoch": 0.6167400881057269, "fcm_dpo/beta": 0.0052343616262078285, "fcm_dpo/delta": 0.007600046694278717, "fcm_dpo/margin": 84.53096771240234, "fcm_dpo/q_t": 0.3991864323616028, "grad_norm": 15.280583381652832, "learning_rate": 1.9401235374032425e-07, "logits/chosen": -2.2325878143310547, "logits/rejected": -1.1336956024169922, "logps/chosen": -290.394287109375, "logps/ref_chosen": -114.30848693847656, "logps/ref_rejected": -75.68355560302734, "logps/rejected": -336.3003234863281, "loss": 1.0709, "margin_dpo/margin_mean": 84.53097534179688, "margin_dpo/margin_std": 110.53395080566406, "step": 420 }, { "KL/chosen_KL_mean": -168.23854064941406, "KL/mean": -215.48219299316406, "KL/rejected_KL_mean": -262.725830078125, "KL/std": 98.87498474121094, "epoch": 0.618208516886931, "fcm_dpo/beta": 0.0051781190559268, "fcm_dpo/delta": -0.041393768042325974, "fcm_dpo/margin": 94.48728942871094, "fcm_dpo/q_t": 0.387340784072876, "grad_norm": 9.249161720275879, "learning_rate": 1.9276236251246653e-07, "logits/chosen": -1.8020110130310059, "logits/rejected": -1.4270470142364502, "logps/chosen": -254.11839294433594, "logps/ref_chosen": -85.87985229492188, "logps/ref_rejected": -96.33648681640625, "logps/rejected": -359.06231689453125, "loss": 1.0307, "margin_dpo/margin_mean": 94.48728942871094, "margin_dpo/margin_std": 108.87814331054688, "step": 421 }, { "KL/chosen_KL_mean": -165.93563842773438, "KL/mean": -210.9785919189453, "KL/rejected_KL_mean": -256.02154541015625, "KL/std": 91.84327697753906, "epoch": 0.6196769456681351, "fcm_dpo/beta": 0.005169377662241459, "fcm_dpo/delta": -0.01647116057574749, "fcm_dpo/margin": 90.0859375, "fcm_dpo/q_t": 0.39310258626937866, "grad_norm": 12.468792915344238, "learning_rate": 1.9151387954958792e-07, "logits/chosen": -2.383885383605957, "logits/rejected": -2.015259265899658, "logps/chosen": -266.4162292480469, "logps/ref_chosen": -100.48060607910156, "logps/ref_rejected": -94.40821838378906, "logps/rejected": -350.4297790527344, "loss": 1.0407, "margin_dpo/margin_mean": 90.0859375, "margin_dpo/margin_std": 104.03911590576172, "step": 422 }, { "KL/chosen_KL_mean": -153.79103088378906, "KL/mean": -199.28440856933594, "KL/rejected_KL_mean": -244.77780151367188, "KL/std": 109.79222869873047, "epoch": 0.6211453744493393, "fcm_dpo/beta": 0.005162882152944803, "fcm_dpo/delta": -0.020984144881367683, "fcm_dpo/margin": 90.98677062988281, "fcm_dpo/q_t": 0.3953096270561218, "grad_norm": 11.123176574707031, "learning_rate": 1.902669377503756e-07, "logits/chosen": -1.9816217422485352, "logits/rejected": -1.8644654750823975, "logps/chosen": -232.240966796875, "logps/ref_chosen": -78.44993591308594, "logps/ref_rejected": -92.04652404785156, "logps/rejected": -336.8243408203125, "loss": 1.0673, "margin_dpo/margin_mean": 90.98677062988281, "margin_dpo/margin_std": 124.88569641113281, "step": 423 }, { "KL/chosen_KL_mean": -173.92189025878906, "KL/mean": -218.35202026367188, "KL/rejected_KL_mean": -262.7821350097656, "KL/std": 97.39276123046875, "epoch": 0.6226138032305433, "fcm_dpo/beta": 0.005129658617079258, "fcm_dpo/delta": -0.006105954758822918, "fcm_dpo/margin": 88.8602523803711, "fcm_dpo/q_t": 0.39420658349990845, "grad_norm": 9.193302154541016, "learning_rate": 1.890215699729057e-07, "logits/chosen": -1.7918660640716553, "logits/rejected": -1.199621319770813, "logps/chosen": -261.5642395019531, "logps/ref_chosen": -87.64236450195312, "logps/ref_rejected": -72.36566162109375, "logps/rejected": -335.1477966308594, "loss": 1.0516, "margin_dpo/margin_mean": 88.86026000976562, "margin_dpo/margin_std": 106.21658325195312, "step": 424 }, { "KL/chosen_KL_mean": -179.52130126953125, "KL/mean": -210.2912139892578, "KL/rejected_KL_mean": -241.06114196777344, "KL/std": 100.02275085449219, "epoch": 0.6240822320117474, "fcm_dpo/beta": 0.00522254966199398, "fcm_dpo/delta": 0.133060485124588, "fcm_dpo/margin": 61.53984451293945, "fcm_dpo/q_t": 0.42998671531677246, "grad_norm": 11.00718879699707, "learning_rate": 1.8777780903377732e-07, "logits/chosen": -1.6064822673797607, "logits/rejected": -1.5371153354644775, "logps/chosen": -258.04107666015625, "logps/ref_chosen": -78.51979064941406, "logps/ref_rejected": -102.74864196777344, "logps/rejected": -343.8097839355469, "loss": 1.1894, "margin_dpo/margin_mean": 61.53984832763672, "margin_dpo/margin_std": 124.87500762939453, "step": 425 }, { "KL/chosen_KL_mean": -153.57254028320312, "KL/mean": -198.11199951171875, "KL/rejected_KL_mean": -242.65147399902344, "KL/std": 92.34453582763672, "epoch": 0.6255506607929515, "fcm_dpo/beta": 0.005266373511403799, "fcm_dpo/delta": -0.020152367651462555, "fcm_dpo/margin": 89.07894897460938, "fcm_dpo/q_t": 0.39211148023605347, "grad_norm": 12.84345817565918, "learning_rate": 1.8653568770724803e-07, "logits/chosen": -1.8006620407104492, "logits/rejected": -1.075664758682251, "logps/chosen": -262.078369140625, "logps/ref_chosen": -108.50582885742188, "logps/ref_rejected": -88.300048828125, "logps/rejected": -330.9515380859375, "loss": 1.0458, "margin_dpo/margin_mean": 89.07894897460938, "margin_dpo/margin_std": 106.60686492919922, "step": 426 }, { "KL/chosen_KL_mean": -126.95327758789062, "KL/mean": -168.5754852294922, "KL/rejected_KL_mean": -210.19769287109375, "KL/std": 88.51153564453125, "epoch": 0.6270190895741556, "fcm_dpo/beta": 0.005263722501695156, "fcm_dpo/delta": 0.012314190156757832, "fcm_dpo/margin": 83.24441528320312, "fcm_dpo/q_t": 0.3982256352901459, "grad_norm": 8.986059188842773, "learning_rate": 1.8529523872436977e-07, "logits/chosen": -2.4963278770446777, "logits/rejected": -1.872001051902771, "logps/chosen": -226.07374572753906, "logps/ref_chosen": -99.12046813964844, "logps/ref_rejected": -85.724609375, "logps/rejected": -295.92230224609375, "loss": 1.0543, "margin_dpo/margin_mean": 83.24441528320312, "margin_dpo/margin_std": 95.05308532714844, "step": 427 }, { "KL/chosen_KL_mean": -179.4477996826172, "KL/mean": -214.71096801757812, "KL/rejected_KL_mean": -249.97412109375, "KL/std": 100.2245101928711, "epoch": 0.6284875183553598, "fcm_dpo/beta": 0.005296891555190086, "fcm_dpo/delta": 0.07929681241512299, "fcm_dpo/margin": 70.52631378173828, "fcm_dpo/q_t": 0.4152657389640808, "grad_norm": 9.958909034729004, "learning_rate": 1.8405649477212697e-07, "logits/chosen": -1.7457547187805176, "logits/rejected": -1.4865880012512207, "logps/chosen": -285.41705322265625, "logps/ref_chosen": -105.96925354003906, "logps/ref_rejected": -109.10218048095703, "logps/rejected": -359.0762939453125, "loss": 1.1305, "margin_dpo/margin_mean": 70.52631378173828, "margin_dpo/margin_std": 110.92942810058594, "step": 428 }, { "KL/chosen_KL_mean": -164.07260131835938, "KL/mean": -209.92584228515625, "KL/rejected_KL_mean": -255.77906799316406, "KL/std": 103.61181640625, "epoch": 0.6299559471365639, "fcm_dpo/beta": 0.005351818632334471, "fcm_dpo/delta": -0.04383639991283417, "fcm_dpo/margin": 91.70643615722656, "fcm_dpo/q_t": 0.3899858593940735, "grad_norm": 9.219000816345215, "learning_rate": 1.828194884925749e-07, "logits/chosen": -1.9351677894592285, "logits/rejected": -1.217651605606079, "logps/chosen": -277.6174621582031, "logps/ref_chosen": -113.54487609863281, "logps/ref_rejected": -98.24201965332031, "logps/rejected": -354.02105712890625, "loss": 1.0496, "margin_dpo/margin_mean": 91.70643615722656, "margin_dpo/margin_std": 116.55728149414062, "step": 429 }, { "KL/chosen_KL_mean": -157.97642517089844, "KL/mean": -192.23570251464844, "KL/rejected_KL_mean": -226.4949951171875, "KL/std": 96.0548324584961, "epoch": 0.631424375917768, "fcm_dpo/beta": 0.0053818728774785995, "fcm_dpo/delta": 0.0838533490896225, "fcm_dpo/margin": 68.51856994628906, "fcm_dpo/q_t": 0.41684645414352417, "grad_norm": 9.969901084899902, "learning_rate": 1.8158425248197928e-07, "logits/chosen": -1.4322922229766846, "logits/rejected": -1.3960630893707275, "logps/chosen": -249.29579162597656, "logps/ref_chosen": -91.31936645507812, "logps/ref_rejected": -110.1096420288086, "logps/rejected": -336.6046447753906, "loss": 1.1364, "margin_dpo/margin_mean": 68.51856994628906, "margin_dpo/margin_std": 111.74356079101562, "step": 430 }, { "KL/chosen_KL_mean": -158.5358123779297, "KL/mean": -199.85360717773438, "KL/rejected_KL_mean": -241.1714324951172, "KL/std": 98.350341796875, "epoch": 0.6328928046989721, "fcm_dpo/beta": 0.005383031442761421, "fcm_dpo/delta": 0.00536933820694685, "fcm_dpo/margin": 82.63562774658203, "fcm_dpo/q_t": 0.3984500765800476, "grad_norm": 11.174623489379883, "learning_rate": 1.8035081928995788e-07, "logits/chosen": -1.9584853649139404, "logits/rejected": -1.574393630027771, "logps/chosen": -251.71702575683594, "logps/ref_chosen": -93.18122100830078, "logps/ref_rejected": -98.13226318359375, "logps/rejected": -339.3037109375, "loss": 1.0751, "margin_dpo/margin_mean": 82.63563537597656, "margin_dpo/margin_std": 111.85409545898438, "step": 431 }, { "KL/chosen_KL_mean": -138.10623168945312, "KL/mean": -188.77899169921875, "KL/rejected_KL_mean": -239.4517364501953, "KL/std": 103.22248840332031, "epoch": 0.6343612334801763, "fcm_dpo/beta": 0.005334785208106041, "fcm_dpo/delta": -0.09574855864048004, "fcm_dpo/margin": 101.34549713134766, "fcm_dpo/q_t": 0.3753151297569275, "grad_norm": 9.233789443969727, "learning_rate": 1.791192214186223e-07, "logits/chosen": -1.7117465734481812, "logits/rejected": -1.1737985610961914, "logps/chosen": -242.541015625, "logps/ref_chosen": -104.43478393554688, "logps/ref_rejected": -105.08955383300781, "logps/rejected": -344.5412902832031, "loss": 0.9866, "margin_dpo/margin_mean": 101.34548950195312, "margin_dpo/margin_std": 99.7818603515625, "step": 432 }, { "KL/chosen_KL_mean": -165.06680297851562, "KL/mean": -204.61166381835938, "KL/rejected_KL_mean": -244.15652465820312, "KL/std": 96.80538940429688, "epoch": 0.6358296622613803, "fcm_dpo/beta": 0.005340633448213339, "fcm_dpo/delta": 0.028125185519456863, "fcm_dpo/margin": 79.08970642089844, "fcm_dpo/q_t": 0.40405088663101196, "grad_norm": 9.684664726257324, "learning_rate": 1.7788949132172193e-07, "logits/chosen": -1.4942045211791992, "logits/rejected": -1.1620519161224365, "logps/chosen": -254.91001892089844, "logps/ref_chosen": -89.84322357177734, "logps/ref_rejected": -101.73345947265625, "logps/rejected": -345.8899841308594, "loss": 1.0911, "margin_dpo/margin_mean": 79.08970642089844, "margin_dpo/margin_std": 110.08870697021484, "step": 433 }, { "KL/chosen_KL_mean": -144.42999267578125, "KL/mean": -189.54330444335938, "KL/rejected_KL_mean": -234.6566162109375, "KL/std": 107.36045837402344, "epoch": 0.6372980910425844, "fcm_dpo/beta": 0.005302520468831062, "fcm_dpo/delta": -0.029848497360944748, "fcm_dpo/margin": 90.22663879394531, "fcm_dpo/q_t": 0.3948814868927002, "grad_norm": 12.283258438110352, "learning_rate": 1.7666166140378853e-07, "logits/chosen": -2.553473949432373, "logits/rejected": -1.9006510972976685, "logps/chosen": -242.12252807617188, "logps/ref_chosen": -97.6925277709961, "logps/ref_rejected": -84.09129333496094, "logps/rejected": -318.7479248046875, "loss": 1.0594, "margin_dpo/margin_mean": 90.22663879394531, "margin_dpo/margin_std": 124.39888763427734, "step": 434 }, { "KL/chosen_KL_mean": -142.01278686523438, "KL/mean": -187.8046875, "KL/rejected_KL_mean": -233.59661865234375, "KL/std": 99.80178833007812, "epoch": 0.6387665198237885, "fcm_dpo/beta": 0.005268946290016174, "fcm_dpo/delta": -0.03419119864702225, "fcm_dpo/margin": 91.58380126953125, "fcm_dpo/q_t": 0.39059340953826904, "grad_norm": 10.709205627441406, "learning_rate": 1.7543576401928218e-07, "logits/chosen": -2.3253371715545654, "logits/rejected": -1.870253086090088, "logps/chosen": -228.18472290039062, "logps/ref_chosen": -86.17192077636719, "logps/ref_rejected": -93.751708984375, "logps/rejected": -327.34832763671875, "loss": 1.0374, "margin_dpo/margin_mean": 91.58380126953125, "margin_dpo/margin_std": 109.91569519042969, "step": 435 }, { "KL/chosen_KL_mean": -118.00169372558594, "KL/mean": -166.27487182617188, "KL/rejected_KL_mean": -214.5480194091797, "KL/std": 85.56976318359375, "epoch": 0.6402349486049926, "fcm_dpo/beta": 0.005205495283007622, "fcm_dpo/delta": -0.05541032925248146, "fcm_dpo/margin": 96.54632568359375, "fcm_dpo/q_t": 0.38232114911079407, "grad_norm": 10.138195037841797, "learning_rate": 1.742118314717391e-07, "logits/chosen": -1.767169713973999, "logits/rejected": -0.9013456106185913, "logps/chosen": -223.78878784179688, "logps/ref_chosen": -105.78710174560547, "logps/ref_rejected": -88.62471008300781, "logps/rejected": -303.1727294921875, "loss": 0.996, "margin_dpo/margin_mean": 96.54632568359375, "margin_dpo/margin_std": 89.16836547851562, "step": 436 }, { "KL/chosen_KL_mean": -138.875, "KL/mean": -177.42764282226562, "KL/rejected_KL_mean": -215.9803009033203, "KL/std": 90.8270263671875, "epoch": 0.6417033773861968, "fcm_dpo/beta": 0.005238938145339489, "fcm_dpo/delta": 0.04788995161652565, "fcm_dpo/margin": 77.10530090332031, "fcm_dpo/q_t": 0.4047277569770813, "grad_norm": 10.809871673583984, "learning_rate": 1.7298989601292036e-07, "logits/chosen": -2.010089874267578, "logits/rejected": -1.3571252822875977, "logps/chosen": -234.93704223632812, "logps/ref_chosen": -96.06204223632812, "logps/ref_rejected": -89.01220703125, "logps/rejected": -304.99249267578125, "loss": 1.0759, "margin_dpo/margin_mean": 77.10530090332031, "margin_dpo/margin_std": 90.0735092163086, "step": 437 }, { "KL/chosen_KL_mean": -141.5880126953125, "KL/mean": -181.06857299804688, "KL/rejected_KL_mean": -220.54913330078125, "KL/std": 91.84729766845703, "epoch": 0.6431718061674009, "fcm_dpo/beta": 0.00525021692737937, "fcm_dpo/delta": 0.036726392805576324, "fcm_dpo/margin": 78.96110534667969, "fcm_dpo/q_t": 0.40397512912750244, "grad_norm": 10.72618579864502, "learning_rate": 1.7176998984196144e-07, "logits/chosen": -2.027184009552002, "logits/rejected": -1.1954039335250854, "logps/chosen": -243.4434051513672, "logps/ref_chosen": -101.85537719726562, "logps/ref_rejected": -89.44766235351562, "logps/rejected": -309.9967956542969, "loss": 1.0791, "margin_dpo/margin_mean": 78.96110534667969, "margin_dpo/margin_std": 98.01394653320312, "step": 438 }, { "KL/chosen_KL_mean": -152.8901824951172, "KL/mean": -192.21527099609375, "KL/rejected_KL_mean": -231.54037475585938, "KL/std": 99.62709045410156, "epoch": 0.644640234948605, "fcm_dpo/beta": 0.005275568459182978, "fcm_dpo/delta": 0.03587264195084572, "fcm_dpo/margin": 78.65019226074219, "fcm_dpo/q_t": 0.40480339527130127, "grad_norm": 10.864400863647461, "learning_rate": 1.7055214510452458e-07, "logits/chosen": -1.7100586891174316, "logits/rejected": -1.4738038778305054, "logps/chosen": -234.64581298828125, "logps/ref_chosen": -81.75563049316406, "logps/ref_rejected": -90.58636474609375, "logps/rejected": -322.1267395019531, "loss": 1.0817, "margin_dpo/margin_mean": 78.65019226074219, "margin_dpo/margin_std": 98.31413269042969, "step": 439 }, { "KL/chosen_KL_mean": -137.06240844726562, "KL/mean": -185.70657348632812, "KL/rejected_KL_mean": -234.35072326660156, "KL/std": 100.66478729248047, "epoch": 0.6461086637298091, "fcm_dpo/beta": 0.005241363774985075, "fcm_dpo/delta": -0.06415960937738419, "fcm_dpo/margin": 97.28831481933594, "fcm_dpo/q_t": 0.38413941860198975, "grad_norm": 8.634608268737793, "learning_rate": 1.6933639389195134e-07, "logits/chosen": -2.493295907974243, "logits/rejected": -2.0014278888702393, "logps/chosen": -242.7034912109375, "logps/ref_chosen": -105.64108276367188, "logps/ref_rejected": -103.4010009765625, "logps/rejected": -337.7517395019531, "loss": 1.0193, "margin_dpo/margin_mean": 97.28831481933594, "margin_dpo/margin_std": 110.69349670410156, "step": 440 }, { "KL/chosen_KL_mean": -174.46405029296875, "KL/mean": -217.0907745361328, "KL/rejected_KL_mean": -259.71746826171875, "KL/std": 95.74724578857422, "epoch": 0.6475770925110133, "fcm_dpo/beta": 0.00524694100022316, "fcm_dpo/delta": 0.0027515720576047897, "fcm_dpo/margin": 85.25341796875, "fcm_dpo/q_t": 0.398668110370636, "grad_norm": 9.658367156982422, "learning_rate": 1.681227682404166e-07, "logits/chosen": -1.7426241636276245, "logits/rejected": -1.3799102306365967, "logps/chosen": -265.99359130859375, "logps/ref_chosen": -91.529541015625, "logps/ref_rejected": -103.619384765625, "logps/rejected": -363.33685302734375, "loss": 1.0535, "margin_dpo/margin_mean": 85.25341796875, "margin_dpo/margin_std": 102.77945709228516, "step": 441 }, { "KL/chosen_KL_mean": -171.5980224609375, "KL/mean": -224.1585235595703, "KL/rejected_KL_mean": -276.71905517578125, "KL/std": 101.61640930175781, "epoch": 0.6490455212922174, "fcm_dpo/beta": 0.005180784035474062, "fcm_dpo/delta": -0.10002824664115906, "fcm_dpo/margin": 105.12101745605469, "fcm_dpo/q_t": 0.37393641471862793, "grad_norm": 11.204852104187012, "learning_rate": 1.669113001300851e-07, "logits/chosen": -1.6903903484344482, "logits/rejected": -1.2218103408813477, "logps/chosen": -256.3755798339844, "logps/ref_chosen": -84.77755737304688, "logps/ref_rejected": -83.82415771484375, "logps/rejected": -360.543212890625, "loss": 0.9777, "margin_dpo/margin_mean": 105.12100982666016, "margin_dpo/margin_std": 101.45738220214844, "step": 442 }, { "KL/chosen_KL_mean": -165.6534423828125, "KL/mean": -199.2949981689453, "KL/rejected_KL_mean": -232.93653869628906, "KL/std": 98.78840637207031, "epoch": 0.6505139500734214, "fcm_dpo/beta": 0.0052094897255301476, "fcm_dpo/delta": 0.10316122323274612, "fcm_dpo/margin": 67.28306579589844, "fcm_dpo/q_t": 0.41906964778900146, "grad_norm": 10.47797679901123, "learning_rate": 1.6570202148426815e-07, "logits/chosen": -1.838462233543396, "logits/rejected": -1.2816200256347656, "logps/chosen": -268.302734375, "logps/ref_chosen": -102.64927673339844, "logps/ref_rejected": -93.03807067871094, "logps/rejected": -325.974609375, "loss": 1.128, "margin_dpo/margin_mean": 67.2830810546875, "margin_dpo/margin_std": 95.50385284423828, "step": 443 }, { "KL/chosen_KL_mean": -172.17324829101562, "KL/mean": -210.02139282226562, "KL/rejected_KL_mean": -247.86953735351562, "KL/std": 96.23963165283203, "epoch": 0.6519823788546255, "fcm_dpo/beta": 0.005304283928126097, "fcm_dpo/delta": 0.05045264586806297, "fcm_dpo/margin": 75.6962890625, "fcm_dpo/q_t": 0.40898263454437256, "grad_norm": 11.841695785522461, "learning_rate": 1.6449496416858282e-07, "logits/chosen": -1.5317130088806152, "logits/rejected": -1.3604544401168823, "logps/chosen": -260.09295654296875, "logps/ref_chosen": -87.91971588134766, "logps/ref_rejected": -103.32345581054688, "logps/rejected": -351.1929931640625, "loss": 1.1063, "margin_dpo/margin_mean": 75.6962890625, "margin_dpo/margin_std": 111.51077270507812, "step": 444 }, { "KL/chosen_KL_mean": -151.49868774414062, "KL/mean": -196.0882110595703, "KL/rejected_KL_mean": -240.67774963378906, "KL/std": 93.3748550415039, "epoch": 0.6534508076358296, "fcm_dpo/beta": 0.005277983844280243, "fcm_dpo/delta": -0.022016234695911407, "fcm_dpo/margin": 89.1790771484375, "fcm_dpo/q_t": 0.3909933865070343, "grad_norm": 9.416054725646973, "learning_rate": 1.6329015999011182e-07, "logits/chosen": -1.9294803142547607, "logits/rejected": -1.5129873752593994, "logps/chosen": -252.89956665039062, "logps/ref_chosen": -101.40086364746094, "logps/ref_rejected": -99.03790283203125, "logps/rejected": -339.71563720703125, "loss": 1.0332, "margin_dpo/margin_mean": 89.1790771484375, "margin_dpo/margin_std": 98.0916976928711, "step": 445 }, { "KL/chosen_KL_mean": -145.10025024414062, "KL/mean": -185.15869140625, "KL/rejected_KL_mean": -225.21714782714844, "KL/std": 99.13099670410156, "epoch": 0.6549192364170338, "fcm_dpo/beta": 0.005317972041666508, "fcm_dpo/delta": 0.024914931505918503, "fcm_dpo/margin": 80.1169204711914, "fcm_dpo/q_t": 0.4015638828277588, "grad_norm": 13.373784065246582, "learning_rate": 1.6208764069656578e-07, "logits/chosen": -1.8549327850341797, "logits/rejected": -1.7110099792480469, "logps/chosen": -232.5225830078125, "logps/ref_chosen": -87.4223403930664, "logps/ref_rejected": -106.70075988769531, "logps/rejected": -331.91790771484375, "loss": 1.0651, "margin_dpo/margin_mean": 80.11691284179688, "margin_dpo/margin_std": 94.8946762084961, "step": 446 }, { "KL/chosen_KL_mean": -142.89254760742188, "KL/mean": -195.46844482421875, "KL/rejected_KL_mean": -248.04434204101562, "KL/std": 102.39170837402344, "epoch": 0.6563876651982379, "fcm_dpo/beta": 0.005230684764683247, "fcm_dpo/delta": -0.10589684545993805, "fcm_dpo/margin": 105.15178680419922, "fcm_dpo/q_t": 0.37490004301071167, "grad_norm": 9.607406616210938, "learning_rate": 1.608874379754465e-07, "logits/chosen": -2.2026915550231934, "logits/rejected": -2.157698631286621, "logps/chosen": -226.5078125, "logps/ref_chosen": -83.6152572631836, "logps/ref_rejected": -104.91239929199219, "logps/rejected": -352.95672607421875, "loss": 0.9964, "margin_dpo/margin_mean": 105.15177917480469, "margin_dpo/margin_std": 116.68319702148438, "step": 447 }, { "KL/chosen_KL_mean": -154.09796142578125, "KL/mean": -204.73121643066406, "KL/rejected_KL_mean": -255.36444091796875, "KL/std": 107.79026794433594, "epoch": 0.657856093979442, "fcm_dpo/beta": 0.005170288495719433, "fcm_dpo/delta": -0.07774815708398819, "fcm_dpo/margin": 101.2665023803711, "fcm_dpo/q_t": 0.3824172616004944, "grad_norm": 10.670835494995117, "learning_rate": 1.5968958345321177e-07, "logits/chosen": -1.6668646335601807, "logits/rejected": -1.3940508365631104, "logps/chosen": -246.6737518310547, "logps/ref_chosen": -92.5757827758789, "logps/ref_rejected": -107.68977355957031, "logps/rejected": -363.0542297363281, "loss": 1.0312, "margin_dpo/margin_mean": 101.2665023803711, "margin_dpo/margin_std": 126.43362426757812, "step": 448 }, { "KL/chosen_KL_mean": -152.4690399169922, "KL/mean": -204.88592529296875, "KL/rejected_KL_mean": -257.30279541015625, "KL/std": 100.44208526611328, "epoch": 0.6593245227606461, "fcm_dpo/beta": 0.005061184987425804, "fcm_dpo/delta": -0.0850830227136612, "fcm_dpo/margin": 104.83377075195312, "fcm_dpo/q_t": 0.3781484067440033, "grad_norm": 9.359891891479492, "learning_rate": 1.584941086944423e-07, "logits/chosen": -1.4257352352142334, "logits/rejected": -0.9795221090316772, "logps/chosen": -254.86798095703125, "logps/ref_chosen": -102.39893341064453, "logps/ref_rejected": -95.14886474609375, "logps/rejected": -352.45166015625, "loss": 1.0022, "margin_dpo/margin_mean": 104.83377075195312, "margin_dpo/margin_std": 114.44087219238281, "step": 449 }, { "KL/chosen_KL_mean": -135.41156005859375, "KL/mean": -184.7716522216797, "KL/rejected_KL_mean": -234.13174438476562, "KL/std": 101.62181854248047, "epoch": 0.6607929515418502, "fcm_dpo/beta": 0.0049993242137134075, "fcm_dpo/delta": -0.04585316777229309, "fcm_dpo/margin": 98.72018432617188, "fcm_dpo/q_t": 0.3859945833683014, "grad_norm": 9.926301956176758, "learning_rate": 1.573010452010098e-07, "logits/chosen": -2.382091522216797, "logits/rejected": -2.1891212463378906, "logps/chosen": -222.4044189453125, "logps/ref_chosen": -86.99285888671875, "logps/ref_rejected": -108.53203582763672, "logps/rejected": -342.6637878417969, "loss": 1.0259, "margin_dpo/margin_mean": 98.72018432617188, "margin_dpo/margin_std": 112.16717529296875, "step": 450 }, { "KL/chosen_KL_mean": -154.05520629882812, "KL/mean": -193.57681274414062, "KL/rejected_KL_mean": -233.09840393066406, "KL/std": 101.50096893310547, "epoch": 0.6622613803230544, "fcm_dpo/beta": 0.005040162242949009, "fcm_dpo/delta": 0.053614094853401184, "fcm_dpo/margin": 79.04322814941406, "fcm_dpo/q_t": 0.4105450510978699, "grad_norm": 14.635862350463867, "learning_rate": 1.5611042441124687e-07, "logits/chosen": -2.411348819732666, "logits/rejected": -1.9070264101028442, "logps/chosen": -240.86648559570312, "logps/ref_chosen": -86.811279296875, "logps/ref_rejected": -79.8555908203125, "logps/rejected": -312.9539794921875, "loss": 1.1218, "margin_dpo/margin_mean": 79.04322814941406, "margin_dpo/margin_std": 126.20407104492188, "step": 451 }, { "KL/chosen_KL_mean": -134.56747436523438, "KL/mean": -179.08526611328125, "KL/rejected_KL_mean": -223.6030731201172, "KL/std": 97.11660766601562, "epoch": 0.6637298091042585, "fcm_dpo/beta": 0.0050099738873541355, "fcm_dpo/delta": 0.0029504671692848206, "fcm_dpo/margin": 89.03560638427734, "fcm_dpo/q_t": 0.39591628313064575, "grad_norm": 8.789139747619629, "learning_rate": 1.549222776991186e-07, "logits/chosen": -1.7120274305343628, "logits/rejected": -1.6523525714874268, "logps/chosen": -213.94711303710938, "logps/ref_chosen": -79.379638671875, "logps/ref_rejected": -103.71539306640625, "logps/rejected": -327.3184814453125, "loss": 1.0422, "margin_dpo/margin_mean": 89.03560638427734, "margin_dpo/margin_std": 88.60342407226562, "step": 452 }, { "KL/chosen_KL_mean": -162.24525451660156, "KL/mean": -215.61207580566406, "KL/rejected_KL_mean": -268.9789123535156, "KL/std": 98.82974243164062, "epoch": 0.6651982378854625, "fcm_dpo/beta": 0.004989871755242348, "fcm_dpo/delta": -0.08715921640396118, "fcm_dpo/margin": 106.73365783691406, "fcm_dpo/q_t": 0.3777143657207489, "grad_norm": 12.883195877075195, "learning_rate": 1.5373663637339584e-07, "logits/chosen": -1.925933599472046, "logits/rejected": -1.4531406164169312, "logps/chosen": -249.94044494628906, "logps/ref_chosen": -87.6951904296875, "logps/ref_rejected": -90.0582275390625, "logps/rejected": -359.0371398925781, "loss": 0.986, "margin_dpo/margin_mean": 106.73365783691406, "margin_dpo/margin_std": 104.16875457763672, "step": 453 }, { "KL/chosen_KL_mean": -170.08926391601562, "KL/mean": -213.6710205078125, "KL/rejected_KL_mean": -257.2528076171875, "KL/std": 101.20454406738281, "epoch": 0.6666666666666666, "fcm_dpo/beta": 0.00499748345464468, "fcm_dpo/delta": 0.014467813074588776, "fcm_dpo/margin": 87.16352844238281, "fcm_dpo/q_t": 0.4020707905292511, "grad_norm": 10.240715026855469, "learning_rate": 1.5255353167683017e-07, "logits/chosen": -1.9337717294692993, "logits/rejected": -1.4788172245025635, "logps/chosen": -259.655517578125, "logps/ref_chosen": -89.56623840332031, "logps/ref_rejected": -92.92105102539062, "logps/rejected": -350.173828125, "loss": 1.0995, "margin_dpo/margin_mean": 87.16352081298828, "margin_dpo/margin_std": 131.09112548828125, "step": 454 }, { "KL/chosen_KL_mean": -160.16110229492188, "KL/mean": -201.14639282226562, "KL/rejected_KL_mean": -242.13168334960938, "KL/std": 99.42855834960938, "epoch": 0.6681350954478708, "fcm_dpo/beta": 0.005022745113819838, "fcm_dpo/delta": 0.039547160267829895, "fcm_dpo/margin": 81.97059631347656, "fcm_dpo/q_t": 0.4057035446166992, "grad_norm": 9.903414726257324, "learning_rate": 1.5137299478533064e-07, "logits/chosen": -1.8397959470748901, "logits/rejected": -1.7978837490081787, "logps/chosen": -237.79107666015625, "logps/ref_chosen": -77.62998962402344, "logps/ref_rejected": -118.97795104980469, "logps/rejected": -361.1096496582031, "loss": 1.0944, "margin_dpo/margin_mean": 81.97059631347656, "margin_dpo/margin_std": 113.62576293945312, "step": 455 }, { "KL/chosen_KL_mean": -161.74935913085938, "KL/mean": -202.62106323242188, "KL/rejected_KL_mean": -243.4927520751953, "KL/std": 101.21768188476562, "epoch": 0.6696035242290749, "fcm_dpo/beta": 0.005054287146776915, "fcm_dpo/delta": 0.03827638179063797, "fcm_dpo/margin": 81.74337768554688, "fcm_dpo/q_t": 0.40470045804977417, "grad_norm": 11.922443389892578, "learning_rate": 1.5019505680714232e-07, "logits/chosen": -1.9774236679077148, "logits/rejected": -1.8444124460220337, "logps/chosen": -251.36624145507812, "logps/ref_chosen": -89.61686706542969, "logps/ref_rejected": -109.5597152709961, "logps/rejected": -353.0524597167969, "loss": 1.0785, "margin_dpo/margin_mean": 81.74337768554688, "margin_dpo/margin_std": 102.49090576171875, "step": 456 }, { "KL/chosen_KL_mean": -132.38502502441406, "KL/mean": -186.63055419921875, "KL/rejected_KL_mean": -240.87606811523438, "KL/std": 101.0234375, "epoch": 0.671071953010279, "fcm_dpo/beta": 0.004944447427988052, "fcm_dpo/delta": -0.09341508150100708, "fcm_dpo/margin": 108.49103546142578, "fcm_dpo/q_t": 0.37410277128219604, "grad_norm": 10.40605354309082, "learning_rate": 1.4901974878202627e-07, "logits/chosen": -2.2007999420166016, "logits/rejected": -1.8904091119766235, "logps/chosen": -219.70672607421875, "logps/ref_chosen": -87.32168579101562, "logps/ref_rejected": -90.76660919189453, "logps/rejected": -331.6426696777344, "loss": 0.9718, "margin_dpo/margin_mean": 108.49102783203125, "margin_dpo/margin_std": 90.35786437988281, "step": 457 }, { "KL/chosen_KL_mean": -153.32867431640625, "KL/mean": -193.28082275390625, "KL/rejected_KL_mean": -233.2329864501953, "KL/std": 100.76258850097656, "epoch": 0.6725403817914831, "fcm_dpo/beta": 0.005011053290218115, "fcm_dpo/delta": 0.05141254514455795, "fcm_dpo/margin": 79.9043197631836, "fcm_dpo/q_t": 0.41007083654403687, "grad_norm": 10.193153381347656, "learning_rate": 1.4784710168044212e-07, "logits/chosen": -1.6329656839370728, "logits/rejected": -1.3935143947601318, "logps/chosen": -246.84912109375, "logps/ref_chosen": -93.52044677734375, "logps/ref_rejected": -103.36898803710938, "logps/rejected": -336.60198974609375, "loss": 1.1041, "margin_dpo/margin_mean": 79.90431213378906, "margin_dpo/margin_std": 116.63407897949219, "step": 458 }, { "KL/chosen_KL_mean": -146.66146850585938, "KL/mean": -196.23553466796875, "KL/rejected_KL_mean": -245.80963134765625, "KL/std": 105.66712951660156, "epoch": 0.6740088105726872, "fcm_dpo/beta": 0.004953456111252308, "fcm_dpo/delta": -0.04402646794915199, "fcm_dpo/margin": 99.14816284179688, "fcm_dpo/q_t": 0.3881978690624237, "grad_norm": 7.7661848068237305, "learning_rate": 1.466771464027316e-07, "logits/chosen": -1.7135825157165527, "logits/rejected": -1.4628551006317139, "logps/chosen": -222.34967041015625, "logps/ref_chosen": -75.68820190429688, "logps/ref_rejected": -92.17048645019531, "logps/rejected": -337.9801025390625, "loss": 1.0283, "margin_dpo/margin_mean": 99.14816284179688, "margin_dpo/margin_std": 112.94024658203125, "step": 459 }, { "KL/chosen_KL_mean": -149.8790283203125, "KL/mean": -195.6499786376953, "KL/rejected_KL_mean": -241.42092895507812, "KL/std": 102.00483703613281, "epoch": 0.6754772393538914, "fcm_dpo/beta": 0.004945947788655758, "fcm_dpo/delta": -0.003225427120923996, "fcm_dpo/margin": 91.54191589355469, "fcm_dpo/q_t": 0.39668217301368713, "grad_norm": 10.641520500183105, "learning_rate": 1.4550991377830423e-07, "logits/chosen": -2.1042251586914062, "logits/rejected": -2.125800848007202, "logps/chosen": -230.99691772460938, "logps/ref_chosen": -81.11788940429688, "logps/ref_rejected": -110.31238555908203, "logps/rejected": -351.73333740234375, "loss": 1.0638, "margin_dpo/margin_mean": 91.54191589355469, "margin_dpo/margin_std": 117.19721221923828, "step": 460 }, { "KL/chosen_KL_mean": -184.72579956054688, "KL/mean": -221.70619201660156, "KL/rejected_KL_mean": -258.68658447265625, "KL/std": 89.47636413574219, "epoch": 0.6769456681350955, "fcm_dpo/beta": 0.005012227222323418, "fcm_dpo/delta": 0.08241581916809082, "fcm_dpo/margin": 73.9607925415039, "fcm_dpo/q_t": 0.4152829051017761, "grad_norm": 9.35671615600586, "learning_rate": 1.4434543456482518e-07, "logits/chosen": -1.468023657798767, "logits/rejected": -1.2711260318756104, "logps/chosen": -266.309326171875, "logps/ref_chosen": -81.58352661132812, "logps/ref_rejected": -93.87710571289062, "logps/rejected": -352.5636901855469, "loss": 1.1178, "margin_dpo/margin_mean": 73.9607925415039, "margin_dpo/margin_std": 105.83638000488281, "step": 461 }, { "KL/chosen_KL_mean": -140.54736328125, "KL/mean": -188.308837890625, "KL/rejected_KL_mean": -236.0703125, "KL/std": 96.39823913574219, "epoch": 0.6784140969162996, "fcm_dpo/beta": 0.005031496752053499, "fcm_dpo/delta": -0.03217264637351036, "fcm_dpo/margin": 95.52291870117188, "fcm_dpo/q_t": 0.38922619819641113, "grad_norm": 9.564581871032715, "learning_rate": 1.4318373944740484e-07, "logits/chosen": -2.441433906555176, "logits/rejected": -1.9524383544921875, "logps/chosen": -234.74591064453125, "logps/ref_chosen": -94.19854736328125, "logps/ref_rejected": -85.63162994384766, "logps/rejected": -321.7019348144531, "loss": 1.0259, "margin_dpo/margin_mean": 95.52291870117188, "margin_dpo/margin_std": 103.81068420410156, "step": 462 }, { "KL/chosen_KL_mean": -136.15145874023438, "KL/mean": -192.84652709960938, "KL/rejected_KL_mean": -249.5416259765625, "KL/std": 103.62898254394531, "epoch": 0.6798825256975036, "fcm_dpo/beta": 0.004961706697940826, "fcm_dpo/delta": -0.11949680745601654, "fcm_dpo/margin": 113.39016723632812, "fcm_dpo/q_t": 0.3697696626186371, "grad_norm": 14.86490535736084, "learning_rate": 1.4202485903778976e-07, "logits/chosen": -1.8930439949035645, "logits/rejected": -1.4674456119537354, "logps/chosen": -222.07620239257812, "logps/ref_chosen": -85.92474365234375, "logps/ref_rejected": -96.90184020996094, "logps/rejected": -346.4434814453125, "loss": 0.9788, "margin_dpo/margin_mean": 113.39016723632812, "margin_dpo/margin_std": 110.37509155273438, "step": 463 }, { "KL/chosen_KL_mean": -167.87515258789062, "KL/mean": -213.98040771484375, "KL/rejected_KL_mean": -260.0856628417969, "KL/std": 102.98896789550781, "epoch": 0.6813509544787077, "fcm_dpo/beta": 0.004885343834757805, "fcm_dpo/delta": -0.0006006844341754913, "fcm_dpo/margin": 92.21051025390625, "fcm_dpo/q_t": 0.3967575430870056, "grad_norm": 12.482551574707031, "learning_rate": 1.4086882387355658e-07, "logits/chosen": -1.7639579772949219, "logits/rejected": -1.8439664840698242, "logps/chosen": -247.56434631347656, "logps/ref_chosen": -79.68919372558594, "logps/ref_rejected": -107.29232025146484, "logps/rejected": -367.37799072265625, "loss": 1.0616, "margin_dpo/margin_mean": 92.21051025390625, "margin_dpo/margin_std": 117.0827865600586, "step": 464 }, { "KL/chosen_KL_mean": -159.87017822265625, "KL/mean": -213.8851318359375, "KL/rejected_KL_mean": -267.900146484375, "KL/std": 111.44023132324219, "epoch": 0.6828193832599119, "fcm_dpo/beta": 0.004832090809941292, "fcm_dpo/delta": -0.07617706060409546, "fcm_dpo/margin": 108.02995300292969, "fcm_dpo/q_t": 0.37889420986175537, "grad_norm": 9.909037590026855, "learning_rate": 1.3971566441730714e-07, "logits/chosen": -1.5861811637878418, "logits/rejected": -1.2803863286972046, "logps/chosen": -251.73040771484375, "logps/ref_chosen": -91.8602294921875, "logps/ref_rejected": -118.71001434326172, "logps/rejected": -386.6101379394531, "loss": 0.9912, "margin_dpo/margin_mean": 108.02996063232422, "margin_dpo/margin_std": 105.53227996826172, "step": 465 }, { "KL/chosen_KL_mean": -142.86985778808594, "KL/mean": -198.246826171875, "KL/rejected_KL_mean": -253.62379455566406, "KL/std": 107.02445983886719, "epoch": 0.684287812041116, "fcm_dpo/beta": 0.004758065566420555, "fcm_dpo/delta": -0.0813850685954094, "fcm_dpo/margin": 110.75393676757812, "fcm_dpo/q_t": 0.37828460335731506, "grad_norm": 7.444459438323975, "learning_rate": 1.3856541105586545e-07, "logits/chosen": -1.5047101974487305, "logits/rejected": -1.1812858581542969, "logps/chosen": -227.57125854492188, "logps/ref_chosen": -84.70140838623047, "logps/ref_rejected": -96.05084228515625, "logps/rejected": -349.67462158203125, "loss": 0.9928, "margin_dpo/margin_mean": 110.75393676757812, "margin_dpo/margin_std": 112.03373718261719, "step": 466 }, { "KL/chosen_KL_mean": -181.15975952148438, "KL/mean": -227.86563110351562, "KL/rejected_KL_mean": -274.57147216796875, "KL/std": 111.49720764160156, "epoch": 0.6857562408223201, "fcm_dpo/beta": 0.004727067891508341, "fcm_dpo/delta": 0.008390307426452637, "fcm_dpo/margin": 93.41171264648438, "fcm_dpo/q_t": 0.39889955520629883, "grad_norm": 13.292122840881348, "learning_rate": 1.3741809409947729e-07, "logits/chosen": -2.382991313934326, "logits/rejected": -1.8103981018066406, "logps/chosen": -290.45806884765625, "logps/ref_chosen": -109.29832458496094, "logps/ref_rejected": -108.8436508178711, "logps/rejected": -383.4151306152344, "loss": 1.0777, "margin_dpo/margin_mean": 93.41171264648438, "margin_dpo/margin_std": 125.93314361572266, "step": 467 }, { "KL/chosen_KL_mean": -141.70941162109375, "KL/mean": -183.89451599121094, "KL/rejected_KL_mean": -226.07962036132812, "KL/std": 108.10533905029297, "epoch": 0.6872246696035242, "fcm_dpo/beta": 0.004781381692737341, "fcm_dpo/delta": 0.04858104884624481, "fcm_dpo/margin": 84.37019348144531, "fcm_dpo/q_t": 0.4094661772251129, "grad_norm": 10.17148494720459, "learning_rate": 1.362737437810114e-07, "logits/chosen": -1.821883201599121, "logits/rejected": -1.6187019348144531, "logps/chosen": -240.03106689453125, "logps/ref_chosen": -98.32164764404297, "logps/ref_rejected": -106.68048095703125, "logps/rejected": -332.76007080078125, "loss": 1.1036, "margin_dpo/margin_mean": 84.37019348144531, "margin_dpo/margin_std": 123.54734802246094, "step": 468 }, { "KL/chosen_KL_mean": -174.17282104492188, "KL/mean": -226.3063201904297, "KL/rejected_KL_mean": -278.4398193359375, "KL/std": 108.27603149414062, "epoch": 0.6886930983847284, "fcm_dpo/beta": 0.004737929441034794, "fcm_dpo/delta": -0.047216739505529404, "fcm_dpo/margin": 104.26698303222656, "fcm_dpo/q_t": 0.3866550028324127, "grad_norm": 11.116669654846191, "learning_rate": 1.351323902551631e-07, "logits/chosen": -2.1750669479370117, "logits/rejected": -1.7511928081512451, "logps/chosen": -270.9370422363281, "logps/ref_chosen": -96.76420593261719, "logps/ref_rejected": -109.59500885009766, "logps/rejected": -388.0347900390625, "loss": 1.0267, "margin_dpo/margin_mean": 104.26698303222656, "margin_dpo/margin_std": 117.83578491210938, "step": 469 }, { "KL/chosen_KL_mean": -160.32565307617188, "KL/mean": -208.27127075195312, "KL/rejected_KL_mean": -256.2169189453125, "KL/std": 101.9165267944336, "epoch": 0.6901615271659325, "fcm_dpo/beta": 0.004747654311358929, "fcm_dpo/delta": -0.005513817071914673, "fcm_dpo/margin": 95.89126586914062, "fcm_dpo/q_t": 0.39419037103652954, "grad_norm": 9.690295219421387, "learning_rate": 1.339940635976592e-07, "logits/chosen": -1.669811487197876, "logits/rejected": -1.2485895156860352, "logps/chosen": -243.82229614257812, "logps/ref_chosen": -83.49665832519531, "logps/ref_rejected": -88.48578643798828, "logps/rejected": -344.70269775390625, "loss": 1.0372, "margin_dpo/margin_mean": 95.89126586914062, "margin_dpo/margin_std": 102.33804321289062, "step": 470 }, { "KL/chosen_KL_mean": -185.2115020751953, "KL/mean": -230.9696044921875, "KL/rejected_KL_mean": -276.72772216796875, "KL/std": 100.63410949707031, "epoch": 0.6916299559471366, "fcm_dpo/beta": 0.00475767906755209, "fcm_dpo/delta": 0.01526210643351078, "fcm_dpo/margin": 91.51620483398438, "fcm_dpo/q_t": 0.39869433641433716, "grad_norm": 9.37916088104248, "learning_rate": 1.3285879380446563e-07, "logits/chosen": -1.6311432123184204, "logits/rejected": -1.2209393978118896, "logps/chosen": -273.685791015625, "logps/ref_chosen": -88.47430419921875, "logps/ref_rejected": -90.48171997070312, "logps/rejected": -367.20941162109375, "loss": 1.0516, "margin_dpo/margin_mean": 91.51620483398438, "margin_dpo/margin_std": 101.12043762207031, "step": 471 }, { "KL/chosen_KL_mean": -171.4448699951172, "KL/mean": -226.10107421875, "KL/rejected_KL_mean": -280.75726318359375, "KL/std": 100.78599548339844, "epoch": 0.6930983847283406, "fcm_dpo/beta": 0.0047001466155052185, "fcm_dpo/delta": -0.06764907389879227, "fcm_dpo/margin": 109.31238555908203, "fcm_dpo/q_t": 0.3814803957939148, "grad_norm": 8.265682220458984, "learning_rate": 1.317266107909975e-07, "logits/chosen": -2.17086124420166, "logits/rejected": -1.472998023033142, "logps/chosen": -296.6785888671875, "logps/ref_chosen": -125.23370361328125, "logps/ref_rejected": -121.05349731445312, "logps/rejected": -401.810791015625, "loss": 0.9951, "margin_dpo/margin_mean": 109.3123779296875, "margin_dpo/margin_std": 105.84391784667969, "step": 472 }, { "KL/chosen_KL_mean": -194.54928588867188, "KL/mean": -228.7971954345703, "KL/rejected_KL_mean": -263.04510498046875, "KL/std": 102.35105895996094, "epoch": 0.6945668135095447, "fcm_dpo/beta": 0.0047929780557751656, "fcm_dpo/delta": 0.1256594955921173, "fcm_dpo/margin": 68.49583435058594, "fcm_dpo/q_t": 0.4262773096561432, "grad_norm": 10.695194244384766, "learning_rate": 1.3059754439133002e-07, "logits/chosen": -1.9057073593139648, "logits/rejected": -1.4209513664245605, "logps/chosen": -290.16064453125, "logps/ref_chosen": -95.61137390136719, "logps/ref_rejected": -88.15115356445312, "logps/rejected": -351.1962585449219, "loss": 1.1592, "margin_dpo/margin_mean": 68.49583435058594, "margin_dpo/margin_std": 118.30596923828125, "step": 473 }, { "KL/chosen_KL_mean": -182.2709503173828, "KL/mean": -217.48263549804688, "KL/rejected_KL_mean": -252.69432067871094, "KL/std": 96.60692596435547, "epoch": 0.6960352422907489, "fcm_dpo/beta": 0.00481058144941926, "fcm_dpo/delta": -0.015532290562987328, "fcm_dpo/margin": 70.4233627319336, "fcm_dpo/q_t": 0.42211002111434937, "grad_norm": 13.555959701538086, "learning_rate": 1.2947162435741277e-07, "logits/chosen": -1.5289095640182495, "logits/rejected": -1.3379920721054077, "logps/chosen": -263.7507019042969, "logps/ref_chosen": -81.47975158691406, "logps/ref_rejected": -96.46562957763672, "logps/rejected": -349.1599426269531, "loss": 1.1575, "margin_dpo/margin_mean": 70.4233627319336, "margin_dpo/margin_std": 116.6693115234375, "step": 474 }, { "KL/chosen_KL_mean": -137.7569580078125, "KL/mean": -191.47225952148438, "KL/rejected_KL_mean": -245.18753051757812, "KL/std": 103.98016357421875, "epoch": 0.697503671071953, "fcm_dpo/beta": 0.0047598229721188545, "fcm_dpo/delta": -0.06462739408016205, "fcm_dpo/margin": 107.43055725097656, "fcm_dpo/q_t": 0.38313812017440796, "grad_norm": 8.57970142364502, "learning_rate": 1.2834888035828596e-07, "logits/chosen": -2.1804757118225098, "logits/rejected": -2.1264116764068604, "logps/chosen": -211.95294189453125, "logps/ref_chosen": -74.19598388671875, "logps/ref_rejected": -94.69242095947266, "logps/rejected": -339.87994384765625, "loss": 1.0099, "margin_dpo/margin_mean": 107.43057250976562, "margin_dpo/margin_std": 116.82521057128906, "step": 475 }, { "KL/chosen_KL_mean": -161.96005249023438, "KL/mean": -209.29946899414062, "KL/rejected_KL_mean": -256.6388854980469, "KL/std": 105.60200500488281, "epoch": 0.6989720998531571, "fcm_dpo/beta": 0.004727785475552082, "fcm_dpo/delta": 0.0022844448685646057, "fcm_dpo/margin": 94.67884826660156, "fcm_dpo/q_t": 0.3981456160545349, "grad_norm": 11.078962326049805, "learning_rate": 1.2722934197929802e-07, "logits/chosen": -2.0900261402130127, "logits/rejected": -1.7165247201919556, "logps/chosen": -233.93115234375, "logps/ref_chosen": -71.97109985351562, "logps/ref_rejected": -80.26224517822266, "logps/rejected": -336.901123046875, "loss": 1.057, "margin_dpo/margin_mean": 94.67884063720703, "margin_dpo/margin_std": 114.95457458496094, "step": 476 }, { "KL/chosen_KL_mean": -161.87371826171875, "KL/mean": -209.43356323242188, "KL/rejected_KL_mean": -256.993408203125, "KL/std": 93.7550048828125, "epoch": 0.7004405286343612, "fcm_dpo/beta": 0.004747455008327961, "fcm_dpo/delta": -0.001672297716140747, "fcm_dpo/margin": 95.11968231201172, "fcm_dpo/q_t": 0.3954678177833557, "grad_norm": 9.578988075256348, "learning_rate": 1.2611303872132631e-07, "logits/chosen": -2.1565287113189697, "logits/rejected": -1.2394235134124756, "logps/chosen": -266.8792724609375, "logps/ref_chosen": -105.00555419921875, "logps/ref_rejected": -81.87843322753906, "logps/rejected": -338.871826171875, "loss": 1.0576, "margin_dpo/margin_mean": 95.11968231201172, "margin_dpo/margin_std": 117.13556671142578, "step": 477 }, { "KL/chosen_KL_mean": -156.97903442382812, "KL/mean": -206.59579467773438, "KL/rejected_KL_mean": -256.21258544921875, "KL/std": 101.80718994140625, "epoch": 0.7019089574155654, "fcm_dpo/beta": 0.004728690255433321, "fcm_dpo/delta": -0.020189039409160614, "fcm_dpo/margin": 99.23353576660156, "fcm_dpo/q_t": 0.39149191975593567, "grad_norm": 8.809531211853027, "learning_rate": 1.2500000000000005e-07, "logits/chosen": -1.5445786714553833, "logits/rejected": -1.3673609495162964, "logps/chosen": -233.76724243164062, "logps/ref_chosen": -76.7882080078125, "logps/ref_rejected": -90.43994140625, "logps/rejected": -346.65252685546875, "loss": 1.0369, "margin_dpo/margin_mean": 99.23353576660156, "margin_dpo/margin_std": 112.28427124023438, "step": 478 }, { "KL/chosen_KL_mean": -161.14065551757812, "KL/mean": -209.66729736328125, "KL/rejected_KL_mean": -258.1939697265625, "KL/std": 101.9322509765625, "epoch": 0.7033773861967695, "fcm_dpo/beta": 0.004733966663479805, "fcm_dpo/delta": -0.010165683925151825, "fcm_dpo/margin": 97.0533218383789, "fcm_dpo/q_t": 0.39276695251464844, "grad_norm": 12.275886535644531, "learning_rate": 1.2389025514492456e-07, "logits/chosen": -1.386313796043396, "logits/rejected": -1.243363857269287, "logps/chosen": -242.50296020507812, "logps/ref_chosen": -81.3623046875, "logps/ref_rejected": -101.09114074707031, "logps/rejected": -359.28509521484375, "loss": 1.0381, "margin_dpo/margin_mean": 97.0533218383789, "margin_dpo/margin_std": 104.44366455078125, "step": 479 }, { "KL/chosen_KL_mean": -185.82891845703125, "KL/mean": -224.86044311523438, "KL/rejected_KL_mean": -263.8919677734375, "KL/std": 97.73199462890625, "epoch": 0.7048458149779736, "fcm_dpo/beta": 0.0047708675265312195, "fcm_dpo/delta": 0.08056198805570602, "fcm_dpo/margin": 78.06306457519531, "fcm_dpo/q_t": 0.41418278217315674, "grad_norm": 10.773375511169434, "learning_rate": 1.227838333989088e-07, "logits/chosen": -1.8190544843673706, "logits/rejected": -1.1443976163864136, "logps/chosen": -282.6028137207031, "logps/ref_chosen": -96.7739028930664, "logps/ref_rejected": -86.40473937988281, "logps/rejected": -350.29669189453125, "loss": 1.1116, "margin_dpo/margin_mean": 78.06306457519531, "margin_dpo/margin_std": 108.42257690429688, "step": 480 }, { "KL/chosen_KL_mean": -181.53097534179688, "KL/mean": -227.4097900390625, "KL/rejected_KL_mean": -273.28863525390625, "KL/std": 101.75514221191406, "epoch": 0.7063142437591777, "fcm_dpo/beta": 0.004794671665877104, "fcm_dpo/delta": 0.01051416527479887, "fcm_dpo/margin": 91.75765991210938, "fcm_dpo/q_t": 0.3983638286590576, "grad_norm": 12.513395309448242, "learning_rate": 1.2168076391719489e-07, "logits/chosen": -2.058073043823242, "logits/rejected": -1.628892183303833, "logps/chosen": -273.2011413574219, "logps/ref_chosen": -91.670166015625, "logps/ref_rejected": -98.69490051269531, "logps/rejected": -371.9835205078125, "loss": 1.0633, "margin_dpo/margin_mean": 91.7576675415039, "margin_dpo/margin_std": 113.5112075805664, "step": 481 }, { "KL/chosen_KL_mean": -175.49473571777344, "KL/mean": -220.43032836914062, "KL/rejected_KL_mean": -265.36590576171875, "KL/std": 99.30690002441406, "epoch": 0.7077826725403817, "fcm_dpo/beta": 0.004801459610462189, "fcm_dpo/delta": 0.019236396998167038, "fcm_dpo/margin": 89.87120056152344, "fcm_dpo/q_t": 0.3995790481567383, "grad_norm": 13.622885704040527, "learning_rate": 1.2058107576668938e-07, "logits/chosen": -1.4896271228790283, "logits/rejected": -1.1253504753112793, "logps/chosen": -274.01483154296875, "logps/ref_chosen": -98.52011108398438, "logps/ref_rejected": -94.82945251464844, "logps/rejected": -360.19537353515625, "loss": 1.0672, "margin_dpo/margin_mean": 89.87120056152344, "margin_dpo/margin_std": 110.11418914794922, "step": 482 }, { "KL/chosen_KL_mean": -180.67239379882812, "KL/mean": -228.32342529296875, "KL/rejected_KL_mean": -275.9744567871094, "KL/std": 106.0830078125, "epoch": 0.7092511013215859, "fcm_dpo/beta": 0.004840238951146603, "fcm_dpo/delta": -0.01257888600230217, "fcm_dpo/margin": 95.30207824707031, "fcm_dpo/q_t": 0.39278697967529297, "grad_norm": 8.121712684631348, "learning_rate": 1.194847979251979e-07, "logits/chosen": -2.209134101867676, "logits/rejected": -1.5848324298858643, "logps/chosen": -287.7909851074219, "logps/ref_chosen": -107.11860656738281, "logps/ref_rejected": -101.11499786376953, "logps/rejected": -377.0894775390625, "loss": 1.0416, "margin_dpo/margin_mean": 95.30207824707031, "margin_dpo/margin_std": 105.41844177246094, "step": 483 }, { "KL/chosen_KL_mean": -166.78041076660156, "KL/mean": -219.15415954589844, "KL/rejected_KL_mean": -271.52789306640625, "KL/std": 107.94287109375, "epoch": 0.71071953010279, "fcm_dpo/beta": 0.004783853888511658, "fcm_dpo/delta": -0.053837914019823074, "fcm_dpo/margin": 104.74748229980469, "fcm_dpo/q_t": 0.38438552618026733, "grad_norm": 8.95742130279541, "learning_rate": 1.1839195928066101e-07, "logits/chosen": -2.4437952041625977, "logits/rejected": -1.9667483568191528, "logps/chosen": -253.7603302001953, "logps/ref_chosen": -86.97991943359375, "logps/ref_rejected": -90.72367095947266, "logps/rejected": -362.2515869140625, "loss": 1.0214, "margin_dpo/margin_mean": 104.74748229980469, "margin_dpo/margin_std": 117.56710815429688, "step": 484 }, { "KL/chosen_KL_mean": -157.0498504638672, "KL/mean": -214.79684448242188, "KL/rejected_KL_mean": -272.5438537597656, "KL/std": 111.50321197509766, "epoch": 0.7121879588839941, "fcm_dpo/beta": 0.004689244087785482, "fcm_dpo/delta": -0.09678236395120621, "fcm_dpo/margin": 115.4940185546875, "fcm_dpo/q_t": 0.37508007884025574, "grad_norm": 9.589292526245117, "learning_rate": 1.1730258863039347e-07, "logits/chosen": -1.5288968086242676, "logits/rejected": -1.0940437316894531, "logps/chosen": -251.10861206054688, "logps/ref_chosen": -94.05874633789062, "logps/ref_rejected": -108.56298828125, "logps/rejected": -381.1068420410156, "loss": 0.9801, "margin_dpo/margin_mean": 115.49401092529297, "margin_dpo/margin_std": 112.31433868408203, "step": 485 }, { "KL/chosen_KL_mean": -180.47369384765625, "KL/mean": -230.94326782226562, "KL/rejected_KL_mean": -281.412841796875, "KL/std": 107.85102081298828, "epoch": 0.7136563876651982, "fcm_dpo/beta": 0.004661254584789276, "fcm_dpo/delta": -0.02160482108592987, "fcm_dpo/margin": 100.93917846679688, "fcm_dpo/q_t": 0.39127978682518005, "grad_norm": 10.805781364440918, "learning_rate": 1.1621671468032493e-07, "logits/chosen": -2.053313970565796, "logits/rejected": -1.3463963270187378, "logps/chosen": -274.2195739746094, "logps/ref_chosen": -93.74588012695312, "logps/ref_rejected": -98.07064819335938, "logps/rejected": -379.4834899902344, "loss": 1.0399, "margin_dpo/margin_mean": 100.93917083740234, "margin_dpo/margin_std": 115.87477111816406, "step": 486 }, { "KL/chosen_KL_mean": -174.2039794921875, "KL/mean": -220.03860473632812, "KL/rejected_KL_mean": -265.87322998046875, "KL/std": 103.42660522460938, "epoch": 0.7151248164464024, "fcm_dpo/beta": 0.0046575660817325115, "fcm_dpo/delta": 0.02407930977642536, "fcm_dpo/margin": 91.66924285888672, "fcm_dpo/q_t": 0.4011977016925812, "grad_norm": 10.19849681854248, "learning_rate": 1.1513436604424378e-07, "logits/chosen": -2.1839797496795654, "logits/rejected": -1.8756451606750488, "logps/chosen": -262.237548828125, "logps/ref_chosen": -88.0335693359375, "logps/ref_rejected": -98.47209930419922, "logps/rejected": -364.3453369140625, "loss": 1.0651, "margin_dpo/margin_mean": 91.66925048828125, "margin_dpo/margin_std": 109.32331085205078, "step": 487 }, { "KL/chosen_KL_mean": -158.35450744628906, "KL/mean": -208.90673828125, "KL/rejected_KL_mean": -259.4589538574219, "KL/std": 109.08303833007812, "epoch": 0.7165932452276065, "fcm_dpo/beta": 0.004656763281673193, "fcm_dpo/delta": -0.02185317687690258, "fcm_dpo/margin": 101.10443878173828, "fcm_dpo/q_t": 0.38953208923339844, "grad_norm": 9.925406455993652, "learning_rate": 1.1405557124304335e-07, "logits/chosen": -1.607795238494873, "logits/rejected": -1.3447988033294678, "logps/chosen": -243.1441650390625, "logps/ref_chosen": -84.78964233398438, "logps/ref_rejected": -90.2734603881836, "logps/rejected": -349.732421875, "loss": 1.02, "margin_dpo/margin_mean": 101.10444641113281, "margin_dpo/margin_std": 96.80280303955078, "step": 488 }, { "KL/chosen_KL_mean": -177.58462524414062, "KL/mean": -223.75230407714844, "KL/rejected_KL_mean": -269.9199523925781, "KL/std": 99.37748718261719, "epoch": 0.7180616740088106, "fcm_dpo/beta": 0.004661159589886665, "fcm_dpo/delta": 0.020484985783696175, "fcm_dpo/margin": 92.3353271484375, "fcm_dpo/q_t": 0.4008216857910156, "grad_norm": 10.168931007385254, "learning_rate": 1.1298035870396985e-07, "logits/chosen": -1.5888497829437256, "logits/rejected": -1.0927530527114868, "logps/chosen": -268.0539245605469, "logps/ref_chosen": -90.46929931640625, "logps/ref_rejected": -86.39761352539062, "logps/rejected": -356.31756591796875, "loss": 1.0639, "margin_dpo/margin_mean": 92.33534240722656, "margin_dpo/margin_std": 110.99066162109375, "step": 489 }, { "KL/chosen_KL_mean": -179.28955078125, "KL/mean": -222.39990234375, "KL/rejected_KL_mean": -265.51025390625, "KL/std": 107.74891662597656, "epoch": 0.7195301027900147, "fcm_dpo/beta": 0.004729720763862133, "fcm_dpo/delta": 0.04253225028514862, "fcm_dpo/margin": 86.22069549560547, "fcm_dpo/q_t": 0.40913423895835876, "grad_norm": 9.120133399963379, "learning_rate": 1.1190875675987355e-07, "logits/chosen": -2.0534164905548096, "logits/rejected": -2.1311709880828857, "logps/chosen": -264.60968017578125, "logps/ref_chosen": -85.32012939453125, "logps/ref_rejected": -115.99385070800781, "logps/rejected": -381.50408935546875, "loss": 1.1287, "margin_dpo/margin_mean": 86.22069549560547, "margin_dpo/margin_std": 141.7015838623047, "step": 490 }, { "KL/chosen_KL_mean": -182.6004180908203, "KL/mean": -222.75865173339844, "KL/rejected_KL_mean": -262.9168701171875, "KL/std": 109.66326141357422, "epoch": 0.7209985315712188, "fcm_dpo/beta": 0.004752345383167267, "fcm_dpo/delta": 0.07104957848787308, "fcm_dpo/margin": 80.31645202636719, "fcm_dpo/q_t": 0.4117836356163025, "grad_norm": 10.851274490356445, "learning_rate": 1.1084079364846241e-07, "logits/chosen": -1.7933106422424316, "logits/rejected": -1.29533052444458, "logps/chosen": -268.74395751953125, "logps/ref_chosen": -86.14352416992188, "logps/ref_rejected": -80.67945861816406, "logps/rejected": -343.5963439941406, "loss": 1.1068, "margin_dpo/margin_mean": 80.31645202636719, "margin_dpo/margin_std": 111.28205871582031, "step": 491 }, { "KL/chosen_KL_mean": -169.64657592773438, "KL/mean": -201.59054565429688, "KL/rejected_KL_mean": -233.53451538085938, "KL/std": 107.24102783203125, "epoch": 0.7224669603524229, "fcm_dpo/beta": 0.004870216827839613, "fcm_dpo/delta": 0.14343884587287903, "fcm_dpo/margin": 63.887962341308594, "fcm_dpo/q_t": 0.4300411343574524, "grad_norm": 14.212003707885742, "learning_rate": 1.097764975115576e-07, "logits/chosen": -1.9620296955108643, "logits/rejected": -1.5676814317703247, "logps/chosen": -250.754150390625, "logps/ref_chosen": -81.10757446289062, "logps/ref_rejected": -80.75199890136719, "logps/rejected": -314.2864990234375, "loss": 1.1993, "margin_dpo/margin_mean": 63.887962341308594, "margin_dpo/margin_std": 133.98292541503906, "step": 492 }, { "KL/chosen_KL_mean": -167.80459594726562, "KL/mean": -210.98052978515625, "KL/rejected_KL_mean": -254.15646362304688, "KL/std": 97.94412231445312, "epoch": 0.723935389133627, "fcm_dpo/beta": 0.004906080663204193, "fcm_dpo/delta": 0.02713683247566223, "fcm_dpo/margin": 86.35186767578125, "fcm_dpo/q_t": 0.40144893527030945, "grad_norm": 15.690646171569824, "learning_rate": 1.0871589639435203e-07, "logits/chosen": -2.3523173332214355, "logits/rejected": -1.6103328466415405, "logps/chosen": -280.0119323730469, "logps/ref_chosen": -112.20733642578125, "logps/ref_rejected": -93.60719299316406, "logps/rejected": -347.763671875, "loss": 1.0698, "margin_dpo/margin_mean": 86.35186767578125, "margin_dpo/margin_std": 102.87307739257812, "step": 493 }, { "KL/chosen_KL_mean": -163.60784912109375, "KL/mean": -219.94192504882812, "KL/rejected_KL_mean": -276.2760009765625, "KL/std": 108.10519409179688, "epoch": 0.7254038179148311, "fcm_dpo/beta": 0.004834240768104792, "fcm_dpo/delta": -0.10148841887712479, "fcm_dpo/margin": 112.66815185546875, "fcm_dpo/q_t": 0.3745891749858856, "grad_norm": 10.725040435791016, "learning_rate": 1.0765901824467166e-07, "logits/chosen": -1.388137936592102, "logits/rejected": -1.3093554973602295, "logps/chosen": -236.72274780273438, "logps/ref_chosen": -73.11489868164062, "logps/ref_rejected": -92.16300201416016, "logps/rejected": -368.4389953613281, "loss": 0.9891, "margin_dpo/margin_mean": 112.66815185546875, "margin_dpo/margin_std": 116.00218200683594, "step": 494 }, { "KL/chosen_KL_mean": -167.45440673828125, "KL/mean": -216.466552734375, "KL/rejected_KL_mean": -265.47869873046875, "KL/std": 103.0387954711914, "epoch": 0.7268722466960352, "fcm_dpo/beta": 0.00481772143393755, "fcm_dpo/delta": -0.02338775247335434, "fcm_dpo/margin": 98.02427673339844, "fcm_dpo/q_t": 0.3920443654060364, "grad_norm": 11.115226745605469, "learning_rate": 1.0660589091223854e-07, "logits/chosen": -2.2392048835754395, "logits/rejected": -1.775106430053711, "logps/chosen": -266.9747314453125, "logps/ref_chosen": -99.52031707763672, "logps/ref_rejected": -97.93089294433594, "logps/rejected": -363.4095764160156, "loss": 1.0489, "margin_dpo/margin_mean": 98.02427673339844, "margin_dpo/margin_std": 122.67973327636719, "step": 495 }, { "KL/chosen_KL_mean": -186.36358642578125, "KL/mean": -227.69281005859375, "KL/rejected_KL_mean": -269.02203369140625, "KL/std": 103.54518127441406, "epoch": 0.7283406754772394, "fcm_dpo/beta": 0.004865934606641531, "fcm_dpo/delta": 0.04945981130003929, "fcm_dpo/margin": 82.658447265625, "fcm_dpo/q_t": 0.4063739478588104, "grad_norm": 9.600869178771973, "learning_rate": 1.0555654214793722e-07, "logits/chosen": -2.279493808746338, "logits/rejected": -1.647784948348999, "logps/chosen": -294.2203369140625, "logps/ref_chosen": -107.85675048828125, "logps/ref_rejected": -92.77056121826172, "logps/rejected": -361.7926025390625, "loss": 1.0865, "margin_dpo/margin_mean": 82.65843963623047, "margin_dpo/margin_std": 103.96592712402344, "step": 496 }, { "KL/chosen_KL_mean": -200.2681884765625, "KL/mean": -227.3326416015625, "KL/rejected_KL_mean": -254.39710998535156, "KL/std": 105.10977172851562, "epoch": 0.7298091042584435, "fcm_dpo/beta": 0.004926739726215601, "fcm_dpo/delta": 0.06447851657867432, "fcm_dpo/margin": 54.12892150878906, "fcm_dpo/q_t": 0.43916094303131104, "grad_norm": 9.265952110290527, "learning_rate": 1.0451099960308374e-07, "logits/chosen": -1.6075884103775024, "logits/rejected": -1.146307349205017, "logps/chosen": -292.3514099121094, "logps/ref_chosen": -92.08322143554688, "logps/ref_rejected": -81.79503631591797, "logps/rejected": -336.192138671875, "loss": 1.2122, "margin_dpo/margin_mean": 54.12892150878906, "margin_dpo/margin_std": 112.56846618652344, "step": 497 }, { "KL/chosen_KL_mean": -168.26063537597656, "KL/mean": -213.84844970703125, "KL/rejected_KL_mean": -259.436279296875, "KL/std": 101.29026794433594, "epoch": 0.7312775330396476, "fcm_dpo/beta": 0.00492420420050621, "fcm_dpo/delta": 0.0010725930333137512, "fcm_dpo/margin": 91.17562103271484, "fcm_dpo/q_t": 0.396063894033432, "grad_norm": 11.956170082092285, "learning_rate": 1.0346929082869641e-07, "logits/chosen": -1.6750918626785278, "logits/rejected": -1.1646183729171753, "logps/chosen": -266.45501708984375, "logps/ref_chosen": -98.19437408447266, "logps/ref_rejected": -90.68745422363281, "logps/rejected": -350.12371826171875, "loss": 1.0544, "margin_dpo/margin_mean": 91.17562103271484, "margin_dpo/margin_std": 109.13645935058594, "step": 498 }, { "KL/chosen_KL_mean": -177.30838012695312, "KL/mean": -224.7327423095703, "KL/rejected_KL_mean": -272.1571044921875, "KL/std": 111.17930603027344, "epoch": 0.7327459618208517, "fcm_dpo/beta": 0.004920176230370998, "fcm_dpo/delta": -0.01749299466609955, "fcm_dpo/margin": 94.84874725341797, "fcm_dpo/q_t": 0.3935285806655884, "grad_norm": 11.57456111907959, "learning_rate": 1.0243144327477013e-07, "logits/chosen": -2.0922465324401855, "logits/rejected": -2.0417070388793945, "logps/chosen": -258.3482666015625, "logps/ref_chosen": -81.0399169921875, "logps/ref_rejected": -106.92170715332031, "logps/rejected": -379.07879638671875, "loss": 1.0514, "margin_dpo/margin_mean": 94.8487548828125, "margin_dpo/margin_std": 118.310302734375, "step": 499 }, { "KL/chosen_KL_mean": -196.89361572265625, "KL/mean": -237.7458953857422, "KL/rejected_KL_mean": -278.5981750488281, "KL/std": 107.0694580078125, "epoch": 0.7342143906020558, "fcm_dpo/beta": 0.004939177073538303, "fcm_dpo/delta": 0.048427581787109375, "fcm_dpo/margin": 81.70455169677734, "fcm_dpo/q_t": 0.40973156690597534, "grad_norm": 12.509004592895508, "learning_rate": 1.0139748428955333e-07, "logits/chosen": -1.6324963569641113, "logits/rejected": -1.4601924419403076, "logps/chosen": -286.14166259765625, "logps/ref_chosen": -89.248046875, "logps/ref_rejected": -100.41021728515625, "logps/rejected": -379.0083923339844, "loss": 1.1061, "margin_dpo/margin_mean": 81.70455932617188, "margin_dpo/margin_std": 121.18946075439453, "step": 500 }, { "KL/chosen_KL_mean": -172.25335693359375, "KL/mean": -216.698974609375, "KL/rejected_KL_mean": -261.14459228515625, "KL/std": 116.15937805175781, "epoch": 0.73568281938326, "fcm_dpo/beta": 0.005002699792385101, "fcm_dpo/delta": 0.004122689366340637, "fcm_dpo/margin": 88.89126586914062, "fcm_dpo/q_t": 0.40019088983535767, "grad_norm": 10.078176498413086, "learning_rate": 1.0036744111882672e-07, "logits/chosen": -1.3775291442871094, "logits/rejected": -0.9249249696731567, "logps/chosen": -262.85028076171875, "logps/ref_chosen": -90.596923828125, "logps/ref_rejected": -85.2972640991211, "logps/rejected": -346.4418640136719, "loss": 1.0941, "margin_dpo/margin_mean": 88.8912582397461, "margin_dpo/margin_std": 129.65127563476562, "step": 501 }, { "KL/chosen_KL_mean": -173.2459716796875, "KL/mean": -213.20974731445312, "KL/rejected_KL_mean": -253.1735382080078, "KL/std": 96.82359313964844, "epoch": 0.737151248164464, "fcm_dpo/beta": 0.0050225649029016495, "fcm_dpo/delta": 0.049928393214941025, "fcm_dpo/margin": 79.92756652832031, "fcm_dpo/q_t": 0.40794628858566284, "grad_norm": 10.836005210876465, "learning_rate": 9.934134090518592e-08, "logits/chosen": -1.8895217180252075, "logits/rejected": -1.1493624448776245, "logps/chosen": -276.7167053222656, "logps/ref_chosen": -103.47073364257812, "logps/ref_rejected": -89.04415893554688, "logps/rejected": -342.21771240234375, "loss": 1.1121, "margin_dpo/margin_mean": 79.92756652832031, "margin_dpo/margin_std": 119.62471008300781, "step": 502 }, { "KL/chosen_KL_mean": -170.5072021484375, "KL/mean": -216.42495727539062, "KL/rejected_KL_mean": -262.34271240234375, "KL/std": 103.584228515625, "epoch": 0.7386196769456681, "fcm_dpo/beta": 0.005027182400226593, "fcm_dpo/delta": -0.012657150626182556, "fcm_dpo/margin": 91.83548736572266, "fcm_dpo/q_t": 0.3943704068660736, "grad_norm": 9.954883575439453, "learning_rate": 9.831921068732571e-08, "logits/chosen": -1.704979419708252, "logits/rejected": -1.1992563009262085, "logps/chosen": -264.2371826171875, "logps/ref_chosen": -93.72998809814453, "logps/ref_rejected": -87.60896301269531, "logps/rejected": -349.95166015625, "loss": 1.0565, "margin_dpo/margin_mean": 91.83548736572266, "margin_dpo/margin_std": 114.19137573242188, "step": 503 }, { "KL/chosen_KL_mean": -173.15093994140625, "KL/mean": -231.43263244628906, "KL/rejected_KL_mean": -289.7143249511719, "KL/std": 107.3223648071289, "epoch": 0.7400881057268722, "fcm_dpo/beta": 0.004910448100417852, "fcm_dpo/delta": -0.12967948615550995, "fcm_dpo/margin": 116.56340026855469, "fcm_dpo/q_t": 0.36714255809783936, "grad_norm": 10.799518585205078, "learning_rate": 9.730107739932805e-08, "logits/chosen": -2.193516254425049, "logits/rejected": -1.863387107849121, "logps/chosen": -265.44281005859375, "logps/ref_chosen": -92.2918701171875, "logps/ref_rejected": -109.03765106201172, "logps/rejected": -398.7519836425781, "loss": 0.9587, "margin_dpo/margin_mean": 116.56340026855469, "margin_dpo/margin_std": 107.65071105957031, "step": 504 }, { "KL/chosen_KL_mean": -191.77728271484375, "KL/mean": -233.93405151367188, "KL/rejected_KL_mean": -276.0908203125, "KL/std": 106.50577545166016, "epoch": 0.7415565345080763, "fcm_dpo/beta": 0.004897596314549446, "fcm_dpo/delta": 0.03867579996585846, "fcm_dpo/margin": 84.31356811523438, "fcm_dpo/q_t": 0.4046417474746704, "grad_norm": 9.086536407470703, "learning_rate": 9.628696786995188e-08, "logits/chosen": -2.4688057899475098, "logits/rejected": -1.9173367023468018, "logps/chosen": -298.05419921875, "logps/ref_chosen": -106.27693176269531, "logps/ref_rejected": -94.20611572265625, "logps/rejected": -370.29693603515625, "loss": 1.0777, "margin_dpo/margin_mean": 84.31356811523438, "margin_dpo/margin_std": 104.20630645751953, "step": 505 }, { "KL/chosen_KL_mean": -186.02706909179688, "KL/mean": -232.8184051513672, "KL/rejected_KL_mean": -279.6097717285156, "KL/std": 99.92256164550781, "epoch": 0.7430249632892805, "fcm_dpo/beta": 0.004884321242570877, "fcm_dpo/delta": -0.007781848311424255, "fcm_dpo/margin": 93.58268737792969, "fcm_dpo/q_t": 0.3927346169948578, "grad_norm": 9.635473251342773, "learning_rate": 9.527690882192635e-08, "logits/chosen": -2.3033764362335205, "logits/rejected": -1.9620904922485352, "logps/chosen": -267.4679260253906, "logps/ref_chosen": -81.44086456298828, "logps/ref_rejected": -85.66439819335938, "logps/rejected": -365.274169921875, "loss": 1.0439, "margin_dpo/margin_mean": 93.58268737792969, "margin_dpo/margin_std": 104.6795654296875, "step": 506 }, { "KL/chosen_KL_mean": -193.2694091796875, "KL/mean": -241.5196533203125, "KL/rejected_KL_mean": -289.7698974609375, "KL/std": 105.75428771972656, "epoch": 0.7444933920704846, "fcm_dpo/beta": 0.004886812996119261, "fcm_dpo/delta": -0.022642912343144417, "fcm_dpo/margin": 96.50050354003906, "fcm_dpo/q_t": 0.39291203022003174, "grad_norm": 10.023951530456543, "learning_rate": 9.427092687124691e-08, "logits/chosen": -2.1121931076049805, "logits/rejected": -1.6658493280410767, "logps/chosen": -292.69635009765625, "logps/ref_chosen": -99.42694854736328, "logps/ref_rejected": -102.25296020507812, "logps/rejected": -392.02288818359375, "loss": 1.0569, "margin_dpo/margin_mean": 96.50050354003906, "margin_dpo/margin_std": 124.43849182128906, "step": 507 }, { "KL/chosen_KL_mean": -188.124755859375, "KL/mean": -228.52621459960938, "KL/rejected_KL_mean": -268.92767333984375, "KL/std": 113.13078308105469, "epoch": 0.7459618208516887, "fcm_dpo/beta": 0.004920968785881996, "fcm_dpo/delta": 0.05452117323875427, "fcm_dpo/margin": 80.8029556274414, "fcm_dpo/q_t": 0.41117680072784424, "grad_norm": 17.02985191345215, "learning_rate": 9.326904852647344e-08, "logits/chosen": -1.8558951616287231, "logits/rejected": -1.3565008640289307, "logps/chosen": -286.0404052734375, "logps/ref_chosen": -97.9156494140625, "logps/ref_rejected": -103.79782104492188, "logps/rejected": -372.72552490234375, "loss": 1.1276, "margin_dpo/margin_mean": 80.8029556274414, "margin_dpo/margin_std": 130.61660766601562, "step": 508 }, { "KL/chosen_KL_mean": -176.96371459960938, "KL/mean": -214.24057006835938, "KL/rejected_KL_mean": -251.5174102783203, "KL/std": 87.99352264404297, "epoch": 0.7474302496328928, "fcm_dpo/beta": 0.005000775679945946, "fcm_dpo/delta": 0.07996828854084015, "fcm_dpo/margin": 74.55369567871094, "fcm_dpo/q_t": 0.4130539000034332, "grad_norm": 16.53234100341797, "learning_rate": 9.227130018803195e-08, "logits/chosen": -1.9470932483673096, "logits/rejected": -1.2735739946365356, "logps/chosen": -277.6935729980469, "logps/ref_chosen": -100.72984313964844, "logps/ref_rejected": -86.943359375, "logps/rejected": -338.46075439453125, "loss": 1.1296, "margin_dpo/margin_mean": 74.55369567871094, "margin_dpo/margin_std": 116.4517822265625, "step": 509 }, { "KL/chosen_KL_mean": -207.454833984375, "KL/mean": -252.71673583984375, "KL/rejected_KL_mean": -297.9786682128906, "KL/std": 107.10193634033203, "epoch": 0.748898678414097, "fcm_dpo/beta": 0.0050178710371255875, "fcm_dpo/delta": -0.004490138962864876, "fcm_dpo/margin": 90.52381896972656, "fcm_dpo/q_t": 0.39535173773765564, "grad_norm": 12.655274391174316, "learning_rate": 9.127770814751932e-08, "logits/chosen": -1.4448351860046387, "logits/rejected": -1.2549282312393188, "logps/chosen": -296.62420654296875, "logps/ref_chosen": -89.16938781738281, "logps/ref_rejected": -107.91939544677734, "logps/rejected": -405.8980712890625, "loss": 1.0537, "margin_dpo/margin_mean": 90.52381896972656, "margin_dpo/margin_std": 107.55709838867188, "step": 510 }, { "KL/chosen_KL_mean": -189.1050262451172, "KL/mean": -237.92193603515625, "KL/rejected_KL_mean": -286.7388610839844, "KL/std": 105.441650390625, "epoch": 0.750367107195301, "fcm_dpo/beta": 0.004991387017071247, "fcm_dpo/delta": -0.039268966764211655, "fcm_dpo/margin": 97.63385009765625, "fcm_dpo/q_t": 0.3893653154373169, "grad_norm": 11.885235786437988, "learning_rate": 9.028829858700973e-08, "logits/chosen": -2.0825769901275635, "logits/rejected": -1.5090551376342773, "logps/chosen": -278.85479736328125, "logps/ref_chosen": -89.74979400634766, "logps/ref_rejected": -97.73127746582031, "logps/rejected": -384.47015380859375, "loss": 1.0356, "margin_dpo/margin_mean": 97.63385009765625, "margin_dpo/margin_std": 116.70793151855469, "step": 511 }, { "KL/chosen_KL_mean": -180.0770263671875, "KL/mean": -227.23626708984375, "KL/rejected_KL_mean": -274.3955078125, "KL/std": 101.90090942382812, "epoch": 0.7518355359765051, "fcm_dpo/beta": 0.004981291014701128, "fcm_dpo/delta": -0.021333176642656326, "fcm_dpo/margin": 94.3184814453125, "fcm_dpo/q_t": 0.39237287640571594, "grad_norm": 14.846067428588867, "learning_rate": 8.930309757836516e-08, "logits/chosen": -1.7191128730773926, "logits/rejected": -1.2665915489196777, "logps/chosen": -265.81219482421875, "logps/ref_chosen": -85.73515319824219, "logps/ref_rejected": -87.6248779296875, "logps/rejected": -362.0203857421875, "loss": 1.0597, "margin_dpo/margin_mean": 94.31848907470703, "margin_dpo/margin_std": 121.84278869628906, "step": 512 }, { "KL/chosen_KL_mean": -191.56814575195312, "KL/mean": -231.19024658203125, "KL/rejected_KL_mean": -270.8123779296875, "KL/std": 98.29747772216797, "epoch": 0.7533039647577092, "fcm_dpo/beta": 0.004967710003256798, "fcm_dpo/delta": 0.058465782552957535, "fcm_dpo/margin": 79.24421691894531, "fcm_dpo/q_t": 0.40833455324172974, "grad_norm": 12.429883003234863, "learning_rate": 8.832213108254863e-08, "logits/chosen": -2.1757917404174805, "logits/rejected": -1.460874080657959, "logps/chosen": -287.7586669921875, "logps/ref_chosen": -96.19050598144531, "logps/ref_rejected": -82.91773986816406, "logps/rejected": -353.7301025390625, "loss": 1.1046, "margin_dpo/margin_mean": 79.24421691894531, "margin_dpo/margin_std": 109.68754577636719, "step": 513 }, { "KL/chosen_KL_mean": -185.51446533203125, "KL/mean": -228.64682006835938, "KL/rejected_KL_mean": -271.7791748046875, "KL/std": 98.27638244628906, "epoch": 0.7547723935389133, "fcm_dpo/beta": 0.005019359290599823, "fcm_dpo/delta": 0.01777580752968788, "fcm_dpo/margin": 86.26473236083984, "fcm_dpo/q_t": 0.40216636657714844, "grad_norm": 12.15334415435791, "learning_rate": 8.734542494893954e-08, "logits/chosen": -1.8832634687423706, "logits/rejected": -1.3261592388153076, "logps/chosen": -283.3381042480469, "logps/ref_chosen": -97.82363891601562, "logps/ref_rejected": -85.36982727050781, "logps/rejected": -357.14898681640625, "loss": 1.0787, "margin_dpo/margin_mean": 86.26473999023438, "margin_dpo/margin_std": 116.4493408203125, "step": 514 }, { "KL/chosen_KL_mean": -178.60165405273438, "KL/mean": -220.3987579345703, "KL/rejected_KL_mean": -262.19586181640625, "KL/std": 102.03836059570312, "epoch": 0.7562408223201175, "fcm_dpo/beta": 0.005045060068368912, "fcm_dpo/delta": 0.02951134368777275, "fcm_dpo/margin": 83.5942153930664, "fcm_dpo/q_t": 0.4032410979270935, "grad_norm": 13.163448333740234, "learning_rate": 8.637300491465272e-08, "logits/chosen": -1.918165683746338, "logits/rejected": -1.6911985874176025, "logps/chosen": -260.53521728515625, "logps/ref_chosen": -81.93355560302734, "logps/ref_rejected": -93.57656860351562, "logps/rejected": -355.7724304199219, "loss": 1.0837, "margin_dpo/margin_mean": 83.59422302246094, "margin_dpo/margin_std": 110.89984893798828, "step": 515 }, { "KL/chosen_KL_mean": -182.73809814453125, "KL/mean": -223.7190399169922, "KL/rejected_KL_mean": -264.6999816894531, "KL/std": 117.15553283691406, "epoch": 0.7577092511013216, "fcm_dpo/beta": 0.005087654106318951, "fcm_dpo/delta": 0.0343351885676384, "fcm_dpo/margin": 81.96188354492188, "fcm_dpo/q_t": 0.40796324610710144, "grad_norm": 12.83326530456543, "learning_rate": 8.540489660386064e-08, "logits/chosen": -2.3145430088043213, "logits/rejected": -2.1350057125091553, "logps/chosen": -279.82977294921875, "logps/ref_chosen": -97.09166717529297, "logps/ref_rejected": -118.55667114257812, "logps/rejected": -383.25665283203125, "loss": 1.126, "margin_dpo/margin_mean": 81.96188354492188, "margin_dpo/margin_std": 138.1246337890625, "step": 516 }, { "KL/chosen_KL_mean": -194.80770874023438, "KL/mean": -244.35067749023438, "KL/rejected_KL_mean": -293.8936767578125, "KL/std": 105.68379211425781, "epoch": 0.7591776798825257, "fcm_dpo/beta": 0.0050135329365730286, "fcm_dpo/delta": -0.050679806619882584, "fcm_dpo/margin": 99.08596801757812, "fcm_dpo/q_t": 0.38733309507369995, "grad_norm": 9.785173416137695, "learning_rate": 8.444112552711752e-08, "logits/chosen": -2.079163074493408, "logits/rejected": -1.6662458181381226, "logps/chosen": -287.9942626953125, "logps/ref_chosen": -93.18656921386719, "logps/ref_rejected": -95.5927963256836, "logps/rejected": -389.4864501953125, "loss": 1.041, "margin_dpo/margin_mean": 99.08596801757812, "margin_dpo/margin_std": 120.85176849365234, "step": 517 }, { "KL/chosen_KL_mean": -203.78955078125, "KL/mean": -240.6020965576172, "KL/rejected_KL_mean": -277.4146423339844, "KL/std": 99.24911499023438, "epoch": 0.7606461086637298, "fcm_dpo/beta": 0.005094345659017563, "fcm_dpo/delta": 0.07787081599235535, "fcm_dpo/margin": 73.62509155273438, "fcm_dpo/q_t": 0.4158647656440735, "grad_norm": 11.506685256958008, "learning_rate": 8.348171708068747e-08, "logits/chosen": -1.72408926486969, "logits/rejected": -1.3833379745483398, "logps/chosen": -290.8109130859375, "logps/ref_chosen": -87.0213623046875, "logps/ref_rejected": -91.24456024169922, "logps/rejected": -368.6592102050781, "loss": 1.1378, "margin_dpo/margin_mean": 73.62509155273438, "margin_dpo/margin_std": 123.48959350585938, "step": 518 }, { "KL/chosen_KL_mean": -188.56741333007812, "KL/mean": -228.94049072265625, "KL/rejected_KL_mean": -269.3135681152344, "KL/std": 93.03599548339844, "epoch": 0.762114537444934, "fcm_dpo/beta": 0.005131518002599478, "fcm_dpo/delta": 0.03715251758694649, "fcm_dpo/margin": 80.74616241455078, "fcm_dpo/q_t": 0.4046279191970825, "grad_norm": 11.094258308410645, "learning_rate": 8.25266965458755e-08, "logits/chosen": -1.780432939529419, "logits/rejected": -1.3956313133239746, "logps/chosen": -289.25494384765625, "logps/ref_chosen": -100.68755340576172, "logps/ref_rejected": -109.03042602539062, "logps/rejected": -378.343994140625, "loss": 1.0718, "margin_dpo/margin_mean": 80.74616241455078, "margin_dpo/margin_std": 96.15090942382812, "step": 519 }, { "KL/chosen_KL_mean": -174.54107666015625, "KL/mean": -222.8872833251953, "KL/rejected_KL_mean": -271.2334899902344, "KL/std": 104.92181396484375, "epoch": 0.7635829662261381, "fcm_dpo/beta": 0.005110214464366436, "fcm_dpo/delta": -0.04654761776328087, "fcm_dpo/margin": 96.69241333007812, "fcm_dpo/q_t": 0.38531070947647095, "grad_norm": 10.005337715148926, "learning_rate": 8.15760890883607e-08, "logits/chosen": -1.5855275392532349, "logits/rejected": -1.1460142135620117, "logps/chosen": -273.3187255859375, "logps/ref_chosen": -98.77762603759766, "logps/ref_rejected": -106.9457015991211, "logps/rejected": -378.17919921875, "loss": 1.0157, "margin_dpo/margin_mean": 96.69241333007812, "margin_dpo/margin_std": 101.92752838134766, "step": 520 }, { "KL/chosen_KL_mean": -176.4241485595703, "KL/mean": -225.93972778320312, "KL/rejected_KL_mean": -275.4552917480469, "KL/std": 98.50906372070312, "epoch": 0.7650513950073421, "fcm_dpo/beta": 0.00508786691352725, "fcm_dpo/delta": -0.056909676641225815, "fcm_dpo/margin": 99.03115844726562, "fcm_dpo/q_t": 0.38355040550231934, "grad_norm": 11.63822078704834, "learning_rate": 8.062991975753378e-08, "logits/chosen": -1.934647560119629, "logits/rejected": -1.595494031906128, "logps/chosen": -262.7752380371094, "logps/ref_chosen": -86.35108947753906, "logps/ref_rejected": -90.39317321777344, "logps/rejected": -365.84844970703125, "loss": 1.0063, "margin_dpo/margin_mean": 99.03115844726562, "margin_dpo/margin_std": 99.65690612792969, "step": 521 }, { "KL/chosen_KL_mean": -184.30136108398438, "KL/mean": -229.09103393554688, "KL/rejected_KL_mean": -273.8807373046875, "KL/std": 106.60728454589844, "epoch": 0.7665198237885462, "fcm_dpo/beta": 0.005056227091699839, "fcm_dpo/delta": -0.0031437724828720093, "fcm_dpo/margin": 89.57937622070312, "fcm_dpo/q_t": 0.3963169455528259, "grad_norm": 11.193562507629395, "learning_rate": 7.968821348583643e-08, "logits/chosen": -1.5549134016036987, "logits/rejected": -1.177118182182312, "logps/chosen": -260.3094177246094, "logps/ref_chosen": -76.008056640625, "logps/ref_rejected": -74.44125366210938, "logps/rejected": -348.3219909667969, "loss": 1.0591, "margin_dpo/margin_mean": 89.57936096191406, "margin_dpo/margin_std": 112.14762878417969, "step": 522 }, { "KL/chosen_KL_mean": -203.88681030273438, "KL/mean": -236.28988647460938, "KL/rejected_KL_mean": -268.6929626464844, "KL/std": 105.2451171875, "epoch": 0.7679882525697503, "fcm_dpo/beta": 0.005130934529006481, "fcm_dpo/delta": 0.12165539711713791, "fcm_dpo/margin": 64.80615234375, "fcm_dpo/q_t": 0.42573124170303345, "grad_norm": 12.027227401733398, "learning_rate": 7.875099508810484e-08, "logits/chosen": -1.9186699390411377, "logits/rejected": -1.2036707401275635, "logps/chosen": -300.3830871582031, "logps/ref_chosen": -96.49627685546875, "logps/ref_rejected": -91.0152816772461, "logps/rejected": -359.708251953125, "loss": 1.1731, "margin_dpo/margin_mean": 64.80615234375, "margin_dpo/margin_std": 122.15135192871094, "step": 523 }, { "KL/chosen_KL_mean": -185.89892578125, "KL/mean": -231.63909912109375, "KL/rejected_KL_mean": -277.37933349609375, "KL/std": 105.57075500488281, "epoch": 0.7694566813509545, "fcm_dpo/beta": 0.005148780532181263, "fcm_dpo/delta": -0.02208206057548523, "fcm_dpo/margin": 91.48039245605469, "fcm_dpo/q_t": 0.39336147904396057, "grad_norm": 14.017488479614258, "learning_rate": 7.781828926091535e-08, "logits/chosen": -2.125394821166992, "logits/rejected": -1.4248558282852173, "logps/chosen": -289.304443359375, "logps/ref_chosen": -103.40550994873047, "logps/ref_rejected": -87.91015625, "logps/rejected": -365.2894592285156, "loss": 1.0462, "margin_dpo/margin_mean": 91.48039245605469, "margin_dpo/margin_std": 112.13156127929688, "step": 524 }, { "KL/chosen_KL_mean": -175.44296264648438, "KL/mean": -226.84909057617188, "KL/rejected_KL_mean": -278.2552490234375, "KL/std": 106.01425170898438, "epoch": 0.7709251101321586, "fcm_dpo/beta": 0.005037224851548672, "fcm_dpo/delta": -0.07464565336704254, "fcm_dpo/margin": 102.8122787475586, "fcm_dpo/q_t": 0.3849982023239136, "grad_norm": 11.277276039123535, "learning_rate": 7.689012058193384e-08, "logits/chosen": -1.5974867343902588, "logits/rejected": -1.5107810497283936, "logps/chosen": -259.6494445800781, "logps/ref_chosen": -84.20648193359375, "logps/ref_rejected": -106.26158905029297, "logps/rejected": -384.516845703125, "loss": 1.0291, "margin_dpo/margin_mean": 102.8122787475586, "margin_dpo/margin_std": 122.57896423339844, "step": 525 }, { "KL/chosen_KL_mean": -181.08453369140625, "KL/mean": -228.5096893310547, "KL/rejected_KL_mean": -275.934814453125, "KL/std": 100.11372375488281, "epoch": 0.7723935389133627, "fcm_dpo/beta": 0.005048790015280247, "fcm_dpo/delta": -0.03032749518752098, "fcm_dpo/margin": 94.85028839111328, "fcm_dpo/q_t": 0.3906853199005127, "grad_norm": 9.261003494262695, "learning_rate": 7.596651350926836e-08, "logits/chosen": -1.7839946746826172, "logits/rejected": -1.1341900825500488, "logps/chosen": -283.93865966796875, "logps/ref_chosen": -102.85411071777344, "logps/ref_rejected": -93.65145874023438, "logps/rejected": -369.5863037109375, "loss": 1.0454, "margin_dpo/margin_mean": 94.85028839111328, "margin_dpo/margin_std": 117.34310913085938, "step": 526 }, { "KL/chosen_KL_mean": -182.96224975585938, "KL/mean": -230.83517456054688, "KL/rejected_KL_mean": -278.70806884765625, "KL/std": 105.57845306396484, "epoch": 0.7738619676945668, "fcm_dpo/beta": 0.005016027949750423, "fcm_dpo/delta": -0.031792886555194855, "fcm_dpo/margin": 95.7458267211914, "fcm_dpo/q_t": 0.38758933544158936, "grad_norm": 12.210872650146484, "learning_rate": 7.504749238082414e-08, "logits/chosen": -3.099153518676758, "logits/rejected": -2.2796692848205566, "logps/chosen": -279.7276916503906, "logps/ref_chosen": -96.76543426513672, "logps/ref_rejected": -85.39518737792969, "logps/rejected": -364.103271484375, "loss": 1.0137, "margin_dpo/margin_mean": 95.74581909179688, "margin_dpo/margin_std": 93.15516662597656, "step": 527 }, { "KL/chosen_KL_mean": -170.5989227294922, "KL/mean": -217.8131103515625, "KL/rejected_KL_mean": -265.02734375, "KL/std": 106.9075927734375, "epoch": 0.775330396475771, "fcm_dpo/beta": 0.004999192431569099, "fcm_dpo/delta": -0.023176094517111778, "fcm_dpo/margin": 94.42839050292969, "fcm_dpo/q_t": 0.3923468589782715, "grad_norm": 10.352832794189453, "learning_rate": 7.413308141366254e-08, "logits/chosen": -2.2386622428894043, "logits/rejected": -1.7365646362304688, "logps/chosen": -264.76385498046875, "logps/ref_chosen": -94.1649169921875, "logps/ref_rejected": -100.372314453125, "logps/rejected": -365.399658203125, "loss": 1.0529, "margin_dpo/margin_mean": 94.42839050292969, "margin_dpo/margin_std": 120.28085327148438, "step": 528 }, { "KL/chosen_KL_mean": -198.13063049316406, "KL/mean": -240.8045654296875, "KL/rejected_KL_mean": -283.478515625, "KL/std": 105.39470672607422, "epoch": 0.7767988252569751, "fcm_dpo/beta": 0.005012898240238428, "fcm_dpo/delta": 0.023006722331047058, "fcm_dpo/margin": 85.34791564941406, "fcm_dpo/q_t": 0.40393248200416565, "grad_norm": 11.763914108276367, "learning_rate": 7.322330470336313e-08, "logits/chosen": -1.6822190284729004, "logits/rejected": -1.5023728609085083, "logps/chosen": -278.90740966796875, "logps/ref_chosen": -80.77680206298828, "logps/ref_rejected": -97.191162109375, "logps/rejected": -380.669677734375, "loss": 1.0935, "margin_dpo/margin_mean": 85.34791564941406, "margin_dpo/margin_std": 124.56035614013672, "step": 529 }, { "KL/chosen_KL_mean": -181.14682006835938, "KL/mean": -223.98953247070312, "KL/rejected_KL_mean": -266.8322448730469, "KL/std": 102.71823120117188, "epoch": 0.7782672540381792, "fcm_dpo/beta": 0.005006622523069382, "fcm_dpo/delta": 0.02176816761493683, "fcm_dpo/margin": 85.68543243408203, "fcm_dpo/q_t": 0.4013262987136841, "grad_norm": 10.490839958190918, "learning_rate": 7.231818622338822e-08, "logits/chosen": -1.5628221035003662, "logits/rejected": -1.2402976751327515, "logps/chosen": -260.92254638671875, "logps/ref_chosen": -79.7757339477539, "logps/ref_rejected": -92.05390167236328, "logps/rejected": -358.8861389160156, "loss": 1.0592, "margin_dpo/margin_mean": 85.68543243408203, "margin_dpo/margin_std": 98.09611511230469, "step": 530 }, { "KL/chosen_KL_mean": -175.47500610351562, "KL/mean": -228.99588012695312, "KL/rejected_KL_mean": -282.5167541503906, "KL/std": 106.51791381835938, "epoch": 0.7797356828193832, "fcm_dpo/beta": 0.004961233586072922, "fcm_dpo/delta": -0.08571291714906693, "fcm_dpo/margin": 107.04176330566406, "fcm_dpo/q_t": 0.3762897849082947, "grad_norm": 11.941046714782715, "learning_rate": 7.141774982445147e-08, "logits/chosen": -1.834104299545288, "logits/rejected": -1.229844570159912, "logps/chosen": -254.794189453125, "logps/ref_chosen": -79.31919860839844, "logps/ref_rejected": -78.10832214355469, "logps/rejected": -360.62506103515625, "loss": 0.9892, "margin_dpo/margin_mean": 107.04176330566406, "margin_dpo/margin_std": 106.6690673828125, "step": 531 }, { "KL/chosen_KL_mean": -189.42636108398438, "KL/mean": -247.23992919921875, "KL/rejected_KL_mean": -305.0534973144531, "KL/std": 102.47343444824219, "epoch": 0.7812041116005873, "fcm_dpo/beta": 0.004868891090154648, "fcm_dpo/delta": -0.11954863369464874, "fcm_dpo/margin": 115.62712097167969, "fcm_dpo/q_t": 0.37052595615386963, "grad_norm": 11.486917495727539, "learning_rate": 7.052201923388953e-08, "logits/chosen": -1.5253279209136963, "logits/rejected": -0.8838618993759155, "logps/chosen": -291.06329345703125, "logps/ref_chosen": -101.63691711425781, "logps/ref_rejected": -93.34539031982422, "logps/rejected": -398.39886474609375, "loss": 0.9723, "margin_dpo/margin_mean": 115.62712860107422, "margin_dpo/margin_std": 112.86474609375, "step": 532 }, { "KL/chosen_KL_mean": -183.12327575683594, "KL/mean": -216.97735595703125, "KL/rejected_KL_mean": -250.8314208984375, "KL/std": 108.60610961914062, "epoch": 0.7826725403817915, "fcm_dpo/beta": 0.004921652842313051, "fcm_dpo/delta": 0.12059812247753143, "fcm_dpo/margin": 67.70816040039062, "fcm_dpo/q_t": 0.42560046911239624, "grad_norm": 11.506623268127441, "learning_rate": 6.963101805503646e-08, "logits/chosen": -2.447031259536743, "logits/rejected": -1.8521575927734375, "logps/chosen": -273.86993408203125, "logps/ref_chosen": -90.74664306640625, "logps/ref_rejected": -82.88848876953125, "logps/rejected": -333.71990966796875, "loss": 1.1571, "margin_dpo/margin_mean": 67.70816040039062, "margin_dpo/margin_std": 116.53468322753906, "step": 533 }, { "KL/chosen_KL_mean": -163.21951293945312, "KL/mean": -213.57797241210938, "KL/rejected_KL_mean": -263.93646240234375, "KL/std": 102.16084289550781, "epoch": 0.7841409691629956, "fcm_dpo/beta": 0.004917072132229805, "fcm_dpo/delta": -0.0475621297955513, "fcm_dpo/margin": 100.7169418334961, "fcm_dpo/q_t": 0.3843523859977722, "grad_norm": 9.64101505279541, "learning_rate": 6.874476976660184e-08, "logits/chosen": -1.8881375789642334, "logits/rejected": -1.487180233001709, "logps/chosen": -249.850341796875, "logps/ref_chosen": -86.6308364868164, "logps/ref_rejected": -84.87196350097656, "logps/rejected": -348.80841064453125, "loss": 1.0065, "margin_dpo/margin_mean": 100.71693420410156, "margin_dpo/margin_std": 98.48625946044922, "step": 534 }, { "KL/chosen_KL_mean": -164.7309112548828, "KL/mean": -216.9906768798828, "KL/rejected_KL_mean": -269.2504577636719, "KL/std": 102.95307922363281, "epoch": 0.7856093979441997, "fcm_dpo/beta": 0.0048759700730443, "fcm_dpo/delta": -0.06299391388893127, "fcm_dpo/margin": 104.51951599121094, "fcm_dpo/q_t": 0.3836818337440491, "grad_norm": 8.676311492919922, "learning_rate": 6.786329772205246e-08, "logits/chosen": -1.4965322017669678, "logits/rejected": -1.3038082122802734, "logps/chosen": -255.0762939453125, "logps/ref_chosen": -90.34539031982422, "logps/ref_rejected": -100.7779769897461, "logps/rejected": -370.0284423828125, "loss": 1.0178, "margin_dpo/margin_mean": 104.51952362060547, "margin_dpo/margin_std": 117.58296203613281, "step": 535 }, { "KL/chosen_KL_mean": -165.7255859375, "KL/mean": -217.365478515625, "KL/rejected_KL_mean": -269.00537109375, "KL/std": 103.90904998779297, "epoch": 0.7870778267254038, "fcm_dpo/beta": 0.004792565479874611, "fcm_dpo/delta": -0.04752416908740997, "fcm_dpo/margin": 103.27981567382812, "fcm_dpo/q_t": 0.38502782583236694, "grad_norm": 9.45933723449707, "learning_rate": 6.698662514899638e-08, "logits/chosen": -1.9056837558746338, "logits/rejected": -1.6558258533477783, "logps/chosen": -244.68515014648438, "logps/ref_chosen": -78.95956420898438, "logps/ref_rejected": -93.08779907226562, "logps/rejected": -362.09320068359375, "loss": 1.0155, "margin_dpo/margin_mean": 103.27983093261719, "margin_dpo/margin_std": 108.91937255859375, "step": 536 }, { "KL/chosen_KL_mean": -142.94801330566406, "KL/mean": -198.394775390625, "KL/rejected_KL_mean": -253.84153747558594, "KL/std": 107.80245971679688, "epoch": 0.788546255506608, "fcm_dpo/beta": 0.004764278419315815, "fcm_dpo/delta": -0.08304021507501602, "fcm_dpo/margin": 110.89352416992188, "fcm_dpo/q_t": 0.37737858295440674, "grad_norm": 9.695345878601074, "learning_rate": 6.611477514857114e-08, "logits/chosen": -1.4823005199432373, "logits/rejected": -0.7776378393173218, "logps/chosen": -229.65740966796875, "logps/ref_chosen": -86.70939636230469, "logps/ref_rejected": -78.90766906738281, "logps/rejected": -332.74920654296875, "loss": 0.9888, "margin_dpo/margin_mean": 110.89352416992188, "margin_dpo/margin_std": 104.5932846069336, "step": 537 }, { "KL/chosen_KL_mean": -194.40371704101562, "KL/mean": -239.60638427734375, "KL/rejected_KL_mean": -284.80908203125, "KL/std": 95.72799682617188, "epoch": 0.7900146842878121, "fcm_dpo/beta": 0.004731073975563049, "fcm_dpo/delta": 0.0232668649405241, "fcm_dpo/margin": 90.4053726196289, "fcm_dpo/q_t": 0.400529146194458, "grad_norm": 9.736135482788086, "learning_rate": 6.524777069483525e-08, "logits/chosen": -1.6960458755493164, "logits/rejected": -1.1449663639068604, "logps/chosen": -292.8458557128906, "logps/ref_chosen": -98.44214630126953, "logps/ref_rejected": -91.09465026855469, "logps/rejected": -375.90374755859375, "loss": 1.0584, "margin_dpo/margin_mean": 90.40536499023438, "margin_dpo/margin_std": 101.99767303466797, "step": 538 }, { "KL/chosen_KL_mean": -170.5214080810547, "KL/mean": -217.82464599609375, "KL/rejected_KL_mean": -265.12786865234375, "KL/std": 105.82832336425781, "epoch": 0.7914831130690162, "fcm_dpo/beta": 0.004737939685583115, "fcm_dpo/delta": 0.0018382698763161898, "fcm_dpo/margin": 94.60646057128906, "fcm_dpo/q_t": 0.39711710810661316, "grad_norm": 8.939530372619629, "learning_rate": 6.438563463416221e-08, "logits/chosen": -2.4470839500427246, "logits/rejected": -1.9688284397125244, "logps/chosen": -261.4535217285156, "logps/ref_chosen": -90.93211364746094, "logps/ref_rejected": -97.44546508789062, "logps/rejected": -362.5733337402344, "loss": 1.0634, "margin_dpo/margin_mean": 94.60646057128906, "margin_dpo/margin_std": 120.57238006591797, "step": 539 }, { "KL/chosen_KL_mean": -180.2310333251953, "KL/mean": -237.65408325195312, "KL/rejected_KL_mean": -295.0771484375, "KL/std": 114.18052673339844, "epoch": 0.7929515418502202, "fcm_dpo/beta": 0.004691114649176598, "fcm_dpo/delta": -0.09378757327795029, "fcm_dpo/margin": 114.84609985351562, "fcm_dpo/q_t": 0.37873169779777527, "grad_norm": 10.716968536376953, "learning_rate": 6.352838968463919e-08, "logits/chosen": -2.0232460498809814, "logits/rejected": -1.7843815088272095, "logps/chosen": -288.1009521484375, "logps/ref_chosen": -107.86990356445312, "logps/ref_rejected": -121.39350128173828, "logps/rejected": -416.47064208984375, "loss": 1.0075, "margin_dpo/margin_mean": 114.84609985351562, "margin_dpo/margin_std": 132.90914916992188, "step": 540 }, { "KL/chosen_KL_mean": -200.82887268066406, "KL/mean": -236.84466552734375, "KL/rejected_KL_mean": -272.8604431152344, "KL/std": 95.08665466308594, "epoch": 0.7944199706314243, "fcm_dpo/beta": 0.004642473068088293, "fcm_dpo/delta": -0.006293103098869324, "fcm_dpo/margin": 72.03156280517578, "fcm_dpo/q_t": 0.42235273122787476, "grad_norm": 10.101605415344238, "learning_rate": 6.267605843546767e-08, "logits/chosen": -2.330595016479492, "logits/rejected": -1.8627632856369019, "logps/chosen": -307.3963623046875, "logps/ref_chosen": -106.56748962402344, "logps/ref_rejected": -109.06633758544922, "logps/rejected": -381.9267578125, "loss": 1.1382, "margin_dpo/margin_mean": 72.03156280517578, "margin_dpo/margin_std": 101.48991394042969, "step": 541 }, { "KL/chosen_KL_mean": -170.8132781982422, "KL/mean": -225.95448303222656, "KL/rejected_KL_mean": -281.095703125, "KL/std": 106.34501647949219, "epoch": 0.7958883994126285, "fcm_dpo/beta": 0.004601244814693928, "fcm_dpo/delta": -0.060502491891384125, "fcm_dpo/margin": 110.28240966796875, "fcm_dpo/q_t": 0.38187992572784424, "grad_norm": 10.427596092224121, "learning_rate": 6.182866334636888e-08, "logits/chosen": -2.3982412815093994, "logits/rejected": -2.2766671180725098, "logps/chosen": -258.18255615234375, "logps/ref_chosen": -87.36929321289062, "logps/ref_rejected": -102.82034301757812, "logps/rejected": -383.9160461425781, "loss": 1.0078, "margin_dpo/margin_mean": 110.28240966796875, "margin_dpo/margin_std": 114.96615600585938, "step": 542 }, { "KL/chosen_KL_mean": -195.9742431640625, "KL/mean": -225.4713897705078, "KL/rejected_KL_mean": -254.96853637695312, "KL/std": 112.53939819335938, "epoch": 0.7973568281938326, "fcm_dpo/beta": 0.004633826203644276, "fcm_dpo/delta": 0.05118121951818466, "fcm_dpo/margin": 58.994293212890625, "fcm_dpo/q_t": 0.43852630257606506, "grad_norm": 12.674715042114258, "learning_rate": 6.098622674699147e-08, "logits/chosen": -1.7109944820404053, "logits/rejected": -1.7543551921844482, "logps/chosen": -283.1268310546875, "logps/ref_chosen": -87.152587890625, "logps/ref_rejected": -111.53172302246094, "logps/rejected": -366.5002746582031, "loss": 1.2258, "margin_dpo/margin_mean": 58.994300842285156, "margin_dpo/margin_std": 135.8321533203125, "step": 543 }, { "KL/chosen_KL_mean": -203.7451171875, "KL/mean": -240.53851318359375, "KL/rejected_KL_mean": -277.3319091796875, "KL/std": 105.84512329101562, "epoch": 0.7988252569750367, "fcm_dpo/beta": 0.004706394858658314, "fcm_dpo/delta": 0.10743935406208038, "fcm_dpo/margin": 73.58678436279297, "fcm_dpo/q_t": 0.4208550453186035, "grad_norm": 9.800400733947754, "learning_rate": 6.01487708363232e-08, "logits/chosen": -1.6806445121765137, "logits/rejected": -1.5842194557189941, "logps/chosen": -289.89202880859375, "logps/ref_chosen": -86.14691162109375, "logps/ref_rejected": -106.3939208984375, "logps/rejected": -383.725830078125, "loss": 1.1315, "margin_dpo/margin_mean": 73.58677673339844, "margin_dpo/margin_std": 109.59210205078125, "step": 544 }, { "KL/chosen_KL_mean": -182.01885986328125, "KL/mean": -227.2618408203125, "KL/rejected_KL_mean": -272.50482177734375, "KL/std": 104.80342102050781, "epoch": 0.8002936857562408, "fcm_dpo/beta": 0.004756860435009003, "fcm_dpo/delta": 0.02035892754793167, "fcm_dpo/margin": 90.48593139648438, "fcm_dpo/q_t": 0.4016454219818115, "grad_norm": 9.674954414367676, "learning_rate": 5.9316317682106294e-08, "logits/chosen": -1.644040584564209, "logits/rejected": -1.2250263690948486, "logps/chosen": -278.443115234375, "logps/ref_chosen": -96.42424011230469, "logps/ref_rejected": -100.73405456542969, "logps/rejected": -373.2388610839844, "loss": 1.0912, "margin_dpo/margin_mean": 90.48593139648438, "margin_dpo/margin_std": 129.81398010253906, "step": 545 }, { "KL/chosen_KL_mean": -187.50372314453125, "KL/mean": -228.64816284179688, "KL/rejected_KL_mean": -269.7926025390625, "KL/std": 106.960693359375, "epoch": 0.801762114537445, "fcm_dpo/beta": 0.004786289297044277, "fcm_dpo/delta": 0.0584806352853775, "fcm_dpo/margin": 82.28887939453125, "fcm_dpo/q_t": 0.4085407853126526, "grad_norm": 9.608528137207031, "learning_rate": 5.848888922025552e-08, "logits/chosen": -1.3605380058288574, "logits/rejected": -0.9943258762359619, "logps/chosen": -273.52484130859375, "logps/ref_chosen": -86.02112579345703, "logps/ref_rejected": -87.37263488769531, "logps/rejected": -357.16522216796875, "loss": 1.1057, "margin_dpo/margin_mean": 82.28887939453125, "margin_dpo/margin_std": 117.43290710449219, "step": 546 }, { "KL/chosen_KL_mean": -180.25697326660156, "KL/mean": -229.6207275390625, "KL/rejected_KL_mean": -278.98443603515625, "KL/std": 104.44168090820312, "epoch": 0.8032305433186491, "fcm_dpo/beta": 0.004806514363735914, "fcm_dpo/delta": -0.02588154375553131, "fcm_dpo/margin": 98.72744750976562, "fcm_dpo/q_t": 0.3885681927204132, "grad_norm": 9.542840957641602, "learning_rate": 5.7666507254280265e-08, "logits/chosen": -1.700653076171875, "logits/rejected": -1.2276029586791992, "logps/chosen": -280.907958984375, "logps/ref_chosen": -100.6510009765625, "logps/ref_rejected": -97.9117431640625, "logps/rejected": -376.89617919921875, "loss": 1.0144, "margin_dpo/margin_mean": 98.72744750976562, "margin_dpo/margin_std": 92.12249755859375, "step": 547 }, { "KL/chosen_KL_mean": -177.18148803710938, "KL/mean": -216.97726440429688, "KL/rejected_KL_mean": -256.77301025390625, "KL/std": 106.55033874511719, "epoch": 0.8046989720998532, "fcm_dpo/beta": 0.00483623705804348, "fcm_dpo/delta": 0.06763452291488647, "fcm_dpo/margin": 79.59153747558594, "fcm_dpo/q_t": 0.412605345249176, "grad_norm": 11.051373481750488, "learning_rate": 5.684919345471029e-08, "logits/chosen": -2.2747554779052734, "logits/rejected": -1.9185969829559326, "logps/chosen": -273.21160888671875, "logps/ref_chosen": -96.03010559082031, "logps/ref_rejected": -98.41322326660156, "logps/rejected": -355.1862487792969, "loss": 1.1371, "margin_dpo/margin_mean": 79.59153747558594, "margin_dpo/margin_std": 133.7232666015625, "step": 548 }, { "KL/chosen_KL_mean": -166.967529296875, "KL/mean": -208.77001953125, "KL/rejected_KL_mean": -250.57254028320312, "KL/std": 100.32382202148438, "epoch": 0.8061674008810573, "fcm_dpo/beta": 0.004903590306639671, "fcm_dpo/delta": 0.04106505215167999, "fcm_dpo/margin": 83.60503387451172, "fcm_dpo/q_t": 0.40571844577789307, "grad_norm": 11.730501174926758, "learning_rate": 5.603696935852426e-08, "logits/chosen": -1.8173472881317139, "logits/rejected": -1.3947107791900635, "logps/chosen": -250.41848754882812, "logps/ref_chosen": -83.45095825195312, "logps/ref_rejected": -81.38023376464844, "logps/rejected": -331.9527587890625, "loss": 1.1079, "margin_dpo/margin_mean": 83.60504150390625, "margin_dpo/margin_std": 123.45405578613281, "step": 549 }, { "KL/chosen_KL_mean": -186.98297119140625, "KL/mean": -227.52963256835938, "KL/rejected_KL_mean": -268.0762939453125, "KL/std": 96.12370300292969, "epoch": 0.8076358296622613, "fcm_dpo/beta": 0.004938596859574318, "fcm_dpo/delta": 0.051362667232751846, "fcm_dpo/margin": 81.09334564208984, "fcm_dpo/q_t": 0.4077901840209961, "grad_norm": 9.947425842285156, "learning_rate": 5.5229856368582376e-08, "logits/chosen": -1.6357877254486084, "logits/rejected": -1.4200901985168457, "logps/chosen": -276.57733154296875, "logps/ref_chosen": -89.59434509277344, "logps/ref_rejected": -101.31549835205078, "logps/rejected": -369.39178466796875, "loss": 1.0886, "margin_dpo/margin_mean": 81.09333801269531, "margin_dpo/margin_std": 102.67247009277344, "step": 550 }, { "KL/chosen_KL_mean": -166.31878662109375, "KL/mean": -215.8612518310547, "KL/rejected_KL_mean": -265.40374755859375, "KL/std": 109.61463928222656, "epoch": 0.8091042584434655, "fcm_dpo/beta": 0.00488814152777195, "fcm_dpo/delta": -0.0368325300514698, "fcm_dpo/margin": 99.08496856689453, "fcm_dpo/q_t": 0.3900824189186096, "grad_norm": 12.51354694366455, "learning_rate": 5.4427875753062734e-08, "logits/chosen": -1.5582746267318726, "logits/rejected": -1.4134386777877808, "logps/chosen": -261.40093994140625, "logps/ref_chosen": -95.08216094970703, "logps/ref_rejected": -117.7192611694336, "logps/rejected": -383.1230163574219, "loss": 1.0274, "margin_dpo/margin_mean": 99.0849609375, "margin_dpo/margin_std": 110.4867172241211, "step": 551 }, { "KL/chosen_KL_mean": -157.32858276367188, "KL/mean": -227.43338012695312, "KL/rejected_KL_mean": -297.5381774902344, "KL/std": 118.01866912841797, "epoch": 0.8105726872246696, "fcm_dpo/beta": 0.004703568760305643, "fcm_dpo/delta": -0.22745440900325775, "fcm_dpo/margin": 140.20960998535156, "fcm_dpo/q_t": 0.3480021357536316, "grad_norm": 11.194291114807129, "learning_rate": 5.363104864490034e-08, "logits/chosen": -2.0567474365234375, "logits/rejected": -1.6932432651519775, "logps/chosen": -258.7377624511719, "logps/ref_chosen": -101.40918731689453, "logps/ref_rejected": -110.83843994140625, "logps/rejected": -408.3766174316406, "loss": 0.9101, "margin_dpo/margin_mean": 140.20960998535156, "margin_dpo/margin_std": 121.55499267578125, "step": 552 }, { "KL/chosen_KL_mean": -172.49346923828125, "KL/mean": -214.55206298828125, "KL/rejected_KL_mean": -256.61065673828125, "KL/std": 109.88848114013672, "epoch": 0.8120411160058737, "fcm_dpo/beta": 0.0047313496470451355, "fcm_dpo/delta": 0.054036304354667664, "fcm_dpo/margin": 84.1171646118164, "fcm_dpo/q_t": 0.4077776074409485, "grad_norm": 9.854809761047363, "learning_rate": 5.2839396041230415e-08, "logits/chosen": -1.5239918231964111, "logits/rejected": -1.1946778297424316, "logps/chosen": -270.4573669433594, "logps/ref_chosen": -97.96388244628906, "logps/ref_rejected": -103.57719421386719, "logps/rejected": -360.1878356933594, "loss": 1.1011, "margin_dpo/margin_mean": 84.1171646118164, "margin_dpo/margin_std": 117.28703308105469, "step": 553 }, { "KL/chosen_KL_mean": -181.18539428710938, "KL/mean": -227.14401245117188, "KL/rejected_KL_mean": -273.1026611328125, "KL/std": 110.34535217285156, "epoch": 0.8135095447870778, "fcm_dpo/beta": 0.0047657424584031105, "fcm_dpo/delta": 0.012077197432518005, "fcm_dpo/margin": 91.9172592163086, "fcm_dpo/q_t": 0.4010778069496155, "grad_norm": 10.021539688110352, "learning_rate": 5.205293880283551e-08, "logits/chosen": -1.9108518362045288, "logits/rejected": -1.1373028755187988, "logps/chosen": -284.8721923828125, "logps/ref_chosen": -103.68680572509766, "logps/ref_rejected": -92.99210357666016, "logps/rejected": -366.0947570800781, "loss": 1.1034, "margin_dpo/margin_mean": 91.91726684570312, "margin_dpo/margin_std": 141.96827697753906, "step": 554 }, { "KL/chosen_KL_mean": -188.48377990722656, "KL/mean": -231.75, "KL/rejected_KL_mean": -275.0162353515625, "KL/std": 114.02422332763672, "epoch": 0.8149779735682819, "fcm_dpo/beta": 0.004774688743054867, "fcm_dpo/delta": 0.038429953157901764, "fcm_dpo/margin": 86.5324478149414, "fcm_dpo/q_t": 0.4072245955467224, "grad_norm": 10.103257179260254, "learning_rate": 5.127169765359515e-08, "logits/chosen": -1.7459869384765625, "logits/rejected": -1.6526474952697754, "logps/chosen": -274.6874084472656, "logps/ref_chosen": -86.20362854003906, "logps/ref_rejected": -114.54719543457031, "logps/rejected": -389.5634460449219, "loss": 1.1053, "margin_dpo/margin_mean": 86.53244018554688, "margin_dpo/margin_std": 130.62356567382812, "step": 555 }, { "KL/chosen_KL_mean": -169.15457153320312, "KL/mean": -215.77145385742188, "KL/rejected_KL_mean": -262.38836669921875, "KL/std": 101.57382202148438, "epoch": 0.8164464023494861, "fcm_dpo/beta": 0.004775206092745066, "fcm_dpo/delta": 0.0048639290034770966, "fcm_dpo/margin": 93.2337646484375, "fcm_dpo/q_t": 0.3975391983985901, "grad_norm": 10.624995231628418, "learning_rate": 5.049569317994012e-08, "logits/chosen": -1.567216157913208, "logits/rejected": -1.261490821838379, "logps/chosen": -257.55804443359375, "logps/ref_chosen": -88.40348815917969, "logps/ref_rejected": -106.63911437988281, "logps/rejected": -369.0274658203125, "loss": 1.0616, "margin_dpo/margin_mean": 93.2337646484375, "margin_dpo/margin_std": 115.73711395263672, "step": 556 }, { "KL/chosen_KL_mean": -180.4205780029297, "KL/mean": -234.20274353027344, "KL/rejected_KL_mean": -287.98492431640625, "KL/std": 116.70541381835938, "epoch": 0.8179148311306902, "fcm_dpo/beta": 0.004723517224192619, "fcm_dpo/delta": -0.061944834887981415, "fcm_dpo/margin": 107.5643539428711, "fcm_dpo/q_t": 0.385007381439209, "grad_norm": 8.434736251831055, "learning_rate": 4.9724945830310144e-08, "logits/chosen": -1.986304759979248, "logits/rejected": -1.8124982118606567, "logps/chosen": -279.25579833984375, "logps/ref_chosen": -98.83523559570312, "logps/ref_rejected": -115.64888763427734, "logps/rejected": -403.6338195800781, "loss": 1.025, "margin_dpo/margin_mean": 107.5643539428711, "margin_dpo/margin_std": 126.51203155517578, "step": 557 }, { "KL/chosen_KL_mean": -179.14883422851562, "KL/mean": -235.4542999267578, "KL/rejected_KL_mean": -291.7597351074219, "KL/std": 100.75602722167969, "epoch": 0.8193832599118943, "fcm_dpo/beta": 0.004664961248636246, "fcm_dpo/delta": -0.07977241277694702, "fcm_dpo/margin": 112.61088562011719, "fcm_dpo/q_t": 0.37782907485961914, "grad_norm": 11.25912094116211, "learning_rate": 4.8959475914614554e-08, "logits/chosen": -2.3992674350738525, "logits/rejected": -1.7925952672958374, "logps/chosen": -285.1606750488281, "logps/ref_chosen": -106.01183319091797, "logps/ref_rejected": -108.39131927490234, "logps/rejected": -400.15106201171875, "loss": 0.9847, "margin_dpo/margin_mean": 112.61088562011719, "margin_dpo/margin_std": 104.0299072265625, "step": 558 }, { "KL/chosen_KL_mean": -193.26144409179688, "KL/mean": -238.98703002929688, "KL/rejected_KL_mean": -284.71258544921875, "KL/std": 103.2884292602539, "epoch": 0.8208516886930984, "fcm_dpo/beta": 0.004676289856433868, "fcm_dpo/delta": 0.02331012487411499, "fcm_dpo/margin": 91.45114135742188, "fcm_dpo/q_t": 0.400562584400177, "grad_norm": 11.394143104553223, "learning_rate": 4.8199303603697614e-08, "logits/chosen": -2.356294631958008, "logits/rejected": -1.8778878450393677, "logps/chosen": -293.6861877441406, "logps/ref_chosen": -100.42474365234375, "logps/ref_rejected": -100.41270446777344, "logps/rejected": -385.12530517578125, "loss": 1.0747, "margin_dpo/margin_mean": 91.45114135742188, "margin_dpo/margin_std": 116.84442138671875, "step": 559 }, { "KL/chosen_KL_mean": -185.6719970703125, "KL/mean": -229.6200714111328, "KL/rejected_KL_mean": -273.568115234375, "KL/std": 101.43780517578125, "epoch": 0.8223201174743024, "fcm_dpo/beta": 0.004700183868408203, "fcm_dpo/delta": 0.03847907856106758, "fcm_dpo/margin": 87.89613342285156, "fcm_dpo/q_t": 0.404948353767395, "grad_norm": 12.233609199523926, "learning_rate": 4.7444448928806615e-08, "logits/chosen": -1.284921407699585, "logits/rejected": -0.8312946557998657, "logps/chosen": -273.3075256347656, "logps/ref_chosen": -87.63551330566406, "logps/ref_rejected": -87.37985229492188, "logps/rejected": -360.947998046875, "loss": 1.0867, "margin_dpo/margin_mean": 87.89612579345703, "margin_dpo/margin_std": 117.25211334228516, "step": 560 }, { "KL/chosen_KL_mean": -193.4197540283203, "KL/mean": -234.80206298828125, "KL/rejected_KL_mean": -276.18438720703125, "KL/std": 100.51805114746094, "epoch": 0.8237885462555066, "fcm_dpo/beta": 0.0047654202207922935, "fcm_dpo/delta": 0.05763612687587738, "fcm_dpo/margin": 82.7646255493164, "fcm_dpo/q_t": 0.40828776359558105, "grad_norm": 8.805339813232422, "learning_rate": 4.669493178106432e-08, "logits/chosen": -2.100630760192871, "logits/rejected": -2.0700228214263916, "logps/chosen": -268.3011474609375, "logps/ref_chosen": -74.88140106201172, "logps/ref_rejected": -106.04796600341797, "logps/rejected": -382.23236083984375, "loss": 1.0802, "margin_dpo/margin_mean": 82.76463317871094, "margin_dpo/margin_std": 96.5006103515625, "step": 561 }, { "KL/chosen_KL_mean": -175.75299072265625, "KL/mean": -217.31918334960938, "KL/rejected_KL_mean": -258.8854064941406, "KL/std": 101.22796630859375, "epoch": 0.8252569750367107, "fcm_dpo/beta": 0.004790683276951313, "fcm_dpo/delta": 0.05383138731122017, "fcm_dpo/margin": 83.1324234008789, "fcm_dpo/q_t": 0.40960556268692017, "grad_norm": 13.344965934753418, "learning_rate": 4.5950771910944596e-08, "logits/chosen": -2.152010440826416, "logits/rejected": -1.6993939876556396, "logps/chosen": -276.41619873046875, "logps/ref_chosen": -100.66322326660156, "logps/ref_rejected": -102.24087524414062, "logps/rejected": -361.12628173828125, "loss": 1.1124, "margin_dpo/margin_mean": 83.13243103027344, "margin_dpo/margin_std": 126.220947265625, "step": 562 }, { "KL/chosen_KL_mean": -186.78192138671875, "KL/mean": -221.53622436523438, "KL/rejected_KL_mean": -256.29052734375, "KL/std": 101.7255859375, "epoch": 0.8267254038179148, "fcm_dpo/beta": 0.004918321035802364, "fcm_dpo/delta": 0.11130814254283905, "fcm_dpo/margin": 69.50858306884766, "fcm_dpo/q_t": 0.4213360846042633, "grad_norm": 11.235312461853027, "learning_rate": 4.521198892775202e-08, "logits/chosen": -1.5184147357940674, "logits/rejected": -1.170419454574585, "logps/chosen": -283.0411376953125, "logps/ref_chosen": -96.25921630859375, "logps/ref_rejected": -101.20503997802734, "logps/rejected": -357.49554443359375, "loss": 1.1416, "margin_dpo/margin_mean": 69.50858306884766, "margin_dpo/margin_std": 108.3563232421875, "step": 563 }, { "KL/chosen_KL_mean": -181.8468780517578, "KL/mean": -220.7062530517578, "KL/rejected_KL_mean": -259.5656433105469, "KL/std": 102.5434341430664, "epoch": 0.8281938325991189, "fcm_dpo/beta": 0.004989357199519873, "fcm_dpo/delta": 0.06443215906620026, "fcm_dpo/margin": 77.71874237060547, "fcm_dpo/q_t": 0.4094778597354889, "grad_norm": 12.664972305297852, "learning_rate": 4.447860229910544e-08, "logits/chosen": -2.2479751110076904, "logits/rejected": -1.6669130325317383, "logps/chosen": -294.8072814941406, "logps/ref_chosen": -112.96040344238281, "logps/ref_rejected": -99.52803802490234, "logps/rejected": -359.09368896484375, "loss": 1.0882, "margin_dpo/margin_mean": 77.71875, "margin_dpo/margin_std": 93.1473159790039, "step": 564 }, { "KL/chosen_KL_mean": -177.90982055664062, "KL/mean": -226.44406127929688, "KL/rejected_KL_mean": -274.9783020019531, "KL/std": 110.97595977783203, "epoch": 0.8296622613803231, "fcm_dpo/beta": 0.0049600861966609955, "fcm_dpo/delta": -0.03316520154476166, "fcm_dpo/margin": 97.06849670410156, "fcm_dpo/q_t": 0.39050543308258057, "grad_norm": 9.854287147521973, "learning_rate": 4.375063135042445e-08, "logits/chosen": -1.5753204822540283, "logits/rejected": -1.057618260383606, "logps/chosen": -273.275390625, "logps/ref_chosen": -95.36558532714844, "logps/ref_rejected": -92.21737670898438, "logps/rejected": -367.1956787109375, "loss": 1.0441, "margin_dpo/margin_mean": 97.06849670410156, "margin_dpo/margin_std": 120.12101745605469, "step": 565 }, { "KL/chosen_KL_mean": -185.07666015625, "KL/mean": -231.44873046875, "KL/rejected_KL_mean": -277.82080078125, "KL/std": 113.49041748046875, "epoch": 0.8311306901615272, "fcm_dpo/beta": 0.004964838735759258, "fcm_dpo/delta": -0.011084163561463356, "fcm_dpo/margin": 92.74412536621094, "fcm_dpo/q_t": 0.39338457584381104, "grad_norm": 9.84110164642334, "learning_rate": 4.3028095264420525e-08, "logits/chosen": -1.8871031999588013, "logits/rejected": -1.452309489250183, "logps/chosen": -284.04986572265625, "logps/ref_chosen": -98.97320556640625, "logps/ref_rejected": -110.82090759277344, "logps/rejected": -388.6416931152344, "loss": 1.0524, "margin_dpo/margin_mean": 92.74413299560547, "margin_dpo/margin_std": 112.541259765625, "step": 566 }, { "KL/chosen_KL_mean": -182.21493530273438, "KL/mean": -223.1317138671875, "KL/rejected_KL_mean": -264.0484924316406, "KL/std": 102.00434112548828, "epoch": 0.8325991189427313, "fcm_dpo/beta": 0.004959041718393564, "fcm_dpo/delta": 0.04581092298030853, "fcm_dpo/margin": 81.83354187011719, "fcm_dpo/q_t": 0.40543872117996216, "grad_norm": 10.911598205566406, "learning_rate": 4.231101308059165e-08, "logits/chosen": -2.275144100189209, "logits/rejected": -1.6030158996582031, "logps/chosen": -274.91851806640625, "logps/ref_chosen": -92.7035903930664, "logps/ref_rejected": -91.22431945800781, "logps/rejected": -355.2728271484375, "loss": 1.0913, "margin_dpo/margin_mean": 81.83353424072266, "margin_dpo/margin_std": 107.32051086425781, "step": 567 }, { "KL/chosen_KL_mean": -173.46554565429688, "KL/mean": -226.94647216796875, "KL/rejected_KL_mean": -280.4273986816406, "KL/std": 107.77141571044922, "epoch": 0.8340675477239354, "fcm_dpo/beta": 0.004917928948998451, "fcm_dpo/delta": -0.08075903356075287, "fcm_dpo/margin": 106.96184539794922, "fcm_dpo/q_t": 0.3789040446281433, "grad_norm": 9.328248977661133, "learning_rate": 4.1599403694720145e-08, "logits/chosen": -1.3077096939086914, "logits/rejected": -1.1737713813781738, "logps/chosen": -246.59780883789062, "logps/ref_chosen": -73.13226318359375, "logps/ref_rejected": -94.95265197753906, "logps/rejected": -375.38006591796875, "loss": 0.9935, "margin_dpo/margin_mean": 106.96184539794922, "margin_dpo/margin_std": 108.15916442871094, "step": 568 }, { "KL/chosen_KL_mean": -183.51248168945312, "KL/mean": -226.96575927734375, "KL/rejected_KL_mean": -270.4190673828125, "KL/std": 108.67988586425781, "epoch": 0.8355359765051396, "fcm_dpo/beta": 0.004938815720379353, "fcm_dpo/delta": 0.021612131968140602, "fcm_dpo/margin": 86.90658569335938, "fcm_dpo/q_t": 0.4017311930656433, "grad_norm": 15.93490982055664, "learning_rate": 4.089328585837512e-08, "logits/chosen": -1.8864727020263672, "logits/rejected": -1.5055018663406372, "logps/chosen": -272.72955322265625, "logps/ref_chosen": -89.21708679199219, "logps/ref_rejected": -86.41105651855469, "logps/rejected": -356.8301086425781, "loss": 1.1006, "margin_dpo/margin_mean": 86.90658569335938, "margin_dpo/margin_std": 129.9566650390625, "step": 569 }, { "KL/chosen_KL_mean": -163.5234375, "KL/mean": -220.2521514892578, "KL/rejected_KL_mean": -276.9808654785156, "KL/std": 103.90315246582031, "epoch": 0.8370044052863436, "fcm_dpo/beta": 0.0048536863178014755, "fcm_dpo/delta": -0.10666178166866302, "fcm_dpo/margin": 113.45742797851562, "fcm_dpo/q_t": 0.3717441260814667, "grad_norm": 9.970564842224121, "learning_rate": 4.019267817841834e-08, "logits/chosen": -2.5078535079956055, "logits/rejected": -1.846949577331543, "logps/chosen": -258.28619384765625, "logps/ref_chosen": -94.76277160644531, "logps/ref_rejected": -88.846923828125, "logps/rejected": -365.8277893066406, "loss": 0.9737, "margin_dpo/margin_mean": 113.45742797851562, "margin_dpo/margin_std": 108.15154266357422, "step": 570 }, { "KL/chosen_KL_mean": -170.45916748046875, "KL/mean": -220.59762573242188, "KL/rejected_KL_mean": -270.736083984375, "KL/std": 102.62117004394531, "epoch": 0.8384728340675477, "fcm_dpo/beta": 0.004818316549062729, "fcm_dpo/delta": -0.034890636801719666, "fcm_dpo/margin": 100.27690887451172, "fcm_dpo/q_t": 0.3898862898349762, "grad_norm": 12.527082443237305, "learning_rate": 3.9497599116513705e-08, "logits/chosen": -1.6255197525024414, "logits/rejected": -1.4647748470306396, "logps/chosen": -251.28045654296875, "logps/ref_chosen": -80.8212890625, "logps/ref_rejected": -97.01528930664062, "logps/rejected": -367.7513732910156, "loss": 1.0439, "margin_dpo/margin_mean": 100.27690887451172, "margin_dpo/margin_std": 123.81684875488281, "step": 571 }, { "KL/chosen_KL_mean": -174.47024536132812, "KL/mean": -225.146484375, "KL/rejected_KL_mean": -275.82269287109375, "KL/std": 108.68096160888672, "epoch": 0.8399412628487518, "fcm_dpo/beta": 0.004766533151268959, "fcm_dpo/delta": -0.034862007945775986, "fcm_dpo/margin": 101.35246276855469, "fcm_dpo/q_t": 0.39039546251296997, "grad_norm": 10.338237762451172, "learning_rate": 3.880806698864086e-08, "logits/chosen": -1.8114025592803955, "logits/rejected": -1.6437764167785645, "logps/chosen": -251.8501739501953, "logps/ref_chosen": -77.37992858886719, "logps/ref_rejected": -90.56597900390625, "logps/rejected": -366.388671875, "loss": 1.0323, "margin_dpo/margin_mean": 101.35246276855469, "margin_dpo/margin_std": 117.46690368652344, "step": 572 }, { "KL/chosen_KL_mean": -168.44476318359375, "KL/mean": -216.23353576660156, "KL/rejected_KL_mean": -264.0223388671875, "KL/std": 108.87879943847656, "epoch": 0.8414096916299559, "fcm_dpo/beta": 0.0047918884083628654, "fcm_dpo/delta": -0.009356088936328888, "fcm_dpo/margin": 95.57757568359375, "fcm_dpo/q_t": 0.3938070237636566, "grad_norm": 9.493922233581543, "learning_rate": 3.812409996461275e-08, "logits/chosen": -2.253699541091919, "logits/rejected": -2.0283048152923584, "logps/chosen": -252.1508331298828, "logps/ref_chosen": -83.7060775756836, "logps/ref_rejected": -90.69746398925781, "logps/rejected": -354.71978759765625, "loss": 1.0547, "margin_dpo/margin_mean": 95.57757568359375, "margin_dpo/margin_std": 114.06787872314453, "step": 573 }, { "KL/chosen_KL_mean": -178.24514770507812, "KL/mean": -228.97540283203125, "KL/rejected_KL_mean": -279.7056884765625, "KL/std": 107.54293823242188, "epoch": 0.8428781204111601, "fcm_dpo/beta": 0.0047480324283242226, "fcm_dpo/delta": -0.03346884623169899, "fcm_dpo/margin": 101.46055603027344, "fcm_dpo/q_t": 0.38947343826293945, "grad_norm": 10.1945161819458, "learning_rate": 3.74457160675965e-08, "logits/chosen": -2.303833484649658, "logits/rejected": -1.9847769737243652, "logps/chosen": -263.3746337890625, "logps/ref_chosen": -85.12948608398438, "logps/ref_rejected": -99.05403137207031, "logps/rejected": -378.75970458984375, "loss": 1.0407, "margin_dpo/margin_mean": 101.46055603027344, "margin_dpo/margin_std": 121.93394470214844, "step": 574 }, { "KL/chosen_KL_mean": -184.06289672851562, "KL/mean": -231.4376220703125, "KL/rejected_KL_mean": -278.8123779296875, "KL/std": 109.81668090820312, "epoch": 0.8443465491923642, "fcm_dpo/beta": 0.00472361221909523, "fcm_dpo/delta": 0.002552381716668606, "fcm_dpo/margin": 94.74946594238281, "fcm_dpo/q_t": 0.3979812562465668, "grad_norm": 12.434805870056152, "learning_rate": 3.677293317363864e-08, "logits/chosen": -1.3767006397247314, "logits/rejected": -0.9102492928504944, "logps/chosen": -289.833740234375, "logps/ref_chosen": -105.77084350585938, "logps/ref_rejected": -100.94944763183594, "logps/rejected": -379.7618103027344, "loss": 1.0723, "margin_dpo/margin_mean": 94.74946594238281, "margin_dpo/margin_std": 127.75698852539062, "step": 575 }, { "KL/chosen_KL_mean": -182.92852783203125, "KL/mean": -228.7220001220703, "KL/rejected_KL_mean": -274.5154724121094, "KL/std": 101.88496398925781, "epoch": 0.8458149779735683, "fcm_dpo/beta": 0.004745191428810358, "fcm_dpo/delta": 0.016020432114601135, "fcm_dpo/margin": 91.58695220947266, "fcm_dpo/q_t": 0.4000887870788574, "grad_norm": 11.390907287597656, "learning_rate": 3.6105769011194224e-08, "logits/chosen": -1.8920578956604004, "logits/rejected": -1.8284693956375122, "logps/chosen": -264.133056640625, "logps/ref_chosen": -81.20452880859375, "logps/ref_rejected": -105.40940856933594, "logps/rejected": -379.92486572265625, "loss": 1.078, "margin_dpo/margin_mean": 91.58695983886719, "margin_dpo/margin_std": 122.36265563964844, "step": 576 }, { "KL/chosen_KL_mean": -163.3242950439453, "KL/mean": -220.65541076660156, "KL/rejected_KL_mean": -277.98651123046875, "KL/std": 110.72624206542969, "epoch": 0.8472834067547724, "fcm_dpo/beta": 0.004659945145249367, "fcm_dpo/delta": -0.08975732326507568, "fcm_dpo/margin": 114.6622085571289, "fcm_dpo/q_t": 0.3772873878479004, "grad_norm": 8.622326850891113, "learning_rate": 3.5444241160659304e-08, "logits/chosen": -1.8065013885498047, "logits/rejected": -1.0841326713562012, "logps/chosen": -264.7884826660156, "logps/ref_chosen": -101.46419525146484, "logps/ref_rejected": -89.32637023925781, "logps/rejected": -367.3128662109375, "loss": 1.0027, "margin_dpo/margin_mean": 114.6622085571289, "margin_dpo/margin_std": 126.03982543945312, "step": 577 }, { "KL/chosen_KL_mean": -195.02908325195312, "KL/mean": -236.84829711914062, "KL/rejected_KL_mean": -278.66754150390625, "KL/std": 105.22526550292969, "epoch": 0.8487518355359766, "fcm_dpo/beta": 0.004698064178228378, "fcm_dpo/delta": 0.05939852446317673, "fcm_dpo/margin": 83.638427734375, "fcm_dpo/q_t": 0.40966176986694336, "grad_norm": 14.506484031677246, "learning_rate": 3.478836705390808e-08, "logits/chosen": -1.154599666595459, "logits/rejected": -0.907585859298706, "logps/chosen": -272.3215026855469, "logps/ref_chosen": -77.29241943359375, "logps/ref_rejected": -88.24701690673828, "logps/rejected": -366.91455078125, "loss": 1.1063, "margin_dpo/margin_mean": 83.638427734375, "margin_dpo/margin_std": 120.07523345947266, "step": 578 }, { "KL/chosen_KL_mean": -174.91900634765625, "KL/mean": -217.19815063476562, "KL/rejected_KL_mean": -259.4773254394531, "KL/std": 105.2657699584961, "epoch": 0.8502202643171806, "fcm_dpo/beta": 0.004728085361421108, "fcm_dpo/delta": 0.052112363278865814, "fcm_dpo/margin": 84.5583267211914, "fcm_dpo/q_t": 0.408389151096344, "grad_norm": 8.933969497680664, "learning_rate": 3.41381639738331e-08, "logits/chosen": -1.7135779857635498, "logits/rejected": -1.408900260925293, "logps/chosen": -262.2620544433594, "logps/ref_chosen": -87.34305572509766, "logps/ref_rejected": -102.12100982666016, "logps/rejected": -361.59832763671875, "loss": 1.092, "margin_dpo/margin_mean": 84.5583267211914, "margin_dpo/margin_std": 107.45379638671875, "step": 579 }, { "KL/chosen_KL_mean": -167.37413024902344, "KL/mean": -218.71963500976562, "KL/rejected_KL_mean": -270.06512451171875, "KL/std": 103.72488403320312, "epoch": 0.8516886930983847, "fcm_dpo/beta": 0.004716012626886368, "fcm_dpo/delta": -0.036564163863658905, "fcm_dpo/margin": 102.69097900390625, "fcm_dpo/q_t": 0.39027202129364014, "grad_norm": 10.67998218536377, "learning_rate": 3.349364905389032e-08, "logits/chosen": -1.3836811780929565, "logits/rejected": -1.0624589920043945, "logps/chosen": -256.30291748046875, "logps/ref_chosen": -88.92879486083984, "logps/ref_rejected": -88.69661712646484, "logps/rejected": -358.76171875, "loss": 1.0399, "margin_dpo/margin_mean": 102.69097900390625, "margin_dpo/margin_std": 124.42339324951172, "step": 580 }, { "KL/chosen_KL_mean": -177.17019653320312, "KL/mean": -223.97393798828125, "KL/rejected_KL_mean": -270.77764892578125, "KL/std": 106.09624481201172, "epoch": 0.8531571218795888, "fcm_dpo/beta": 0.004737949930131435, "fcm_dpo/delta": 0.006753142923116684, "fcm_dpo/margin": 93.60745239257812, "fcm_dpo/q_t": 0.39859747886657715, "grad_norm": 11.960638046264648, "learning_rate": 3.285483927764726e-08, "logits/chosen": -2.027475357055664, "logits/rejected": -1.6816093921661377, "logps/chosen": -269.3934326171875, "logps/ref_chosen": -92.22323608398438, "logps/ref_rejected": -99.38943481445312, "logps/rejected": -370.1670837402344, "loss": 1.0847, "margin_dpo/margin_mean": 93.60745239257812, "margin_dpo/margin_std": 132.91754150390625, "step": 581 }, { "KL/chosen_KL_mean": -156.19517517089844, "KL/mean": -198.79200744628906, "KL/rejected_KL_mean": -241.3888397216797, "KL/std": 109.99616241455078, "epoch": 0.8546255506607929, "fcm_dpo/beta": 0.004808083176612854, "fcm_dpo/delta": 0.039822474122047424, "fcm_dpo/margin": 85.19365692138672, "fcm_dpo/q_t": 0.40805190801620483, "grad_norm": 10.671210289001465, "learning_rate": 3.222175147833556e-08, "logits/chosen": -2.2726540565490723, "logits/rejected": -2.2626798152923584, "logps/chosen": -243.53079223632812, "logps/ref_chosen": -87.33561706542969, "logps/ref_rejected": -115.1544189453125, "logps/rejected": -356.54327392578125, "loss": 1.1027, "margin_dpo/margin_mean": 85.19365692138672, "margin_dpo/margin_std": 120.42509460449219, "step": 582 }, { "KL/chosen_KL_mean": -189.59402465820312, "KL/mean": -236.19659423828125, "KL/rejected_KL_mean": -282.7991943359375, "KL/std": 103.25798034667969, "epoch": 0.856093979441997, "fcm_dpo/beta": 0.00475682970136404, "fcm_dpo/delta": 0.006634358316659927, "fcm_dpo/margin": 93.20515441894531, "fcm_dpo/q_t": 0.3995400667190552, "grad_norm": 11.125332832336426, "learning_rate": 3.159440233840763e-08, "logits/chosen": -1.5351135730743408, "logits/rejected": -1.3023037910461426, "logps/chosen": -273.31109619140625, "logps/ref_chosen": -83.71707153320312, "logps/ref_rejected": -94.58465576171875, "logps/rejected": -377.38385009765625, "loss": 1.0703, "margin_dpo/margin_mean": 93.20515441894531, "margin_dpo/margin_std": 121.99015045166016, "step": 583 }, { "KL/chosen_KL_mean": -167.42575073242188, "KL/mean": -216.7974853515625, "KL/rejected_KL_mean": -266.169189453125, "KL/std": 107.61104583740234, "epoch": 0.8575624082232012, "fcm_dpo/beta": 0.0047964793629944324, "fcm_dpo/delta": -0.025810755789279938, "fcm_dpo/margin": 98.74342346191406, "fcm_dpo/q_t": 0.3911040425300598, "grad_norm": 8.54472827911377, "learning_rate": 3.0972808389096635e-08, "logits/chosen": -1.9103858470916748, "logits/rejected": -1.4478344917297363, "logps/chosen": -261.685302734375, "logps/ref_chosen": -94.25955963134766, "logps/ref_rejected": -104.48269653320312, "logps/rejected": -370.6518859863281, "loss": 1.054, "margin_dpo/margin_mean": 98.7434310913086, "margin_dpo/margin_std": 122.18767547607422, "step": 584 }, { "KL/chosen_KL_mean": -178.62423706054688, "KL/mean": -234.43539428710938, "KL/rejected_KL_mean": -290.2465515136719, "KL/std": 122.98223876953125, "epoch": 0.8590308370044053, "fcm_dpo/beta": 0.0047151679173111916, "fcm_dpo/delta": -0.08055678009986877, "fcm_dpo/margin": 111.62234497070312, "fcm_dpo/q_t": 0.38218066096305847, "grad_norm": 9.863591194152832, "learning_rate": 3.035698600998121e-08, "logits/chosen": -2.1924378871917725, "logits/rejected": -1.9288148880004883, "logps/chosen": -264.260986328125, "logps/ref_chosen": -85.63676452636719, "logps/ref_rejected": -90.28547668457031, "logps/rejected": -380.53204345703125, "loss": 1.0326, "margin_dpo/margin_mean": 111.62234497070312, "margin_dpo/margin_std": 143.08486938476562, "step": 585 }, { "KL/chosen_KL_mean": -198.4064178466797, "KL/mean": -236.27288818359375, "KL/rejected_KL_mean": -274.1393737792969, "KL/std": 107.932861328125, "epoch": 0.8604992657856094, "fcm_dpo/beta": 0.004743432626128197, "fcm_dpo/delta": 0.0941477119922638, "fcm_dpo/margin": 75.73295593261719, "fcm_dpo/q_t": 0.4166196584701538, "grad_norm": 11.513298988342285, "learning_rate": 2.974695142855388e-08, "logits/chosen": -1.8965815305709839, "logits/rejected": -1.6332364082336426, "logps/chosen": -282.0875549316406, "logps/ref_chosen": -83.68114471435547, "logps/ref_rejected": -98.35916137695312, "logps/rejected": -372.49853515625, "loss": 1.1148, "margin_dpo/margin_mean": 75.73295593261719, "margin_dpo/margin_std": 101.49545288085938, "step": 586 }, { "KL/chosen_KL_mean": -160.5615234375, "KL/mean": -201.52297973632812, "KL/rejected_KL_mean": -242.48443603515625, "KL/std": 104.24542999267578, "epoch": 0.8619676945668135, "fcm_dpo/beta": 0.004813813604414463, "fcm_dpo/delta": 0.057809218764305115, "fcm_dpo/margin": 81.92289733886719, "fcm_dpo/q_t": 0.4092627763748169, "grad_norm": 11.267950057983398, "learning_rate": 2.9142720719793122e-08, "logits/chosen": -2.5131096839904785, "logits/rejected": -2.232171058654785, "logps/chosen": -238.1390380859375, "logps/ref_chosen": -77.57750701904297, "logps/ref_rejected": -89.32032775878906, "logps/rejected": -331.80474853515625, "loss": 1.0939, "margin_dpo/margin_mean": 81.92289733886719, "margin_dpo/margin_std": 108.14523315429688, "step": 587 }, { "KL/chosen_KL_mean": -174.6376953125, "KL/mean": -212.6331329345703, "KL/rejected_KL_mean": -250.62857055664062, "KL/std": 106.32521057128906, "epoch": 0.8634361233480177, "fcm_dpo/beta": 0.004871168173849583, "fcm_dpo/delta": 0.0829886794090271, "fcm_dpo/margin": 75.9908676147461, "fcm_dpo/q_t": 0.41710951924324036, "grad_norm": 13.760732650756836, "learning_rate": 2.8544309805740018e-08, "logits/chosen": -1.6966710090637207, "logits/rejected": -1.6548397541046143, "logps/chosen": -258.44195556640625, "logps/ref_chosen": -83.80426025390625, "logps/ref_rejected": -113.11943054199219, "logps/rejected": -363.74798583984375, "loss": 1.1403, "margin_dpo/margin_mean": 75.99087524414062, "margin_dpo/margin_std": 127.1422119140625, "step": 588 }, { "KL/chosen_KL_mean": -187.17181396484375, "KL/mean": -237.55117797851562, "KL/rejected_KL_mean": -287.9305725097656, "KL/std": 104.54691314697266, "epoch": 0.8649045521292217, "fcm_dpo/beta": 0.004872431047260761, "fcm_dpo/delta": -0.043043069541454315, "fcm_dpo/margin": 100.75877380371094, "fcm_dpo/q_t": 0.38554611802101135, "grad_norm": 9.566523551940918, "learning_rate": 2.7951734455078786e-08, "logits/chosen": -1.542040228843689, "logits/rejected": -1.1396864652633667, "logps/chosen": -283.6368408203125, "logps/ref_chosen": -96.46501922607422, "logps/ref_rejected": -116.03719329833984, "logps/rejected": -403.9677734375, "loss": 1.0104, "margin_dpo/margin_mean": 100.75877380371094, "margin_dpo/margin_std": 100.22196960449219, "step": 589 }, { "KL/chosen_KL_mean": -169.7957305908203, "KL/mean": -220.34649658203125, "KL/rejected_KL_mean": -270.89727783203125, "KL/std": 97.92941284179688, "epoch": 0.8663729809104258, "fcm_dpo/beta": 0.004851190373301506, "fcm_dpo/delta": -0.0427684523165226, "fcm_dpo/margin": 101.10150146484375, "fcm_dpo/q_t": 0.3867577016353607, "grad_norm": 11.905020713806152, "learning_rate": 2.736501028272095e-08, "logits/chosen": -1.7764450311660767, "logits/rejected": -1.6228710412979126, "logps/chosen": -258.22357177734375, "logps/ref_chosen": -88.4278564453125, "logps/ref_rejected": -110.19876861572266, "logps/rejected": -381.09600830078125, "loss": 1.0131, "margin_dpo/margin_mean": 101.10150909423828, "margin_dpo/margin_std": 102.35348510742188, "step": 590 }, { "KL/chosen_KL_mean": -163.75808715820312, "KL/mean": -216.78436279296875, "KL/rejected_KL_mean": -269.81060791015625, "KL/std": 105.61711120605469, "epoch": 0.8678414096916299, "fcm_dpo/beta": 0.004765205085277557, "fcm_dpo/delta": -0.05858120694756508, "fcm_dpo/margin": 106.05250549316406, "fcm_dpo/q_t": 0.3826689124107361, "grad_norm": 7.80182409286499, "learning_rate": 2.678415274939408e-08, "logits/chosen": -1.9163053035736084, "logits/rejected": -1.2959301471710205, "logps/chosen": -251.6077880859375, "logps/ref_chosen": -87.84968566894531, "logps/ref_rejected": -89.57919311523438, "logps/rejected": -359.38983154296875, "loss": 1.0001, "margin_dpo/margin_mean": 106.05252075195312, "margin_dpo/margin_std": 102.91415405273438, "step": 591 }, { "KL/chosen_KL_mean": -193.43685913085938, "KL/mean": -235.18594360351562, "KL/rejected_KL_mean": -276.93505859375, "KL/std": 101.48924255371094, "epoch": 0.869309838472834, "fcm_dpo/beta": 0.004801520146429539, "fcm_dpo/delta": 0.05110149085521698, "fcm_dpo/margin": 83.4981689453125, "fcm_dpo/q_t": 0.4080692529678345, "grad_norm": 12.897024154663086, "learning_rate": 2.6209177161234442e-08, "logits/chosen": -1.7987284660339355, "logits/rejected": -1.470049262046814, "logps/chosen": -268.41259765625, "logps/ref_chosen": -74.97573852539062, "logps/ref_rejected": -81.55262756347656, "logps/rejected": -358.4876708984375, "loss": 1.1104, "margin_dpo/margin_mean": 83.49818420410156, "margin_dpo/margin_std": 125.49330139160156, "step": 592 }, { "KL/chosen_KL_mean": -177.61285400390625, "KL/mean": -212.94195556640625, "KL/rejected_KL_mean": -248.27108764648438, "KL/std": 106.93603515625, "epoch": 0.8707782672540382, "fcm_dpo/beta": 0.004873002879321575, "fcm_dpo/delta": 0.10957694798707962, "fcm_dpo/margin": 70.65821075439453, "fcm_dpo/q_t": 0.42214667797088623, "grad_norm": 15.375000953674316, "learning_rate": 2.564009866938349e-08, "logits/chosen": -0.8637688159942627, "logits/rejected": -0.5610991716384888, "logps/chosen": -249.85189819335938, "logps/ref_chosen": -72.23902893066406, "logps/ref_rejected": -70.09652709960938, "logps/rejected": -318.36761474609375, "loss": 1.1502, "margin_dpo/margin_mean": 70.65821075439453, "margin_dpo/margin_std": 117.49882507324219, "step": 593 }, { "KL/chosen_KL_mean": -159.781005859375, "KL/mean": -214.12362670898438, "KL/rejected_KL_mean": -268.4662780761719, "KL/std": 104.10148620605469, "epoch": 0.8722466960352423, "fcm_dpo/beta": 0.0048754652962088585, "fcm_dpo/delta": -0.08433500677347183, "fcm_dpo/margin": 108.68526458740234, "fcm_dpo/q_t": 0.37713533639907837, "grad_norm": 9.886177062988281, "learning_rate": 2.5076932269588708e-08, "logits/chosen": -2.007129192352295, "logits/rejected": -1.487847924232483, "logps/chosen": -250.21871948242188, "logps/ref_chosen": -90.43771362304688, "logps/ref_rejected": -92.267578125, "logps/rejected": -360.7338562011719, "loss": 0.9819, "margin_dpo/margin_mean": 108.68526458740234, "margin_dpo/margin_std": 100.19976043701172, "step": 594 }, { "KL/chosen_KL_mean": -171.06468200683594, "KL/mean": -215.6085205078125, "KL/rejected_KL_mean": -260.15234375, "KL/std": 103.46780395507812, "epoch": 0.8737151248164464, "fcm_dpo/beta": 0.004862664267420769, "fcm_dpo/delta": 0.017333049327135086, "fcm_dpo/margin": 89.08767700195312, "fcm_dpo/q_t": 0.4019402265548706, "grad_norm": 10.475020408630371, "learning_rate": 2.451969280180849e-08, "logits/chosen": -1.879788875579834, "logits/rejected": -1.472987413406372, "logps/chosen": -252.57949829101562, "logps/ref_chosen": -81.51480102539062, "logps/ref_rejected": -86.52047729492188, "logps/rejected": -346.6728210449219, "loss": 1.0782, "margin_dpo/margin_mean": 89.08767700195312, "margin_dpo/margin_std": 119.22312927246094, "step": 595 }, { "KL/chosen_KL_mean": -178.04481506347656, "KL/mean": -217.61434936523438, "KL/rejected_KL_mean": -257.1838684082031, "KL/std": 102.07959747314453, "epoch": 0.8751835535976505, "fcm_dpo/beta": 0.004903014749288559, "fcm_dpo/delta": 0.06442783027887344, "fcm_dpo/margin": 79.13906860351562, "fcm_dpo/q_t": 0.41271764039993286, "grad_norm": 13.890960693359375, "learning_rate": 2.396839494982103e-08, "logits/chosen": -1.5041656494140625, "logits/rejected": -0.9086236953735352, "logps/chosen": -266.25286865234375, "logps/ref_chosen": -88.20805358886719, "logps/ref_rejected": -86.09375, "logps/rejected": -343.27764892578125, "loss": 1.1292, "margin_dpo/margin_mean": 79.13906860351562, "margin_dpo/margin_std": 128.70896911621094, "step": 596 }, { "KL/chosen_KL_mean": -172.63336181640625, "KL/mean": -225.65191650390625, "KL/rejected_KL_mean": -278.67047119140625, "KL/std": 110.42037963867188, "epoch": 0.8766519823788547, "fcm_dpo/beta": 0.004855680279433727, "fcm_dpo/delta": -0.06863737106323242, "fcm_dpo/margin": 106.037109375, "fcm_dpo/q_t": 0.382799357175827, "grad_norm": 9.550575256347656, "learning_rate": 2.3423053240837514e-08, "logits/chosen": -1.3084278106689453, "logits/rejected": -1.1904325485229492, "logps/chosen": -256.22332763671875, "logps/ref_chosen": -83.58998107910156, "logps/ref_rejected": -106.94439697265625, "logps/rejected": -385.6148681640625, "loss": 1.0145, "margin_dpo/margin_mean": 106.037109375, "margin_dpo/margin_std": 120.2269058227539, "step": 597 }, { "KL/chosen_KL_mean": -164.1527099609375, "KL/mean": -208.7948760986328, "KL/rejected_KL_mean": -253.43702697753906, "KL/std": 103.03628540039062, "epoch": 0.8781204111600588, "fcm_dpo/beta": 0.004840881563723087, "fcm_dpo/delta": 0.018280409276485443, "fcm_dpo/margin": 89.28433227539062, "fcm_dpo/q_t": 0.39983421564102173, "grad_norm": 10.328418731689453, "learning_rate": 2.2883682045119062e-08, "logits/chosen": -1.920729398727417, "logits/rejected": -1.6049250364303589, "logps/chosen": -254.33534240722656, "logps/ref_chosen": -90.18262481689453, "logps/ref_rejected": -94.79856872558594, "logps/rejected": -348.235595703125, "loss": 1.078, "margin_dpo/margin_mean": 89.28433227539062, "margin_dpo/margin_std": 116.40673828125, "step": 598 }, { "KL/chosen_KL_mean": -167.89181518554688, "KL/mean": -207.4963836669922, "KL/rejected_KL_mean": -247.1009521484375, "KL/std": 107.64773559570312, "epoch": 0.8795888399412628, "fcm_dpo/beta": 0.004888907074928284, "fcm_dpo/delta": 0.06507787853479385, "fcm_dpo/margin": 79.20914459228516, "fcm_dpo/q_t": 0.411202609539032, "grad_norm": 10.175901412963867, "learning_rate": 2.2350295575598367e-08, "logits/chosen": -1.636303186416626, "logits/rejected": -1.5111571550369263, "logps/chosen": -245.7340850830078, "logps/ref_chosen": -77.84226989746094, "logps/ref_rejected": -89.1976318359375, "logps/rejected": -336.298583984375, "loss": 1.1178, "margin_dpo/margin_mean": 79.20914459228516, "margin_dpo/margin_std": 117.85272216796875, "step": 599 }, { "KL/chosen_KL_mean": -186.15249633789062, "KL/mean": -230.818115234375, "KL/rejected_KL_mean": -275.48370361328125, "KL/std": 101.63578033447266, "epoch": 0.8810572687224669, "fcm_dpo/beta": 0.004939359147101641, "fcm_dpo/delta": 0.00915601383894682, "fcm_dpo/margin": 89.33125305175781, "fcm_dpo/q_t": 0.3990960717201233, "grad_norm": 10.114336013793945, "learning_rate": 2.1822907887504932e-08, "logits/chosen": -2.3316235542297363, "logits/rejected": -1.980991244316101, "logps/chosen": -278.3403015136719, "logps/ref_chosen": -92.18781280517578, "logps/ref_rejected": -92.95429992675781, "logps/rejected": -368.4380187988281, "loss": 1.0634, "margin_dpo/margin_mean": 89.33124542236328, "margin_dpo/margin_std": 110.2298583984375, "step": 600 }, { "KL/chosen_KL_mean": -187.6708984375, "KL/mean": -234.78570556640625, "KL/rejected_KL_mean": -281.9005432128906, "KL/std": 103.02455139160156, "epoch": 0.882525697503671, "fcm_dpo/beta": 0.004912571050226688, "fcm_dpo/delta": -0.013815362006425858, "fcm_dpo/margin": 94.22964477539062, "fcm_dpo/q_t": 0.39269837737083435, "grad_norm": 10.507804870605469, "learning_rate": 2.1301532877994742e-08, "logits/chosen": -1.7895655632019043, "logits/rejected": -1.4660348892211914, "logps/chosen": -276.88702392578125, "logps/ref_chosen": -89.21614074707031, "logps/ref_rejected": -100.17054748535156, "logps/rejected": -382.07110595703125, "loss": 1.0319, "margin_dpo/margin_mean": 94.22964477539062, "margin_dpo/margin_std": 98.8106460571289, "step": 601 }, { "KL/chosen_KL_mean": -148.8975830078125, "KL/mean": -210.4900360107422, "KL/rejected_KL_mean": -272.08245849609375, "KL/std": 113.03854370117188, "epoch": 0.8839941262848752, "fcm_dpo/beta": 0.0048156240954995155, "fcm_dpo/delta": -0.15224555134773254, "fcm_dpo/margin": 123.18489074707031, "fcm_dpo/q_t": 0.36536622047424316, "grad_norm": 10.005538940429688, "learning_rate": 2.0786184285784298e-08, "logits/chosen": -2.169020652770996, "logits/rejected": -1.9122167825698853, "logps/chosen": -228.9552001953125, "logps/ref_chosen": -80.05760192871094, "logps/ref_rejected": -93.197509765625, "logps/rejected": -365.27996826171875, "loss": 0.965, "margin_dpo/margin_mean": 123.18489074707031, "margin_dpo/margin_std": 128.56874084472656, "step": 602 }, { "KL/chosen_KL_mean": -159.88824462890625, "KL/mean": -210.6729278564453, "KL/rejected_KL_mean": -261.4576110839844, "KL/std": 102.02827453613281, "epoch": 0.8854625550660793, "fcm_dpo/beta": 0.00475830864161253, "fcm_dpo/delta": -0.03497551381587982, "fcm_dpo/margin": 101.5693359375, "fcm_dpo/q_t": 0.3890826106071472, "grad_norm": 9.110335350036621, "learning_rate": 2.0276875690788204e-08, "logits/chosen": -2.11802339553833, "logits/rejected": -1.5342588424682617, "logps/chosen": -262.1978454589844, "logps/ref_chosen": -102.30958557128906, "logps/ref_rejected": -108.06884765625, "logps/rejected": -369.5264587402344, "loss": 1.0311, "margin_dpo/margin_mean": 101.56934356689453, "margin_dpo/margin_std": 117.06777954101562, "step": 603 }, { "KL/chosen_KL_mean": -165.91738891601562, "KL/mean": -218.85491943359375, "KL/rejected_KL_mean": -271.79241943359375, "KL/std": 107.22111511230469, "epoch": 0.8869309838472834, "fcm_dpo/beta": 0.0046936641447246075, "fcm_dpo/delta": -0.049801453948020935, "fcm_dpo/margin": 105.87506103515625, "fcm_dpo/q_t": 0.3833632469177246, "grad_norm": 9.115262031555176, "learning_rate": 1.977362051376158e-08, "logits/chosen": -1.6962742805480957, "logits/rejected": -1.5405266284942627, "logps/chosen": -244.0914764404297, "logps/ref_chosen": -78.17408752441406, "logps/ref_rejected": -99.49610900878906, "logps/rejected": -371.28857421875, "loss": 1.0123, "margin_dpo/margin_mean": 105.87506103515625, "margin_dpo/margin_std": 108.82525634765625, "step": 604 }, { "KL/chosen_KL_mean": -181.64035034179688, "KL/mean": -224.87034606933594, "KL/rejected_KL_mean": -268.100341796875, "KL/std": 96.00192260742188, "epoch": 0.8883994126284875, "fcm_dpo/beta": 0.004725199192762375, "fcm_dpo/delta": 0.043243639171123505, "fcm_dpo/margin": 86.45996856689453, "fcm_dpo/q_t": 0.40586990118026733, "grad_norm": 10.408443450927734, "learning_rate": 1.9276432015946446e-08, "logits/chosen": -1.8498953580856323, "logits/rejected": -1.3510491847991943, "logps/chosen": -276.4136962890625, "logps/ref_chosen": -94.77332305908203, "logps/ref_rejected": -107.30490112304688, "logps/rejected": -375.40521240234375, "loss": 1.0805, "margin_dpo/margin_mean": 86.45997619628906, "margin_dpo/margin_std": 108.26947784423828, "step": 605 }, { "KL/chosen_KL_mean": -174.79664611816406, "KL/mean": -219.79824829101562, "KL/rejected_KL_mean": -264.7998352050781, "KL/std": 104.88798522949219, "epoch": 0.8898678414096917, "fcm_dpo/beta": 0.004780953750014305, "fcm_dpo/delta": 0.019732438027858734, "fcm_dpo/margin": 90.00321960449219, "fcm_dpo/q_t": 0.39998355507850647, "grad_norm": 9.32400894165039, "learning_rate": 1.8785323298722093e-08, "logits/chosen": -1.779496192932129, "logits/rejected": -1.3828221559524536, "logps/chosen": -262.550048828125, "logps/ref_chosen": -87.7533950805664, "logps/ref_rejected": -104.2422103881836, "logps/rejected": -369.04205322265625, "loss": 1.0716, "margin_dpo/margin_mean": 90.00321960449219, "margin_dpo/margin_std": 111.46886444091797, "step": 606 }, { "KL/chosen_KL_mean": -170.30282592773438, "KL/mean": -221.07574462890625, "KL/rejected_KL_mean": -271.8486633300781, "KL/std": 111.82902526855469, "epoch": 0.8913362701908958, "fcm_dpo/beta": 0.004737128037959337, "fcm_dpo/delta": -0.03259171172976494, "fcm_dpo/margin": 101.54586029052734, "fcm_dpo/q_t": 0.3909189701080322, "grad_norm": 12.469743728637695, "learning_rate": 1.8300307303259904e-08, "logits/chosen": -1.8413944244384766, "logits/rejected": -1.3157618045806885, "logps/chosen": -258.6318664550781, "logps/ref_chosen": -88.32904052734375, "logps/ref_rejected": -86.76810455322266, "logps/rejected": -358.61676025390625, "loss": 1.0608, "margin_dpo/margin_mean": 101.54585266113281, "margin_dpo/margin_std": 137.57362365722656, "step": 607 }, { "KL/chosen_KL_mean": -155.44029235839844, "KL/mean": -208.53402709960938, "KL/rejected_KL_mean": -261.6277770996094, "KL/std": 110.51072692871094, "epoch": 0.8928046989720999, "fcm_dpo/beta": 0.004659214057028294, "fcm_dpo/delta": -0.048452552407979965, "fcm_dpo/margin": 106.18748474121094, "fcm_dpo/q_t": 0.38495171070098877, "grad_norm": 10.721970558166504, "learning_rate": 1.7821396810182437e-08, "logits/chosen": -1.8137047290802002, "logits/rejected": -1.5158920288085938, "logps/chosen": -241.20965576171875, "logps/ref_chosen": -85.76937866210938, "logps/ref_rejected": -100.23281860351562, "logps/rejected": -361.860595703125, "loss": 1.0298, "margin_dpo/margin_mean": 106.18748474121094, "margin_dpo/margin_std": 119.422119140625, "step": 608 }, { "KL/chosen_KL_mean": -172.4480438232422, "KL/mean": -222.6395263671875, "KL/rejected_KL_mean": -272.83099365234375, "KL/std": 110.90241241455078, "epoch": 0.8942731277533039, "fcm_dpo/beta": 0.004647298250347376, "fcm_dpo/delta": -0.017813783138990402, "fcm_dpo/margin": 100.38296508789062, "fcm_dpo/q_t": 0.3932035565376282, "grad_norm": 11.612581253051758, "learning_rate": 1.7348604439226617e-08, "logits/chosen": -2.2478439807891846, "logits/rejected": -1.8245940208435059, "logps/chosen": -265.41461181640625, "logps/ref_chosen": -92.9665756225586, "logps/ref_rejected": -95.91818237304688, "logps/rejected": -368.74920654296875, "loss": 1.0486, "margin_dpo/margin_mean": 100.3829574584961, "margin_dpo/margin_std": 117.288818359375, "step": 609 }, { "KL/chosen_KL_mean": -158.64759826660156, "KL/mean": -212.06503295898438, "KL/rejected_KL_mean": -265.48248291015625, "KL/std": 106.18761444091797, "epoch": 0.895741556534508, "fcm_dpo/beta": 0.004621714353561401, "fcm_dpo/delta": -0.04623851925134659, "fcm_dpo/margin": 106.83487701416016, "fcm_dpo/q_t": 0.3845939040184021, "grad_norm": 8.957317352294922, "learning_rate": 1.6881942648911074e-08, "logits/chosen": -1.76459801197052, "logits/rejected": -1.2192771434783936, "logps/chosen": -253.34788513183594, "logps/ref_chosen": -94.70028686523438, "logps/ref_rejected": -89.68739318847656, "logps/rejected": -355.16986083984375, "loss": 1.0159, "margin_dpo/margin_mean": 106.83486938476562, "margin_dpo/margin_std": 112.31874084472656, "step": 610 }, { "KL/chosen_KL_mean": -167.4735565185547, "KL/mean": -219.63662719726562, "KL/rejected_KL_mean": -271.7997131347656, "KL/std": 110.14964294433594, "epoch": 0.8972099853157122, "fcm_dpo/beta": 0.004620520398020744, "fcm_dpo/delta": -0.03399227187037468, "fcm_dpo/margin": 104.3261489868164, "fcm_dpo/q_t": 0.3893621563911438, "grad_norm": 10.346162796020508, "learning_rate": 1.6421423736208e-08, "logits/chosen": -1.9958183765411377, "logits/rejected": -1.6436913013458252, "logps/chosen": -254.25689697265625, "logps/ref_chosen": -86.78334045410156, "logps/ref_rejected": -89.84307861328125, "logps/rejected": -361.64276123046875, "loss": 1.0256, "margin_dpo/margin_mean": 104.3261489868164, "margin_dpo/margin_std": 112.11479187011719, "step": 611 }, { "KL/chosen_KL_mean": -183.2674560546875, "KL/mean": -236.60855102539062, "KL/rejected_KL_mean": -289.9496765136719, "KL/std": 104.09312438964844, "epoch": 0.8986784140969163, "fcm_dpo/beta": 0.004556679166853428, "fcm_dpo/delta": -0.03807242587208748, "fcm_dpo/margin": 106.6822280883789, "fcm_dpo/q_t": 0.3870040774345398, "grad_norm": 9.27346134185791, "learning_rate": 1.5967059836219042e-08, "logits/chosen": -2.1088085174560547, "logits/rejected": -1.267046570777893, "logps/chosen": -284.28759765625, "logps/ref_chosen": -101.02015686035156, "logps/ref_rejected": -93.78302764892578, "logps/rejected": -383.7326965332031, "loss": 1.014, "margin_dpo/margin_mean": 106.6822280883789, "margin_dpo/margin_std": 107.85067749023438, "step": 612 }, { "KL/chosen_KL_mean": -146.6988525390625, "KL/mean": -206.68246459960938, "KL/rejected_KL_mean": -266.66607666015625, "KL/std": 106.4085693359375, "epoch": 0.9001468428781204, "fcm_dpo/beta": 0.004484009929001331, "fcm_dpo/delta": -0.09317655861377716, "fcm_dpo/margin": 119.96723175048828, "fcm_dpo/q_t": 0.37405645847320557, "grad_norm": 8.81155776977539, "learning_rate": 1.551886292185553e-08, "logits/chosen": -2.016073703765869, "logits/rejected": -1.8203673362731934, "logps/chosen": -235.68748474121094, "logps/ref_chosen": -88.98861694335938, "logps/ref_rejected": -109.99551391601562, "logps/rejected": -376.6615905761719, "loss": 0.9694, "margin_dpo/margin_mean": 119.96722412109375, "margin_dpo/margin_std": 104.03485107421875, "step": 613 }, { "KL/chosen_KL_mean": -185.25738525390625, "KL/mean": -233.3673858642578, "KL/rejected_KL_mean": -281.4773864746094, "KL/std": 100.46821594238281, "epoch": 0.9016152716593245, "fcm_dpo/beta": 0.0044782571494579315, "fcm_dpo/delta": 0.019953060895204544, "fcm_dpo/margin": 96.2199935913086, "fcm_dpo/q_t": 0.40007302165031433, "grad_norm": 11.269189834594727, "learning_rate": 1.507684480352292e-08, "logits/chosen": -1.4590733051300049, "logits/rejected": -1.4495806694030762, "logps/chosen": -265.45745849609375, "logps/ref_chosen": -80.20005798339844, "logps/ref_rejected": -109.86239624023438, "logps/rejected": -391.33978271484375, "loss": 1.0738, "margin_dpo/margin_mean": 96.2199935913086, "margin_dpo/margin_std": 122.71536254882812, "step": 614 }, { "KL/chosen_KL_mean": -173.26766967773438, "KL/mean": -213.72019958496094, "KL/rejected_KL_mean": -254.17269897460938, "KL/std": 101.97980499267578, "epoch": 0.9030837004405287, "fcm_dpo/beta": 0.004574804566800594, "fcm_dpo/delta": 0.0814293697476387, "fcm_dpo/margin": 80.9050521850586, "fcm_dpo/q_t": 0.41295474767684937, "grad_norm": 9.873544692993164, "learning_rate": 1.4641017128809801e-08, "logits/chosen": -1.590479850769043, "logits/rejected": -1.1801893711090088, "logps/chosen": -273.70294189453125, "logps/ref_chosen": -100.43525695800781, "logps/ref_rejected": -101.1800537109375, "logps/rejected": -355.352783203125, "loss": 1.1009, "margin_dpo/margin_mean": 80.9050521850586, "margin_dpo/margin_std": 96.39158630371094, "step": 615 }, { "KL/chosen_KL_mean": -175.1005401611328, "KL/mean": -230.30368041992188, "KL/rejected_KL_mean": -285.5068054199219, "KL/std": 102.12892150878906, "epoch": 0.9045521292217328, "fcm_dpo/beta": 0.004543361719697714, "fcm_dpo/delta": -0.054439835250377655, "fcm_dpo/margin": 110.40625762939453, "fcm_dpo/q_t": 0.38406243920326233, "grad_norm": 9.328760147094727, "learning_rate": 1.4211391382180637e-08, "logits/chosen": -1.753568410873413, "logits/rejected": -1.2510175704956055, "logps/chosen": -267.5934753417969, "logps/ref_chosen": -92.49292755126953, "logps/ref_rejected": -82.06065368652344, "logps/rejected": -367.56744384765625, "loss": 1.0146, "margin_dpo/margin_mean": 110.40625762939453, "margin_dpo/margin_std": 118.6685791015625, "step": 616 }, { "KL/chosen_KL_mean": -182.56533813476562, "KL/mean": -226.22821044921875, "KL/rejected_KL_mean": -269.89111328125, "KL/std": 99.32344055175781, "epoch": 0.9060205580029369, "fcm_dpo/beta": 0.004548664204776287, "fcm_dpo/delta": 0.054983749985694885, "fcm_dpo/margin": 87.32576751708984, "fcm_dpo/q_t": 0.40742790699005127, "grad_norm": 9.524311065673828, "learning_rate": 1.378797888467345e-08, "logits/chosen": -1.422196388244629, "logits/rejected": -0.7640622854232788, "logps/chosen": -273.6623229980469, "logps/ref_chosen": -91.09699249267578, "logps/ref_rejected": -70.41004943847656, "logps/rejected": -340.3011474609375, "loss": 1.0898, "margin_dpo/margin_mean": 87.32576751708984, "margin_dpo/margin_std": 111.27055358886719, "step": 617 }, { "KL/chosen_KL_mean": -184.1827392578125, "KL/mean": -232.53775024414062, "KL/rejected_KL_mean": -280.89276123046875, "KL/std": 109.46292877197266, "epoch": 0.9074889867841409, "fcm_dpo/beta": 0.00456183310598135, "fcm_dpo/delta": 0.009194123558700085, "fcm_dpo/margin": 96.71001434326172, "fcm_dpo/q_t": 0.3969927430152893, "grad_norm": 9.151403427124023, "learning_rate": 1.3370790793601371e-08, "logits/chosen": -2.015610456466675, "logits/rejected": -1.6541812419891357, "logps/chosen": -286.2033386230469, "logps/ref_chosen": -102.02059936523438, "logps/ref_rejected": -99.80119323730469, "logps/rejected": -380.6939697265625, "loss": 1.0661, "margin_dpo/margin_mean": 96.71001434326172, "margin_dpo/margin_std": 121.06686401367188, "step": 618 }, { "KL/chosen_KL_mean": -178.2374267578125, "KL/mean": -221.03900146484375, "KL/rejected_KL_mean": -263.8406066894531, "KL/std": 108.02735137939453, "epoch": 0.908957415565345, "fcm_dpo/beta": 0.004609095398336649, "fcm_dpo/delta": 0.05774322897195816, "fcm_dpo/margin": 85.60318756103516, "fcm_dpo/q_t": 0.4113660454750061, "grad_norm": 9.244808197021484, "learning_rate": 1.2959838102258535e-08, "logits/chosen": -1.4586336612701416, "logits/rejected": -1.0880571603775024, "logps/chosen": -267.9787902832031, "logps/ref_chosen": -89.74136352539062, "logps/ref_rejected": -99.90138244628906, "logps/rejected": -363.74200439453125, "loss": 1.1125, "margin_dpo/margin_mean": 85.60319519042969, "margin_dpo/margin_std": 129.63429260253906, "step": 619 }, { "KL/chosen_KL_mean": -170.60426330566406, "KL/mean": -218.12603759765625, "KL/rejected_KL_mean": -265.6478271484375, "KL/std": 113.73348999023438, "epoch": 0.9104258443465492, "fcm_dpo/beta": 0.004587803967297077, "fcm_dpo/delta": 0.012716904282569885, "fcm_dpo/margin": 95.04354858398438, "fcm_dpo/q_t": 0.4000408947467804, "grad_norm": 10.134498596191406, "learning_rate": 1.2555131639630567e-08, "logits/chosen": -2.3685035705566406, "logits/rejected": -1.856990098953247, "logps/chosen": -255.72857666015625, "logps/ref_chosen": -85.12431335449219, "logps/ref_rejected": -85.41253662109375, "logps/rejected": -351.06036376953125, "loss": 1.081, "margin_dpo/margin_mean": 95.04354858398438, "margin_dpo/margin_std": 122.09893798828125, "step": 620 }, { "KL/chosen_KL_mean": -173.086669921875, "KL/mean": -219.64083862304688, "KL/rejected_KL_mean": -266.1950378417969, "KL/std": 109.73408508300781, "epoch": 0.9118942731277533, "fcm_dpo/beta": 0.004689943976700306, "fcm_dpo/delta": 0.01188938319683075, "fcm_dpo/margin": 93.10836791992188, "fcm_dpo/q_t": 0.4004325270652771, "grad_norm": 9.9964017868042, "learning_rate": 1.2156682070109086e-08, "logits/chosen": -2.0491368770599365, "logits/rejected": -1.7892625331878662, "logps/chosen": -262.3350830078125, "logps/ref_chosen": -89.24842071533203, "logps/ref_rejected": -95.46463775634766, "logps/rejected": -361.65966796875, "loss": 1.09, "margin_dpo/margin_mean": 93.10836791992188, "margin_dpo/margin_std": 127.75167846679688, "step": 621 }, { "KL/chosen_KL_mean": -171.01986694335938, "KL/mean": -220.12286376953125, "KL/rejected_KL_mean": -269.225830078125, "KL/std": 97.43096160888672, "epoch": 0.9133627019089574, "fcm_dpo/beta": 0.004632966592907906, "fcm_dpo/delta": -0.005341339856386185, "fcm_dpo/margin": 98.20596313476562, "fcm_dpo/q_t": 0.39399614930152893, "grad_norm": 11.154865264892578, "learning_rate": 1.1764499893210878e-08, "logits/chosen": -1.434091329574585, "logits/rejected": -0.7677087783813477, "logps/chosen": -270.8139953613281, "logps/ref_chosen": -99.79412841796875, "logps/ref_rejected": -90.82821655273438, "logps/rejected": -360.0540466308594, "loss": 1.0392, "margin_dpo/margin_mean": 98.20596313476562, "margin_dpo/margin_std": 105.42474365234375, "step": 622 }, { "KL/chosen_KL_mean": -178.2598876953125, "KL/mean": -212.8975372314453, "KL/rejected_KL_mean": -247.53517150878906, "KL/std": 105.79509735107422, "epoch": 0.9148311306901615, "fcm_dpo/beta": 0.004716922529041767, "fcm_dpo/delta": 0.1275968998670578, "fcm_dpo/margin": 69.27526092529297, "fcm_dpo/q_t": 0.4265623986721039, "grad_norm": 10.9679594039917, "learning_rate": 1.1378595443300998e-08, "logits/chosen": -2.231057643890381, "logits/rejected": -1.790826678276062, "logps/chosen": -268.7154541015625, "logps/ref_chosen": -90.45555114746094, "logps/ref_rejected": -91.32276153564453, "logps/rejected": -338.85791015625, "loss": 1.1756, "margin_dpo/margin_mean": 69.27526092529297, "margin_dpo/margin_std": 129.77325439453125, "step": 623 }, { "KL/chosen_KL_mean": -182.99085998535156, "KL/mean": -238.72291564941406, "KL/rejected_KL_mean": -294.4549865722656, "KL/std": 104.68896484375, "epoch": 0.9162995594713657, "fcm_dpo/beta": 0.004705128259956837, "fcm_dpo/delta": -0.0786077231168747, "fcm_dpo/margin": 111.46412658691406, "fcm_dpo/q_t": 0.379744291305542, "grad_norm": 10.653809547424316, "learning_rate": 1.0998978889320582e-08, "logits/chosen": -2.5369553565979004, "logits/rejected": -1.6156611442565918, "logps/chosen": -292.8660888671875, "logps/ref_chosen": -109.87522888183594, "logps/ref_rejected": -104.77320861816406, "logps/rejected": -399.22821044921875, "loss": 1.003, "margin_dpo/margin_mean": 111.46412658691406, "margin_dpo/margin_std": 120.23019409179688, "step": 624 }, { "KL/chosen_KL_mean": -191.24447631835938, "KL/mean": -231.90008544921875, "KL/rejected_KL_mean": -272.55572509765625, "KL/std": 104.50779724121094, "epoch": 0.9177679882525698, "fcm_dpo/beta": 0.004745472222566605, "fcm_dpo/delta": 0.06645552814006805, "fcm_dpo/margin": 81.31123352050781, "fcm_dpo/q_t": 0.4112913906574249, "grad_norm": 9.284390449523926, "learning_rate": 1.0625660234518913e-08, "logits/chosen": -1.7679665088653564, "logits/rejected": -1.3943238258361816, "logps/chosen": -278.41265869140625, "logps/ref_chosen": -87.16815948486328, "logps/ref_rejected": -91.86148071289062, "logps/rejected": -364.41717529296875, "loss": 1.1024, "margin_dpo/margin_mean": 81.31123352050781, "margin_dpo/margin_std": 110.92190551757812, "step": 625 }, { "KL/chosen_KL_mean": -181.08592224121094, "KL/mean": -221.35281372070312, "KL/rejected_KL_mean": -261.61968994140625, "KL/std": 115.74142456054688, "epoch": 0.9192364170337739, "fcm_dpo/beta": 0.004809065721929073, "fcm_dpo/delta": 0.0649387314915657, "fcm_dpo/margin": 80.53378295898438, "fcm_dpo/q_t": 0.4128105342388153, "grad_norm": 11.609611511230469, "learning_rate": 1.0258649316189721e-08, "logits/chosen": -1.4680793285369873, "logits/rejected": -1.0361180305480957, "logps/chosen": -285.31011962890625, "logps/ref_chosen": -104.22421264648438, "logps/ref_rejected": -104.1774673461914, "logps/rejected": -365.79718017578125, "loss": 1.1224, "margin_dpo/margin_mean": 80.53378295898438, "margin_dpo/margin_std": 126.5710220336914, "step": 626 }, { "KL/chosen_KL_mean": -161.65158081054688, "KL/mean": -212.0081787109375, "KL/rejected_KL_mean": -262.36474609375, "KL/std": 118.21295166015625, "epoch": 0.920704845814978, "fcm_dpo/beta": 0.004788564518094063, "fcm_dpo/delta": -0.033907003700733185, "fcm_dpo/margin": 100.71316528320312, "fcm_dpo/q_t": 0.3918336033821106, "grad_norm": 8.047354698181152, "learning_rate": 9.897955805412e-09, "logits/chosen": -1.5927295684814453, "logits/rejected": -1.737905740737915, "logps/chosen": -236.58621215820312, "logps/ref_chosen": -74.93461608886719, "logps/ref_rejected": -112.57289123535156, "logps/rejected": -374.9376525878906, "loss": 1.0484, "margin_dpo/margin_mean": 100.71316528320312, "margin_dpo/margin_std": 129.9547576904297, "step": 627 }, { "KL/chosen_KL_mean": -169.31735229492188, "KL/mean": -222.12353515625, "KL/rejected_KL_mean": -274.9296875, "KL/std": 106.74333953857422, "epoch": 0.922173274596182, "fcm_dpo/beta": 0.004742040298879147, "fcm_dpo/delta": -0.053501617163419724, "fcm_dpo/margin": 105.61235046386719, "fcm_dpo/q_t": 0.38509994745254517, "grad_norm": 10.507683753967285, "learning_rate": 9.543589206795238e-09, "logits/chosen": -1.8934911489486694, "logits/rejected": -1.6267330646514893, "logps/chosen": -263.0084228515625, "logps/ref_chosen": -93.69107818603516, "logps/ref_rejected": -107.34395599365234, "logps/rejected": -382.2736511230469, "loss": 1.0197, "margin_dpo/margin_mean": 105.61235046386719, "margin_dpo/margin_std": 117.55007934570312, "step": 628 }, { "KL/chosen_KL_mean": -182.3243408203125, "KL/mean": -229.99411010742188, "KL/rejected_KL_mean": -277.66387939453125, "KL/std": 106.87051391601562, "epoch": 0.9236417033773862, "fcm_dpo/beta": 0.004714585840702057, "fcm_dpo/delta": 0.00033011287450790405, "fcm_dpo/margin": 95.33956909179688, "fcm_dpo/q_t": 0.39539289474487305, "grad_norm": 10.074275970458984, "learning_rate": 9.19555885822887e-09, "logits/chosen": -2.1480307579040527, "logits/rejected": -1.7198066711425781, "logps/chosen": -285.5547180175781, "logps/ref_chosen": -103.23037719726562, "logps/ref_rejected": -97.16841888427734, "logps/rejected": -374.83233642578125, "loss": 1.0553, "margin_dpo/margin_mean": 95.33956909179688, "margin_dpo/margin_std": 113.55947875976562, "step": 629 }, { "KL/chosen_KL_mean": -166.311279296875, "KL/mean": -197.57908630371094, "KL/rejected_KL_mean": -228.8468780517578, "KL/std": 99.85243225097656, "epoch": 0.9251101321585903, "fcm_dpo/beta": 0.004856232088059187, "fcm_dpo/delta": 0.14996640384197235, "fcm_dpo/margin": 62.53557586669922, "fcm_dpo/q_t": 0.43007415533065796, "grad_norm": 13.54204273223877, "learning_rate": 8.85387393063622e-09, "logits/chosen": -1.973757266998291, "logits/rejected": -1.4395477771759033, "logps/chosen": -260.2088317871094, "logps/ref_chosen": -93.89755249023438, "logps/ref_rejected": -89.3743896484375, "logps/rejected": -318.22125244140625, "loss": 1.1781, "margin_dpo/margin_mean": 62.53557586669922, "margin_dpo/margin_std": 112.05850219726562, "step": 630 }, { "KL/chosen_KL_mean": -164.4595489501953, "KL/mean": -210.38082885742188, "KL/rejected_KL_mean": -256.3020935058594, "KL/std": 98.89360046386719, "epoch": 0.9265785609397944, "fcm_dpo/beta": 0.004870392847806215, "fcm_dpo/delta": 0.0027950862422585487, "fcm_dpo/margin": 91.842529296875, "fcm_dpo/q_t": 0.39640283584594727, "grad_norm": 9.133491516113281, "learning_rate": 8.518543427732949e-09, "logits/chosen": -2.471451759338379, "logits/rejected": -2.041480779647827, "logps/chosen": -252.23037719726562, "logps/ref_chosen": -87.77082061767578, "logps/ref_rejected": -88.68241882324219, "logps/rejected": -344.9844970703125, "loss": 1.059, "margin_dpo/margin_mean": 91.842529296875, "margin_dpo/margin_std": 112.66258239746094, "step": 631 }, { "KL/chosen_KL_mean": -161.1116180419922, "KL/mean": -211.72911071777344, "KL/rejected_KL_mean": -262.34661865234375, "KL/std": 104.71236419677734, "epoch": 0.9280469897209985, "fcm_dpo/beta": 0.004819872789084911, "fcm_dpo/delta": -0.040760889649391174, "fcm_dpo/margin": 101.23500061035156, "fcm_dpo/q_t": 0.3892359733581543, "grad_norm": 12.062975883483887, "learning_rate": 8.189576185789637e-09, "logits/chosen": -2.054488182067871, "logits/rejected": -1.6294389963150024, "logps/chosen": -249.7381591796875, "logps/ref_chosen": -88.62652587890625, "logps/ref_rejected": -91.45091247558594, "logps/rejected": -353.79754638671875, "loss": 1.0482, "margin_dpo/margin_mean": 101.2349853515625, "margin_dpo/margin_std": 128.82888793945312, "step": 632 }, { "KL/chosen_KL_mean": -200.8941650390625, "KL/mean": -230.75674438476562, "KL/rejected_KL_mean": -260.6193542480469, "KL/std": 106.32029724121094, "epoch": 0.9295154185022027, "fcm_dpo/beta": 0.004847935400903225, "fcm_dpo/delta": 0.022854354232549667, "fcm_dpo/margin": 59.72520446777344, "fcm_dpo/q_t": 0.4360605776309967, "grad_norm": 11.445959091186523, "learning_rate": 7.866980873399015e-09, "logits/chosen": -2.161868095397949, "logits/rejected": -1.9704504013061523, "logps/chosen": -282.2685852050781, "logps/ref_chosen": -81.37442016601562, "logps/ref_rejected": -98.62571716308594, "logps/rejected": -359.2450866699219, "loss": 1.2294, "margin_dpo/margin_mean": 59.72520446777344, "margin_dpo/margin_std": 141.68458557128906, "step": 633 }, { "KL/chosen_KL_mean": -184.75729370117188, "KL/mean": -223.542236328125, "KL/rejected_KL_mean": -262.3271484375, "KL/std": 95.78713989257812, "epoch": 0.9309838472834068, "fcm_dpo/beta": 0.0048352256417274475, "fcm_dpo/delta": -0.04920076206326485, "fcm_dpo/margin": 77.5698471069336, "fcm_dpo/q_t": 0.41228485107421875, "grad_norm": 11.62212085723877, "learning_rate": 7.550765991247654e-09, "logits/chosen": -1.3433046340942383, "logits/rejected": -1.166331171989441, "logps/chosen": -280.880126953125, "logps/ref_chosen": -96.12284851074219, "logps/ref_rejected": -112.84780883789062, "logps/rejected": -375.1749572753906, "loss": 1.105, "margin_dpo/margin_mean": 77.5698471069336, "margin_dpo/margin_std": 97.00047302246094, "step": 634 }, { "KL/chosen_KL_mean": -192.25848388671875, "KL/mean": -222.4342803955078, "KL/rejected_KL_mean": -252.61007690429688, "KL/std": 103.95018005371094, "epoch": 0.9324522760646109, "fcm_dpo/beta": 0.004827727563679218, "fcm_dpo/delta": 0.033745329827070236, "fcm_dpo/margin": 60.351585388183594, "fcm_dpo/q_t": 0.43377456068992615, "grad_norm": 13.313103675842285, "learning_rate": 7.240939871891699e-09, "logits/chosen": -1.741532564163208, "logits/rejected": -1.2016403675079346, "logps/chosen": -290.9425964355469, "logps/ref_chosen": -98.68411254882812, "logps/ref_rejected": -89.89920043945312, "logps/rejected": -342.50927734375, "loss": 1.2027, "margin_dpo/margin_mean": 60.351585388183594, "margin_dpo/margin_std": 124.63910675048828, "step": 635 }, { "KL/chosen_KL_mean": -172.7763671875, "KL/mean": -223.48602294921875, "KL/rejected_KL_mean": -274.1956787109375, "KL/std": 107.63565826416016, "epoch": 0.933920704845815, "fcm_dpo/beta": 0.004821810871362686, "fcm_dpo/delta": -0.04101572185754776, "fcm_dpo/margin": 101.41928100585938, "fcm_dpo/q_t": 0.38723403215408325, "grad_norm": 10.1853609085083, "learning_rate": 6.937510679537628e-09, "logits/chosen": -1.5749695301055908, "logits/rejected": -1.09731924533844, "logps/chosen": -263.1943359375, "logps/ref_chosen": -90.41796112060547, "logps/ref_rejected": -87.70687866210938, "logps/rejected": -361.90252685546875, "loss": 1.019, "margin_dpo/margin_mean": 101.41928100585938, "margin_dpo/margin_std": 107.80775451660156, "step": 636 }, { "KL/chosen_KL_mean": -174.0252685546875, "KL/mean": -228.01861572265625, "KL/rejected_KL_mean": -282.011962890625, "KL/std": 119.41990661621094, "epoch": 0.9353891336270191, "fcm_dpo/beta": 0.004764461889863014, "fcm_dpo/delta": -0.06794070452451706, "fcm_dpo/margin": 107.98666381835938, "fcm_dpo/q_t": 0.38107621669769287, "grad_norm": 8.683420181274414, "learning_rate": 6.640486409826785e-09, "logits/chosen": -2.010817050933838, "logits/rejected": -1.9075961112976074, "logps/chosen": -256.4749755859375, "logps/ref_chosen": -82.44971466064453, "logps/ref_rejected": -104.02860260009766, "logps/rejected": -386.04052734375, "loss": 1.0056, "margin_dpo/margin_mean": 107.98666381835938, "margin_dpo/margin_std": 112.98416137695312, "step": 637 }, { "KL/chosen_KL_mean": -161.8271484375, "KL/mean": -215.07395935058594, "KL/rejected_KL_mean": -268.3207702636719, "KL/std": 105.83755493164062, "epoch": 0.9368575624082232, "fcm_dpo/beta": 0.004725996404886246, "fcm_dpo/delta": -0.056419726461172104, "fcm_dpo/margin": 106.49361419677734, "fcm_dpo/q_t": 0.3823361396789551, "grad_norm": 8.91274642944336, "learning_rate": 6.349874889624962e-09, "logits/chosen": -1.4914363622665405, "logits/rejected": -0.7988900542259216, "logps/chosen": -253.75213623046875, "logps/ref_chosen": -91.92498779296875, "logps/ref_rejected": -86.28703308105469, "logps/rejected": -354.6077880859375, "loss": 0.9972, "margin_dpo/margin_mean": 106.49362182617188, "margin_dpo/margin_std": 95.94612121582031, "step": 638 }, { "KL/chosen_KL_mean": -214.31985473632812, "KL/mean": -239.8204345703125, "KL/rejected_KL_mean": -265.321044921875, "KL/std": 99.06515502929688, "epoch": 0.9383259911894273, "fcm_dpo/beta": 0.004721669480204582, "fcm_dpo/delta": 0.07464981079101562, "fcm_dpo/margin": 51.00115203857422, "fcm_dpo/q_t": 0.4439978003501892, "grad_norm": 10.234736442565918, "learning_rate": 6.065683776815933e-09, "logits/chosen": -1.3730717897415161, "logits/rejected": -0.6060410141944885, "logps/chosen": -318.847412109375, "logps/ref_chosen": -104.52755737304688, "logps/ref_rejected": -81.4803466796875, "logps/rejected": -346.8013916015625, "loss": 1.2413, "margin_dpo/margin_mean": 51.00115203857422, "margin_dpo/margin_std": 123.33016967773438, "step": 639 }, { "KL/chosen_KL_mean": -180.89990234375, "KL/mean": -228.81802368164062, "KL/rejected_KL_mean": -276.73614501953125, "KL/std": 106.22747802734375, "epoch": 0.9397944199706314, "fcm_dpo/beta": 0.004792827181518078, "fcm_dpo/delta": -0.011392727494239807, "fcm_dpo/margin": 95.83621978759766, "fcm_dpo/q_t": 0.39340639114379883, "grad_norm": 11.994951248168945, "learning_rate": 5.7879205600998296e-09, "logits/chosen": -1.8697707653045654, "logits/rejected": -1.3506278991699219, "logps/chosen": -278.78515625, "logps/ref_chosen": -97.88526916503906, "logps/ref_rejected": -112.70502471923828, "logps/rejected": -389.441162109375, "loss": 1.0486, "margin_dpo/margin_mean": 95.83621978759766, "margin_dpo/margin_std": 105.78228759765625, "step": 640 }, { "KL/chosen_KL_mean": -166.28106689453125, "KL/mean": -216.40504455566406, "KL/rejected_KL_mean": -266.529052734375, "KL/std": 123.32797241210938, "epoch": 0.9412628487518355, "fcm_dpo/beta": 0.004731766879558563, "fcm_dpo/delta": -0.025555146858096123, "fcm_dpo/margin": 100.24797058105469, "fcm_dpo/q_t": 0.3950076103210449, "grad_norm": 9.189568519592285, "learning_rate": 5.516592558795746e-09, "logits/chosen": -2.4536731243133545, "logits/rejected": -1.9285693168640137, "logps/chosen": -262.72674560546875, "logps/ref_chosen": -96.4456787109375, "logps/ref_rejected": -95.13568878173828, "logps/rejected": -361.66473388671875, "loss": 1.058, "margin_dpo/margin_mean": 100.24797058105469, "margin_dpo/margin_std": 135.9686737060547, "step": 641 }, { "KL/chosen_KL_mean": -184.8050537109375, "KL/mean": -231.78439331054688, "KL/rejected_KL_mean": -278.76373291015625, "KL/std": 106.00778198242188, "epoch": 0.9427312775330396, "fcm_dpo/beta": 0.004737743176519871, "fcm_dpo/delta": 0.004920683801174164, "fcm_dpo/margin": 93.9587173461914, "fcm_dpo/q_t": 0.40042415261268616, "grad_norm": 11.429861068725586, "learning_rate": 5.251706922648868e-09, "logits/chosen": -1.5664832592010498, "logits/rejected": -1.2332630157470703, "logps/chosen": -285.56488037109375, "logps/ref_chosen": -100.75984954833984, "logps/ref_rejected": -114.7076416015625, "logps/rejected": -393.47137451171875, "loss": 1.0766, "margin_dpo/margin_mean": 93.9587173461914, "margin_dpo/margin_std": 131.00064086914062, "step": 642 }, { "KL/chosen_KL_mean": -173.32237243652344, "KL/mean": -218.06170654296875, "KL/rejected_KL_mean": -262.801025390625, "KL/std": 98.66541290283203, "epoch": 0.9441997063142438, "fcm_dpo/beta": 0.004710989538580179, "fcm_dpo/delta": 0.028752140700817108, "fcm_dpo/margin": 89.47865295410156, "fcm_dpo/q_t": 0.4023323059082031, "grad_norm": 11.671807289123535, "learning_rate": 4.993270631642038e-09, "logits/chosen": -2.3143203258514404, "logits/rejected": -1.9429935216903687, "logps/chosen": -258.0660400390625, "logps/ref_chosen": -84.74365997314453, "logps/ref_rejected": -94.31842041015625, "logps/rejected": -357.11944580078125, "loss": 1.083, "margin_dpo/margin_mean": 89.47865295410156, "margin_dpo/margin_std": 114.07974243164062, "step": 643 }, { "KL/chosen_KL_mean": -179.53750610351562, "KL/mean": -216.66766357421875, "KL/rejected_KL_mean": -253.79782104492188, "KL/std": 99.16422271728516, "epoch": 0.9456681350954479, "fcm_dpo/beta": 0.004822202026844025, "fcm_dpo/delta": 0.09530861675739288, "fcm_dpo/margin": 74.26031494140625, "fcm_dpo/q_t": 0.4172069728374481, "grad_norm": 12.127134323120117, "learning_rate": 4.741290495811873e-09, "logits/chosen": -1.798338770866394, "logits/rejected": -1.526397466659546, "logps/chosen": -264.8602600097656, "logps/ref_chosen": -85.32274627685547, "logps/ref_rejected": -94.60861206054688, "logps/rejected": -348.40643310546875, "loss": 1.1223, "margin_dpo/margin_mean": 74.26031494140625, "margin_dpo/margin_std": 106.45948791503906, "step": 644 }, { "KL/chosen_KL_mean": -192.14700317382812, "KL/mean": -216.12217712402344, "KL/rejected_KL_mean": -240.0973358154297, "KL/std": 108.17072296142578, "epoch": 0.947136563876652, "fcm_dpo/beta": 0.004887172486633062, "fcm_dpo/delta": 0.08570849895477295, "fcm_dpo/margin": 47.950321197509766, "fcm_dpo/q_t": 0.4466143846511841, "grad_norm": 11.438575744628906, "learning_rate": 4.495773155069299e-09, "logits/chosen": -1.275451898574829, "logits/rejected": -1.1009612083435059, "logps/chosen": -274.73724365234375, "logps/ref_chosen": -82.59024047851562, "logps/ref_rejected": -103.00375366210938, "logps/rejected": -343.10107421875, "loss": 1.2523, "margin_dpo/margin_mean": 47.950321197509766, "margin_dpo/margin_std": 122.57881164550781, "step": 645 }, { "KL/chosen_KL_mean": -167.93702697753906, "KL/mean": -215.34378051757812, "KL/rejected_KL_mean": -262.75054931640625, "KL/std": 94.5502700805664, "epoch": 0.9486049926578561, "fcm_dpo/beta": 0.004896007478237152, "fcm_dpo/delta": -0.015372931957244873, "fcm_dpo/margin": 94.81353759765625, "fcm_dpo/q_t": 0.392168253660202, "grad_norm": 9.645984649658203, "learning_rate": 4.256725079024553e-09, "logits/chosen": -1.759879231452942, "logits/rejected": -1.1478692293167114, "logps/chosen": -259.1290588378906, "logps/ref_chosen": -91.1920394897461, "logps/ref_rejected": -83.77833557128906, "logps/rejected": -346.52886962890625, "loss": 1.0355, "margin_dpo/margin_mean": 94.81353759765625, "margin_dpo/margin_std": 102.04331970214844, "step": 646 }, { "KL/chosen_KL_mean": -183.03317260742188, "KL/mean": -224.60198974609375, "KL/rejected_KL_mean": -266.1707763671875, "KL/std": 114.03018188476562, "epoch": 0.9500734214390602, "fcm_dpo/beta": 0.0049431947991251945, "fcm_dpo/delta": 0.04071224108338356, "fcm_dpo/margin": 83.13764190673828, "fcm_dpo/q_t": 0.40789031982421875, "grad_norm": 10.713689804077148, "learning_rate": 4.024152566816791e-09, "logits/chosen": -1.5028190612792969, "logits/rejected": -1.3653578758239746, "logps/chosen": -271.87762451171875, "logps/ref_chosen": -88.8444595336914, "logps/ref_rejected": -99.49833679199219, "logps/rejected": -365.66912841796875, "loss": 1.1079, "margin_dpo/margin_mean": 83.13764953613281, "margin_dpo/margin_std": 126.62890625, "step": 647 }, { "KL/chosen_KL_mean": -169.91302490234375, "KL/mean": -227.63442993164062, "KL/rejected_KL_mean": -285.3558349609375, "KL/std": 116.15348815917969, "epoch": 0.9515418502202643, "fcm_dpo/beta": 0.004880198277533054, "fcm_dpo/delta": -0.11997868120670319, "fcm_dpo/margin": 115.44281005859375, "fcm_dpo/q_t": 0.3711271584033966, "grad_norm": 10.906291961669922, "learning_rate": 3.798061746947995e-09, "logits/chosen": -1.8949060440063477, "logits/rejected": -1.8207893371582031, "logps/chosen": -257.7611389160156, "logps/ref_chosen": -87.84810638427734, "logps/ref_rejected": -104.67005920410156, "logps/rejected": -390.02587890625, "loss": 0.983, "margin_dpo/margin_mean": 115.44281005859375, "margin_dpo/margin_std": 122.52619171142578, "step": 648 }, { "KL/chosen_KL_mean": -155.88717651367188, "KL/mean": -204.36273193359375, "KL/rejected_KL_mean": -252.83828735351562, "KL/std": 101.7880859375, "epoch": 0.9530102790014684, "fcm_dpo/beta": 0.00479974877089262, "fcm_dpo/delta": -0.0167253278195858, "fcm_dpo/margin": 96.95110321044922, "fcm_dpo/q_t": 0.39167076349258423, "grad_norm": 11.536678314208984, "learning_rate": 3.5784585771215235e-09, "logits/chosen": -2.4893527030944824, "logits/rejected": -2.091926097869873, "logps/chosen": -245.57968139648438, "logps/ref_chosen": -89.6925048828125, "logps/ref_rejected": -88.70658111572266, "logps/rejected": -341.54486083984375, "loss": 1.0292, "margin_dpo/margin_mean": 96.95109558105469, "margin_dpo/margin_std": 97.85920715332031, "step": 649 }, { "KL/chosen_KL_mean": -184.9298553466797, "KL/mean": -230.22842407226562, "KL/rejected_KL_mean": -275.5269775390625, "KL/std": 109.57231140136719, "epoch": 0.9544787077826725, "fcm_dpo/beta": 0.004835030063986778, "fcm_dpo/delta": 0.012466302141547203, "fcm_dpo/margin": 90.59712982177734, "fcm_dpo/q_t": 0.40001872181892395, "grad_norm": 10.516207695007324, "learning_rate": 3.3653488440851253e-09, "logits/chosen": -1.15730881690979, "logits/rejected": -0.9545855522155762, "logps/chosen": -274.8604736328125, "logps/ref_chosen": -89.93060302734375, "logps/ref_rejected": -102.61282348632812, "logps/rejected": -378.1398010253906, "loss": 1.0848, "margin_dpo/margin_mean": 90.59713745117188, "margin_dpo/margin_std": 129.09523010253906, "step": 650 }, { "KL/chosen_KL_mean": -153.58432006835938, "KL/mean": -209.2176971435547, "KL/rejected_KL_mean": -264.85107421875, "KL/std": 106.31681823730469, "epoch": 0.9559471365638766, "fcm_dpo/beta": 0.00476833013817668, "fcm_dpo/delta": -0.08514226973056793, "fcm_dpo/margin": 111.2667236328125, "fcm_dpo/q_t": 0.37607014179229736, "grad_norm": 9.568754196166992, "learning_rate": 3.158738163478475e-09, "logits/chosen": -2.108980655670166, "logits/rejected": -1.9345710277557373, "logps/chosen": -232.7716522216797, "logps/ref_chosen": -79.18731689453125, "logps/ref_rejected": -105.93333435058594, "logps/rejected": -370.7843933105469, "loss": 0.9835, "margin_dpo/margin_mean": 111.2667236328125, "margin_dpo/margin_std": 104.88247680664062, "step": 651 }, { "KL/chosen_KL_mean": -160.6580047607422, "KL/mean": -207.53530883789062, "KL/rejected_KL_mean": -254.41259765625, "KL/std": 98.65438079833984, "epoch": 0.9574155653450808, "fcm_dpo/beta": 0.0047644879668951035, "fcm_dpo/delta": 0.003341231495141983, "fcm_dpo/margin": 93.75457763671875, "fcm_dpo/q_t": 0.39559128880500793, "grad_norm": 9.580215454101562, "learning_rate": 2.9586319796851555e-09, "logits/chosen": -2.5949487686157227, "logits/rejected": -2.302154541015625, "logps/chosen": -262.4482421875, "logps/ref_chosen": -101.79022216796875, "logps/ref_rejected": -116.3245849609375, "logps/rejected": -370.7371826171875, "loss": 1.0354, "margin_dpo/margin_mean": 93.75457000732422, "margin_dpo/margin_std": 93.6308822631836, "step": 652 }, { "KL/chosen_KL_mean": -184.45904541015625, "KL/mean": -229.0028076171875, "KL/rejected_KL_mean": -273.54656982421875, "KL/std": 107.53547668457031, "epoch": 0.9588839941262849, "fcm_dpo/beta": 0.004780316725373268, "fcm_dpo/delta": 0.025099385529756546, "fcm_dpo/margin": 89.08750915527344, "fcm_dpo/q_t": 0.3997681736946106, "grad_norm": 10.551021575927734, "learning_rate": 2.7650355656892166e-09, "logits/chosen": -2.2270355224609375, "logits/rejected": -1.978057622909546, "logps/chosen": -277.8126220703125, "logps/ref_chosen": -93.35359191894531, "logps/ref_rejected": -109.12324523925781, "logps/rejected": -382.6697998046875, "loss": 1.066, "margin_dpo/margin_mean": 89.08751678466797, "margin_dpo/margin_std": 105.1566390991211, "step": 653 }, { "KL/chosen_KL_mean": -184.26467895507812, "KL/mean": -223.2406005859375, "KL/rejected_KL_mean": -262.2165222167969, "KL/std": 104.10440826416016, "epoch": 0.960352422907489, "fcm_dpo/beta": 0.004833141341805458, "fcm_dpo/delta": 0.07610604166984558, "fcm_dpo/margin": 77.95184326171875, "fcm_dpo/q_t": 0.4165007472038269, "grad_norm": 13.57054615020752, "learning_rate": 2.577954022936174e-09, "logits/chosen": -2.2561111450195312, "logits/rejected": -2.0729568004608154, "logps/chosen": -273.3802185058594, "logps/ref_chosen": -89.11553955078125, "logps/ref_rejected": -104.91995239257812, "logps/rejected": -367.136474609375, "loss": 1.1369, "margin_dpo/margin_mean": 77.95184326171875, "margin_dpo/margin_std": 131.37371826171875, "step": 654 }, { "KL/chosen_KL_mean": -168.5019073486328, "KL/mean": -215.442138671875, "KL/rejected_KL_mean": -262.38238525390625, "KL/std": 112.25436401367188, "epoch": 0.9618208516886931, "fcm_dpo/beta": 0.004854114260524511, "fcm_dpo/delta": -0.006014498881995678, "fcm_dpo/margin": 93.88047790527344, "fcm_dpo/q_t": 0.394818514585495, "grad_norm": 9.471657752990723, "learning_rate": 2.397392281198729e-09, "logits/chosen": -1.9707748889923096, "logits/rejected": -1.971581220626831, "logps/chosen": -249.5380096435547, "logps/ref_chosen": -81.03610229492188, "logps/ref_rejected": -102.80233764648438, "logps/rejected": -365.1847229003906, "loss": 1.0617, "margin_dpo/margin_mean": 93.88047790527344, "margin_dpo/margin_std": 119.57864379882812, "step": 655 }, { "KL/chosen_KL_mean": -168.82737731933594, "KL/mean": -228.6781005859375, "KL/rejected_KL_mean": -288.5288391113281, "KL/std": 113.01327514648438, "epoch": 0.9632892804698973, "fcm_dpo/beta": 0.004747756756842136, "fcm_dpo/delta": -0.125717893242836, "fcm_dpo/margin": 119.70145416259766, "fcm_dpo/q_t": 0.3699204921722412, "grad_norm": 10.317848205566406, "learning_rate": 2.223355098446622e-09, "logits/chosen": -1.5403485298156738, "logits/rejected": -1.6090104579925537, "logps/chosen": -254.15272521972656, "logps/ref_chosen": -85.32534790039062, "logps/ref_rejected": -118.33866119384766, "logps/rejected": -406.86749267578125, "loss": 0.965, "margin_dpo/margin_mean": 119.70146179199219, "margin_dpo/margin_std": 116.03785705566406, "step": 656 }, { "KL/chosen_KL_mean": -184.0067138671875, "KL/mean": -227.7652587890625, "KL/rejected_KL_mean": -271.5237731933594, "KL/std": 103.92236328125, "epoch": 0.9647577092511013, "fcm_dpo/beta": 0.004729727283120155, "fcm_dpo/delta": 0.03736189752817154, "fcm_dpo/margin": 87.51704406738281, "fcm_dpo/q_t": 0.4064503312110901, "grad_norm": 12.323017120361328, "learning_rate": 2.055847060721566e-09, "logits/chosen": -1.886023998260498, "logits/rejected": -1.7156312465667725, "logps/chosen": -264.2044372558594, "logps/ref_chosen": -80.19772338867188, "logps/ref_rejected": -102.581298828125, "logps/rejected": -374.1050720214844, "loss": 1.099, "margin_dpo/margin_mean": 87.51704406738281, "margin_dpo/margin_std": 125.48452758789062, "step": 657 }, { "KL/chosen_KL_mean": -183.02366638183594, "KL/mean": -223.7471923828125, "KL/rejected_KL_mean": -264.470703125, "KL/std": 98.57740020751953, "epoch": 0.9662261380323054, "fcm_dpo/beta": 0.0048185959458351135, "fcm_dpo/delta": 0.0594821460545063, "fcm_dpo/margin": 81.4470443725586, "fcm_dpo/q_t": 0.4092613160610199, "grad_norm": 9.629891395568848, "learning_rate": 1.8948725820160662e-09, "logits/chosen": -2.125157356262207, "logits/rejected": -1.6637425422668457, "logps/chosen": -277.658203125, "logps/ref_chosen": -94.634521484375, "logps/ref_rejected": -101.6316146850586, "logps/rejected": -366.1023254394531, "loss": 1.0951, "margin_dpo/margin_mean": 81.4470443725586, "margin_dpo/margin_std": 106.2724380493164, "step": 658 }, { "KL/chosen_KL_mean": -165.2835693359375, "KL/mean": -214.57528686523438, "KL/rejected_KL_mean": -263.86700439453125, "KL/std": 106.11569213867188, "epoch": 0.9676945668135095, "fcm_dpo/beta": 0.004780924879014492, "fcm_dpo/delta": -0.022840596735477448, "fcm_dpo/margin": 98.58344268798828, "fcm_dpo/q_t": 0.39388275146484375, "grad_norm": 9.142254829406738, "learning_rate": 1.7404359041573723e-09, "logits/chosen": -2.28271484375, "logits/rejected": -1.3397457599639893, "logps/chosen": -277.8394470214844, "logps/ref_chosen": -112.55587005615234, "logps/ref_rejected": -93.9216079711914, "logps/rejected": -357.78863525390625, "loss": 1.075, "margin_dpo/margin_mean": 98.58345031738281, "margin_dpo/margin_std": 138.21209716796875, "step": 659 }, { "KL/chosen_KL_mean": -172.90357971191406, "KL/mean": -223.55447387695312, "KL/rejected_KL_mean": -274.20538330078125, "KL/std": 103.32798767089844, "epoch": 0.9691629955947136, "fcm_dpo/beta": 0.00476967915892601, "fcm_dpo/delta": -0.034870874136686325, "fcm_dpo/margin": 101.30177307128906, "fcm_dpo/q_t": 0.38880249857902527, "grad_norm": 12.850919723510742, "learning_rate": 1.592541096695571e-09, "logits/chosen": -1.628364086151123, "logits/rejected": -0.9756385087966919, "logps/chosen": -266.281005859375, "logps/ref_chosen": -93.37742614746094, "logps/ref_rejected": -81.39482116699219, "logps/rejected": -355.6001892089844, "loss": 1.0288, "margin_dpo/margin_mean": 101.30177307128906, "margin_dpo/margin_std": 114.35575866699219, "step": 660 }, { "KL/chosen_KL_mean": -172.91943359375, "KL/mean": -230.09494018554688, "KL/rejected_KL_mean": -287.27044677734375, "KL/std": 115.08345031738281, "epoch": 0.9706314243759178, "fcm_dpo/beta": 0.004706782754510641, "fcm_dpo/delta": -0.09313911199569702, "fcm_dpo/margin": 114.35101318359375, "fcm_dpo/q_t": 0.3779047727584839, "grad_norm": 10.328747749328613, "learning_rate": 1.4511920567963908e-09, "logits/chosen": -1.8205652236938477, "logits/rejected": -1.3034087419509888, "logps/chosen": -260.77459716796875, "logps/ref_chosen": -87.85516357421875, "logps/ref_rejected": -92.40330505371094, "logps/rejected": -379.67376708984375, "loss": 1.003, "margin_dpo/margin_mean": 114.35101318359375, "margin_dpo/margin_std": 127.89285278320312, "step": 661 }, { "KL/chosen_KL_mean": -168.3148193359375, "KL/mean": -220.00526428222656, "KL/rejected_KL_mean": -271.6957092285156, "KL/std": 100.7904052734375, "epoch": 0.9720998531571219, "fcm_dpo/beta": 0.0046593910083174706, "fcm_dpo/delta": -0.033299483358860016, "fcm_dpo/margin": 103.380859375, "fcm_dpo/q_t": 0.3881474733352661, "grad_norm": 10.51960563659668, "learning_rate": 1.3163925091384532e-09, "logits/chosen": -1.7792580127716064, "logits/rejected": -1.2135748863220215, "logps/chosen": -271.0946044921875, "logps/ref_chosen": -102.77980041503906, "logps/ref_rejected": -95.22531127929688, "logps/rejected": -366.9210205078125, "loss": 1.0198, "margin_dpo/margin_mean": 103.38087463378906, "margin_dpo/margin_std": 105.8804702758789, "step": 662 }, { "KL/chosen_KL_mean": -169.60647583007812, "KL/mean": -216.4501953125, "KL/rejected_KL_mean": -263.2939147949219, "KL/std": 104.20580291748047, "epoch": 0.973568281938326, "fcm_dpo/beta": 0.004663016647100449, "fcm_dpo/delta": 0.01360559556633234, "fcm_dpo/margin": 93.68742370605469, "fcm_dpo/q_t": 0.39977845549583435, "grad_norm": 11.331018447875977, "learning_rate": 1.1881460058152382e-09, "logits/chosen": -2.1741013526916504, "logits/rejected": -1.9527440071105957, "logps/chosen": -265.95306396484375, "logps/ref_chosen": -96.34658813476562, "logps/ref_rejected": -120.52645874023438, "logps/rejected": -383.82037353515625, "loss": 1.0722, "margin_dpo/margin_mean": 93.68743133544922, "margin_dpo/margin_std": 121.16324615478516, "step": 663 }, { "KL/chosen_KL_mean": -187.57080078125, "KL/mean": -230.36416625976562, "KL/rejected_KL_mean": -273.15753173828125, "KL/std": 106.96551513671875, "epoch": 0.9750367107195301, "fcm_dpo/beta": 0.0047010621055960655, "fcm_dpo/delta": 0.04943276569247246, "fcm_dpo/margin": 85.58675384521484, "fcm_dpo/q_t": 0.40661299228668213, "grad_norm": 9.259799003601074, "learning_rate": 1.066455926241383e-09, "logits/chosen": -1.527221441268921, "logits/rejected": -1.3484258651733398, "logps/chosen": -279.4132080078125, "logps/ref_chosen": -91.84242248535156, "logps/ref_rejected": -111.8366928100586, "logps/rejected": -384.9942321777344, "loss": 1.0844, "margin_dpo/margin_mean": 85.58674621582031, "margin_dpo/margin_std": 105.78905487060547, "step": 664 }, { "KL/chosen_KL_mean": -161.17608642578125, "KL/mean": -208.01638793945312, "KL/rejected_KL_mean": -254.85671997070312, "KL/std": 103.48817443847656, "epoch": 0.9765051395007343, "fcm_dpo/beta": 0.004718531854450703, "fcm_dpo/delta": 0.00821390375494957, "fcm_dpo/margin": 93.68061828613281, "fcm_dpo/q_t": 0.39649200439453125, "grad_norm": 8.78691291809082, "learning_rate": 9.513254770636137e-10, "logits/chosen": -2.5400588512420654, "logits/rejected": -2.1195805072784424, "logps/chosen": -249.36227416992188, "logps/ref_chosen": -88.18618774414062, "logps/ref_rejected": -91.9120101928711, "logps/rejected": -346.76873779296875, "loss": 1.0511, "margin_dpo/margin_mean": 93.68061828613281, "margin_dpo/margin_std": 105.55032348632812, "step": 665 }, { "KL/chosen_KL_mean": -177.7834014892578, "KL/mean": -225.45401000976562, "KL/rejected_KL_mean": -273.12457275390625, "KL/std": 101.31594848632812, "epoch": 0.9779735682819384, "fcm_dpo/beta": 0.004697824828326702, "fcm_dpo/delta": 0.00203627347946167, "fcm_dpo/margin": 95.34117126464844, "fcm_dpo/q_t": 0.39769503474235535, "grad_norm": 11.992183685302734, "learning_rate": 8.427576920763956e-10, "logits/chosen": -1.56244695186615, "logits/rejected": -1.1785318851470947, "logps/chosen": -278.75799560546875, "logps/ref_chosen": -100.97459411621094, "logps/ref_rejected": -101.24992370605469, "logps/rejected": -374.37451171875, "loss": 1.0639, "margin_dpo/margin_mean": 95.34117126464844, "margin_dpo/margin_std": 119.4270248413086, "step": 666 }, { "KL/chosen_KL_mean": -192.60086059570312, "KL/mean": -241.48414611816406, "KL/rejected_KL_mean": -290.3674621582031, "KL/std": 105.29362487792969, "epoch": 0.9794419970631424, "fcm_dpo/beta": 0.004698245320469141, "fcm_dpo/delta": -0.009837804362177849, "fcm_dpo/margin": 97.76659393310547, "fcm_dpo/q_t": 0.3927449584007263, "grad_norm": 10.372552871704102, "learning_rate": 7.407554321417764e-10, "logits/chosen": -1.4981027841567993, "logits/rejected": -0.839004635810852, "logps/chosen": -290.1720275878906, "logps/ref_chosen": -97.5711669921875, "logps/ref_rejected": -93.58476257324219, "logps/rejected": -383.95220947265625, "loss": 1.0336, "margin_dpo/margin_mean": 97.76659393310547, "margin_dpo/margin_std": 102.42034912109375, "step": 667 }, { "KL/chosen_KL_mean": -170.088623046875, "KL/mean": -227.27218627929688, "KL/rejected_KL_mean": -284.45574951171875, "KL/std": 113.23973083496094, "epoch": 0.9809104258443465, "fcm_dpo/beta": 0.004648497793823481, "fcm_dpo/delta": -0.08614526689052582, "fcm_dpo/margin": 114.36713409423828, "fcm_dpo/q_t": 0.37957486510276794, "grad_norm": 11.999695777893066, "learning_rate": 6.453213851142225e-10, "logits/chosen": -2.3595683574676514, "logits/rejected": -1.8413339853286743, "logps/chosen": -272.6636657714844, "logps/ref_chosen": -102.5750503540039, "logps/ref_rejected": -108.81768798828125, "logps/rejected": -393.2734375, "loss": 0.9981, "margin_dpo/margin_mean": 114.36714172363281, "margin_dpo/margin_std": 123.38951110839844, "step": 668 }, { "KL/chosen_KL_mean": -155.29486083984375, "KL/mean": -210.3527374267578, "KL/rejected_KL_mean": -265.41064453125, "KL/std": 109.57008361816406, "epoch": 0.9823788546255506, "fcm_dpo/beta": 0.004584559705108404, "fcm_dpo/delta": -0.0577290877699852, "fcm_dpo/margin": 110.11578369140625, "fcm_dpo/q_t": 0.3834017217159271, "grad_norm": 10.260527610778809, "learning_rate": 5.564580657695939e-10, "logits/chosen": -1.5053339004516602, "logits/rejected": -1.0941473245620728, "logps/chosen": -244.78964233398438, "logps/ref_chosen": -89.49478149414062, "logps/ref_rejected": -82.51950073242188, "logps/rejected": -347.93011474609375, "loss": 1.0095, "margin_dpo/margin_mean": 110.11578369140625, "margin_dpo/margin_std": 115.93708801269531, "step": 669 }, { "KL/chosen_KL_mean": -155.6254119873047, "KL/mean": -216.21726989746094, "KL/rejected_KL_mean": -276.80914306640625, "KL/std": 112.19461822509766, "epoch": 0.9838472834067548, "fcm_dpo/beta": 0.004506596364080906, "fcm_dpo/delta": -0.10158378630876541, "fcm_dpo/margin": 121.18373107910156, "fcm_dpo/q_t": 0.3722996711730957, "grad_norm": 8.068968772888184, "learning_rate": 4.741678157389739e-10, "logits/chosen": -1.727135419845581, "logits/rejected": -1.1860239505767822, "logps/chosen": -251.08001708984375, "logps/ref_chosen": -95.45459747314453, "logps/ref_rejected": -101.53292846679688, "logps/rejected": -378.342041015625, "loss": 0.9783, "margin_dpo/margin_mean": 121.18373107910156, "margin_dpo/margin_std": 116.87356567382812, "step": 670 }, { "KL/chosen_KL_mean": -190.99586486816406, "KL/mean": -231.87551879882812, "KL/rejected_KL_mean": -272.7551574707031, "KL/std": 103.49320983886719, "epoch": 0.9853157121879589, "fcm_dpo/beta": 0.004556992091238499, "fcm_dpo/delta": 0.07927247136831284, "fcm_dpo/margin": 81.7593002319336, "fcm_dpo/q_t": 0.4144529700279236, "grad_norm": 10.071465492248535, "learning_rate": 3.9845280344705245e-10, "logits/chosen": -1.6471831798553467, "logits/rejected": -1.4217215776443481, "logps/chosen": -273.1189880371094, "logps/ref_chosen": -82.12312316894531, "logps/ref_rejected": -90.21969604492188, "logps/rejected": -362.974853515625, "loss": 1.1147, "margin_dpo/margin_mean": 81.7593002319336, "margin_dpo/margin_std": 114.15276336669922, "step": 671 }, { "KL/chosen_KL_mean": -189.8311767578125, "KL/mean": -225.12796020507812, "KL/rejected_KL_mean": -260.42474365234375, "KL/std": 99.92588806152344, "epoch": 0.986784140969163, "fcm_dpo/beta": 0.004630607552826405, "fcm_dpo/delta": 0.127395898103714, "fcm_dpo/margin": 70.59355926513672, "fcm_dpo/q_t": 0.42451173067092896, "grad_norm": 12.29115104675293, "learning_rate": 3.293150240547549e-10, "logits/chosen": -1.893174648284912, "logits/rejected": -1.5534356832504272, "logps/chosen": -279.8930969238281, "logps/ref_chosen": -90.0619125366211, "logps/ref_rejected": -100.45323181152344, "logps/rejected": -360.87799072265625, "loss": 1.1456, "margin_dpo/margin_mean": 70.59355926513672, "margin_dpo/margin_std": 104.85562133789062, "step": 672 }, { "KL/chosen_KL_mean": -170.20123291015625, "KL/mean": -210.61099243164062, "KL/rejected_KL_mean": -251.02072143554688, "KL/std": 102.75851440429688, "epoch": 0.9882525697503671, "fcm_dpo/beta": 0.004727584309875965, "fcm_dpo/delta": 0.0703030526638031, "fcm_dpo/margin": 80.8194808959961, "fcm_dpo/q_t": 0.4116077125072479, "grad_norm": 11.87732982635498, "learning_rate": 2.6675629940689504e-10, "logits/chosen": -1.9416654109954834, "logits/rejected": -1.5728824138641357, "logps/chosen": -249.46438598632812, "logps/ref_chosen": -79.26315307617188, "logps/ref_rejected": -91.34925079345703, "logps/rejected": -342.3699645996094, "loss": 1.0986, "margin_dpo/margin_mean": 80.81948852539062, "margin_dpo/margin_std": 104.83180236816406, "step": 673 }, { "KL/chosen_KL_mean": -159.9476318359375, "KL/mean": -216.77557373046875, "KL/rejected_KL_mean": -273.60357666015625, "KL/std": 103.0810546875, "epoch": 0.9897209985315712, "fcm_dpo/beta": 0.004667171277105808, "fcm_dpo/delta": -0.08509520441293716, "fcm_dpo/margin": 113.65593719482422, "fcm_dpo/q_t": 0.37813296914100647, "grad_norm": 8.982342720031738, "learning_rate": 2.1077827798404725e-10, "logits/chosen": -1.3266725540161133, "logits/rejected": -0.9054714441299438, "logps/chosen": -235.40594482421875, "logps/ref_chosen": -75.45831298828125, "logps/ref_rejected": -76.20362854003906, "logps/rejected": -349.80718994140625, "loss": 0.9866, "margin_dpo/margin_mean": 113.65594482421875, "margin_dpo/margin_std": 112.28579711914062, "step": 674 }, { "KL/chosen_KL_mean": -175.3204803466797, "KL/mean": -225.6072235107422, "KL/rejected_KL_mean": -275.8939514160156, "KL/std": 111.3902359008789, "epoch": 0.9911894273127754, "fcm_dpo/beta": 0.004610551055520773, "fcm_dpo/delta": -0.015415102243423462, "fcm_dpo/margin": 100.573486328125, "fcm_dpo/q_t": 0.3937044143676758, "grad_norm": 10.685237884521484, "learning_rate": 1.6138243485910863e-10, "logits/chosen": -1.6183019876480103, "logits/rejected": -1.2845849990844727, "logps/chosen": -255.23001098632812, "logps/ref_chosen": -79.90953063964844, "logps/ref_rejected": -81.21824645996094, "logps/rejected": -357.1121826171875, "loss": 1.0705, "margin_dpo/margin_mean": 100.573486328125, "margin_dpo/margin_std": 135.77354431152344, "step": 675 }, { "KL/chosen_KL_mean": -188.84844970703125, "KL/mean": -238.12591552734375, "KL/rejected_KL_mean": -287.40338134765625, "KL/std": 106.96812438964844, "epoch": 0.9926578560939795, "fcm_dpo/beta": 0.0046224407851696014, "fcm_dpo/delta": -0.005975460633635521, "fcm_dpo/margin": 98.5549545288086, "fcm_dpo/q_t": 0.3946434259414673, "grad_norm": 9.166580200195312, "learning_rate": 1.1857007165852472e-10, "logits/chosen": -2.1155519485473633, "logits/rejected": -1.5424535274505615, "logps/chosen": -287.0195617675781, "logps/ref_chosen": -98.17111206054688, "logps/ref_rejected": -95.024658203125, "logps/rejected": -382.42803955078125, "loss": 1.0555, "margin_dpo/margin_mean": 98.55494689941406, "margin_dpo/margin_std": 119.53730010986328, "step": 676 }, { "KL/chosen_KL_mean": -168.68438720703125, "KL/mean": -218.90933227539062, "KL/rejected_KL_mean": -269.13427734375, "KL/std": 105.43244934082031, "epoch": 0.9941262848751835, "fcm_dpo/beta": 0.00461141113191843, "fcm_dpo/delta": -0.01402292400598526, "fcm_dpo/margin": 100.44989013671875, "fcm_dpo/q_t": 0.39210790395736694, "grad_norm": 8.679218292236328, "learning_rate": 8.23423165278725e-11, "logits/chosen": -1.9051822423934937, "logits/rejected": -1.2468986511230469, "logps/chosen": -260.06365966796875, "logps/ref_chosen": -91.37928771972656, "logps/ref_rejected": -82.87776947021484, "logps/rejected": -352.01202392578125, "loss": 1.0304, "margin_dpo/margin_mean": 100.44989013671875, "margin_dpo/margin_std": 102.97470092773438, "step": 677 }, { "KL/chosen_KL_mean": -159.788330078125, "KL/mean": -219.01699829101562, "KL/rejected_KL_mean": -278.24566650390625, "KL/std": 112.53511047363281, "epoch": 0.9955947136563876, "fcm_dpo/beta": 0.004575745202600956, "fcm_dpo/delta": -0.09731646627187729, "fcm_dpo/margin": 118.45732116699219, "fcm_dpo/q_t": 0.3743739724159241, "grad_norm": 10.344512939453125, "learning_rate": 5.270012410216185e-11, "logits/chosen": -1.8819496631622314, "logits/rejected": -1.5630353689193726, "logps/chosen": -235.4342041015625, "logps/ref_chosen": -75.6458740234375, "logps/ref_rejected": -86.96611022949219, "logps/rejected": -365.2117919921875, "loss": 0.9814, "margin_dpo/margin_mean": 118.45732116699219, "margin_dpo/margin_std": 113.800537109375, "step": 678 }, { "KL/chosen_KL_mean": -183.46322631835938, "KL/mean": -232.7026824951172, "KL/rejected_KL_mean": -281.942138671875, "KL/std": 93.56512451171875, "epoch": 0.9970631424375918, "fcm_dpo/beta": 0.0045309788547456264, "fcm_dpo/delta": 0.003967920318245888, "fcm_dpo/margin": 98.47888946533203, "fcm_dpo/q_t": 0.3962134122848511, "grad_norm": 9.133407592773438, "learning_rate": 2.9644275480772416e-11, "logits/chosen": -2.3770525455474854, "logits/rejected": -1.8721058368682861, "logps/chosen": -264.2366943359375, "logps/ref_chosen": -80.77344512939453, "logps/ref_rejected": -82.87850189208984, "logps/rejected": -364.82061767578125, "loss": 1.0359, "margin_dpo/margin_mean": 98.4788818359375, "margin_dpo/margin_std": 99.60039520263672, "step": 679 }, { "KL/chosen_KL_mean": -185.55514526367188, "KL/mean": -238.05531311035156, "KL/rejected_KL_mean": -290.55548095703125, "KL/std": 111.19891357421875, "epoch": 0.9985315712187959, "fcm_dpo/beta": 0.00454926211386919, "fcm_dpo/delta": -0.030054979026317596, "fcm_dpo/margin": 105.0003433227539, "fcm_dpo/q_t": 0.39070942997932434, "grad_norm": 7.670646667480469, "learning_rate": 1.31753782067201e-11, "logits/chosen": -2.0947909355163574, "logits/rejected": -1.77396559715271, "logps/chosen": -293.23809814453125, "logps/ref_chosen": -107.68292999267578, "logps/ref_rejected": -116.0948715209961, "logps/rejected": -406.6503601074219, "loss": 1.0388, "margin_dpo/margin_mean": 105.00033569335938, "margin_dpo/margin_std": 119.4825668334961, "step": 680 }, { "KL/chosen_KL_mean": -168.48878479003906, "KL/mean": -214.45407104492188, "KL/rejected_KL_mean": -260.41937255859375, "KL/std": 105.0776138305664, "epoch": 1.0, "fcm_dpo/beta": 0.0045382184907794, "fcm_dpo/delta": 0.03409893810749054, "fcm_dpo/margin": 91.9305648803711, "fcm_dpo/q_t": 0.40387219190597534, "grad_norm": 10.59050464630127, "learning_rate": 3.2938662507808745e-12, "logits/chosen": -2.4693641662597656, "logits/rejected": -2.1671481132507324, "logps/chosen": -261.4998474121094, "logps/ref_chosen": -93.01106262207031, "logps/ref_rejected": -94.8221664428711, "logps/rejected": -355.24151611328125, "loss": 1.0761, "margin_dpo/margin_mean": 91.9305648803711, "margin_dpo/margin_std": 114.75448608398438, "step": 681 }, { "epoch": 1.0, "step": 681, "total_flos": 0.0, "train_loss": 1.0783974358513841, "train_runtime": 2022.8503, "train_samples_per_second": 21.553, "train_steps_per_second": 0.337 } ], "logging_steps": 1, "max_steps": 681, "num_input_tokens_seen": 0, "num_train_epochs": 1, "save_steps": 50, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": false, "should_training_stop": false }, "attributes": {} } }, "total_flos": 0.0, "train_batch_size": 8, "trial_name": null, "trial_params": null }