| { |
| "best_global_step": null, |
| "best_metric": null, |
| "best_model_checkpoint": null, |
| "epoch": 1.0, |
| "eval_steps": 100, |
| "global_step": 681, |
| "is_hyper_param_search": false, |
| "is_local_process_zero": true, |
| "is_world_process_zero": true, |
| "log_history": [ |
| { |
| "epoch": 0.0014684287812041115, |
| "fcm_dpo/beta": 0.10000000149011612, |
| "fcm_dpo/delta": 0.0, |
| "fcm_dpo/margin": 0.06123492121696472, |
| "fcm_dpo/q_t": 0.4984707534313202, |
| "grad_norm": 38.81780242919922, |
| "learning_rate": 0.0, |
| "logits/chosen": 0.2677389979362488, |
| "logits/rejected": 0.6191319227218628, |
| "logps/chosen": -85.37664031982422, |
| "logps/ref_chosen": -85.43083190917969, |
| "logps/ref_rejected": -79.90458679199219, |
| "logps/rejected": -79.91163635253906, |
| "loss": 1.3807, |
| "margin_dpo/margin_mean": 0.061235010623931885, |
| "margin_dpo/margin_std": 0.4424324035644531, |
| "step": 1 |
| }, |
| { |
| "epoch": 0.002936857562408223, |
| "fcm_dpo/beta": 0.10000000149011612, |
| "fcm_dpo/delta": 0.0, |
| "fcm_dpo/margin": -0.12442219257354736, |
| "fcm_dpo/q_t": 0.5031089186668396, |
| "grad_norm": 29.75464630126953, |
| "learning_rate": 7.246376811594203e-09, |
| "logits/chosen": 0.47771453857421875, |
| "logits/rejected": 0.5908545851707458, |
| "logps/chosen": -82.11383056640625, |
| "logps/ref_chosen": -82.06893157958984, |
| "logps/ref_rejected": -81.65457916259766, |
| "logps/rejected": -81.57505798339844, |
| "loss": 1.3992, |
| "margin_dpo/margin_mean": -0.12442246079444885, |
| "margin_dpo/margin_std": 0.41358208656311035, |
| "step": 2 |
| }, |
| { |
| "epoch": 0.004405286343612335, |
| "fcm_dpo/beta": 0.10000000149011612, |
| "fcm_dpo/delta": 0.0, |
| "fcm_dpo/margin": 0.014965459704399109, |
| "fcm_dpo/q_t": 0.4996257424354553, |
| "grad_norm": 26.07487678527832, |
| "learning_rate": 1.4492753623188406e-08, |
| "logits/chosen": 0.4268788695335388, |
| "logits/rejected": 0.6501424312591553, |
| "logps/chosen": -93.79121398925781, |
| "logps/ref_chosen": -93.81098937988281, |
| "logps/ref_rejected": -74.22950744628906, |
| "logps/rejected": -74.22470092773438, |
| "loss": 1.3853, |
| "margin_dpo/margin_mean": 0.014965400099754333, |
| "margin_dpo/margin_std": 0.45240455865859985, |
| "step": 3 |
| }, |
| { |
| "epoch": 0.005873715124816446, |
| "fcm_dpo/beta": 0.10000000149011612, |
| "fcm_dpo/delta": 0.0, |
| "fcm_dpo/margin": 0.02085861563682556, |
| "fcm_dpo/q_t": 0.4994773864746094, |
| "grad_norm": 32.71585464477539, |
| "learning_rate": 2.1739130434782606e-08, |
| "logits/chosen": 0.6463205814361572, |
| "logits/rejected": 0.7779839038848877, |
| "logps/chosen": -87.30850219726562, |
| "logps/ref_chosen": -87.29246520996094, |
| "logps/ref_rejected": -93.82425689697266, |
| "logps/rejected": -93.86114501953125, |
| "loss": 1.3846, |
| "margin_dpo/margin_mean": 0.020858705043792725, |
| "margin_dpo/margin_std": 0.39117318391799927, |
| "step": 4 |
| }, |
| { |
| "epoch": 0.007342143906020558, |
| "fcm_dpo/beta": 0.10000000149011612, |
| "fcm_dpo/delta": 0.0, |
| "fcm_dpo/margin": 0.050249382853507996, |
| "fcm_dpo/q_t": 0.4987432062625885, |
| "grad_norm": 35.20225143432617, |
| "learning_rate": 2.898550724637681e-08, |
| "logits/chosen": 0.4468464255332947, |
| "logits/rejected": 0.6582976579666138, |
| "logps/chosen": -89.32503509521484, |
| "logps/ref_chosen": -89.33676147460938, |
| "logps/ref_rejected": -88.74783325195312, |
| "logps/rejected": -88.78636169433594, |
| "loss": 1.3818, |
| "margin_dpo/margin_mean": 0.050250083208084106, |
| "margin_dpo/margin_std": 0.44784289598464966, |
| "step": 5 |
| }, |
| { |
| "epoch": 0.00881057268722467, |
| "fcm_dpo/beta": 0.10000000149011612, |
| "fcm_dpo/delta": 0.0, |
| "fcm_dpo/margin": -0.01285526156425476, |
| "fcm_dpo/q_t": 0.5003200173377991, |
| "grad_norm": 36.15748596191406, |
| "learning_rate": 3.6231884057971014e-08, |
| "logits/chosen": 0.48705005645751953, |
| "logits/rejected": 0.803049623966217, |
| "logps/chosen": -97.32769775390625, |
| "logps/ref_chosen": -97.32147216796875, |
| "logps/ref_rejected": -97.88345336914062, |
| "logps/rejected": -97.87681579589844, |
| "loss": 1.3881, |
| "margin_dpo/margin_mean": -0.012855678796768188, |
| "margin_dpo/margin_std": 0.42675212025642395, |
| "step": 6 |
| }, |
| { |
| "epoch": 0.010279001468428781, |
| "fcm_dpo/beta": 0.10000000149011612, |
| "fcm_dpo/delta": 0.0, |
| "fcm_dpo/margin": 0.06436635553836823, |
| "fcm_dpo/q_t": 0.49839529395103455, |
| "grad_norm": 37.96323776245117, |
| "learning_rate": 4.347826086956521e-08, |
| "logits/chosen": 0.49750369787216187, |
| "logits/rejected": 0.5453470945358276, |
| "logps/chosen": -86.62813568115234, |
| "logps/ref_chosen": -86.64852905273438, |
| "logps/ref_rejected": -109.61618041992188, |
| "logps/rejected": -109.66015625, |
| "loss": 1.3806, |
| "margin_dpo/margin_mean": 0.06436613202095032, |
| "margin_dpo/margin_std": 0.5262112021446228, |
| "step": 7 |
| }, |
| { |
| "epoch": 0.011747430249632892, |
| "fcm_dpo/beta": 0.10000000149011612, |
| "fcm_dpo/delta": 0.0, |
| "fcm_dpo/margin": -0.1121322363615036, |
| "fcm_dpo/q_t": 0.5028019547462463, |
| "grad_norm": 32.900169372558594, |
| "learning_rate": 5.0724637681159424e-08, |
| "logits/chosen": 0.436749666929245, |
| "logits/rejected": 0.7514491677284241, |
| "logps/chosen": -90.00808715820312, |
| "logps/ref_chosen": -89.9236831665039, |
| "logps/ref_rejected": -86.22804260253906, |
| "logps/rejected": -86.20030975341797, |
| "loss": 1.398, |
| "margin_dpo/margin_mean": -0.11213211715221405, |
| "margin_dpo/margin_std": 0.4109366536140442, |
| "step": 8 |
| }, |
| { |
| "epoch": 0.013215859030837005, |
| "fcm_dpo/beta": 0.10000000149011612, |
| "fcm_dpo/delta": 0.0, |
| "fcm_dpo/margin": 0.02080167829990387, |
| "fcm_dpo/q_t": 0.499479740858078, |
| "grad_norm": 36.028873443603516, |
| "learning_rate": 5.797101449275362e-08, |
| "logits/chosen": 0.661954939365387, |
| "logits/rejected": 0.8412786722183228, |
| "logps/chosen": -103.87676239013672, |
| "logps/ref_chosen": -103.85713195800781, |
| "logps/ref_rejected": -104.31932067871094, |
| "logps/rejected": -104.35975646972656, |
| "loss": 1.3848, |
| "margin_dpo/margin_mean": 0.020801380276679993, |
| "margin_dpo/margin_std": 0.4883061647415161, |
| "step": 9 |
| }, |
| { |
| "epoch": 0.014684287812041116, |
| "fcm_dpo/beta": 0.10000000149011612, |
| "fcm_dpo/delta": 0.0, |
| "fcm_dpo/margin": -0.07037302851676941, |
| "fcm_dpo/q_t": 0.5017586350440979, |
| "grad_norm": 33.41664505004883, |
| "learning_rate": 6.521739130434782e-08, |
| "logits/chosen": 0.6513626575469971, |
| "logits/rejected": 0.870223879814148, |
| "logps/chosen": -76.17118072509766, |
| "logps/ref_chosen": -76.20436096191406, |
| "logps/ref_rejected": -87.152099609375, |
| "logps/rejected": -87.04855346679688, |
| "loss": 1.3939, |
| "margin_dpo/margin_mean": -0.07037210464477539, |
| "margin_dpo/margin_std": 0.4575299322605133, |
| "step": 10 |
| }, |
| { |
| "epoch": 0.016152716593245228, |
| "fcm_dpo/beta": 0.10000000149011612, |
| "fcm_dpo/delta": 0.0, |
| "fcm_dpo/margin": 0.053068071603775024, |
| "fcm_dpo/q_t": 0.49867451190948486, |
| "grad_norm": 36.69655990600586, |
| "learning_rate": 7.246376811594203e-08, |
| "logits/chosen": 0.277323842048645, |
| "logits/rejected": 0.4923889636993408, |
| "logps/chosen": -82.32759094238281, |
| "logps/ref_chosen": -82.36650085449219, |
| "logps/ref_rejected": -94.26461791992188, |
| "logps/rejected": -94.2787857055664, |
| "loss": 1.3814, |
| "margin_dpo/margin_mean": 0.053068608045578, |
| "margin_dpo/margin_std": 0.41766154766082764, |
| "step": 11 |
| }, |
| { |
| "epoch": 0.01762114537444934, |
| "fcm_dpo/beta": 0.10000000149011612, |
| "fcm_dpo/delta": 0.0, |
| "fcm_dpo/margin": 0.01172947883605957, |
| "fcm_dpo/q_t": 0.4997062683105469, |
| "grad_norm": 40.26313781738281, |
| "learning_rate": 7.971014492753623e-08, |
| "logits/chosen": 0.40038958191871643, |
| "logits/rejected": 0.5286105871200562, |
| "logps/chosen": -99.0933837890625, |
| "logps/ref_chosen": -99.10549926757812, |
| "logps/ref_rejected": -110.27140808105469, |
| "logps/rejected": -110.27102661132812, |
| "loss": 1.3858, |
| "margin_dpo/margin_mean": 0.011729925870895386, |
| "margin_dpo/margin_std": 0.5183511972427368, |
| "step": 12 |
| }, |
| { |
| "epoch": 0.01908957415565345, |
| "fcm_dpo/beta": 0.10000000149011612, |
| "fcm_dpo/delta": 0.0, |
| "fcm_dpo/margin": -0.059926122426986694, |
| "fcm_dpo/q_t": 0.501498281955719, |
| "grad_norm": 41.93380355834961, |
| "learning_rate": 8.695652173913042e-08, |
| "logits/chosen": 0.5050075054168701, |
| "logits/rejected": 0.7639053463935852, |
| "logps/chosen": -90.6314697265625, |
| "logps/ref_chosen": -90.55972290039062, |
| "logps/ref_rejected": -93.69110107421875, |
| "logps/rejected": -93.70291137695312, |
| "loss": 1.3928, |
| "margin_dpo/margin_mean": -0.05992648005485535, |
| "margin_dpo/margin_std": 0.4534745216369629, |
| "step": 13 |
| }, |
| { |
| "epoch": 0.020558002936857563, |
| "fcm_dpo/beta": 0.10000000149011612, |
| "fcm_dpo/delta": 0.0, |
| "fcm_dpo/margin": -0.019990622997283936, |
| "fcm_dpo/q_t": 0.500498354434967, |
| "grad_norm": 35.67725372314453, |
| "learning_rate": 9.420289855072464e-08, |
| "logits/chosen": 1.0646815299987793, |
| "logits/rejected": 1.1590429544448853, |
| "logps/chosen": -99.85965728759766, |
| "logps/ref_chosen": -99.82717895507812, |
| "logps/ref_rejected": -108.94200134277344, |
| "logps/rejected": -108.95449829101562, |
| "loss": 1.3888, |
| "margin_dpo/margin_mean": -0.01999080181121826, |
| "margin_dpo/margin_std": 0.4524303078651428, |
| "step": 14 |
| }, |
| { |
| "epoch": 0.022026431718061675, |
| "fcm_dpo/beta": 0.10000000149011612, |
| "fcm_dpo/delta": 0.0, |
| "fcm_dpo/margin": 0.06978121399879456, |
| "fcm_dpo/q_t": 0.49825772643089294, |
| "grad_norm": 31.65287971496582, |
| "learning_rate": 1.0144927536231885e-07, |
| "logits/chosen": 0.4620896875858307, |
| "logits/rejected": 0.6329752802848816, |
| "logps/chosen": -78.88731384277344, |
| "logps/ref_chosen": -78.90997314453125, |
| "logps/ref_rejected": -90.06234741210938, |
| "logps/rejected": -90.10946655273438, |
| "loss": 1.3799, |
| "margin_dpo/margin_mean": 0.06978106498718262, |
| "margin_dpo/margin_std": 0.4777568280696869, |
| "step": 15 |
| }, |
| { |
| "epoch": 0.023494860499265784, |
| "fcm_dpo/beta": 0.10000000149011612, |
| "fcm_dpo/delta": 0.0, |
| "fcm_dpo/margin": 0.01587851345539093, |
| "fcm_dpo/q_t": 0.49960121512413025, |
| "grad_norm": 34.593482971191406, |
| "learning_rate": 1.0869565217391303e-07, |
| "logits/chosen": 0.7190598249435425, |
| "logits/rejected": 0.9916927814483643, |
| "logps/chosen": -97.41981506347656, |
| "logps/ref_chosen": -97.42327880859375, |
| "logps/ref_rejected": -90.59945678710938, |
| "logps/rejected": -90.61186981201172, |
| "loss": 1.3855, |
| "margin_dpo/margin_mean": 0.015878498554229736, |
| "margin_dpo/margin_std": 0.5510396361351013, |
| "step": 16 |
| }, |
| { |
| "epoch": 0.024963289280469897, |
| "fcm_dpo/beta": 0.10000000149011612, |
| "fcm_dpo/delta": 0.0, |
| "fcm_dpo/margin": 0.08168734610080719, |
| "fcm_dpo/q_t": 0.49796023964881897, |
| "grad_norm": 35.512939453125, |
| "learning_rate": 1.1594202898550725e-07, |
| "logits/chosen": 0.4151402413845062, |
| "logits/rejected": 0.6361797451972961, |
| "logps/chosen": -104.31832122802734, |
| "logps/ref_chosen": -104.36430358886719, |
| "logps/ref_rejected": -90.46772766113281, |
| "logps/rejected": -90.50341796875, |
| "loss": 1.3787, |
| "margin_dpo/margin_mean": 0.08168637752532959, |
| "margin_dpo/margin_std": 0.47371718287467957, |
| "step": 17 |
| }, |
| { |
| "epoch": 0.02643171806167401, |
| "fcm_dpo/beta": 0.10000000149011612, |
| "fcm_dpo/delta": 0.0, |
| "fcm_dpo/margin": -0.03281433880329132, |
| "fcm_dpo/q_t": 0.5008206367492676, |
| "grad_norm": 42.083641052246094, |
| "learning_rate": 1.2318840579710146e-07, |
| "logits/chosen": 0.47482335567474365, |
| "logits/rejected": 0.8290694355964661, |
| "logps/chosen": -87.07789611816406, |
| "logps/ref_chosen": -87.09195709228516, |
| "logps/ref_rejected": -81.85072326660156, |
| "logps/rejected": -81.80384826660156, |
| "loss": 1.3903, |
| "margin_dpo/margin_mean": -0.03281354904174805, |
| "margin_dpo/margin_std": 0.543687105178833, |
| "step": 18 |
| }, |
| { |
| "epoch": 0.027900146842878122, |
| "fcm_dpo/beta": 0.10000000149011612, |
| "fcm_dpo/delta": 0.0, |
| "fcm_dpo/margin": -0.07938456535339355, |
| "fcm_dpo/q_t": 0.501983642578125, |
| "grad_norm": 31.655494689941406, |
| "learning_rate": 1.3043478260869563e-07, |
| "logits/chosen": 0.5405235886573792, |
| "logits/rejected": 0.8114136457443237, |
| "logps/chosen": -105.96595764160156, |
| "logps/ref_chosen": -105.87354278564453, |
| "logps/ref_rejected": -96.93023681640625, |
| "logps/rejected": -96.94326782226562, |
| "loss": 1.3946, |
| "margin_dpo/margin_mean": -0.07938426733016968, |
| "margin_dpo/margin_std": 0.3811599910259247, |
| "step": 19 |
| }, |
| { |
| "epoch": 0.02936857562408223, |
| "fcm_dpo/beta": 0.10000000149011612, |
| "fcm_dpo/delta": 0.0, |
| "fcm_dpo/margin": 0.025069117546081543, |
| "fcm_dpo/q_t": 0.4993741512298584, |
| "grad_norm": 34.66242599487305, |
| "learning_rate": 1.3768115942028986e-07, |
| "logits/chosen": 0.23079314827919006, |
| "logits/rejected": 0.5058917999267578, |
| "logps/chosen": -90.76351928710938, |
| "logps/ref_chosen": -90.75811767578125, |
| "logps/ref_rejected": -85.91232299804688, |
| "logps/rejected": -85.94279479980469, |
| "loss": 1.3844, |
| "margin_dpo/margin_mean": 0.025068938732147217, |
| "margin_dpo/margin_std": 0.4899085462093353, |
| "step": 20 |
| }, |
| { |
| "epoch": 0.030837004405286344, |
| "fcm_dpo/beta": 0.10000000149011612, |
| "fcm_dpo/delta": 0.0, |
| "fcm_dpo/margin": 0.091633141040802, |
| "fcm_dpo/q_t": 0.4977106750011444, |
| "grad_norm": 31.416053771972656, |
| "learning_rate": 1.4492753623188405e-07, |
| "logits/chosen": 0.5428293347358704, |
| "logits/rejected": 0.7209779620170593, |
| "logps/chosen": -80.27669525146484, |
| "logps/ref_chosen": -80.33345794677734, |
| "logps/ref_rejected": -83.93373107910156, |
| "logps/rejected": -83.9686050415039, |
| "loss": 1.3776, |
| "margin_dpo/margin_mean": 0.09163343906402588, |
| "margin_dpo/margin_std": 0.40345489978790283, |
| "step": 21 |
| }, |
| { |
| "epoch": 0.032305433186490456, |
| "fcm_dpo/beta": 0.10000000149011612, |
| "fcm_dpo/delta": 0.0, |
| "fcm_dpo/margin": 0.0237371027469635, |
| "fcm_dpo/q_t": 0.4994067847728729, |
| "grad_norm": 42.76321029663086, |
| "learning_rate": 1.5217391304347825e-07, |
| "logits/chosen": 0.4888147711753845, |
| "logits/rejected": 0.6019254326820374, |
| "logps/chosen": -95.3747329711914, |
| "logps/ref_chosen": -95.39530181884766, |
| "logps/ref_rejected": -103.47351837158203, |
| "logps/rejected": -103.47669219970703, |
| "loss": 1.3843, |
| "margin_dpo/margin_mean": 0.02373659610748291, |
| "margin_dpo/margin_std": 0.41101372241973877, |
| "step": 22 |
| }, |
| { |
| "epoch": 0.033773861967694566, |
| "fcm_dpo/beta": 0.10000000149011612, |
| "fcm_dpo/delta": 0.0, |
| "fcm_dpo/margin": -0.054152026772499084, |
| "fcm_dpo/q_t": 0.5013521909713745, |
| "grad_norm": 31.53017807006836, |
| "learning_rate": 1.5942028985507245e-07, |
| "logits/chosen": 0.7403199076652527, |
| "logits/rejected": 0.9192092418670654, |
| "logps/chosen": -90.61726379394531, |
| "logps/ref_chosen": -90.63751220703125, |
| "logps/ref_rejected": -86.59425354003906, |
| "logps/rejected": -86.51985168457031, |
| "loss": 1.3921, |
| "margin_dpo/margin_mean": -0.05415254831314087, |
| "margin_dpo/margin_std": 0.4045397639274597, |
| "step": 23 |
| }, |
| { |
| "epoch": 0.03524229074889868, |
| "fcm_dpo/beta": 0.10000000149011612, |
| "fcm_dpo/delta": 0.0, |
| "fcm_dpo/margin": 0.038981035351753235, |
| "fcm_dpo/q_t": 0.4990260601043701, |
| "grad_norm": 44.209590911865234, |
| "learning_rate": 1.6666666666666665e-07, |
| "logits/chosen": 0.6239376068115234, |
| "logits/rejected": 0.6544588804244995, |
| "logps/chosen": -69.91582489013672, |
| "logps/ref_chosen": -69.91728973388672, |
| "logps/ref_rejected": -106.60990142822266, |
| "logps/rejected": -106.64741516113281, |
| "loss": 1.383, |
| "margin_dpo/margin_mean": 0.03898155689239502, |
| "margin_dpo/margin_std": 0.4552841782569885, |
| "step": 24 |
| }, |
| { |
| "epoch": 0.03671071953010279, |
| "fcm_dpo/beta": 0.10000000149011612, |
| "fcm_dpo/delta": 0.0, |
| "fcm_dpo/margin": 0.050283268094062805, |
| "fcm_dpo/q_t": 0.4987441301345825, |
| "grad_norm": 36.85416030883789, |
| "learning_rate": 1.7391304347826085e-07, |
| "logits/chosen": 0.5321314334869385, |
| "logits/rejected": 0.7012242078781128, |
| "logps/chosen": -80.85140991210938, |
| "logps/ref_chosen": -80.82548522949219, |
| "logps/ref_rejected": -95.95710754394531, |
| "logps/rejected": -96.0333251953125, |
| "loss": 1.3816, |
| "margin_dpo/margin_mean": 0.05028343200683594, |
| "margin_dpo/margin_std": 0.37170085310935974, |
| "step": 25 |
| }, |
| { |
| "epoch": 0.0381791483113069, |
| "fcm_dpo/beta": 0.10000000149011612, |
| "fcm_dpo/delta": 0.0, |
| "fcm_dpo/margin": 0.07252025604248047, |
| "fcm_dpo/q_t": 0.4981876611709595, |
| "grad_norm": 40.67619705200195, |
| "learning_rate": 1.8115942028985507e-07, |
| "logits/chosen": 0.36407941579818726, |
| "logits/rejected": 0.5750868320465088, |
| "logps/chosen": -88.88731384277344, |
| "logps/ref_chosen": -88.90116882324219, |
| "logps/ref_rejected": -109.82819366455078, |
| "logps/rejected": -109.88685607910156, |
| "loss": 1.3795, |
| "margin_dpo/margin_mean": 0.072520911693573, |
| "margin_dpo/margin_std": 0.4113643765449524, |
| "step": 26 |
| }, |
| { |
| "epoch": 0.039647577092511016, |
| "fcm_dpo/beta": 0.10000000149011612, |
| "fcm_dpo/delta": 0.0, |
| "fcm_dpo/margin": 0.0897430032491684, |
| "fcm_dpo/q_t": 0.4977561831474304, |
| "grad_norm": 44.181434631347656, |
| "learning_rate": 1.8840579710144927e-07, |
| "logits/chosen": 0.2009836733341217, |
| "logits/rejected": 0.38905927538871765, |
| "logps/chosen": -77.58397674560547, |
| "logps/ref_chosen": -77.59600830078125, |
| "logps/ref_rejected": -103.93850708007812, |
| "logps/rejected": -104.0162124633789, |
| "loss": 1.3779, |
| "margin_dpo/margin_mean": 0.08974292874336243, |
| "margin_dpo/margin_std": 0.4670272469520569, |
| "step": 27 |
| }, |
| { |
| "epoch": 0.041116005873715125, |
| "fcm_dpo/beta": 0.10000000149011612, |
| "fcm_dpo/delta": 0.0, |
| "fcm_dpo/margin": 0.12937551736831665, |
| "fcm_dpo/q_t": 0.49676674604415894, |
| "grad_norm": 36.718238830566406, |
| "learning_rate": 1.9565217391304347e-07, |
| "logits/chosen": 0.7008620500564575, |
| "logits/rejected": 0.9329044818878174, |
| "logps/chosen": -102.15823364257812, |
| "logps/ref_chosen": -102.22856903076172, |
| "logps/ref_rejected": -96.9594955444336, |
| "logps/rejected": -97.01853942871094, |
| "loss": 1.3738, |
| "margin_dpo/margin_mean": 0.12937626242637634, |
| "margin_dpo/margin_std": 0.42088428139686584, |
| "step": 28 |
| }, |
| { |
| "epoch": 0.042584434654919234, |
| "fcm_dpo/beta": 0.10000000149011612, |
| "fcm_dpo/delta": 0.0, |
| "fcm_dpo/margin": 0.11628834903240204, |
| "fcm_dpo/q_t": 0.49709412455558777, |
| "grad_norm": 42.49556350708008, |
| "learning_rate": 2.028985507246377e-07, |
| "logits/chosen": 0.37622809410095215, |
| "logits/rejected": 0.638656497001648, |
| "logps/chosen": -88.61930084228516, |
| "logps/ref_chosen": -88.64704895019531, |
| "logps/ref_rejected": -102.96011352539062, |
| "logps/rejected": -103.04864501953125, |
| "loss": 1.3753, |
| "margin_dpo/margin_mean": 0.11628936231136322, |
| "margin_dpo/margin_std": 0.48608508706092834, |
| "step": 29 |
| }, |
| { |
| "epoch": 0.04405286343612335, |
| "fcm_dpo/beta": 0.10000000149011612, |
| "fcm_dpo/delta": 0.0, |
| "fcm_dpo/margin": 0.11767734587192535, |
| "fcm_dpo/q_t": 0.49706411361694336, |
| "grad_norm": 39.036170959472656, |
| "learning_rate": 2.1014492753623187e-07, |
| "logits/chosen": 0.8277263045310974, |
| "logits/rejected": 0.9531229734420776, |
| "logps/chosen": -88.38198852539062, |
| "logps/ref_chosen": -88.38838195800781, |
| "logps/ref_rejected": -102.31889343261719, |
| "logps/rejected": -102.43018341064453, |
| "loss": 1.3752, |
| "margin_dpo/margin_mean": 0.11767733097076416, |
| "margin_dpo/margin_std": 0.5070103406906128, |
| "step": 30 |
| }, |
| { |
| "epoch": 0.04552129221732746, |
| "fcm_dpo/beta": 0.10000000149011612, |
| "fcm_dpo/delta": 0.0, |
| "fcm_dpo/margin": 0.05853770673274994, |
| "fcm_dpo/q_t": 0.49853795766830444, |
| "grad_norm": 31.66895866394043, |
| "learning_rate": 2.1739130434782607e-07, |
| "logits/chosen": 0.718494713306427, |
| "logits/rejected": 0.9628149271011353, |
| "logps/chosen": -101.16627502441406, |
| "logps/ref_chosen": -101.12565612792969, |
| "logps/ref_rejected": -79.8584213256836, |
| "logps/rejected": -79.95758056640625, |
| "loss": 1.381, |
| "margin_dpo/margin_mean": 0.05853772163391113, |
| "margin_dpo/margin_std": 0.4458237588405609, |
| "step": 31 |
| }, |
| { |
| "epoch": 0.04698972099853157, |
| "fcm_dpo/beta": 0.10000000149011612, |
| "fcm_dpo/delta": 0.0, |
| "fcm_dpo/margin": 0.14585043489933014, |
| "fcm_dpo/q_t": 0.4963546395301819, |
| "grad_norm": 35.37748718261719, |
| "learning_rate": 2.2463768115942027e-07, |
| "logits/chosen": 0.3660997748374939, |
| "logits/rejected": 0.5037564635276794, |
| "logps/chosen": -96.66069030761719, |
| "logps/ref_chosen": -96.61702728271484, |
| "logps/ref_rejected": -96.224365234375, |
| "logps/rejected": -96.41387176513672, |
| "loss": 1.3723, |
| "margin_dpo/margin_mean": 0.14585140347480774, |
| "margin_dpo/margin_std": 0.44825515151023865, |
| "step": 32 |
| }, |
| { |
| "epoch": 0.048458149779735685, |
| "fcm_dpo/beta": 0.10000000149011612, |
| "fcm_dpo/delta": 0.0, |
| "fcm_dpo/margin": 0.16191767156124115, |
| "fcm_dpo/q_t": 0.4959545135498047, |
| "grad_norm": 33.53567123413086, |
| "learning_rate": 2.318840579710145e-07, |
| "logits/chosen": 0.5710635185241699, |
| "logits/rejected": 0.7510501742362976, |
| "logps/chosen": -81.49163055419922, |
| "logps/ref_chosen": -81.52101135253906, |
| "logps/ref_rejected": -93.80595397949219, |
| "logps/rejected": -93.93849182128906, |
| "loss": 1.3709, |
| "margin_dpo/margin_mean": 0.1619182527065277, |
| "margin_dpo/margin_std": 0.5155723094940186, |
| "step": 33 |
| }, |
| { |
| "epoch": 0.049926578560939794, |
| "fcm_dpo/beta": 0.10000000149011612, |
| "fcm_dpo/delta": 0.0, |
| "fcm_dpo/margin": 0.19678400456905365, |
| "fcm_dpo/q_t": 0.4950863718986511, |
| "grad_norm": 40.77206802368164, |
| "learning_rate": 2.391304347826087e-07, |
| "logits/chosen": 0.5938290357589722, |
| "logits/rejected": 0.7664215564727783, |
| "logps/chosen": -77.15574645996094, |
| "logps/ref_chosen": -77.20204162597656, |
| "logps/ref_rejected": -106.71876525878906, |
| "logps/rejected": -106.86925506591797, |
| "loss": 1.3673, |
| "margin_dpo/margin_mean": 0.19678422808647156, |
| "margin_dpo/margin_std": 0.4836421608924866, |
| "step": 34 |
| }, |
| { |
| "epoch": 0.0513950073421439, |
| "fcm_dpo/beta": 0.10000000149011612, |
| "fcm_dpo/delta": 0.0, |
| "fcm_dpo/margin": 0.1951250433921814, |
| "fcm_dpo/q_t": 0.4951246976852417, |
| "grad_norm": 42.55626678466797, |
| "learning_rate": 2.463768115942029e-07, |
| "logits/chosen": 0.1299372911453247, |
| "logits/rejected": 0.2869094908237457, |
| "logps/chosen": -77.56372833251953, |
| "logps/ref_chosen": -77.57035064697266, |
| "logps/ref_rejected": -112.18855285644531, |
| "logps/rejected": -112.3770523071289, |
| "loss": 1.3673, |
| "margin_dpo/margin_mean": 0.1951252818107605, |
| "margin_dpo/margin_std": 0.41524142026901245, |
| "step": 35 |
| }, |
| { |
| "epoch": 0.05286343612334802, |
| "fcm_dpo/beta": 0.10000000149011612, |
| "fcm_dpo/delta": 0.0, |
| "fcm_dpo/margin": 0.12170365452766418, |
| "fcm_dpo/q_t": 0.4969629645347595, |
| "grad_norm": 32.65164566040039, |
| "learning_rate": 2.536231884057971e-07, |
| "logits/chosen": 0.40253281593322754, |
| "logits/rejected": 0.5915440320968628, |
| "logps/chosen": -83.32685089111328, |
| "logps/ref_chosen": -83.28823852539062, |
| "logps/ref_rejected": -91.75741577148438, |
| "logps/rejected": -91.91773223876953, |
| "loss": 1.375, |
| "margin_dpo/margin_mean": 0.12170398235321045, |
| "margin_dpo/margin_std": 0.55162113904953, |
| "step": 36 |
| }, |
| { |
| "epoch": 0.05433186490455213, |
| "fcm_dpo/beta": 0.10000000149011612, |
| "fcm_dpo/delta": 0.0, |
| "fcm_dpo/margin": 0.3155759572982788, |
| "fcm_dpo/q_t": 0.4921198785305023, |
| "grad_norm": 36.134918212890625, |
| "learning_rate": 2.6086956521739126e-07, |
| "logits/chosen": 0.5907583236694336, |
| "logits/rejected": 0.7441189885139465, |
| "logps/chosen": -94.78392028808594, |
| "logps/ref_chosen": -94.77108764648438, |
| "logps/ref_rejected": -85.71726989746094, |
| "logps/rejected": -86.04566955566406, |
| "loss": 1.3558, |
| "margin_dpo/margin_mean": 0.3155764639377594, |
| "margin_dpo/margin_std": 0.5700342655181885, |
| "step": 37 |
| }, |
| { |
| "epoch": 0.055800293685756244, |
| "fcm_dpo/beta": 0.10000000149011612, |
| "fcm_dpo/delta": 0.0, |
| "fcm_dpo/margin": 0.19760102033615112, |
| "fcm_dpo/q_t": 0.4950709939002991, |
| "grad_norm": 37.12024688720703, |
| "learning_rate": 2.681159420289855e-07, |
| "logits/chosen": 0.5169904232025146, |
| "logits/rejected": 0.6563962697982788, |
| "logps/chosen": -75.88797760009766, |
| "logps/ref_chosen": -75.97850036621094, |
| "logps/ref_rejected": -104.10401916503906, |
| "logps/rejected": -104.21109008789062, |
| "loss": 1.3675, |
| "margin_dpo/margin_mean": 0.197601780295372, |
| "margin_dpo/margin_std": 0.5769727230072021, |
| "step": 38 |
| }, |
| { |
| "epoch": 0.05726872246696035, |
| "fcm_dpo/beta": 0.10000000149011612, |
| "fcm_dpo/delta": 0.0, |
| "fcm_dpo/margin": 0.3096907138824463, |
| "fcm_dpo/q_t": 0.492266446352005, |
| "grad_norm": 36.1661376953125, |
| "learning_rate": 2.753623188405797e-07, |
| "logits/chosen": 0.6212589740753174, |
| "logits/rejected": 0.818722128868103, |
| "logps/chosen": -81.21092987060547, |
| "logps/ref_chosen": -81.18577575683594, |
| "logps/ref_rejected": -84.1959228515625, |
| "logps/rejected": -84.53076171875, |
| "loss": 1.3563, |
| "margin_dpo/margin_mean": 0.30969101190567017, |
| "margin_dpo/margin_std": 0.5184357762336731, |
| "step": 39 |
| }, |
| { |
| "epoch": 0.05873715124816446, |
| "fcm_dpo/beta": 0.10000000149011612, |
| "fcm_dpo/delta": 0.0, |
| "fcm_dpo/margin": 0.24696290493011475, |
| "fcm_dpo/q_t": 0.49383074045181274, |
| "grad_norm": 29.007984161376953, |
| "learning_rate": 2.8260869565217386e-07, |
| "logits/chosen": 0.6268089413642883, |
| "logits/rejected": 0.8235390186309814, |
| "logps/chosen": -83.29570007324219, |
| "logps/ref_chosen": -83.33257293701172, |
| "logps/ref_rejected": -80.25591278076172, |
| "logps/rejected": -80.46599578857422, |
| "loss": 1.3624, |
| "margin_dpo/margin_mean": 0.24696248769760132, |
| "margin_dpo/margin_std": 0.489422082901001, |
| "step": 40 |
| }, |
| { |
| "epoch": 0.06020558002936858, |
| "fcm_dpo/beta": 0.10000000149011612, |
| "fcm_dpo/delta": 0.0, |
| "fcm_dpo/margin": 0.3290652930736542, |
| "fcm_dpo/q_t": 0.4917981028556824, |
| "grad_norm": 33.2258415222168, |
| "learning_rate": 2.898550724637681e-07, |
| "logits/chosen": 0.6201198697090149, |
| "logits/rejected": 0.8052632808685303, |
| "logps/chosen": -93.12638854980469, |
| "logps/ref_chosen": -93.14866638183594, |
| "logps/ref_rejected": -102.07920837402344, |
| "logps/rejected": -102.38600158691406, |
| "loss": 1.3551, |
| "margin_dpo/margin_mean": 0.32906582951545715, |
| "margin_dpo/margin_std": 0.7185872197151184, |
| "step": 41 |
| }, |
| { |
| "epoch": 0.06167400881057269, |
| "fcm_dpo/beta": 0.10000000149011612, |
| "fcm_dpo/delta": 0.0, |
| "fcm_dpo/margin": 0.4246216416358948, |
| "fcm_dpo/q_t": 0.4893936514854431, |
| "grad_norm": 40.833499908447266, |
| "learning_rate": 2.971014492753623e-07, |
| "logits/chosen": 0.3932719826698303, |
| "logits/rejected": 0.5345179438591003, |
| "logps/chosen": -90.62908172607422, |
| "logps/ref_chosen": -90.70162200927734, |
| "logps/ref_rejected": -113.8646469116211, |
| "logps/rejected": -114.21672821044922, |
| "loss": 1.3449, |
| "margin_dpo/margin_mean": 0.4246218204498291, |
| "margin_dpo/margin_std": 0.5078005790710449, |
| "step": 42 |
| }, |
| { |
| "epoch": 0.0631424375917768, |
| "fcm_dpo/beta": 0.10000000149011612, |
| "fcm_dpo/delta": 0.0, |
| "fcm_dpo/margin": 0.4403374493122101, |
| "fcm_dpo/q_t": 0.4890177845954895, |
| "grad_norm": 37.972869873046875, |
| "learning_rate": 3.043478260869565e-07, |
| "logits/chosen": 0.5793946981430054, |
| "logits/rejected": 0.7695983052253723, |
| "logps/chosen": -89.6876220703125, |
| "logps/ref_chosen": -89.64402770996094, |
| "logps/ref_rejected": -103.95185852050781, |
| "logps/rejected": -104.435791015625, |
| "loss": 1.3441, |
| "margin_dpo/margin_mean": 0.440337598323822, |
| "margin_dpo/margin_std": 0.7237739562988281, |
| "step": 43 |
| }, |
| { |
| "epoch": 0.06461086637298091, |
| "fcm_dpo/beta": 0.10000000149011612, |
| "fcm_dpo/delta": 0.0, |
| "fcm_dpo/margin": 0.3891373872756958, |
| "fcm_dpo/q_t": 0.49028751254081726, |
| "grad_norm": 34.31340026855469, |
| "learning_rate": 3.115942028985507e-07, |
| "logits/chosen": 0.8794394135475159, |
| "logits/rejected": 1.000215768814087, |
| "logps/chosen": -81.93470764160156, |
| "logps/ref_chosen": -81.8783187866211, |
| "logps/ref_rejected": -113.421630859375, |
| "logps/rejected": -113.86715698242188, |
| "loss": 1.3487, |
| "margin_dpo/margin_mean": 0.3891381621360779, |
| "margin_dpo/margin_std": 0.6131525039672852, |
| "step": 44 |
| }, |
| { |
| "epoch": 0.06607929515418502, |
| "fcm_dpo/beta": 0.10000000149011612, |
| "fcm_dpo/delta": 0.0, |
| "fcm_dpo/margin": 0.1822262555360794, |
| "fcm_dpo/q_t": 0.49545037746429443, |
| "grad_norm": 24.454357147216797, |
| "learning_rate": 3.188405797101449e-07, |
| "logits/chosen": 0.6332857608795166, |
| "logits/rejected": 0.7449449300765991, |
| "logps/chosen": -77.44635009765625, |
| "logps/ref_chosen": -77.34459686279297, |
| "logps/ref_rejected": -84.24774169921875, |
| "logps/rejected": -84.53172302246094, |
| "loss": 1.3691, |
| "margin_dpo/margin_mean": 0.18222709000110626, |
| "margin_dpo/margin_std": 0.5951623916625977, |
| "step": 45 |
| }, |
| { |
| "epoch": 0.06754772393538913, |
| "fcm_dpo/beta": 0.10000000149011612, |
| "fcm_dpo/delta": 0.0, |
| "fcm_dpo/margin": 0.48641544580459595, |
| "fcm_dpo/q_t": 0.4878547787666321, |
| "grad_norm": 32.93342208862305, |
| "learning_rate": 3.260869565217391e-07, |
| "logits/chosen": 0.5085247755050659, |
| "logits/rejected": 0.7518465518951416, |
| "logps/chosen": -90.32666015625, |
| "logps/ref_chosen": -90.3408203125, |
| "logps/ref_rejected": -100.676513671875, |
| "logps/rejected": -101.14876556396484, |
| "loss": 1.339, |
| "margin_dpo/margin_mean": 0.4864157736301422, |
| "margin_dpo/margin_std": 0.5446327328681946, |
| "step": 46 |
| }, |
| { |
| "epoch": 0.06901615271659324, |
| "fcm_dpo/beta": 0.10000000149011612, |
| "fcm_dpo/delta": 0.0, |
| "fcm_dpo/margin": 0.3937411308288574, |
| "fcm_dpo/q_t": 0.49017661809921265, |
| "grad_norm": 34.04849624633789, |
| "learning_rate": 3.333333333333333e-07, |
| "logits/chosen": 0.45205074548721313, |
| "logits/rejected": 0.6340922117233276, |
| "logps/chosen": -104.42767333984375, |
| "logps/ref_chosen": -104.41130065917969, |
| "logps/ref_rejected": -101.25489807128906, |
| "logps/rejected": -101.6650161743164, |
| "loss": 1.3487, |
| "margin_dpo/margin_mean": 0.39373981952667236, |
| "margin_dpo/margin_std": 0.7378332614898682, |
| "step": 47 |
| }, |
| { |
| "epoch": 0.07048458149779736, |
| "fcm_dpo/beta": 0.10000000149011612, |
| "fcm_dpo/delta": 0.0, |
| "fcm_dpo/margin": 0.6404982805252075, |
| "fcm_dpo/q_t": 0.4840244948863983, |
| "grad_norm": 38.5645751953125, |
| "learning_rate": 3.4057971014492755e-07, |
| "logits/chosen": 0.29659873247146606, |
| "logits/rejected": 0.5025741457939148, |
| "logps/chosen": -91.0909423828125, |
| "logps/ref_chosen": -91.10027313232422, |
| "logps/ref_rejected": -95.08058166503906, |
| "logps/rejected": -95.71174621582031, |
| "loss": 1.3251, |
| "margin_dpo/margin_mean": 0.6404985189437866, |
| "margin_dpo/margin_std": 0.856590986251831, |
| "step": 48 |
| }, |
| { |
| "epoch": 0.07195301027900147, |
| "fcm_dpo/beta": 0.10000000149011612, |
| "fcm_dpo/delta": 0.0, |
| "fcm_dpo/margin": 0.7378369569778442, |
| "fcm_dpo/q_t": 0.4816128611564636, |
| "grad_norm": 39.67837905883789, |
| "learning_rate": 3.478260869565217e-07, |
| "logits/chosen": 0.10201732814311981, |
| "logits/rejected": 0.5736548900604248, |
| "logps/chosen": -93.02400207519531, |
| "logps/ref_chosen": -93.00367736816406, |
| "logps/ref_rejected": -91.74899291992188, |
| "logps/rejected": -92.50715637207031, |
| "loss": 1.316, |
| "margin_dpo/margin_mean": 0.7378373742103577, |
| "margin_dpo/margin_std": 0.9256041646003723, |
| "step": 49 |
| }, |
| { |
| "epoch": 0.07342143906020558, |
| "fcm_dpo/beta": 0.10000000149011612, |
| "fcm_dpo/delta": 0.0, |
| "fcm_dpo/margin": 0.7001360654830933, |
| "fcm_dpo/q_t": 0.4825558662414551, |
| "grad_norm": 33.148658752441406, |
| "learning_rate": 3.5507246376811595e-07, |
| "logits/chosen": 0.5446861982345581, |
| "logits/rejected": 0.7148386240005493, |
| "logps/chosen": -94.72885131835938, |
| "logps/ref_chosen": -94.62681579589844, |
| "logps/ref_rejected": -103.57435607910156, |
| "logps/rejected": -104.37653350830078, |
| "loss": 1.3197, |
| "margin_dpo/margin_mean": 0.7001360654830933, |
| "margin_dpo/margin_std": 0.9148711562156677, |
| "step": 50 |
| }, |
| { |
| "epoch": 0.07488986784140969, |
| "fcm_dpo/beta": 0.10000000149011612, |
| "fcm_dpo/delta": 0.0, |
| "fcm_dpo/margin": 0.6637753248214722, |
| "fcm_dpo/q_t": 0.48347365856170654, |
| "grad_norm": 30.014331817626953, |
| "learning_rate": 3.6231884057971015e-07, |
| "logits/chosen": 0.7485964298248291, |
| "logits/rejected": 0.8959745168685913, |
| "logps/chosen": -87.53605651855469, |
| "logps/ref_chosen": -87.50727844238281, |
| "logps/ref_rejected": -83.47235107421875, |
| "logps/rejected": -84.16490936279297, |
| "loss": 1.3236, |
| "margin_dpo/margin_mean": 0.6637749671936035, |
| "margin_dpo/margin_std": 1.0224616527557373, |
| "step": 51 |
| }, |
| { |
| "epoch": 0.0763582966226138, |
| "fcm_dpo/beta": 0.10000000149011612, |
| "fcm_dpo/delta": 0.0, |
| "fcm_dpo/margin": 0.873137354850769, |
| "fcm_dpo/q_t": 0.47824588418006897, |
| "grad_norm": 32.00587844848633, |
| "learning_rate": 3.695652173913043e-07, |
| "logits/chosen": 0.49871087074279785, |
| "logits/rejected": 0.7171226739883423, |
| "logps/chosen": -90.64942169189453, |
| "logps/ref_chosen": -90.63026428222656, |
| "logps/ref_rejected": -87.0390625, |
| "logps/rejected": -87.93136596679688, |
| "loss": 1.3029, |
| "margin_dpo/margin_mean": 0.8731374740600586, |
| "margin_dpo/margin_std": 0.9018590450286865, |
| "step": 52 |
| }, |
| { |
| "epoch": 0.07782672540381791, |
| "fcm_dpo/beta": 0.10000000149011612, |
| "fcm_dpo/delta": 0.0, |
| "fcm_dpo/margin": 1.1096992492675781, |
| "fcm_dpo/q_t": 0.47249969840049744, |
| "grad_norm": 39.590824127197266, |
| "learning_rate": 3.7681159420289855e-07, |
| "logits/chosen": 0.07211381196975708, |
| "logits/rejected": 0.1840796172618866, |
| "logps/chosen": -81.551025390625, |
| "logps/ref_chosen": -81.58307647705078, |
| "logps/ref_rejected": -95.66152954101562, |
| "logps/rejected": -96.73918151855469, |
| "loss": 1.2831, |
| "margin_dpo/margin_mean": 1.1096991300582886, |
| "margin_dpo/margin_std": 1.3239152431488037, |
| "step": 53 |
| }, |
| { |
| "epoch": 0.07929515418502203, |
| "fcm_dpo/beta": 0.10000000149011612, |
| "fcm_dpo/delta": 0.0, |
| "fcm_dpo/margin": 1.1452147960662842, |
| "fcm_dpo/q_t": 0.4715806841850281, |
| "grad_norm": 39.977752685546875, |
| "learning_rate": 3.8405797101449274e-07, |
| "logits/chosen": 0.47239190340042114, |
| "logits/rejected": 0.6545497179031372, |
| "logps/chosen": -88.91471099853516, |
| "logps/ref_chosen": -88.91017150878906, |
| "logps/ref_rejected": -99.1175537109375, |
| "logps/rejected": -100.2673110961914, |
| "loss": 1.2792, |
| "margin_dpo/margin_mean": 1.1452147960662842, |
| "margin_dpo/margin_std": 1.2928842306137085, |
| "step": 54 |
| }, |
| { |
| "epoch": 0.08076358296622614, |
| "fcm_dpo/beta": 0.10000000149011612, |
| "fcm_dpo/delta": 0.0, |
| "fcm_dpo/margin": 1.180344820022583, |
| "fcm_dpo/q_t": 0.4708316922187805, |
| "grad_norm": 34.956790924072266, |
| "learning_rate": 3.9130434782608694e-07, |
| "logits/chosen": 0.10857650637626648, |
| "logits/rejected": 0.4178081750869751, |
| "logps/chosen": -92.31873321533203, |
| "logps/ref_chosen": -92.45591735839844, |
| "logps/ref_rejected": -92.97093200683594, |
| "logps/rejected": -94.01409149169922, |
| "loss": 1.2787, |
| "margin_dpo/margin_mean": 1.180344581604004, |
| "margin_dpo/margin_std": 1.6435117721557617, |
| "step": 55 |
| }, |
| { |
| "epoch": 0.08223201174743025, |
| "fcm_dpo/beta": 0.10000000149011612, |
| "fcm_dpo/delta": 0.0, |
| "fcm_dpo/margin": 1.060053825378418, |
| "fcm_dpo/q_t": 0.4738434851169586, |
| "grad_norm": 33.11833572387695, |
| "learning_rate": 3.9855072463768114e-07, |
| "logits/chosen": 0.2020319402217865, |
| "logits/rejected": 0.5419068336486816, |
| "logps/chosen": -87.31375122070312, |
| "logps/ref_chosen": -87.23665618896484, |
| "logps/ref_rejected": -100.64553833007812, |
| "logps/rejected": -101.78268432617188, |
| "loss": 1.2901, |
| "margin_dpo/margin_mean": 1.0600528717041016, |
| "margin_dpo/margin_std": 1.6760348081588745, |
| "step": 56 |
| }, |
| { |
| "epoch": 0.08370044052863436, |
| "fcm_dpo/beta": 0.10000000149011612, |
| "fcm_dpo/delta": 0.0, |
| "fcm_dpo/margin": 1.1467325687408447, |
| "fcm_dpo/q_t": 0.4716096520423889, |
| "grad_norm": 33.74794387817383, |
| "learning_rate": 4.057971014492754e-07, |
| "logits/chosen": 0.31785082817077637, |
| "logits/rejected": 0.5342439413070679, |
| "logps/chosen": -98.2012939453125, |
| "logps/ref_chosen": -98.15074157714844, |
| "logps/ref_rejected": -101.05284118652344, |
| "logps/rejected": -102.25011444091797, |
| "loss": 1.2808, |
| "margin_dpo/margin_mean": 1.146733283996582, |
| "margin_dpo/margin_std": 1.5416290760040283, |
| "step": 57 |
| }, |
| { |
| "epoch": 0.08516886930983847, |
| "fcm_dpo/beta": 0.10000000149011612, |
| "fcm_dpo/delta": 0.0, |
| "fcm_dpo/margin": 1.5004881620407104, |
| "fcm_dpo/q_t": 0.46311402320861816, |
| "grad_norm": 37.92626190185547, |
| "learning_rate": 4.1304347826086954e-07, |
| "logits/chosen": 0.29364755749702454, |
| "logits/rejected": 0.6253389120101929, |
| "logps/chosen": -99.32469177246094, |
| "logps/ref_chosen": -99.58097076416016, |
| "logps/ref_rejected": -91.22227478027344, |
| "logps/rejected": -92.46648406982422, |
| "loss": 1.2512, |
| "margin_dpo/margin_mean": 1.50048828125, |
| "margin_dpo/margin_std": 1.8517991304397583, |
| "step": 58 |
| }, |
| { |
| "epoch": 0.08663729809104258, |
| "fcm_dpo/beta": 0.10000000149011612, |
| "fcm_dpo/delta": 0.0, |
| "fcm_dpo/margin": 1.2302589416503906, |
| "fcm_dpo/q_t": 0.469511479139328, |
| "grad_norm": 34.269100189208984, |
| "learning_rate": 4.2028985507246374e-07, |
| "logits/chosen": 0.13759316504001617, |
| "logits/rejected": 0.3323509693145752, |
| "logps/chosen": -89.81268310546875, |
| "logps/ref_chosen": -89.80232238769531, |
| "logps/ref_rejected": -94.27667236328125, |
| "logps/rejected": -95.51730346679688, |
| "loss": 1.2723, |
| "margin_dpo/margin_mean": 1.230258584022522, |
| "margin_dpo/margin_std": 1.4574449062347412, |
| "step": 59 |
| }, |
| { |
| "epoch": 0.0881057268722467, |
| "fcm_dpo/beta": 0.10000000149011612, |
| "fcm_dpo/delta": 0.0, |
| "fcm_dpo/margin": 1.1427745819091797, |
| "fcm_dpo/q_t": 0.4716527462005615, |
| "grad_norm": 28.718473434448242, |
| "learning_rate": 4.2753623188405794e-07, |
| "logits/chosen": 0.3863717317581177, |
| "logits/rejected": 0.6563723087310791, |
| "logps/chosen": -95.13877868652344, |
| "logps/ref_chosen": -95.15571594238281, |
| "logps/ref_rejected": -91.3724365234375, |
| "logps/rejected": -92.4982681274414, |
| "loss": 1.2804, |
| "margin_dpo/margin_mean": 1.1427747011184692, |
| "margin_dpo/margin_std": 1.4296643733978271, |
| "step": 60 |
| }, |
| { |
| "epoch": 0.08957415565345081, |
| "fcm_dpo/beta": 0.10000000149011612, |
| "fcm_dpo/delta": 0.0, |
| "fcm_dpo/margin": 1.1766782999038696, |
| "fcm_dpo/q_t": 0.47107481956481934, |
| "grad_norm": 28.268402099609375, |
| "learning_rate": 4.3478260869565214e-07, |
| "logits/chosen": 0.2091524302959442, |
| "logits/rejected": 0.34982138872146606, |
| "logps/chosen": -85.9404296875, |
| "logps/ref_chosen": -85.73230743408203, |
| "logps/ref_rejected": -97.96575927734375, |
| "logps/rejected": -99.3505630493164, |
| "loss": 1.2807, |
| "margin_dpo/margin_mean": 1.1766784191131592, |
| "margin_dpo/margin_std": 1.8282430171966553, |
| "step": 61 |
| }, |
| { |
| "epoch": 0.09104258443465492, |
| "fcm_dpo/beta": 0.10000000149011612, |
| "fcm_dpo/delta": 0.0, |
| "fcm_dpo/margin": 1.2114932537078857, |
| "fcm_dpo/q_t": 0.47012367844581604, |
| "grad_norm": 28.394248962402344, |
| "learning_rate": 4.420289855072464e-07, |
| "logits/chosen": 0.222091406583786, |
| "logits/rejected": 0.4194578528404236, |
| "logps/chosen": -81.77252197265625, |
| "logps/ref_chosen": -81.63538360595703, |
| "logps/ref_rejected": -84.03831481933594, |
| "logps/rejected": -85.38694763183594, |
| "loss": 1.2787, |
| "margin_dpo/margin_mean": 1.2114924192428589, |
| "margin_dpo/margin_std": 1.857269048690796, |
| "step": 62 |
| }, |
| { |
| "epoch": 0.09251101321585903, |
| "fcm_dpo/beta": 0.10000000149011612, |
| "fcm_dpo/delta": 0.0, |
| "fcm_dpo/margin": 1.4869378805160522, |
| "fcm_dpo/q_t": 0.46316325664520264, |
| "grad_norm": 35.04859924316406, |
| "learning_rate": 4.4927536231884053e-07, |
| "logits/chosen": 0.33780792355537415, |
| "logits/rejected": 0.5467942357063293, |
| "logps/chosen": -103.6246337890625, |
| "logps/ref_chosen": -103.62405395507812, |
| "logps/ref_rejected": -103.40303039550781, |
| "logps/rejected": -104.89054870605469, |
| "loss": 1.2495, |
| "margin_dpo/margin_mean": 1.4869377613067627, |
| "margin_dpo/margin_std": 1.5714600086212158, |
| "step": 63 |
| }, |
| { |
| "epoch": 0.09397944199706314, |
| "fcm_dpo/beta": 0.10389302670955658, |
| "fcm_dpo/delta": 0.19095781445503235, |
| "fcm_dpo/margin": 1.7406915426254272, |
| "fcm_dpo/q_t": 0.45685333013534546, |
| "grad_norm": 35.974021911621094, |
| "learning_rate": 4.5652173913043473e-07, |
| "logits/chosen": 0.3727140724658966, |
| "logits/rejected": 0.584496259689331, |
| "logps/chosen": -87.18478393554688, |
| "logps/ref_chosen": -87.0015869140625, |
| "logps/ref_rejected": -100.5854721069336, |
| "logps/rejected": -102.50936126708984, |
| "loss": 1.2273, |
| "margin_dpo/margin_mean": 1.7406916618347168, |
| "margin_dpo/margin_std": 2.1689281463623047, |
| "step": 64 |
| }, |
| { |
| "epoch": 0.09544787077826726, |
| "fcm_dpo/beta": 0.10389302670955658, |
| "fcm_dpo/delta": 0.0, |
| "fcm_dpo/margin": 1.7801563739776611, |
| "fcm_dpo/q_t": 0.4546605348587036, |
| "grad_norm": 39.58053970336914, |
| "learning_rate": 4.63768115942029e-07, |
| "logits/chosen": 0.08340189605951309, |
| "logits/rejected": 0.22737503051757812, |
| "logps/chosen": -91.27607727050781, |
| "logps/ref_chosen": -91.22191619873047, |
| "logps/ref_rejected": -115.33553314208984, |
| "logps/rejected": -117.16984558105469, |
| "loss": 1.2216, |
| "margin_dpo/margin_mean": 1.780156135559082, |
| "margin_dpo/margin_std": 2.092562437057495, |
| "step": 65 |
| }, |
| { |
| "epoch": 0.09691629955947137, |
| "fcm_dpo/beta": 0.10389302670955658, |
| "fcm_dpo/delta": 0.0, |
| "fcm_dpo/margin": 1.2560508251190186, |
| "fcm_dpo/q_t": 0.46791407465934753, |
| "grad_norm": 26.43943977355957, |
| "learning_rate": 4.7101449275362313e-07, |
| "logits/chosen": 0.25161832571029663, |
| "logits/rejected": 0.460490345954895, |
| "logps/chosen": -84.09396362304688, |
| "logps/ref_chosen": -83.78422546386719, |
| "logps/ref_rejected": -82.7520980834961, |
| "logps/rejected": -84.31788635253906, |
| "loss": 1.2702, |
| "margin_dpo/margin_mean": 1.2560513019561768, |
| "margin_dpo/margin_std": 1.885561466217041, |
| "step": 66 |
| }, |
| { |
| "epoch": 0.09838472834067548, |
| "fcm_dpo/beta": 0.10389302670955658, |
| "fcm_dpo/delta": 0.0, |
| "fcm_dpo/margin": 1.1451356410980225, |
| "fcm_dpo/q_t": 0.47134000062942505, |
| "grad_norm": 25.080429077148438, |
| "learning_rate": 4.782608695652174e-07, |
| "logits/chosen": 0.3801344335079193, |
| "logits/rejected": 0.6603550910949707, |
| "logps/chosen": -88.52891540527344, |
| "logps/ref_chosen": -87.67295837402344, |
| "logps/ref_rejected": -79.08674621582031, |
| "logps/rejected": -81.08783721923828, |
| "loss": 1.2902, |
| "margin_dpo/margin_mean": 1.1451354026794434, |
| "margin_dpo/margin_std": 2.6883256435394287, |
| "step": 67 |
| }, |
| { |
| "epoch": 0.09985315712187959, |
| "fcm_dpo/beta": 0.10986920446157455, |
| "fcm_dpo/delta": 0.3722814917564392, |
| "fcm_dpo/margin": 2.152301788330078, |
| "fcm_dpo/q_t": 0.44556739926338196, |
| "grad_norm": 29.909442901611328, |
| "learning_rate": 4.855072463768116e-07, |
| "logits/chosen": -0.003267938271164894, |
| "logits/rejected": 0.2983437776565552, |
| "logps/chosen": -97.34490966796875, |
| "logps/ref_chosen": -97.07884216308594, |
| "logps/ref_rejected": -84.11872863769531, |
| "logps/rejected": -86.53709411621094, |
| "loss": 1.1887, |
| "margin_dpo/margin_mean": 2.15230131149292, |
| "margin_dpo/margin_std": 2.906233072280884, |
| "step": 68 |
| }, |
| { |
| "epoch": 0.1013215859030837, |
| "fcm_dpo/beta": 0.1179037019610405, |
| "fcm_dpo/delta": 0.3364790081977844, |
| "fcm_dpo/margin": 2.307511806488037, |
| "fcm_dpo/q_t": 0.4372074604034424, |
| "grad_norm": 33.9486083984375, |
| "learning_rate": 4.927536231884058e-07, |
| "logits/chosen": 0.2297275960445404, |
| "logits/rejected": 0.3395928740501404, |
| "logps/chosen": -86.02600860595703, |
| "logps/ref_chosen": -85.71971130371094, |
| "logps/ref_rejected": -109.48020935058594, |
| "logps/rejected": -112.0940170288086, |
| "loss": 1.1623, |
| "margin_dpo/margin_mean": 2.307512044906616, |
| "margin_dpo/margin_std": 3.0080556869506836, |
| "step": 69 |
| }, |
| { |
| "epoch": 0.1027900146842878, |
| "fcm_dpo/beta": 0.12486476451158524, |
| "fcm_dpo/delta": 0.2788800001144409, |
| "fcm_dpo/margin": 2.644542694091797, |
| "fcm_dpo/q_t": 0.4251058101654053, |
| "grad_norm": 33.3270378112793, |
| "learning_rate": 5e-07, |
| "logits/chosen": 0.3216286301612854, |
| "logits/rejected": 0.5021553635597229, |
| "logps/chosen": -95.56564331054688, |
| "logps/ref_chosen": -95.00994873046875, |
| "logps/ref_rejected": -96.21272277832031, |
| "logps/rejected": -99.4129638671875, |
| "loss": 1.1246, |
| "margin_dpo/margin_mean": 2.644542932510376, |
| "margin_dpo/margin_std": 3.2647337913513184, |
| "step": 70 |
| }, |
| { |
| "epoch": 0.10425844346549193, |
| "fcm_dpo/beta": 0.12874360382556915, |
| "fcm_dpo/delta": 0.13646240532398224, |
| "fcm_dpo/margin": 3.6519854068756104, |
| "fcm_dpo/q_t": 0.3953820765018463, |
| "grad_norm": 39.30204772949219, |
| "learning_rate": 4.999967061337492e-07, |
| "logits/chosen": -0.15218183398246765, |
| "logits/rejected": 0.21293151378631592, |
| "logps/chosen": -91.0079345703125, |
| "logps/ref_chosen": -90.97735595703125, |
| "logps/ref_rejected": -102.59103393554688, |
| "logps/rejected": -106.27359771728516, |
| "loss": 1.0304, |
| "margin_dpo/margin_mean": 3.6519851684570312, |
| "margin_dpo/margin_std": 4.069307804107666, |
| "step": 71 |
| }, |
| { |
| "epoch": 0.10572687224669604, |
| "fcm_dpo/beta": 0.13415689766407013, |
| "fcm_dpo/delta": 0.2152521312236786, |
| "fcm_dpo/margin": 2.9328012466430664, |
| "fcm_dpo/q_t": 0.4104134142398834, |
| "grad_norm": 35.07579803466797, |
| "learning_rate": 4.999868246217933e-07, |
| "logits/chosen": -0.1298740804195404, |
| "logits/rejected": 0.19889742136001587, |
| "logps/chosen": -98.41287231445312, |
| "logps/ref_chosen": -97.89152526855469, |
| "logps/ref_rejected": -100.19171142578125, |
| "logps/rejected": -103.64585876464844, |
| "loss": 1.0781, |
| "margin_dpo/margin_mean": 2.932800769805908, |
| "margin_dpo/margin_std": 3.356008529663086, |
| "step": 72 |
| }, |
| { |
| "epoch": 0.10719530102790015, |
| "fcm_dpo/beta": 0.1394435167312622, |
| "fcm_dpo/delta": 0.17690719664096832, |
| "fcm_dpo/margin": 3.0887374877929688, |
| "fcm_dpo/q_t": 0.4099673926830292, |
| "grad_norm": 33.995853424072266, |
| "learning_rate": 4.999703557245192e-07, |
| "logits/chosen": -0.2099972814321518, |
| "logits/rejected": 0.10542194545269012, |
| "logps/chosen": -96.5327377319336, |
| "logps/ref_chosen": -95.7690200805664, |
| "logps/ref_rejected": -95.93243408203125, |
| "logps/rejected": -99.78489685058594, |
| "loss": 1.1024, |
| "margin_dpo/margin_mean": 3.0887370109558105, |
| "margin_dpo/margin_std": 4.92803430557251, |
| "step": 73 |
| }, |
| { |
| "epoch": 0.10866372980910426, |
| "fcm_dpo/beta": 0.14254900813102722, |
| "fcm_dpo/delta": 0.1032918393611908, |
| "fcm_dpo/margin": 3.521709442138672, |
| "fcm_dpo/q_t": 0.3906998634338379, |
| "grad_norm": 36.36503219604492, |
| "learning_rate": 4.999472998758977e-07, |
| "logits/chosen": -0.10034316778182983, |
| "logits/rejected": 0.0426066517829895, |
| "logps/chosen": -79.76939392089844, |
| "logps/ref_chosen": -78.80839538574219, |
| "logps/ref_rejected": -101.64677429199219, |
| "logps/rejected": -106.12947082519531, |
| "loss": 1.0323, |
| "margin_dpo/margin_mean": 3.5217092037200928, |
| "margin_dpo/margin_std": 4.384693145751953, |
| "step": 74 |
| }, |
| { |
| "epoch": 0.11013215859030837, |
| "fcm_dpo/beta": 0.14341631531715393, |
| "fcm_dpo/delta": -0.055621419101953506, |
| "fcm_dpo/margin": 4.54258918762207, |
| "fcm_dpo/q_t": 0.3650940954685211, |
| "grad_norm": 36.794151306152344, |
| "learning_rate": 4.999176576834721e-07, |
| "logits/chosen": -0.20140711963176727, |
| "logits/rejected": -0.000715896487236023, |
| "logps/chosen": -79.31031799316406, |
| "logps/ref_chosen": -78.28185272216797, |
| "logps/ref_rejected": -115.40311431884766, |
| "logps/rejected": -120.97416687011719, |
| "loss": 0.959, |
| "margin_dpo/margin_mean": 4.542590141296387, |
| "margin_dpo/margin_std": 5.413492679595947, |
| "step": 75 |
| }, |
| { |
| "epoch": 0.11160058737151249, |
| "fcm_dpo/beta": 0.14552651345729828, |
| "fcm_dpo/delta": 0.16189850866794586, |
| "fcm_dpo/margin": 3.059915542602539, |
| "fcm_dpo/q_t": 0.40516766905784607, |
| "grad_norm": 30.741098403930664, |
| "learning_rate": 4.998814299283415e-07, |
| "logits/chosen": -0.614639401435852, |
| "logits/rejected": -0.23801718652248383, |
| "logps/chosen": -89.27581787109375, |
| "logps/ref_chosen": -87.87713623046875, |
| "logps/ref_rejected": -85.71968078613281, |
| "logps/rejected": -90.17826843261719, |
| "loss": 1.0865, |
| "margin_dpo/margin_mean": 3.059915542602539, |
| "margin_dpo/margin_std": 4.615878105163574, |
| "step": 76 |
| }, |
| { |
| "epoch": 0.1130690161527166, |
| "fcm_dpo/beta": 0.14861461520195007, |
| "fcm_dpo/delta": 0.10367655009031296, |
| "fcm_dpo/margin": 3.3754987716674805, |
| "fcm_dpo/q_t": 0.3928101658821106, |
| "grad_norm": 37.48255920410156, |
| "learning_rate": 4.998386175651409e-07, |
| "logits/chosen": -0.18063946068286896, |
| "logits/rejected": 0.18999367952346802, |
| "logps/chosen": -101.2906723022461, |
| "logps/ref_chosen": -99.70034790039062, |
| "logps/ref_rejected": -98.20576477050781, |
| "logps/rejected": -103.17158508300781, |
| "loss": 1.0692, |
| "margin_dpo/margin_mean": 3.3754987716674805, |
| "margin_dpo/margin_std": 5.01793098449707, |
| "step": 77 |
| }, |
| { |
| "epoch": 0.1145374449339207, |
| "fcm_dpo/beta": 0.15083855390548706, |
| "fcm_dpo/delta": 0.049115367233753204, |
| "fcm_dpo/margin": 3.670267105102539, |
| "fcm_dpo/q_t": 0.3794415593147278, |
| "grad_norm": 31.02741241455078, |
| "learning_rate": 4.997892217220159e-07, |
| "logits/chosen": 0.14838413894176483, |
| "logits/rejected": 0.38533467054367065, |
| "logps/chosen": -90.91835021972656, |
| "logps/ref_chosen": -90.29670715332031, |
| "logps/ref_rejected": -91.13772583007812, |
| "logps/rejected": -95.42964172363281, |
| "loss": 0.9953, |
| "margin_dpo/margin_mean": 3.670267105102539, |
| "margin_dpo/margin_std": 4.296913146972656, |
| "step": 78 |
| }, |
| { |
| "epoch": 0.11600587371512482, |
| "fcm_dpo/beta": 0.14908111095428467, |
| "fcm_dpo/delta": -0.10986006259918213, |
| "fcm_dpo/margin": 4.710598945617676, |
| "fcm_dpo/q_t": 0.355399489402771, |
| "grad_norm": 32.22175598144531, |
| "learning_rate": 4.997332437005931e-07, |
| "logits/chosen": -0.6625052690505981, |
| "logits/rejected": -0.28548330068588257, |
| "logps/chosen": -87.22914123535156, |
| "logps/ref_chosen": -86.37832641601562, |
| "logps/ref_rejected": -94.10777282714844, |
| "logps/rejected": -99.66917419433594, |
| "loss": 0.9479, |
| "margin_dpo/margin_mean": 4.710598945617676, |
| "margin_dpo/margin_std": 5.8075103759765625, |
| "step": 79 |
| }, |
| { |
| "epoch": 0.11747430249632893, |
| "fcm_dpo/beta": 0.1489923596382141, |
| "fcm_dpo/delta": 0.04605354368686676, |
| "fcm_dpo/margin": 3.7351717948913574, |
| "fcm_dpo/q_t": 0.3830299377441406, |
| "grad_norm": 33.21147537231445, |
| "learning_rate": 4.996706849759452e-07, |
| "logits/chosen": -0.6652204990386963, |
| "logits/rejected": -0.30884838104248047, |
| "logps/chosen": -95.46414947509766, |
| "logps/ref_chosen": -93.97032165527344, |
| "logps/ref_rejected": -92.57441711425781, |
| "logps/rejected": -97.80342102050781, |
| "loss": 1.0157, |
| "margin_dpo/margin_mean": 3.7351717948913574, |
| "margin_dpo/margin_std": 4.898221015930176, |
| "step": 80 |
| }, |
| { |
| "epoch": 0.11894273127753303, |
| "fcm_dpo/beta": 0.14419203996658325, |
| "fcm_dpo/delta": -0.18252533674240112, |
| "fcm_dpo/margin": 5.311590194702148, |
| "fcm_dpo/q_t": 0.3500846028327942, |
| "grad_norm": 32.420406341552734, |
| "learning_rate": 4.996015471965529e-07, |
| "logits/chosen": -0.641904890537262, |
| "logits/rejected": -0.22355474531650543, |
| "logps/chosen": -100.79706573486328, |
| "logps/ref_chosen": -99.83012390136719, |
| "logps/ref_rejected": -133.67247009277344, |
| "logps/rejected": -139.95098876953125, |
| "loss": 0.9325, |
| "margin_dpo/margin_mean": 5.311590194702148, |
| "margin_dpo/margin_std": 6.772265434265137, |
| "step": 81 |
| }, |
| { |
| "epoch": 0.12041116005873716, |
| "fcm_dpo/beta": 0.14681574702262878, |
| "fcm_dpo/delta": 0.07446617633104324, |
| "fcm_dpo/margin": 3.593234062194824, |
| "fcm_dpo/q_t": 0.38746753334999084, |
| "grad_norm": 33.34585952758789, |
| "learning_rate": 4.995258321842611e-07, |
| "logits/chosen": -0.10603200644254684, |
| "logits/rejected": 0.11097083985805511, |
| "logps/chosen": -84.93232727050781, |
| "logps/ref_chosen": -83.04598236083984, |
| "logps/ref_rejected": -94.52595520019531, |
| "logps/rejected": -100.00553131103516, |
| "loss": 1.0399, |
| "margin_dpo/margin_mean": 3.593233823776245, |
| "margin_dpo/margin_std": 4.823699951171875, |
| "step": 82 |
| }, |
| { |
| "epoch": 0.12187958883994127, |
| "fcm_dpo/beta": 0.14609777927398682, |
| "fcm_dpo/delta": -0.0029450468719005585, |
| "fcm_dpo/margin": 4.125781536102295, |
| "fcm_dpo/q_t": 0.38040691614151, |
| "grad_norm": 39.993614196777344, |
| "learning_rate": 4.994435419342304e-07, |
| "logits/chosen": -0.4488382339477539, |
| "logits/rejected": -0.025571677833795547, |
| "logps/chosen": -94.34185028076172, |
| "logps/ref_chosen": -92.17621612548828, |
| "logps/ref_rejected": -107.74464416503906, |
| "logps/rejected": -114.03605651855469, |
| "loss": 1.0479, |
| "margin_dpo/margin_mean": 4.125782012939453, |
| "margin_dpo/margin_std": 6.356573104858398, |
| "step": 83 |
| }, |
| { |
| "epoch": 0.12334801762114538, |
| "fcm_dpo/beta": 0.1460931897163391, |
| "fcm_dpo/delta": 0.0006008504424244165, |
| "fcm_dpo/margin": 2.527191638946533, |
| "fcm_dpo/q_t": 0.422789990901947, |
| "grad_norm": 44.41518020629883, |
| "learning_rate": 4.993546786148857e-07, |
| "logits/chosen": -0.4026920795440674, |
| "logits/rejected": -0.021030515432357788, |
| "logps/chosen": -104.27578735351562, |
| "logps/ref_chosen": -101.5264892578125, |
| "logps/ref_rejected": -92.42608642578125, |
| "logps/rejected": -97.70257568359375, |
| "loss": 1.1681, |
| "margin_dpo/margin_mean": 2.5271923542022705, |
| "margin_dpo/margin_std": 4.771922588348389, |
| "step": 84 |
| }, |
| { |
| "epoch": 0.12481644640234948, |
| "fcm_dpo/beta": 0.1492038369178772, |
| "fcm_dpo/delta": 0.09993791580200195, |
| "fcm_dpo/margin": 3.376296043395996, |
| "fcm_dpo/q_t": 0.3976307511329651, |
| "grad_norm": 42.52838134765625, |
| "learning_rate": 4.992592445678582e-07, |
| "logits/chosen": -0.22517216205596924, |
| "logits/rejected": 0.10873828083276749, |
| "logps/chosen": -98.53033447265625, |
| "logps/ref_chosen": -96.12738037109375, |
| "logps/ref_rejected": -85.05519104003906, |
| "logps/rejected": -90.83444213867188, |
| "loss": 1.1171, |
| "margin_dpo/margin_mean": 3.3762965202331543, |
| "margin_dpo/margin_std": 5.977930545806885, |
| "step": 85 |
| }, |
| { |
| "epoch": 0.1262848751835536, |
| "fcm_dpo/beta": 0.14783477783203125, |
| "fcm_dpo/delta": -0.044383928179740906, |
| "fcm_dpo/margin": 4.338801383972168, |
| "fcm_dpo/q_t": 0.37967485189437866, |
| "grad_norm": 30.416973114013672, |
| "learning_rate": 4.991572423079235e-07, |
| "logits/chosen": -0.4074563682079315, |
| "logits/rejected": -0.16365355253219604, |
| "logps/chosen": -83.54771423339844, |
| "logps/ref_chosen": -81.70426177978516, |
| "logps/ref_rejected": -93.65550231933594, |
| "logps/rejected": -99.8377456665039, |
| "loss": 1.0355, |
| "margin_dpo/margin_mean": 4.338800430297852, |
| "margin_dpo/margin_std": 6.941473960876465, |
| "step": 86 |
| }, |
| { |
| "epoch": 0.1277533039647577, |
| "fcm_dpo/beta": 0.1455276608467102, |
| "fcm_dpo/delta": -0.07910901308059692, |
| "fcm_dpo/margin": 4.626860618591309, |
| "fcm_dpo/q_t": 0.36312583088874817, |
| "grad_norm": 38.379615783691406, |
| "learning_rate": 4.990486745229364e-07, |
| "logits/chosen": -0.8469661474227905, |
| "logits/rejected": -0.48443603515625, |
| "logps/chosen": -94.81890869140625, |
| "logps/ref_chosen": -92.68596649169922, |
| "logps/ref_rejected": -102.91818237304688, |
| "logps/rejected": -109.67797088623047, |
| "loss": 1.0375, |
| "margin_dpo/margin_mean": 4.626861572265625, |
| "margin_dpo/margin_std": 7.198031425476074, |
| "step": 87 |
| }, |
| { |
| "epoch": 0.12922173274596183, |
| "fcm_dpo/beta": 0.14544734358787537, |
| "fcm_dpo/delta": 0.05892753228545189, |
| "fcm_dpo/margin": 3.7355780601501465, |
| "fcm_dpo/q_t": 0.39093679189682007, |
| "grad_norm": 35.78076171875, |
| "learning_rate": 4.989335440737586e-07, |
| "logits/chosen": -0.09812933206558228, |
| "logits/rejected": 0.08861222863197327, |
| "logps/chosen": -103.42122650146484, |
| "logps/ref_chosen": -100.76298522949219, |
| "logps/ref_rejected": -113.15037536621094, |
| "logps/rejected": -119.54420471191406, |
| "loss": 1.087, |
| "margin_dpo/margin_mean": 3.7355780601501465, |
| "margin_dpo/margin_std": 6.172787666320801, |
| "step": 88 |
| }, |
| { |
| "epoch": 0.13069016152716592, |
| "fcm_dpo/beta": 0.147188201546669, |
| "fcm_dpo/delta": 0.028476351872086525, |
| "fcm_dpo/margin": 3.892774820327759, |
| "fcm_dpo/q_t": 0.3894566297531128, |
| "grad_norm": 30.757919311523438, |
| "learning_rate": 4.988118539941847e-07, |
| "logits/chosen": -0.6192222833633423, |
| "logits/rejected": -0.28894031047821045, |
| "logps/chosen": -91.76663208007812, |
| "logps/ref_chosen": -89.69108581542969, |
| "logps/ref_rejected": -88.56832885742188, |
| "logps/rejected": -94.53666687011719, |
| "loss": 1.0588, |
| "margin_dpo/margin_mean": 3.892774820327759, |
| "margin_dpo/margin_std": 6.543310165405273, |
| "step": 89 |
| }, |
| { |
| "epoch": 0.13215859030837004, |
| "fcm_dpo/beta": 0.14558328688144684, |
| "fcm_dpo/delta": -0.14587749540805817, |
| "fcm_dpo/margin": 5.050919532775879, |
| "fcm_dpo/q_t": 0.35632702708244324, |
| "grad_norm": 32.816978454589844, |
| "learning_rate": 4.986836074908615e-07, |
| "logits/chosen": -0.28241512179374695, |
| "logits/rejected": -0.16540709137916565, |
| "logps/chosen": -83.23228454589844, |
| "logps/ref_chosen": -81.38255310058594, |
| "logps/ref_rejected": -117.77714538574219, |
| "logps/rejected": -124.67778015136719, |
| "loss": 0.9688, |
| "margin_dpo/margin_mean": 5.050920486450195, |
| "margin_dpo/margin_std": 6.877577781677246, |
| "step": 90 |
| }, |
| { |
| "epoch": 0.13362701908957417, |
| "fcm_dpo/beta": 0.14302682876586914, |
| "fcm_dpo/delta": -0.03883608058094978, |
| "fcm_dpo/margin": 4.4493865966796875, |
| "fcm_dpo/q_t": 0.37537193298339844, |
| "grad_norm": 35.73554611206055, |
| "learning_rate": 4.985488079432037e-07, |
| "logits/chosen": -0.2560047507286072, |
| "logits/rejected": 0.15471355617046356, |
| "logps/chosen": -99.27584838867188, |
| "logps/ref_chosen": -97.22187805175781, |
| "logps/ref_rejected": -92.97674560546875, |
| "logps/rejected": -99.4801025390625, |
| "loss": 1.0129, |
| "margin_dpo/margin_mean": 4.449386119842529, |
| "margin_dpo/margin_std": 6.499499320983887, |
| "step": 91 |
| }, |
| { |
| "epoch": 0.13509544787077826, |
| "fcm_dpo/beta": 0.14428263902664185, |
| "fcm_dpo/delta": 0.06443801522254944, |
| "fcm_dpo/margin": 3.7344517707824707, |
| "fcm_dpo/q_t": 0.39002060890197754, |
| "grad_norm": 30.80695343017578, |
| "learning_rate": 4.984074589033043e-07, |
| "logits/chosen": -0.8915022611618042, |
| "logits/rejected": -0.5445601344108582, |
| "logps/chosen": -86.91651153564453, |
| "logps/ref_chosen": -84.5302734375, |
| "logps/ref_rejected": -84.5013198852539, |
| "logps/rejected": -90.62200927734375, |
| "loss": 1.0648, |
| "margin_dpo/margin_mean": 3.7344512939453125, |
| "margin_dpo/margin_std": 5.879533290863037, |
| "step": 92 |
| }, |
| { |
| "epoch": 0.13656387665198239, |
| "fcm_dpo/beta": 0.14631253480911255, |
| "fcm_dpo/delta": 0.07103022187948227, |
| "fcm_dpo/margin": 3.6395092010498047, |
| "fcm_dpo/q_t": 0.39281517267227173, |
| "grad_norm": 39.31964111328125, |
| "learning_rate": 4.982595640958425e-07, |
| "logits/chosen": -0.8906035423278809, |
| "logits/rejected": -0.34005117416381836, |
| "logps/chosen": -92.74388122558594, |
| "logps/ref_chosen": -90.25043487548828, |
| "logps/ref_rejected": -84.0942153930664, |
| "logps/rejected": -90.2271728515625, |
| "loss": 1.1067, |
| "margin_dpo/margin_mean": 3.6395092010498047, |
| "margin_dpo/margin_std": 6.321407318115234, |
| "step": 93 |
| }, |
| { |
| "epoch": 0.13803230543318648, |
| "fcm_dpo/beta": 0.14079803228378296, |
| "fcm_dpo/delta": -0.1881101429462433, |
| "fcm_dpo/margin": 5.460287094116211, |
| "fcm_dpo/q_t": 0.3449600040912628, |
| "grad_norm": 34.704139709472656, |
| "learning_rate": 4.98105127417984e-07, |
| "logits/chosen": -0.45207756757736206, |
| "logits/rejected": -0.15455085039138794, |
| "logps/chosen": -94.94246673583984, |
| "logps/ref_chosen": -92.4542236328125, |
| "logps/ref_rejected": -105.2472915649414, |
| "logps/rejected": -113.19581604003906, |
| "loss": 0.954, |
| "margin_dpo/margin_mean": 5.4602861404418945, |
| "margin_dpo/margin_std": 6.94245719909668, |
| "step": 94 |
| }, |
| { |
| "epoch": 0.1395007342143906, |
| "fcm_dpo/beta": 0.14194484055042267, |
| "fcm_dpo/delta": -0.0035935528576374054, |
| "fcm_dpo/margin": 4.245328903198242, |
| "fcm_dpo/q_t": 0.38038820028305054, |
| "grad_norm": 29.971302032470703, |
| "learning_rate": 4.979441529392784e-07, |
| "logits/chosen": -0.8304082155227661, |
| "logits/rejected": -0.5099495053291321, |
| "logps/chosen": -80.91264343261719, |
| "logps/ref_chosen": -78.87370300292969, |
| "logps/ref_rejected": -83.59121704101562, |
| "logps/rejected": -89.87547302246094, |
| "loss": 1.029, |
| "margin_dpo/margin_mean": 4.245327949523926, |
| "margin_dpo/margin_std": 6.151725769042969, |
| "step": 95 |
| }, |
| { |
| "epoch": 0.14096916299559473, |
| "fcm_dpo/beta": 0.13753080368041992, |
| "fcm_dpo/delta": -0.15204164385795593, |
| "fcm_dpo/margin": 5.382932186126709, |
| "fcm_dpo/q_t": 0.35132813453674316, |
| "grad_norm": 29.244211196899414, |
| "learning_rate": 4.977766449015534e-07, |
| "logits/chosen": -0.7205407619476318, |
| "logits/rejected": -0.30231958627700806, |
| "logps/chosen": -108.13746643066406, |
| "logps/ref_chosen": -106.5921630859375, |
| "logps/ref_rejected": -101.76802825927734, |
| "logps/rejected": -108.6962661743164, |
| "loss": 0.9483, |
| "margin_dpo/margin_mean": 5.382932662963867, |
| "margin_dpo/margin_std": 6.747956275939941, |
| "step": 96 |
| }, |
| { |
| "epoch": 0.14243759177679882, |
| "fcm_dpo/beta": 0.1367703527212143, |
| "fcm_dpo/delta": -0.06670509278774261, |
| "fcm_dpo/margin": 4.833524703979492, |
| "fcm_dpo/q_t": 0.36010098457336426, |
| "grad_norm": 32.554283142089844, |
| "learning_rate": 4.976026077188012e-07, |
| "logits/chosen": -0.327857106924057, |
| "logits/rejected": 0.07270251214504242, |
| "logps/chosen": -91.24253845214844, |
| "logps/ref_chosen": -88.67988586425781, |
| "logps/ref_rejected": -84.81229400634766, |
| "logps/rejected": -92.20846557617188, |
| "loss": 0.9517, |
| "margin_dpo/margin_mean": 4.833524227142334, |
| "margin_dpo/margin_std": 5.391106605529785, |
| "step": 97 |
| }, |
| { |
| "epoch": 0.14390602055800295, |
| "fcm_dpo/beta": 0.1318550407886505, |
| "fcm_dpo/delta": -0.11207875609397888, |
| "fcm_dpo/margin": 5.328702926635742, |
| "fcm_dpo/q_t": 0.35097742080688477, |
| "grad_norm": 30.32093620300293, |
| "learning_rate": 4.974220459770639e-07, |
| "logits/chosen": -0.5091695785522461, |
| "logits/rejected": -0.2531248927116394, |
| "logps/chosen": -94.11245727539062, |
| "logps/ref_chosen": -92.24249267578125, |
| "logps/ref_rejected": -101.51948547363281, |
| "logps/rejected": -108.7181396484375, |
| "loss": 0.917, |
| "margin_dpo/margin_mean": 5.328701972961426, |
| "margin_dpo/margin_std": 5.660694122314453, |
| "step": 98 |
| }, |
| { |
| "epoch": 0.14537444933920704, |
| "fcm_dpo/beta": 0.12901289761066437, |
| "fcm_dpo/delta": -0.14893479645252228, |
| "fcm_dpo/margin": 5.719096660614014, |
| "fcm_dpo/q_t": 0.35661765933036804, |
| "grad_norm": 27.99370002746582, |
| "learning_rate": 4.972349644343108e-07, |
| "logits/chosen": -0.3547890782356262, |
| "logits/rejected": -0.14507462084293365, |
| "logps/chosen": -73.60296630859375, |
| "logps/ref_chosen": -72.18464660644531, |
| "logps/ref_rejected": -91.88131713867188, |
| "logps/rejected": -99.01873016357422, |
| "loss": 0.9419, |
| "margin_dpo/margin_mean": 5.719097137451172, |
| "margin_dpo/margin_std": 7.474823474884033, |
| "step": 99 |
| }, |
| { |
| "epoch": 0.14684287812041116, |
| "fcm_dpo/beta": 0.1281561702489853, |
| "fcm_dpo/delta": 0.03391501307487488, |
| "fcm_dpo/margin": 4.430019378662109, |
| "fcm_dpo/q_t": 0.3762722611427307, |
| "grad_norm": 26.248151779174805, |
| "learning_rate": 4.970413680203148e-07, |
| "logits/chosen": -0.6022966504096985, |
| "logits/rejected": -0.14130988717079163, |
| "logps/chosen": -91.0530014038086, |
| "logps/ref_chosen": -89.51382446289062, |
| "logps/ref_rejected": -81.21713256835938, |
| "logps/rejected": -87.18632507324219, |
| "loss": 0.9831, |
| "margin_dpo/margin_mean": 4.430019378662109, |
| "margin_dpo/margin_std": 4.9434709548950195, |
| "step": 100 |
| }, |
| { |
| "epoch": 0.14684287812041116, |
| "eval_fcm_dpo/beta": 0.12897978723049164, |
| "eval_logits/chosen": 0.03100498393177986, |
| "eval_logits/rejected": 0.2742968201637268, |
| "eval_logps/chosen": -103.41716766357422, |
| "eval_logps/ref_chosen": -100.49356842041016, |
| "eval_logps/ref_rejected": -94.06775665283203, |
| "eval_logps/rejected": -100.0186767578125, |
| "eval_loss": 0.5851320028305054, |
| "eval_margin_dpo/margin_mean": 3.027308940887451, |
| "eval_margin_dpo/margin_std": 6.074099063873291, |
| "eval_runtime": 43.7521, |
| "eval_samples_per_second": 53.46, |
| "eval_steps_per_second": 1.691, |
| "step": 100 |
| }, |
| { |
| "epoch": 0.14831130690161526, |
| "fcm_dpo/beta": 0.13015779852867126, |
| "fcm_dpo/delta": 0.09972133487462997, |
| "fcm_dpo/margin": 3.881521224975586, |
| "fcm_dpo/q_t": 0.39363157749176025, |
| "grad_norm": 33.8118896484375, |
| "learning_rate": 4.968412618365215e-07, |
| "logits/chosen": -0.4824643135070801, |
| "logits/rejected": -0.21651335060596466, |
| "logps/chosen": -97.09152221679688, |
| "logps/ref_chosen": -94.4031982421875, |
| "logps/ref_rejected": -89.91001892089844, |
| "logps/rejected": -96.47987365722656, |
| "loss": 1.0872, |
| "margin_dpo/margin_mean": 3.881521224975586, |
| "margin_dpo/margin_std": 6.2039475440979, |
| "step": 101 |
| }, |
| { |
| "epoch": 0.14977973568281938, |
| "fcm_dpo/beta": 0.1360168159008026, |
| "fcm_dpo/delta": 0.1453799605369568, |
| "fcm_dpo/margin": 3.3603551387786865, |
| "fcm_dpo/q_t": 0.4019954204559326, |
| "grad_norm": 43.768089294433594, |
| "learning_rate": 4.966346511559149e-07, |
| "logits/chosen": -0.7954562902450562, |
| "logits/rejected": -0.4590097665786743, |
| "logps/chosen": -94.66258239746094, |
| "logps/ref_chosen": -91.60942077636719, |
| "logps/ref_rejected": -75.7689208984375, |
| "logps/rejected": -82.18241882324219, |
| "loss": 1.128, |
| "margin_dpo/margin_mean": 3.3603553771972656, |
| "margin_dpo/margin_std": 5.937575340270996, |
| "step": 102 |
| }, |
| { |
| "epoch": 0.1512481644640235, |
| "fcm_dpo/beta": 0.13238456845283508, |
| "fcm_dpo/delta": -0.1370110958814621, |
| "fcm_dpo/margin": 5.488382339477539, |
| "fcm_dpo/q_t": 0.34756556153297424, |
| "grad_norm": 32.67784118652344, |
| "learning_rate": 4.964215414228785e-07, |
| "logits/chosen": -0.6738690733909607, |
| "logits/rejected": -0.30804526805877686, |
| "logps/chosen": -99.65579223632812, |
| "logps/ref_chosen": -97.40238952636719, |
| "logps/ref_rejected": -98.981689453125, |
| "logps/rejected": -106.72347259521484, |
| "loss": 0.9394, |
| "margin_dpo/margin_mean": 5.4883832931518555, |
| "margin_dpo/margin_std": 6.411886215209961, |
| "step": 103 |
| }, |
| { |
| "epoch": 0.1527165932452276, |
| "fcm_dpo/beta": 0.12842795252799988, |
| "fcm_dpo/delta": -0.16619333624839783, |
| "fcm_dpo/margin": 5.866588115692139, |
| "fcm_dpo/q_t": 0.34973686933517456, |
| "grad_norm": 25.817495346069336, |
| "learning_rate": 4.96201938253052e-07, |
| "logits/chosen": -0.8139798045158386, |
| "logits/rejected": -0.4907284379005432, |
| "logps/chosen": -96.42156982421875, |
| "logps/ref_chosen": -94.85636138916016, |
| "logps/ref_rejected": -97.02669525146484, |
| "logps/rejected": -104.45848846435547, |
| "loss": 0.9237, |
| "margin_dpo/margin_mean": 5.866589069366455, |
| "margin_dpo/margin_std": 7.296744346618652, |
| "step": 104 |
| }, |
| { |
| "epoch": 0.15418502202643172, |
| "fcm_dpo/beta": 0.12592965364456177, |
| "fcm_dpo/delta": -0.12789560854434967, |
| "fcm_dpo/margin": 5.70680046081543, |
| "fcm_dpo/q_t": 0.3505769968032837, |
| "grad_norm": 30.186168670654297, |
| "learning_rate": 4.959758474331832e-07, |
| "logits/chosen": -0.8461470603942871, |
| "logits/rejected": -0.40368765592575073, |
| "logps/chosen": -98.9314956665039, |
| "logps/ref_chosen": -96.75790405273438, |
| "logps/ref_rejected": -101.5774917602539, |
| "logps/rejected": -109.4578857421875, |
| "loss": 0.9491, |
| "margin_dpo/margin_mean": 5.70680046081543, |
| "margin_dpo/margin_std": 6.950338840484619, |
| "step": 105 |
| }, |
| { |
| "epoch": 0.15565345080763582, |
| "fcm_dpo/beta": 0.12308678776025772, |
| "fcm_dpo/delta": -0.1014535203576088, |
| "fcm_dpo/margin": 5.641180515289307, |
| "fcm_dpo/q_t": 0.3579603433609009, |
| "grad_norm": 26.967708587646484, |
| "learning_rate": 4.957432749209755e-07, |
| "logits/chosen": -0.3590039014816284, |
| "logits/rejected": 0.06661949306726456, |
| "logps/chosen": -94.51701354980469, |
| "logps/ref_chosen": -92.43791198730469, |
| "logps/ref_rejected": -91.00970458984375, |
| "logps/rejected": -98.72999572753906, |
| "loss": 0.9482, |
| "margin_dpo/margin_mean": 5.641180038452148, |
| "margin_dpo/margin_std": 6.846166133880615, |
| "step": 106 |
| }, |
| { |
| "epoch": 0.15712187958883994, |
| "fcm_dpo/beta": 0.11860156059265137, |
| "fcm_dpo/delta": -0.1794869601726532, |
| "fcm_dpo/margin": 6.456604480743408, |
| "fcm_dpo/q_t": 0.3544139862060547, |
| "grad_norm": 27.01357650756836, |
| "learning_rate": 4.955042268449307e-07, |
| "logits/chosen": -0.43201225996017456, |
| "logits/rejected": 0.07057125866413116, |
| "logps/chosen": -106.36869812011719, |
| "logps/ref_chosen": -105.12500762939453, |
| "logps/ref_rejected": -101.1824951171875, |
| "logps/rejected": -108.8827896118164, |
| "loss": 0.9456, |
| "margin_dpo/margin_mean": 6.456604480743408, |
| "margin_dpo/margin_std": 9.23145580291748, |
| "step": 107 |
| }, |
| { |
| "epoch": 0.15859030837004406, |
| "fcm_dpo/beta": 0.11540443450212479, |
| "fcm_dpo/delta": -0.08156667649745941, |
| "fcm_dpo/margin": 5.848906993865967, |
| "fcm_dpo/q_t": 0.3599376976490021, |
| "grad_norm": 25.21524429321289, |
| "learning_rate": 4.952587095041881e-07, |
| "logits/chosen": -0.8443419933319092, |
| "logits/rejected": -0.35467660427093506, |
| "logps/chosen": -91.43235778808594, |
| "logps/ref_chosen": -89.20936584472656, |
| "logps/ref_rejected": -101.83784484863281, |
| "logps/rejected": -109.90974426269531, |
| "loss": 0.9559, |
| "margin_dpo/margin_mean": 5.848907470703125, |
| "margin_dpo/margin_std": 6.974748134613037, |
| "step": 108 |
| }, |
| { |
| "epoch": 0.16005873715124816, |
| "fcm_dpo/beta": 0.11124076694250107, |
| "fcm_dpo/delta": -0.23742878437042236, |
| "fcm_dpo/margin": 7.345666408538818, |
| "fcm_dpo/q_t": 0.33463311195373535, |
| "grad_norm": 25.55326271057129, |
| "learning_rate": 4.95006729368358e-07, |
| "logits/chosen": -0.5116323232650757, |
| "logits/rejected": -0.30790773034095764, |
| "logps/chosen": -101.7799072265625, |
| "logps/ref_chosen": -100.18708801269531, |
| "logps/ref_rejected": -103.53739929199219, |
| "logps/rejected": -112.47588348388672, |
| "loss": 0.895, |
| "margin_dpo/margin_mean": 7.34566593170166, |
| "margin_dpo/margin_std": 8.431303024291992, |
| "step": 109 |
| }, |
| { |
| "epoch": 0.16152716593245228, |
| "fcm_dpo/beta": 0.11126747727394104, |
| "fcm_dpo/delta": 0.09853193908929825, |
| "fcm_dpo/margin": 4.551311492919922, |
| "fcm_dpo/q_t": 0.3907678723335266, |
| "grad_norm": 26.554285049438477, |
| "learning_rate": 4.947482930773511e-07, |
| "logits/chosen": -0.20916850864887238, |
| "logits/rejected": 0.15151086449623108, |
| "logps/chosen": -109.06941223144531, |
| "logps/ref_chosen": -106.89572143554688, |
| "logps/ref_rejected": -85.81120300292969, |
| "logps/rejected": -92.53620910644531, |
| "loss": 1.0485, |
| "margin_dpo/margin_mean": 4.551311492919922, |
| "margin_dpo/margin_std": 6.169561386108398, |
| "step": 110 |
| }, |
| { |
| "epoch": 0.16299559471365638, |
| "fcm_dpo/beta": 0.10644402354955673, |
| "fcm_dpo/delta": -0.31627288460731506, |
| "fcm_dpo/margin": 8.312642097473145, |
| "fcm_dpo/q_t": 0.3186783790588379, |
| "grad_norm": 23.229040145874023, |
| "learning_rate": 4.944834074412042e-07, |
| "logits/chosen": -0.9531800746917725, |
| "logits/rejected": -0.6935971975326538, |
| "logps/chosen": -95.22097778320312, |
| "logps/ref_chosen": -93.3293685913086, |
| "logps/ref_rejected": -104.177490234375, |
| "logps/rejected": -114.38174438476562, |
| "loss": 0.8389, |
| "margin_dpo/margin_mean": 8.312643051147461, |
| "margin_dpo/margin_std": 8.214632034301758, |
| "step": 111 |
| }, |
| { |
| "epoch": 0.1644640234948605, |
| "fcm_dpo/beta": 0.10744242370128632, |
| "fcm_dpo/delta": 0.12501248717308044, |
| "fcm_dpo/margin": 4.4774169921875, |
| "fcm_dpo/q_t": 0.39411237835884094, |
| "grad_norm": 24.20155906677246, |
| "learning_rate": 4.942120794399002e-07, |
| "logits/chosen": -0.9408313632011414, |
| "logits/rejected": -0.488766074180603, |
| "logps/chosen": -84.89042663574219, |
| "logps/ref_chosen": -82.74703979492188, |
| "logps/ref_rejected": -71.5584487915039, |
| "logps/rejected": -78.17925262451172, |
| "loss": 1.0479, |
| "margin_dpo/margin_mean": 4.4774169921875, |
| "margin_dpo/margin_std": 5.751065254211426, |
| "step": 112 |
| }, |
| { |
| "epoch": 0.16593245227606462, |
| "fcm_dpo/beta": 0.10971181094646454, |
| "fcm_dpo/delta": 0.08851583302021027, |
| "fcm_dpo/margin": 4.70300817489624, |
| "fcm_dpo/q_t": 0.38598090410232544, |
| "grad_norm": 28.11321449279785, |
| "learning_rate": 4.939343162231841e-07, |
| "logits/chosen": -0.6415950059890747, |
| "logits/rejected": -0.14356252551078796, |
| "logps/chosen": -100.6945571899414, |
| "logps/ref_chosen": -97.89949035644531, |
| "logps/ref_rejected": -84.41960144042969, |
| "logps/rejected": -91.91766357421875, |
| "loss": 1.0218, |
| "margin_dpo/margin_mean": 4.703007698059082, |
| "margin_dpo/margin_std": 5.460232257843018, |
| "step": 113 |
| }, |
| { |
| "epoch": 0.16740088105726872, |
| "fcm_dpo/beta": 0.10798073559999466, |
| "fcm_dpo/delta": -0.10012990236282349, |
| "fcm_dpo/margin": 6.41129732131958, |
| "fcm_dpo/q_t": 0.36218881607055664, |
| "grad_norm": 28.432401657104492, |
| "learning_rate": 4.936501251103751e-07, |
| "logits/chosen": -1.0188771486282349, |
| "logits/rejected": -0.4950348734855652, |
| "logps/chosen": -99.10498046875, |
| "logps/ref_chosen": -96.4951171875, |
| "logps/ref_rejected": -92.01536560058594, |
| "logps/rejected": -101.0365219116211, |
| "loss": 0.9806, |
| "margin_dpo/margin_mean": 6.411297798156738, |
| "margin_dpo/margin_std": 8.566350936889648, |
| "step": 114 |
| }, |
| { |
| "epoch": 0.16886930983847284, |
| "fcm_dpo/beta": 0.10620786994695663, |
| "fcm_dpo/delta": -0.11520249396562576, |
| "fcm_dpo/margin": 6.658008098602295, |
| "fcm_dpo/q_t": 0.3513981103897095, |
| "grad_norm": 23.633058547973633, |
| "learning_rate": 4.933595135901732e-07, |
| "logits/chosen": -0.5462427139282227, |
| "logits/rejected": -0.1587030589580536, |
| "logps/chosen": -105.50727844238281, |
| "logps/ref_chosen": -103.18013000488281, |
| "logps/ref_rejected": -105.71595764160156, |
| "logps/rejected": -114.70111846923828, |
| "loss": 0.9284, |
| "margin_dpo/margin_mean": 6.658008575439453, |
| "margin_dpo/margin_std": 7.443235874176025, |
| "step": 115 |
| }, |
| { |
| "epoch": 0.17033773861967694, |
| "fcm_dpo/beta": 0.1062956303358078, |
| "fcm_dpo/delta": 0.024708664044737816, |
| "fcm_dpo/margin": 5.4217939376831055, |
| "fcm_dpo/q_t": 0.37565532326698303, |
| "grad_norm": 22.831968307495117, |
| "learning_rate": 4.930624893204624e-07, |
| "logits/chosen": -0.5053553581237793, |
| "logits/rejected": -0.07657349109649658, |
| "logps/chosen": -83.38106536865234, |
| "logps/ref_chosen": -81.15837097167969, |
| "logps/ref_rejected": -86.30586242675781, |
| "logps/rejected": -93.95034790039062, |
| "loss": 0.9908, |
| "margin_dpo/margin_mean": 5.421794414520264, |
| "margin_dpo/margin_std": 6.337811470031738, |
| "step": 116 |
| }, |
| { |
| "epoch": 0.17180616740088106, |
| "fcm_dpo/beta": 0.1065780371427536, |
| "fcm_dpo/delta": 0.09205998480319977, |
| "fcm_dpo/margin": 4.804106712341309, |
| "fcm_dpo/q_t": 0.38822945952415466, |
| "grad_norm": 31.471729278564453, |
| "learning_rate": 4.927590601281083e-07, |
| "logits/chosen": -0.6579692363739014, |
| "logits/rejected": -0.30281364917755127, |
| "logps/chosen": -99.8343505859375, |
| "logps/ref_chosen": -97.0809555053711, |
| "logps/ref_rejected": -75.1399154663086, |
| "logps/rejected": -82.69741821289062, |
| "loss": 1.0457, |
| "margin_dpo/margin_mean": 4.804106712341309, |
| "margin_dpo/margin_std": 6.34288215637207, |
| "step": 117 |
| }, |
| { |
| "epoch": 0.17327459618208516, |
| "fcm_dpo/beta": 0.10656137019395828, |
| "fcm_dpo/delta": -0.0756685882806778, |
| "fcm_dpo/margin": 6.291032791137695, |
| "fcm_dpo/q_t": 0.35226526856422424, |
| "grad_norm": 22.64267349243164, |
| "learning_rate": 4.924492340087524e-07, |
| "logits/chosen": -1.0219924449920654, |
| "logits/rejected": -0.7220288515090942, |
| "logps/chosen": -88.30572509765625, |
| "logps/ref_chosen": -86.53623962402344, |
| "logps/ref_rejected": -82.93563842773438, |
| "logps/rejected": -90.99617004394531, |
| "loss": 0.9216, |
| "margin_dpo/margin_mean": 6.291032791137695, |
| "margin_dpo/margin_std": 6.30666446685791, |
| "step": 118 |
| }, |
| { |
| "epoch": 0.17474302496328928, |
| "fcm_dpo/beta": 0.10483279079198837, |
| "fcm_dpo/delta": -0.08183208853006363, |
| "fcm_dpo/margin": 6.448603630065918, |
| "fcm_dpo/q_t": 0.3583568036556244, |
| "grad_norm": 26.867589950561523, |
| "learning_rate": 4.92133019126601e-07, |
| "logits/chosen": -0.9083903431892395, |
| "logits/rejected": -0.6197102665901184, |
| "logps/chosen": -107.41993713378906, |
| "logps/ref_chosen": -104.60908508300781, |
| "logps/ref_rejected": -109.15780639648438, |
| "logps/rejected": -118.41727447509766, |
| "loss": 0.9898, |
| "margin_dpo/margin_mean": 6.448603630065918, |
| "margin_dpo/margin_std": 8.662832260131836, |
| "step": 119 |
| }, |
| { |
| "epoch": 0.1762114537444934, |
| "fcm_dpo/beta": 0.10017947107553482, |
| "fcm_dpo/delta": -0.2905012369155884, |
| "fcm_dpo/margin": 8.636421203613281, |
| "fcm_dpo/q_t": 0.33309102058410645, |
| "grad_norm": 30.011741638183594, |
| "learning_rate": 4.918104238142103e-07, |
| "logits/chosen": -1.0279607772827148, |
| "logits/rejected": -0.5229378938674927, |
| "logps/chosen": -116.57168579101562, |
| "logps/ref_chosen": -114.68742370605469, |
| "logps/ref_rejected": -111.18586730957031, |
| "logps/rejected": -121.70654296875, |
| "loss": 0.915, |
| "margin_dpo/margin_mean": 8.636421203613281, |
| "margin_dpo/margin_std": 10.763944625854492, |
| "step": 120 |
| }, |
| { |
| "epoch": 0.1776798825256975, |
| "fcm_dpo/beta": 0.0964190736413002, |
| "fcm_dpo/delta": -0.12846185266971588, |
| "fcm_dpo/margin": 7.4496049880981445, |
| "fcm_dpo/q_t": 0.3460380434989929, |
| "grad_norm": 25.585208892822266, |
| "learning_rate": 4.91481456572267e-07, |
| "logits/chosen": -0.7847539186477661, |
| "logits/rejected": -0.4723082184791565, |
| "logps/chosen": -93.91836547851562, |
| "logps/ref_chosen": -92.11666107177734, |
| "logps/ref_rejected": -114.972900390625, |
| "logps/rejected": -124.22420501708984, |
| "loss": 0.9003, |
| "margin_dpo/margin_mean": 7.4496049880981445, |
| "margin_dpo/margin_std": 7.533651351928711, |
| "step": 121 |
| }, |
| { |
| "epoch": 0.17914831130690162, |
| "fcm_dpo/beta": 0.09395934641361237, |
| "fcm_dpo/delta": -0.18832112848758698, |
| "fcm_dpo/margin": 8.238999366760254, |
| "fcm_dpo/q_t": 0.3329077959060669, |
| "grad_norm": 26.141273498535156, |
| "learning_rate": 4.911461260693638e-07, |
| "logits/chosen": -0.47960373759269714, |
| "logits/rejected": -0.35823744535446167, |
| "logps/chosen": -87.03057861328125, |
| "logps/ref_chosen": -84.87757110595703, |
| "logps/ref_rejected": -111.0829849243164, |
| "logps/rejected": -121.47499084472656, |
| "loss": 0.8771, |
| "margin_dpo/margin_mean": 8.238998413085938, |
| "margin_dpo/margin_std": 8.030372619628906, |
| "step": 122 |
| }, |
| { |
| "epoch": 0.18061674008810572, |
| "fcm_dpo/beta": 0.09212273359298706, |
| "fcm_dpo/delta": 0.013646617531776428, |
| "fcm_dpo/margin": 6.361550807952881, |
| "fcm_dpo/q_t": 0.3727787137031555, |
| "grad_norm": 22.423507690429688, |
| "learning_rate": 4.908044411417711e-07, |
| "logits/chosen": -0.9911929368972778, |
| "logits/rejected": -0.7056489586830139, |
| "logps/chosen": -100.84278869628906, |
| "logps/ref_chosen": -98.47874450683594, |
| "logps/ref_rejected": -93.85417175292969, |
| "logps/rejected": -102.57976531982422, |
| "loss": 0.9885, |
| "margin_dpo/margin_mean": 6.361550331115723, |
| "margin_dpo/margin_std": 7.391707420349121, |
| "step": 123 |
| }, |
| { |
| "epoch": 0.18208516886930984, |
| "fcm_dpo/beta": 0.09153375774621964, |
| "fcm_dpo/delta": -0.10555803775787354, |
| "fcm_dpo/margin": 7.628256320953369, |
| "fcm_dpo/q_t": 0.36510169506073, |
| "grad_norm": 22.723674774169922, |
| "learning_rate": 4.904564107932048e-07, |
| "logits/chosen": -0.6390783786773682, |
| "logits/rejected": -0.385051965713501, |
| "logps/chosen": -104.82085418701172, |
| "logps/ref_chosen": -102.76290130615234, |
| "logps/ref_rejected": -123.005615234375, |
| "logps/rejected": -132.69183349609375, |
| "loss": 0.9851, |
| "margin_dpo/margin_mean": 7.628256320953369, |
| "margin_dpo/margin_std": 10.736438751220703, |
| "step": 124 |
| }, |
| { |
| "epoch": 0.18355359765051396, |
| "fcm_dpo/beta": 0.09196379780769348, |
| "fcm_dpo/delta": 0.07185746729373932, |
| "fcm_dpo/margin": 5.784401893615723, |
| "fcm_dpo/q_t": 0.38678333163261414, |
| "grad_norm": 24.231779098510742, |
| "learning_rate": 4.90102044194588e-07, |
| "logits/chosen": -0.6529036164283752, |
| "logits/rejected": -0.4633956253528595, |
| "logps/chosen": -93.97822570800781, |
| "logps/ref_chosen": -90.42627716064453, |
| "logps/ref_rejected": -89.24901580810547, |
| "logps/rejected": -98.58536529541016, |
| "loss": 1.0442, |
| "margin_dpo/margin_mean": 5.784401893615723, |
| "margin_dpo/margin_std": 8.095062255859375, |
| "step": 125 |
| }, |
| { |
| "epoch": 0.18502202643171806, |
| "fcm_dpo/beta": 0.09028153121471405, |
| "fcm_dpo/delta": -0.15866564214229584, |
| "fcm_dpo/margin": 8.273946762084961, |
| "fcm_dpo/q_t": 0.3347667157649994, |
| "grad_norm": 22.696521759033203, |
| "learning_rate": 4.897413506838102e-07, |
| "logits/chosen": -0.7492173910140991, |
| "logits/rejected": -0.49526458978652954, |
| "logps/chosen": -96.82389831542969, |
| "logps/ref_chosen": -94.42288208007812, |
| "logps/ref_rejected": -103.88156127929688, |
| "logps/rejected": -114.55652618408203, |
| "loss": 0.8628, |
| "margin_dpo/margin_mean": 8.273946762084961, |
| "margin_dpo/margin_std": 7.15079402923584, |
| "step": 126 |
| }, |
| { |
| "epoch": 0.18649045521292218, |
| "fcm_dpo/beta": 0.09020580351352692, |
| "fcm_dpo/delta": 0.022558212280273438, |
| "fcm_dpo/margin": 6.407971382141113, |
| "fcm_dpo/q_t": 0.377363920211792, |
| "grad_norm": 23.82652473449707, |
| "learning_rate": 4.89374339765481e-07, |
| "logits/chosen": -0.6647278070449829, |
| "logits/rejected": -0.35185229778289795, |
| "logps/chosen": -87.22705841064453, |
| "logps/ref_chosen": -84.75105285644531, |
| "logps/ref_rejected": -83.54523468017578, |
| "logps/rejected": -92.42921447753906, |
| "loss": 1.0158, |
| "margin_dpo/margin_mean": 6.407971382141113, |
| "margin_dpo/margin_std": 8.475217819213867, |
| "step": 127 |
| }, |
| { |
| "epoch": 0.18795888399412627, |
| "fcm_dpo/beta": 0.09033428132534027, |
| "fcm_dpo/delta": 0.012706171721220016, |
| "fcm_dpo/margin": 6.505819320678711, |
| "fcm_dpo/q_t": 0.37230080366134644, |
| "grad_norm": 25.748743057250977, |
| "learning_rate": 4.890010211106795e-07, |
| "logits/chosen": -0.5167370438575745, |
| "logits/rejected": -0.09917855262756348, |
| "logps/chosen": -93.13040161132812, |
| "logps/ref_chosen": -90.61964416503906, |
| "logps/ref_rejected": -83.10347747802734, |
| "logps/rejected": -92.12004852294922, |
| "loss": 1.0246, |
| "margin_dpo/margin_mean": 6.505819320678711, |
| "margin_dpo/margin_std": 9.225418090820312, |
| "step": 128 |
| }, |
| { |
| "epoch": 0.1894273127753304, |
| "fcm_dpo/beta": 0.09042790532112122, |
| "fcm_dpo/delta": -0.0049272216856479645, |
| "fcm_dpo/margin": 6.681199550628662, |
| "fcm_dpo/q_t": 0.37444210052490234, |
| "grad_norm": 21.542911529541016, |
| "learning_rate": 4.88621404556699e-07, |
| "logits/chosen": -0.42197108268737793, |
| "logits/rejected": -0.1338576376438141, |
| "logps/chosen": -101.74089050292969, |
| "logps/ref_chosen": -98.91870880126953, |
| "logps/ref_rejected": -103.28419494628906, |
| "logps/rejected": -112.78758239746094, |
| "loss": 0.9973, |
| "margin_dpo/margin_mean": 6.681199073791504, |
| "margin_dpo/margin_std": 8.355775833129883, |
| "step": 129 |
| }, |
| { |
| "epoch": 0.19089574155653452, |
| "fcm_dpo/beta": 0.08857293426990509, |
| "fcm_dpo/delta": -0.17602676153182983, |
| "fcm_dpo/margin": 8.605558395385742, |
| "fcm_dpo/q_t": 0.3452799320220947, |
| "grad_norm": 20.744892120361328, |
| "learning_rate": 4.882355001067891e-07, |
| "logits/chosen": -0.8332382440567017, |
| "logits/rejected": -0.5825516581535339, |
| "logps/chosen": -75.53247833251953, |
| "logps/ref_chosen": -73.18368530273438, |
| "logps/ref_rejected": -88.02154541015625, |
| "logps/rejected": -98.97589111328125, |
| "loss": 0.9116, |
| "margin_dpo/margin_mean": 8.605558395385742, |
| "margin_dpo/margin_std": 9.510639190673828, |
| "step": 130 |
| }, |
| { |
| "epoch": 0.19236417033773862, |
| "fcm_dpo/beta": 0.08698062598705292, |
| "fcm_dpo/delta": -0.04303303733468056, |
| "fcm_dpo/margin": 7.350835800170898, |
| "fcm_dpo/q_t": 0.3662865161895752, |
| "grad_norm": 21.23581886291504, |
| "learning_rate": 4.878433179298909e-07, |
| "logits/chosen": -0.792221188545227, |
| "logits/rejected": -0.576511800289154, |
| "logps/chosen": -75.73124694824219, |
| "logps/ref_chosen": -73.9564437866211, |
| "logps/ref_rejected": -94.45582580566406, |
| "logps/rejected": -103.58146667480469, |
| "loss": 0.9788, |
| "margin_dpo/margin_mean": 7.350835800170898, |
| "margin_dpo/margin_std": 9.003894805908203, |
| "step": 131 |
| }, |
| { |
| "epoch": 0.19383259911894274, |
| "fcm_dpo/beta": 0.08460888266563416, |
| "fcm_dpo/delta": -0.08415042608976364, |
| "fcm_dpo/margin": 8.008447647094727, |
| "fcm_dpo/q_t": 0.3516160249710083, |
| "grad_norm": 23.11528205871582, |
| "learning_rate": 4.874448683603694e-07, |
| "logits/chosen": -0.7575106620788574, |
| "logits/rejected": -0.360861212015152, |
| "logps/chosen": -95.101318359375, |
| "logps/ref_chosen": -92.43948364257812, |
| "logps/ref_rejected": -94.13252258300781, |
| "logps/rejected": -104.80279541015625, |
| "loss": 0.9311, |
| "margin_dpo/margin_mean": 8.008447647094727, |
| "margin_dpo/margin_std": 8.533775329589844, |
| "step": 132 |
| }, |
| { |
| "epoch": 0.19530102790014683, |
| "fcm_dpo/beta": 0.08362139761447906, |
| "fcm_dpo/delta": -0.1000952273607254, |
| "fcm_dpo/margin": 8.29056167602539, |
| "fcm_dpo/q_t": 0.35254591703414917, |
| "grad_norm": 21.20782470703125, |
| "learning_rate": 4.870401618977415e-07, |
| "logits/chosen": -1.1061053276062012, |
| "logits/rejected": -0.7356460690498352, |
| "logps/chosen": -94.83768463134766, |
| "logps/ref_chosen": -93.11666107177734, |
| "logps/ref_rejected": -102.09490966796875, |
| "logps/rejected": -112.10649108886719, |
| "loss": 0.9185, |
| "margin_dpo/margin_mean": 8.29056167602539, |
| "margin_dpo/margin_std": 8.746664047241211, |
| "step": 133 |
| }, |
| { |
| "epoch": 0.19676945668135096, |
| "fcm_dpo/beta": 0.08186972141265869, |
| "fcm_dpo/delta": -0.14321613311767578, |
| "fcm_dpo/margin": 8.948598861694336, |
| "fcm_dpo/q_t": 0.3416699171066284, |
| "grad_norm": 19.530412673950195, |
| "learning_rate": 4.866292092063986e-07, |
| "logits/chosen": -1.1816595792770386, |
| "logits/rejected": -0.8657940626144409, |
| "logps/chosen": -85.77832794189453, |
| "logps/ref_chosen": -83.55012512207031, |
| "logps/ref_rejected": -93.8815689086914, |
| "logps/rejected": -105.05838012695312, |
| "loss": 0.8911, |
| "margin_dpo/margin_mean": 8.948599815368652, |
| "margin_dpo/margin_std": 8.551406860351562, |
| "step": 134 |
| }, |
| { |
| "epoch": 0.19823788546255505, |
| "fcm_dpo/beta": 0.0804293155670166, |
| "fcm_dpo/delta": -0.045598309487104416, |
| "fcm_dpo/margin": 7.988539695739746, |
| "fcm_dpo/q_t": 0.3668595552444458, |
| "grad_norm": 26.672277450561523, |
| "learning_rate": 4.862120211153265e-07, |
| "logits/chosen": -0.9140989780426025, |
| "logits/rejected": -0.7617922425270081, |
| "logps/chosen": -79.91949462890625, |
| "logps/ref_chosen": -77.58262634277344, |
| "logps/ref_rejected": -121.05648040771484, |
| "logps/rejected": -131.38189697265625, |
| "loss": 0.9718, |
| "margin_dpo/margin_mean": 7.9885406494140625, |
| "margin_dpo/margin_std": 9.902532577514648, |
| "step": 135 |
| }, |
| { |
| "epoch": 0.19970631424375918, |
| "fcm_dpo/beta": 0.0786333680152893, |
| "fcm_dpo/delta": -0.09831453859806061, |
| "fcm_dpo/margin": 8.791913986206055, |
| "fcm_dpo/q_t": 0.34778937697410583, |
| "grad_norm": 19.27442169189453, |
| "learning_rate": 4.857886086178193e-07, |
| "logits/chosen": -1.0831172466278076, |
| "logits/rejected": -0.6637728214263916, |
| "logps/chosen": -96.29593658447266, |
| "logps/ref_chosen": -94.1727294921875, |
| "logps/ref_rejected": -103.26428985595703, |
| "logps/rejected": -114.17940521240234, |
| "loss": 0.8993, |
| "margin_dpo/margin_mean": 8.791913986206055, |
| "margin_dpo/margin_std": 8.21432113647461, |
| "step": 136 |
| }, |
| { |
| "epoch": 0.2011747430249633, |
| "fcm_dpo/beta": 0.07321903109550476, |
| "fcm_dpo/delta": -0.3953135013580322, |
| "fcm_dpo/margin": 13.009666442871094, |
| "fcm_dpo/q_t": 0.302865207195282, |
| "grad_norm": 21.34975814819336, |
| "learning_rate": 4.853589828711902e-07, |
| "logits/chosen": -0.7808685302734375, |
| "logits/rejected": -0.5483202934265137, |
| "logps/chosen": -79.78280639648438, |
| "logps/ref_chosen": -78.11874389648438, |
| "logps/ref_rejected": -117.92237091064453, |
| "logps/rejected": -132.59609985351562, |
| "loss": 0.7814, |
| "margin_dpo/margin_mean": 13.00966739654541, |
| "margin_dpo/margin_std": 11.704439163208008, |
| "step": 137 |
| }, |
| { |
| "epoch": 0.2026431718061674, |
| "fcm_dpo/beta": 0.07211971282958984, |
| "fcm_dpo/delta": -0.01127841230481863, |
| "fcm_dpo/margin": 8.464805603027344, |
| "fcm_dpo/q_t": 0.36739087104797363, |
| "grad_norm": 18.32305908203125, |
| "learning_rate": 4.849231551964771e-07, |
| "logits/chosen": -0.5027905702590942, |
| "logits/rejected": -0.2673172950744629, |
| "logps/chosen": -96.51560974121094, |
| "logps/ref_chosen": -92.82994079589844, |
| "logps/ref_rejected": -98.64840698242188, |
| "logps/rejected": -110.79888153076172, |
| "loss": 0.96, |
| "margin_dpo/margin_mean": 8.464805603027344, |
| "margin_dpo/margin_std": 9.157770156860352, |
| "step": 138 |
| }, |
| { |
| "epoch": 0.20411160058737152, |
| "fcm_dpo/beta": 0.07099495828151703, |
| "fcm_dpo/delta": -0.09988156706094742, |
| "fcm_dpo/margin": 9.755861282348633, |
| "fcm_dpo/q_t": 0.3473873734474182, |
| "grad_norm": 18.42373275756836, |
| "learning_rate": 4.844811370781446e-07, |
| "logits/chosen": -0.8951112031936646, |
| "logits/rejected": -0.577903687953949, |
| "logps/chosen": -81.29713439941406, |
| "logps/ref_chosen": -79.00301361083984, |
| "logps/ref_rejected": -85.82733154296875, |
| "logps/rejected": -97.87730407714844, |
| "loss": 0.9312, |
| "margin_dpo/margin_mean": 9.755861282348633, |
| "margin_dpo/margin_std": 10.73771858215332, |
| "step": 139 |
| }, |
| { |
| "epoch": 0.2055800293685756, |
| "fcm_dpo/beta": 0.07040712982416153, |
| "fcm_dpo/delta": -0.03950156271457672, |
| "fcm_dpo/margin": 9.047286987304688, |
| "fcm_dpo/q_t": 0.36051124334335327, |
| "grad_norm": 17.700237274169922, |
| "learning_rate": 4.840329401637809e-07, |
| "logits/chosen": -1.4827024936676025, |
| "logits/rejected": -1.119741439819336, |
| "logps/chosen": -90.76173400878906, |
| "logps/ref_chosen": -88.35244750976562, |
| "logps/ref_rejected": -90.57476806640625, |
| "logps/rejected": -102.03134155273438, |
| "loss": 0.9434, |
| "margin_dpo/margin_mean": 9.047286033630371, |
| "margin_dpo/margin_std": 9.427406311035156, |
| "step": 140 |
| }, |
| { |
| "epoch": 0.20704845814977973, |
| "fcm_dpo/beta": 0.06885601580142975, |
| "fcm_dpo/delta": -0.14049942791461945, |
| "fcm_dpo/margin": 10.606624603271484, |
| "fcm_dpo/q_t": 0.3436024785041809, |
| "grad_norm": 19.141874313354492, |
| "learning_rate": 4.83578576263792e-07, |
| "logits/chosen": -1.1690764427185059, |
| "logits/rejected": -0.8353248834609985, |
| "logps/chosen": -110.29083251953125, |
| "logps/ref_chosen": -107.22016143798828, |
| "logps/ref_rejected": -104.71223449707031, |
| "logps/rejected": -118.3895263671875, |
| "loss": 0.8996, |
| "margin_dpo/margin_mean": 10.606623649597168, |
| "margin_dpo/margin_std": 10.912744522094727, |
| "step": 141 |
| }, |
| { |
| "epoch": 0.20851688693098386, |
| "fcm_dpo/beta": 0.067325659096241, |
| "fcm_dpo/delta": -0.07276973128318787, |
| "fcm_dpo/margin": 9.913378715515137, |
| "fcm_dpo/q_t": 0.3580227196216583, |
| "grad_norm": 18.446134567260742, |
| "learning_rate": 4.83118057351089e-07, |
| "logits/chosen": -0.8606932759284973, |
| "logits/rejected": -0.6145029067993164, |
| "logps/chosen": -92.71931457519531, |
| "logps/ref_chosen": -88.72894287109375, |
| "logps/ref_rejected": -99.73038482666016, |
| "logps/rejected": -113.63412475585938, |
| "loss": 0.9319, |
| "margin_dpo/margin_mean": 9.91337776184082, |
| "margin_dpo/margin_std": 10.864763259887695, |
| "step": 142 |
| }, |
| { |
| "epoch": 0.20998531571218795, |
| "fcm_dpo/beta": 0.06714846193790436, |
| "fcm_dpo/delta": 0.021465405821800232, |
| "fcm_dpo/margin": 8.62674331665039, |
| "fcm_dpo/q_t": 0.37440505623817444, |
| "grad_norm": 17.625202178955078, |
| "learning_rate": 4.826513955607734e-07, |
| "logits/chosen": -1.2188315391540527, |
| "logits/rejected": -0.7806566953659058, |
| "logps/chosen": -87.72565460205078, |
| "logps/ref_chosen": -85.50457763671875, |
| "logps/ref_rejected": -85.8056640625, |
| "logps/rejected": -96.65348815917969, |
| "loss": 0.9974, |
| "margin_dpo/margin_mean": 8.62674331665039, |
| "margin_dpo/margin_std": 10.493571281433105, |
| "step": 143 |
| }, |
| { |
| "epoch": 0.21145374449339208, |
| "fcm_dpo/beta": 0.06795937567949295, |
| "fcm_dpo/delta": -0.022204145789146423, |
| "fcm_dpo/margin": 9.117193222045898, |
| "fcm_dpo/q_t": 0.37021785974502563, |
| "grad_norm": 22.30422019958496, |
| "learning_rate": 4.821786031898176e-07, |
| "logits/chosen": -1.0173625946044922, |
| "logits/rejected": -0.5171209573745728, |
| "logps/chosen": -107.23023223876953, |
| "logps/ref_chosen": -104.68708801269531, |
| "logps/ref_rejected": -87.71908569335938, |
| "logps/rejected": -99.37940979003906, |
| "loss": 1.0019, |
| "margin_dpo/margin_mean": 9.117193222045898, |
| "margin_dpo/margin_std": 12.013388633728027, |
| "step": 144 |
| }, |
| { |
| "epoch": 0.21292217327459617, |
| "fcm_dpo/beta": 0.0664219856262207, |
| "fcm_dpo/delta": -0.08367905020713806, |
| "fcm_dpo/margin": 10.206480026245117, |
| "fcm_dpo/q_t": 0.35269540548324585, |
| "grad_norm": 18.23365592956543, |
| "learning_rate": 4.816996926967401e-07, |
| "logits/chosen": -0.9210905432701111, |
| "logits/rejected": -0.36352017521858215, |
| "logps/chosen": -90.94596862792969, |
| "logps/ref_chosen": -88.51988983154297, |
| "logps/ref_rejected": -83.76513671875, |
| "logps/rejected": -96.397705078125, |
| "loss": 0.9285, |
| "margin_dpo/margin_mean": 10.206480979919434, |
| "margin_dpo/margin_std": 10.862861633300781, |
| "step": 145 |
| }, |
| { |
| "epoch": 0.2143906020558003, |
| "fcm_dpo/beta": 0.06422297656536102, |
| "fcm_dpo/delta": -0.15030890703201294, |
| "fcm_dpo/margin": 11.474971771240234, |
| "fcm_dpo/q_t": 0.3387315571308136, |
| "grad_norm": 17.679115295410156, |
| "learning_rate": 4.812146767012779e-07, |
| "logits/chosen": -1.0683330297470093, |
| "logits/rejected": -0.4716656804084778, |
| "logps/chosen": -105.42335510253906, |
| "logps/ref_chosen": -102.90586853027344, |
| "logps/ref_rejected": -88.9493637084961, |
| "logps/rejected": -102.94181823730469, |
| "loss": 0.8824, |
| "margin_dpo/margin_mean": 11.474971771240234, |
| "margin_dpo/margin_std": 10.659229278564453, |
| "step": 146 |
| }, |
| { |
| "epoch": 0.21585903083700442, |
| "fcm_dpo/beta": 0.06264235824346542, |
| "fcm_dpo/delta": -0.1676965206861496, |
| "fcm_dpo/margin": 12.052690505981445, |
| "fcm_dpo/q_t": 0.334587037563324, |
| "grad_norm": 16.055294036865234, |
| "learning_rate": 4.807235679840536e-07, |
| "logits/chosen": -1.0700526237487793, |
| "logits/rejected": -0.5048679113388062, |
| "logps/chosen": -91.99482727050781, |
| "logps/ref_chosen": -89.26219177246094, |
| "logps/ref_rejected": -76.50011444091797, |
| "logps/rejected": -91.28544616699219, |
| "loss": 0.8674, |
| "margin_dpo/margin_mean": 12.052690505981445, |
| "margin_dpo/margin_std": 10.964826583862305, |
| "step": 147 |
| }, |
| { |
| "epoch": 0.2173274596182085, |
| "fcm_dpo/beta": 0.06236677244305611, |
| "fcm_dpo/delta": -0.03803453594446182, |
| "fcm_dpo/margin": 10.167755126953125, |
| "fcm_dpo/q_t": 0.3664420247077942, |
| "grad_norm": 18.700401306152344, |
| "learning_rate": 4.802263794862384e-07, |
| "logits/chosen": -1.050333023071289, |
| "logits/rejected": -0.6528257131576538, |
| "logps/chosen": -93.0657958984375, |
| "logps/ref_chosen": -91.76287841796875, |
| "logps/ref_rejected": -110.38552856445312, |
| "logps/rejected": -121.856201171875, |
| "loss": 0.966, |
| "margin_dpo/margin_mean": 10.167754173278809, |
| "margin_dpo/margin_std": 11.42441177368164, |
| "step": 148 |
| }, |
| { |
| "epoch": 0.21879588839941264, |
| "fcm_dpo/beta": 0.0613025538623333, |
| "fcm_dpo/delta": -0.013467121869325638, |
| "fcm_dpo/margin": 9.98930835723877, |
| "fcm_dpo/q_t": 0.37364453077316284, |
| "grad_norm": 17.756973266601562, |
| "learning_rate": 4.797231243092118e-07, |
| "logits/chosen": -0.9625898003578186, |
| "logits/rejected": -0.6630960702896118, |
| "logps/chosen": -97.20555114746094, |
| "logps/ref_chosen": -93.48401641845703, |
| "logps/ref_rejected": -105.6436996459961, |
| "logps/rejected": -119.35454559326172, |
| "loss": 0.991, |
| "margin_dpo/margin_mean": 9.989309310913086, |
| "margin_dpo/margin_std": 12.843564987182617, |
| "step": 149 |
| }, |
| { |
| "epoch": 0.22026431718061673, |
| "fcm_dpo/beta": 0.060672082006931305, |
| "fcm_dpo/delta": -0.07046893239021301, |
| "fcm_dpo/margin": 10.968360900878906, |
| "fcm_dpo/q_t": 0.35937947034835815, |
| "grad_norm": 15.75436782836914, |
| "learning_rate": 4.792138157142157e-07, |
| "logits/chosen": -0.8642300367355347, |
| "logits/rejected": -0.5881214141845703, |
| "logps/chosen": -78.63943481445312, |
| "logps/ref_chosen": -75.7786865234375, |
| "logps/ref_rejected": -90.02676391601562, |
| "logps/rejected": -103.85586547851562, |
| "loss": 0.9473, |
| "margin_dpo/margin_mean": 10.968360900878906, |
| "margin_dpo/margin_std": 12.808120727539062, |
| "step": 150 |
| }, |
| { |
| "epoch": 0.22173274596182085, |
| "fcm_dpo/beta": 0.06035682559013367, |
| "fcm_dpo/delta": -0.029848817735910416, |
| "fcm_dpo/margin": 10.404455184936523, |
| "fcm_dpo/q_t": 0.3672201931476593, |
| "grad_norm": 19.685895919799805, |
| "learning_rate": 4.786984671220053e-07, |
| "logits/chosen": -1.7692604064941406, |
| "logits/rejected": -1.2120516300201416, |
| "logps/chosen": -106.98040771484375, |
| "logps/ref_chosen": -102.73422241210938, |
| "logps/ref_rejected": -106.03781127929688, |
| "logps/rejected": -120.68846130371094, |
| "loss": 0.9769, |
| "margin_dpo/margin_mean": 10.404455184936523, |
| "margin_dpo/margin_std": 12.831315994262695, |
| "step": 151 |
| }, |
| { |
| "epoch": 0.22320117474302498, |
| "fcm_dpo/beta": 0.05888533219695091, |
| "fcm_dpo/delta": -0.14253246784210205, |
| "fcm_dpo/margin": 12.427541732788086, |
| "fcm_dpo/q_t": 0.34149640798568726, |
| "grad_norm": 17.039073944091797, |
| "learning_rate": 4.78177092112495e-07, |
| "logits/chosen": -1.1869471073150635, |
| "logits/rejected": -0.8218509554862976, |
| "logps/chosen": -99.31748962402344, |
| "logps/ref_chosen": -97.59074401855469, |
| "logps/ref_rejected": -110.49913024902344, |
| "logps/rejected": -124.6534194946289, |
| "loss": 0.8913, |
| "margin_dpo/margin_mean": 12.427543640136719, |
| "margin_dpo/margin_std": 12.471309661865234, |
| "step": 152 |
| }, |
| { |
| "epoch": 0.22466960352422907, |
| "fcm_dpo/beta": 0.05911749601364136, |
| "fcm_dpo/delta": 0.06782151758670807, |
| "fcm_dpo/margin": 9.064018249511719, |
| "fcm_dpo/q_t": 0.3846895396709442, |
| "grad_norm": 17.052255630493164, |
| "learning_rate": 4.776497044244016e-07, |
| "logits/chosen": -1.4898900985717773, |
| "logits/rejected": -1.1749351024627686, |
| "logps/chosen": -91.19923400878906, |
| "logps/ref_chosen": -87.02560424804688, |
| "logps/ref_rejected": -91.06854248046875, |
| "logps/rejected": -104.30618286132812, |
| "loss": 1.0342, |
| "margin_dpo/margin_mean": 9.064018249511719, |
| "margin_dpo/margin_std": 12.064422607421875, |
| "step": 153 |
| }, |
| { |
| "epoch": 0.2261380323054332, |
| "fcm_dpo/beta": 0.05774786323308945, |
| "fcm_dpo/delta": -0.2577056586742401, |
| "fcm_dpo/margin": 14.486139297485352, |
| "fcm_dpo/q_t": 0.319768488407135, |
| "grad_norm": 16.834199905395508, |
| "learning_rate": 4.771163179548808e-07, |
| "logits/chosen": -1.1176913976669312, |
| "logits/rejected": -0.7909313440322876, |
| "logps/chosen": -93.70020294189453, |
| "logps/ref_chosen": -91.10566711425781, |
| "logps/ref_rejected": -111.37238311767578, |
| "logps/rejected": -128.45306396484375, |
| "loss": 0.8249, |
| "margin_dpo/margin_mean": 14.486139297485352, |
| "margin_dpo/margin_std": 12.080253601074219, |
| "step": 154 |
| }, |
| { |
| "epoch": 0.2276064610866373, |
| "fcm_dpo/beta": 0.05554405599832535, |
| "fcm_dpo/delta": -0.05215900391340256, |
| "fcm_dpo/margin": 11.656644821166992, |
| "fcm_dpo/q_t": 0.35945552587509155, |
| "grad_norm": 20.919530868530273, |
| "learning_rate": 4.7657694675916247e-07, |
| "logits/chosen": -1.0245808362960815, |
| "logits/rejected": -0.6745032072067261, |
| "logps/chosen": -98.85678100585938, |
| "logps/ref_chosen": -94.56097412109375, |
| "logps/ref_rejected": -84.77359008789062, |
| "logps/rejected": -100.72604370117188, |
| "loss": 0.9947, |
| "margin_dpo/margin_mean": 11.656644821166992, |
| "margin_dpo/margin_std": 15.20389175415039, |
| "step": 155 |
| }, |
| { |
| "epoch": 0.2290748898678414, |
| "fcm_dpo/beta": 0.055864423513412476, |
| "fcm_dpo/delta": 0.03552495688199997, |
| "fcm_dpo/margin": 10.137336730957031, |
| "fcm_dpo/q_t": 0.37930774688720703, |
| "grad_norm": 18.78835105895996, |
| "learning_rate": 4.7603160505017893e-07, |
| "logits/chosen": -1.3272318840026855, |
| "logits/rejected": -0.8460725545883179, |
| "logps/chosen": -100.33680725097656, |
| "logps/ref_chosen": -94.65538787841797, |
| "logps/ref_rejected": -84.41934204101562, |
| "logps/rejected": -100.23809051513672, |
| "loss": 1.0171, |
| "margin_dpo/margin_mean": 10.137334823608398, |
| "margin_dpo/margin_std": 13.136527061462402, |
| "step": 156 |
| }, |
| { |
| "epoch": 0.2305433186490455, |
| "fcm_dpo/beta": 0.05504686385393143, |
| "fcm_dpo/delta": -0.15157991647720337, |
| "fcm_dpo/margin": 13.453115463256836, |
| "fcm_dpo/q_t": 0.33803731203079224, |
| "grad_norm": 17.70085334777832, |
| "learning_rate": 4.7548030719819154e-07, |
| "logits/chosen": -1.6043157577514648, |
| "logits/rejected": -1.1462740898132324, |
| "logps/chosen": -102.32630920410156, |
| "logps/ref_chosen": -97.01239013671875, |
| "logps/ref_rejected": -111.70133972167969, |
| "logps/rejected": -130.4683837890625, |
| "loss": 0.8766, |
| "margin_dpo/margin_mean": 13.453115463256836, |
| "margin_dpo/margin_std": 12.435213088989258, |
| "step": 157 |
| }, |
| { |
| "epoch": 0.23201174743024963, |
| "fcm_dpo/beta": 0.053367845714092255, |
| "fcm_dpo/delta": -0.1598518192768097, |
| "fcm_dpo/margin": 14.018548011779785, |
| "fcm_dpo/q_t": 0.3366357088088989, |
| "grad_norm": 17.31310272216797, |
| "learning_rate": 4.7492306773041136e-07, |
| "logits/chosen": -1.1085999011993408, |
| "logits/rejected": -0.8174431920051575, |
| "logps/chosen": -99.63768005371094, |
| "logps/ref_chosen": -94.71296691894531, |
| "logps/ref_rejected": -120.93733978271484, |
| "logps/rejected": -139.880615234375, |
| "loss": 0.8714, |
| "margin_dpo/margin_mean": 14.018546104431152, |
| "margin_dpo/margin_std": 12.86544418334961, |
| "step": 158 |
| }, |
| { |
| "epoch": 0.23348017621145375, |
| "fcm_dpo/beta": 0.05200641602277756, |
| "fcm_dpo/delta": -0.09326906502246857, |
| "fcm_dpo/margin": 13.204719543457031, |
| "fcm_dpo/q_t": 0.35249608755111694, |
| "grad_norm": 14.883583068847656, |
| "learning_rate": 4.743599013306165e-07, |
| "logits/chosen": -1.7073493003845215, |
| "logits/rejected": -1.2342015504837036, |
| "logps/chosen": -112.4176025390625, |
| "logps/ref_chosen": -108.2182388305664, |
| "logps/ref_rejected": -94.83901977539062, |
| "logps/rejected": -112.24309539794922, |
| "loss": 0.9328, |
| "margin_dpo/margin_mean": 13.204719543457031, |
| "margin_dpo/margin_std": 14.517641067504883, |
| "step": 159 |
| }, |
| { |
| "epoch": 0.23494860499265785, |
| "fcm_dpo/beta": 0.05130424350500107, |
| "fcm_dpo/delta": -0.05173896625638008, |
| "fcm_dpo/margin": 12.633888244628906, |
| "fcm_dpo/q_t": 0.3558708131313324, |
| "grad_norm": 15.668502807617188, |
| "learning_rate": 4.737908228387656e-07, |
| "logits/chosen": -1.4224272966384888, |
| "logits/rejected": -0.8817993402481079, |
| "logps/chosen": -101.59320068359375, |
| "logps/ref_chosen": -98.16340637207031, |
| "logps/ref_rejected": -102.28097534179688, |
| "logps/rejected": -118.34466552734375, |
| "loss": 0.9332, |
| "margin_dpo/margin_mean": 12.633889198303223, |
| "margin_dpo/margin_std": 12.74842643737793, |
| "step": 160 |
| }, |
| { |
| "epoch": 0.23641703377386197, |
| "fcm_dpo/beta": 0.05100458487868309, |
| "fcm_dpo/delta": -0.0010594986379146576, |
| "fcm_dpo/margin": 11.772229194641113, |
| "fcm_dpo/q_t": 0.3696848750114441, |
| "grad_norm": 16.675426483154297, |
| "learning_rate": 4.7321584725060594e-07, |
| "logits/chosen": -1.661262035369873, |
| "logits/rejected": -1.1878881454467773, |
| "logps/chosen": -92.0523681640625, |
| "logps/ref_chosen": -86.59187316894531, |
| "logps/ref_rejected": -89.01643371582031, |
| "logps/rejected": -106.24916076660156, |
| "loss": 0.9897, |
| "margin_dpo/margin_mean": 11.772229194641113, |
| "margin_dpo/margin_std": 14.080129623413086, |
| "step": 161 |
| }, |
| { |
| "epoch": 0.23788546255506607, |
| "fcm_dpo/beta": 0.050782568752765656, |
| "fcm_dpo/delta": -0.046759188175201416, |
| "fcm_dpo/margin": 12.671504974365234, |
| "fcm_dpo/q_t": 0.357089638710022, |
| "grad_norm": 14.808147430419922, |
| "learning_rate": 4.7263498971727905e-07, |
| "logits/chosen": -0.9641022682189941, |
| "logits/rejected": -0.5625172257423401, |
| "logps/chosen": -102.07867431640625, |
| "logps/ref_chosen": -97.40288543701172, |
| "logps/ref_rejected": -95.86923217773438, |
| "logps/rejected": -113.2165298461914, |
| "loss": 0.9463, |
| "margin_dpo/margin_mean": 12.671504974365234, |
| "margin_dpo/margin_std": 13.506097793579102, |
| "step": 162 |
| }, |
| { |
| "epoch": 0.2393538913362702, |
| "fcm_dpo/beta": 0.04923873394727707, |
| "fcm_dpo/delta": -0.19184833765029907, |
| "fcm_dpo/margin": 15.766294479370117, |
| "fcm_dpo/q_t": 0.3363766372203827, |
| "grad_norm": 13.101168632507324, |
| "learning_rate": 4.720482655449212e-07, |
| "logits/chosen": -1.5308029651641846, |
| "logits/rejected": -1.145806908607483, |
| "logps/chosen": -86.65454864501953, |
| "logps/ref_chosen": -82.1753921508789, |
| "logps/ref_rejected": -82.86473846435547, |
| "logps/rejected": -103.11018371582031, |
| "loss": 0.8757, |
| "margin_dpo/margin_mean": 15.766292572021484, |
| "margin_dpo/margin_std": 16.105995178222656, |
| "step": 163 |
| }, |
| { |
| "epoch": 0.24082232011747431, |
| "fcm_dpo/beta": 0.04908021539449692, |
| "fcm_dpo/delta": 0.08986516296863556, |
| "fcm_dpo/margin": 10.468819618225098, |
| "fcm_dpo/q_t": 0.38547319173812866, |
| "grad_norm": 15.708305358886719, |
| "learning_rate": 4.714556901942599e-07, |
| "logits/chosen": -1.2432619333267212, |
| "logits/rejected": -0.7992422580718994, |
| "logps/chosen": -98.1744613647461, |
| "logps/ref_chosen": -90.47427368164062, |
| "logps/ref_rejected": -87.02809143066406, |
| "logps/rejected": -105.19710540771484, |
| "loss": 1.031, |
| "margin_dpo/margin_mean": 10.468820571899414, |
| "margin_dpo/margin_std": 12.44760799407959, |
| "step": 164 |
| }, |
| { |
| "epoch": 0.2422907488986784, |
| "fcm_dpo/beta": 0.04876025393605232, |
| "fcm_dpo/delta": -0.1661471575498581, |
| "fcm_dpo/margin": 15.461935997009277, |
| "fcm_dpo/q_t": 0.3421006202697754, |
| "grad_norm": 13.452424049377441, |
| "learning_rate": 4.708572792802069e-07, |
| "logits/chosen": -1.3514103889465332, |
| "logits/rejected": -0.941323459148407, |
| "logps/chosen": -92.18637084960938, |
| "logps/ref_chosen": -87.313232421875, |
| "logps/ref_rejected": -81.902587890625, |
| "logps/rejected": -102.23765563964844, |
| "loss": 0.8884, |
| "margin_dpo/margin_mean": 15.461935997009277, |
| "margin_dpo/margin_std": 16.554237365722656, |
| "step": 165 |
| }, |
| { |
| "epoch": 0.24375917767988253, |
| "fcm_dpo/beta": 0.04737546294927597, |
| "fcm_dpo/delta": -0.10841958224773407, |
| "fcm_dpo/margin": 14.792806625366211, |
| "fcm_dpo/q_t": 0.35638824105262756, |
| "grad_norm": 14.667271614074707, |
| "learning_rate": 4.702530485714461e-07, |
| "logits/chosen": -1.409278154373169, |
| "logits/rejected": -1.1956782341003418, |
| "logps/chosen": -90.57990264892578, |
| "logps/ref_chosen": -81.58606719970703, |
| "logps/ref_rejected": -103.16517639160156, |
| "logps/rejected": -126.95182800292969, |
| "loss": 0.9348, |
| "margin_dpo/margin_mean": 14.792806625366211, |
| "margin_dpo/margin_std": 17.73269271850586, |
| "step": 166 |
| }, |
| { |
| "epoch": 0.24522760646108663, |
| "fcm_dpo/beta": 0.0457189604640007, |
| "fcm_dpo/delta": -0.15542298555374146, |
| "fcm_dpo/margin": 16.228336334228516, |
| "fcm_dpo/q_t": 0.33740636706352234, |
| "grad_norm": 14.068784713745117, |
| "learning_rate": 4.6964301399001877e-07, |
| "logits/chosen": -0.9149848222732544, |
| "logits/rejected": -0.7234656810760498, |
| "logps/chosen": -91.24684143066406, |
| "logps/ref_chosen": -83.56167602539062, |
| "logps/ref_rejected": -99.05305480957031, |
| "logps/rejected": -122.966552734375, |
| "loss": 0.8664, |
| "margin_dpo/margin_mean": 16.228334426879883, |
| "margin_dpo/margin_std": 14.279183387756348, |
| "step": 167 |
| }, |
| { |
| "epoch": 0.24669603524229075, |
| "fcm_dpo/beta": 0.04559946060180664, |
| "fcm_dpo/delta": -0.028263315558433533, |
| "fcm_dpo/margin": 13.73760986328125, |
| "fcm_dpo/q_t": 0.3622729182243347, |
| "grad_norm": 15.556920051574707, |
| "learning_rate": 4.690271916109034e-07, |
| "logits/chosen": -1.2671973705291748, |
| "logits/rejected": -0.9255896210670471, |
| "logps/chosen": -87.52298736572266, |
| "logps/ref_chosen": -79.93123626708984, |
| "logps/ref_rejected": -80.10216522216797, |
| "logps/rejected": -101.43153381347656, |
| "loss": 0.9486, |
| "margin_dpo/margin_mean": 13.73760986328125, |
| "margin_dpo/margin_std": 14.369284629821777, |
| "step": 168 |
| }, |
| { |
| "epoch": 0.24816446402349487, |
| "fcm_dpo/beta": 0.0447533056139946, |
| "fcm_dpo/delta": -0.10509349405765533, |
| "fcm_dpo/margin": 15.592907905578613, |
| "fcm_dpo/q_t": 0.3495541214942932, |
| "grad_norm": 14.77795124053955, |
| "learning_rate": 4.6840559766159235e-07, |
| "logits/chosen": -1.474048137664795, |
| "logits/rejected": -1.2569692134857178, |
| "logps/chosen": -89.0520248413086, |
| "logps/ref_chosen": -82.90370178222656, |
| "logps/ref_rejected": -90.12191009521484, |
| "logps/rejected": -111.8631362915039, |
| "loss": 0.9092, |
| "margin_dpo/margin_mean": 15.592906951904297, |
| "margin_dpo/margin_std": 15.848993301391602, |
| "step": 169 |
| }, |
| { |
| "epoch": 0.24963289280469897, |
| "fcm_dpo/beta": 0.044094495475292206, |
| "fcm_dpo/delta": -0.01727929711341858, |
| "fcm_dpo/margin": 13.957527160644531, |
| "fcm_dpo/q_t": 0.3613913655281067, |
| "grad_norm": 15.097552299499512, |
| "learning_rate": 4.6777824852166437e-07, |
| "logits/chosen": -1.403516173362732, |
| "logits/rejected": -1.1804269552230835, |
| "logps/chosen": -82.96324157714844, |
| "logps/ref_chosen": -74.98197174072266, |
| "logps/ref_rejected": -82.16560363769531, |
| "logps/rejected": -104.10440063476562, |
| "loss": 0.9393, |
| "margin_dpo/margin_mean": 13.957527160644531, |
| "margin_dpo/margin_std": 13.041166305541992, |
| "step": 170 |
| }, |
| { |
| "epoch": 0.2511013215859031, |
| "fcm_dpo/beta": 0.043724410235881805, |
| "fcm_dpo/delta": -0.0946701318025589, |
| "fcm_dpo/margin": 15.739529609680176, |
| "fcm_dpo/q_t": 0.36089038848876953, |
| "grad_norm": 16.572017669677734, |
| "learning_rate": 4.6714516072235273e-07, |
| "logits/chosen": -1.3717052936553955, |
| "logits/rejected": -0.8343698978424072, |
| "logps/chosen": -128.6714630126953, |
| "logps/ref_chosen": -118.97853088378906, |
| "logps/ref_rejected": -115.76930236816406, |
| "logps/rejected": -141.2017822265625, |
| "loss": 0.9651, |
| "margin_dpo/margin_mean": 15.73952865600586, |
| "margin_dpo/margin_std": 20.676483154296875, |
| "step": 171 |
| }, |
| { |
| "epoch": 0.2525697503671072, |
| "fcm_dpo/beta": 0.043144889175891876, |
| "fcm_dpo/delta": -0.026401542127132416, |
| "fcm_dpo/margin": 14.471285820007324, |
| "fcm_dpo/q_t": 0.35602670907974243, |
| "grad_norm": 12.957140922546387, |
| "learning_rate": 4.6650635094610966e-07, |
| "logits/chosen": -1.9741286039352417, |
| "logits/rejected": -1.2486335039138794, |
| "logps/chosen": -111.53092956542969, |
| "logps/ref_chosen": -101.15379333496094, |
| "logps/ref_rejected": -91.00568389892578, |
| "logps/rejected": -115.85411071777344, |
| "loss": 0.9095, |
| "margin_dpo/margin_mean": 14.471284866333008, |
| "margin_dpo/margin_std": 10.961095809936523, |
| "step": 172 |
| }, |
| { |
| "epoch": 0.2540381791483113, |
| "fcm_dpo/beta": 0.04372722655534744, |
| "fcm_dpo/delta": 0.06630315631628036, |
| "fcm_dpo/margin": 12.284669876098633, |
| "fcm_dpo/q_t": 0.37949445843696594, |
| "grad_norm": 13.340476989746094, |
| "learning_rate": 4.6586183602616687e-07, |
| "logits/chosen": -1.3271675109863281, |
| "logits/rejected": -0.8779016733169556, |
| "logps/chosen": -108.86444091796875, |
| "logps/ref_chosen": -99.05668640136719, |
| "logps/ref_rejected": -83.8507308959961, |
| "logps/rejected": -105.94315338134766, |
| "loss": 0.9963, |
| "margin_dpo/margin_mean": 12.284669876098633, |
| "margin_dpo/margin_std": 13.309188842773438, |
| "step": 173 |
| }, |
| { |
| "epoch": 0.2555066079295154, |
| "fcm_dpo/beta": 0.04317288100719452, |
| "fcm_dpo/delta": -0.11161762475967407, |
| "fcm_dpo/margin": 16.303455352783203, |
| "fcm_dpo/q_t": 0.34579095244407654, |
| "grad_norm": 15.587017059326172, |
| "learning_rate": 4.652116329460919e-07, |
| "logits/chosen": -1.7453947067260742, |
| "logits/rejected": -1.6798264980316162, |
| "logps/chosen": -89.8681869506836, |
| "logps/ref_chosen": -80.40890502929688, |
| "logps/ref_rejected": -108.20171356201172, |
| "logps/rejected": -133.96444702148438, |
| "loss": 0.904, |
| "margin_dpo/margin_mean": 16.303455352783203, |
| "margin_dpo/margin_std": 16.183778762817383, |
| "step": 174 |
| }, |
| { |
| "epoch": 0.25697503671071953, |
| "fcm_dpo/beta": 0.04247898980975151, |
| "fcm_dpo/delta": -0.09163874387741089, |
| "fcm_dpo/margin": 16.133861541748047, |
| "fcm_dpo/q_t": 0.3471967577934265, |
| "grad_norm": 15.230539321899414, |
| "learning_rate": 4.645557588393406e-07, |
| "logits/chosen": -0.9117776155471802, |
| "logits/rejected": -0.6414705514907837, |
| "logps/chosen": -95.48237609863281, |
| "logps/ref_chosen": -82.71649169921875, |
| "logps/ref_rejected": -96.66712951660156, |
| "logps/rejected": -125.56687927246094, |
| "loss": 0.8972, |
| "margin_dpo/margin_mean": 16.133861541748047, |
| "margin_dpo/margin_std": 14.272943496704102, |
| "step": 175 |
| }, |
| { |
| "epoch": 0.25844346549192365, |
| "fcm_dpo/beta": 0.041571564972400665, |
| "fcm_dpo/delta": -0.08201269805431366, |
| "fcm_dpo/margin": 16.272031784057617, |
| "fcm_dpo/q_t": 0.3564577102661133, |
| "grad_norm": 13.990058898925781, |
| "learning_rate": 4.638942309888058e-07, |
| "logits/chosen": -1.4704281091690063, |
| "logits/rejected": -1.2904351949691772, |
| "logps/chosen": -90.04341888427734, |
| "logps/ref_chosen": -79.62930297851562, |
| "logps/ref_rejected": -102.6885986328125, |
| "logps/rejected": -129.374755859375, |
| "loss": 0.9461, |
| "margin_dpo/margin_mean": 16.272031784057617, |
| "margin_dpo/margin_std": 18.543813705444336, |
| "step": 176 |
| }, |
| { |
| "epoch": 0.2599118942731278, |
| "fcm_dpo/beta": 0.04052150249481201, |
| "fcm_dpo/delta": -0.1876201629638672, |
| "fcm_dpo/margin": 19.08730125427246, |
| "fcm_dpo/q_t": 0.33940553665161133, |
| "grad_norm": 12.637894630432129, |
| "learning_rate": 4.6322706682636137e-07, |
| "logits/chosen": -1.425307273864746, |
| "logits/rejected": -1.0769648551940918, |
| "logps/chosen": -103.93244171142578, |
| "logps/ref_chosen": -92.56109619140625, |
| "logps/ref_rejected": -103.33839416503906, |
| "logps/rejected": -133.79702758789062, |
| "loss": 0.876, |
| "margin_dpo/margin_mean": 19.08730125427246, |
| "margin_dpo/margin_std": 19.590530395507812, |
| "step": 177 |
| }, |
| { |
| "epoch": 0.26138032305433184, |
| "fcm_dpo/beta": 0.03819538652896881, |
| "fcm_dpo/delta": -0.28372639417648315, |
| "fcm_dpo/margin": 22.48978614807129, |
| "fcm_dpo/q_t": 0.31603994965553284, |
| "grad_norm": 14.502700805664062, |
| "learning_rate": 4.6255428393240354e-07, |
| "logits/chosen": -1.4522943496704102, |
| "logits/rejected": -1.1993482112884521, |
| "logps/chosen": -94.813720703125, |
| "logps/ref_chosen": -84.26579284667969, |
| "logps/ref_rejected": -110.79730987548828, |
| "logps/rejected": -143.83502197265625, |
| "loss": 0.8098, |
| "margin_dpo/margin_mean": 22.489788055419922, |
| "margin_dpo/margin_std": 19.596847534179688, |
| "step": 178 |
| }, |
| { |
| "epoch": 0.26284875183553597, |
| "fcm_dpo/beta": 0.03770379722118378, |
| "fcm_dpo/delta": 0.03393785282969475, |
| "fcm_dpo/margin": 15.058501243591309, |
| "fcm_dpo/q_t": 0.3792954683303833, |
| "grad_norm": 14.292525291442871, |
| "learning_rate": 4.6187590003538724e-07, |
| "logits/chosen": -1.4518247842788696, |
| "logits/rejected": -1.0252907276153564, |
| "logps/chosen": -107.04389953613281, |
| "logps/ref_chosen": -93.4479751586914, |
| "logps/ref_rejected": -94.25541687011719, |
| "logps/rejected": -122.90983581542969, |
| "loss": 1.0249, |
| "margin_dpo/margin_mean": 15.058500289916992, |
| "margin_dpo/margin_std": 20.123884201049805, |
| "step": 179 |
| }, |
| { |
| "epoch": 0.2643171806167401, |
| "fcm_dpo/beta": 0.037198685109615326, |
| "fcm_dpo/delta": -0.1041245386004448, |
| "fcm_dpo/margin": 18.714359283447266, |
| "fcm_dpo/q_t": 0.34685850143432617, |
| "grad_norm": 16.52766227722168, |
| "learning_rate": 4.611919330113591e-07, |
| "logits/chosen": -1.3025885820388794, |
| "logits/rejected": -1.0561097860336304, |
| "logps/chosen": -100.08863830566406, |
| "logps/ref_chosen": -88.83554077148438, |
| "logps/ref_rejected": -103.90696716308594, |
| "logps/rejected": -133.87442016601562, |
| "loss": 0.894, |
| "margin_dpo/margin_mean": 18.714359283447266, |
| "margin_dpo/margin_std": 17.619686126708984, |
| "step": 180 |
| }, |
| { |
| "epoch": 0.2657856093979442, |
| "fcm_dpo/beta": 0.037504084408283234, |
| "fcm_dpo/delta": 0.05481020361185074, |
| "fcm_dpo/margin": 14.614221572875977, |
| "fcm_dpo/q_t": 0.3806135952472687, |
| "grad_norm": 13.32461166381836, |
| "learning_rate": 4.605024008834863e-07, |
| "logits/chosen": -1.3358888626098633, |
| "logits/rejected": -0.8776803612709045, |
| "logps/chosen": -90.83370208740234, |
| "logps/ref_chosen": -79.23396301269531, |
| "logps/ref_rejected": -69.17442321777344, |
| "logps/rejected": -95.38838195800781, |
| "loss": 1.0004, |
| "margin_dpo/margin_mean": 14.614219665527344, |
| "margin_dpo/margin_std": 17.222267150878906, |
| "step": 181 |
| }, |
| { |
| "epoch": 0.26725403817914833, |
| "fcm_dpo/beta": 0.036797843873500824, |
| "fcm_dpo/delta": -0.10142774879932404, |
| "fcm_dpo/margin": 18.840681076049805, |
| "fcm_dpo/q_t": 0.3518170118331909, |
| "grad_norm": 13.206196784973145, |
| "learning_rate": 4.598073218215817e-07, |
| "logits/chosen": -0.9532027244567871, |
| "logits/rejected": -0.7038797736167908, |
| "logps/chosen": -90.80711364746094, |
| "logps/ref_chosen": -76.08218383789062, |
| "logps/ref_rejected": -94.9686050415039, |
| "logps/rejected": -128.53421020507812, |
| "loss": 0.9321, |
| "margin_dpo/margin_mean": 18.840679168701172, |
| "margin_dpo/margin_std": 20.858131408691406, |
| "step": 182 |
| }, |
| { |
| "epoch": 0.2687224669603524, |
| "fcm_dpo/beta": 0.03612806648015976, |
| "fcm_dpo/delta": -0.09574753046035767, |
| "fcm_dpo/margin": 19.054080963134766, |
| "fcm_dpo/q_t": 0.3482775390148163, |
| "grad_norm": 13.609108924865723, |
| "learning_rate": 4.5910671414162484e-07, |
| "logits/chosen": -1.2038804292678833, |
| "logits/rejected": -0.5809817314147949, |
| "logps/chosen": -108.66018676757812, |
| "logps/ref_chosen": -93.90790557861328, |
| "logps/ref_rejected": -83.56061553955078, |
| "logps/rejected": -117.36697387695312, |
| "loss": 0.9029, |
| "margin_dpo/margin_mean": 19.054080963134766, |
| "margin_dpo/margin_std": 17.8356990814209, |
| "step": 183 |
| }, |
| { |
| "epoch": 0.2701908957415565, |
| "fcm_dpo/beta": 0.03689391911029816, |
| "fcm_dpo/delta": 0.14321771264076233, |
| "fcm_dpo/margin": 12.56014633178711, |
| "fcm_dpo/q_t": 0.3973623514175415, |
| "grad_norm": 13.919024467468262, |
| "learning_rate": 4.5840059630527985e-07, |
| "logits/chosen": -1.4166245460510254, |
| "logits/rejected": -1.154975175857544, |
| "logps/chosen": -100.36515045166016, |
| "logps/ref_chosen": -86.86790466308594, |
| "logps/ref_rejected": -84.28456115722656, |
| "logps/rejected": -110.34194946289062, |
| "loss": 1.0914, |
| "margin_dpo/margin_mean": 12.56014633178711, |
| "margin_dpo/margin_std": 19.446250915527344, |
| "step": 184 |
| }, |
| { |
| "epoch": 0.27165932452276065, |
| "fcm_dpo/beta": 0.03748057410120964, |
| "fcm_dpo/delta": 0.027648886665701866, |
| "fcm_dpo/margin": 15.299049377441406, |
| "fcm_dpo/q_t": 0.3777276873588562, |
| "grad_norm": 15.781747817993164, |
| "learning_rate": 4.5768898691940836e-07, |
| "logits/chosen": -1.9094171524047852, |
| "logits/rejected": -1.3005274534225464, |
| "logps/chosen": -108.41390991210938, |
| "logps/ref_chosen": -94.02142333984375, |
| "logps/ref_rejected": -81.45668029785156, |
| "logps/rejected": -111.1482162475586, |
| "loss": 1.0174, |
| "margin_dpo/margin_mean": 15.299049377441406, |
| "margin_dpo/margin_std": 20.224950790405273, |
| "step": 185 |
| }, |
| { |
| "epoch": 0.27312775330396477, |
| "fcm_dpo/beta": 0.03701728954911232, |
| "fcm_dpo/delta": -0.03318379819393158, |
| "fcm_dpo/margin": 17.034767150878906, |
| "fcm_dpo/q_t": 0.3616868853569031, |
| "grad_norm": 15.530001640319824, |
| "learning_rate": 4.5697190473557947e-07, |
| "logits/chosen": -2.2073938846588135, |
| "logits/rejected": -1.3490548133850098, |
| "logps/chosen": -131.6390380859375, |
| "logps/ref_chosen": -114.19168090820312, |
| "logps/ref_rejected": -93.33912658691406, |
| "logps/rejected": -127.82122802734375, |
| "loss": 0.9536, |
| "margin_dpo/margin_mean": 17.03476905822754, |
| "margin_dpo/margin_std": 18.32429313659668, |
| "step": 186 |
| }, |
| { |
| "epoch": 0.2745961820851689, |
| "fcm_dpo/beta": 0.03734853118658066, |
| "fcm_dpo/delta": 0.018614832311868668, |
| "fcm_dpo/margin": 15.587629318237305, |
| "fcm_dpo/q_t": 0.37575846910476685, |
| "grad_norm": 13.691910743713379, |
| "learning_rate": 4.5624936864957555e-07, |
| "logits/chosen": -1.4853415489196777, |
| "logits/rejected": -1.2423460483551025, |
| "logps/chosen": -92.11011505126953, |
| "logps/ref_chosen": -76.86955261230469, |
| "logps/ref_rejected": -88.16494750976562, |
| "logps/rejected": -118.99314880371094, |
| "loss": 1.013, |
| "margin_dpo/margin_mean": 15.587630271911621, |
| "margin_dpo/margin_std": 20.420238494873047, |
| "step": 187 |
| }, |
| { |
| "epoch": 0.27606461086637296, |
| "fcm_dpo/beta": 0.0366520918905735, |
| "fcm_dpo/delta": -0.07102610170841217, |
| "fcm_dpo/margin": 18.150962829589844, |
| "fcm_dpo/q_t": 0.35110145807266235, |
| "grad_norm": 13.728299140930176, |
| "learning_rate": 4.5552139770089454e-07, |
| "logits/chosen": -1.6838798522949219, |
| "logits/rejected": -1.4296061992645264, |
| "logps/chosen": -90.44649505615234, |
| "logps/ref_chosen": -79.1948013305664, |
| "logps/ref_rejected": -96.82889556884766, |
| "logps/rejected": -126.23155212402344, |
| "loss": 0.9272, |
| "margin_dpo/margin_mean": 18.150962829589844, |
| "margin_dpo/margin_std": 17.76366424560547, |
| "step": 188 |
| }, |
| { |
| "epoch": 0.2775330396475771, |
| "fcm_dpo/beta": 0.0365501269698143, |
| "fcm_dpo/delta": -0.09013816714286804, |
| "fcm_dpo/margin": 18.695587158203125, |
| "fcm_dpo/q_t": 0.3573761284351349, |
| "grad_norm": 12.32153606414795, |
| "learning_rate": 4.5478801107224794e-07, |
| "logits/chosen": -1.5201849937438965, |
| "logits/rejected": -0.8749784231185913, |
| "logps/chosen": -99.00448608398438, |
| "logps/ref_chosen": -83.8927993774414, |
| "logps/ref_rejected": -79.46589660644531, |
| "logps/rejected": -113.2731704711914, |
| "loss": 0.9442, |
| "margin_dpo/margin_mean": 18.695587158203125, |
| "margin_dpo/margin_std": 21.5902042388916, |
| "step": 189 |
| }, |
| { |
| "epoch": 0.2790014684287812, |
| "fcm_dpo/beta": 0.03506568446755409, |
| "fcm_dpo/delta": -0.180100679397583, |
| "fcm_dpo/margin": 21.843788146972656, |
| "fcm_dpo/q_t": 0.331352174282074, |
| "grad_norm": 13.670723915100098, |
| "learning_rate": 4.5404922808905543e-07, |
| "logits/chosen": -1.5921630859375, |
| "logits/rejected": -0.9873641133308411, |
| "logps/chosen": -115.09228515625, |
| "logps/ref_chosen": -103.44146728515625, |
| "logps/ref_rejected": -109.21502685546875, |
| "logps/rejected": -142.70962524414062, |
| "loss": 0.8555, |
| "margin_dpo/margin_mean": 21.843786239624023, |
| "margin_dpo/margin_std": 19.022666931152344, |
| "step": 190 |
| }, |
| { |
| "epoch": 0.28046989720998533, |
| "fcm_dpo/beta": 0.03367793187499046, |
| "fcm_dpo/delta": -0.23534753918647766, |
| "fcm_dpo/margin": 24.244056701660156, |
| "fcm_dpo/q_t": 0.3260498046875, |
| "grad_norm": 14.045138359069824, |
| "learning_rate": 4.5330506821893565e-07, |
| "logits/chosen": -1.439011812210083, |
| "logits/rejected": -1.1144874095916748, |
| "logps/chosen": -110.62017059326172, |
| "logps/ref_chosen": -99.78551483154297, |
| "logps/ref_rejected": -116.00028228759766, |
| "logps/rejected": -151.0789794921875, |
| "loss": 0.8445, |
| "margin_dpo/margin_mean": 24.244056701660156, |
| "margin_dpo/margin_std": 22.547290802001953, |
| "step": 191 |
| }, |
| { |
| "epoch": 0.28193832599118945, |
| "fcm_dpo/beta": 0.03333042562007904, |
| "fcm_dpo/delta": 0.03673925995826721, |
| "fcm_dpo/margin": 16.96189308166504, |
| "fcm_dpo/q_t": 0.37764817476272583, |
| "grad_norm": 15.383365631103516, |
| "learning_rate": 4.5255555107119336e-07, |
| "logits/chosen": -1.293067455291748, |
| "logits/rejected": -1.030430793762207, |
| "logps/chosen": -120.11058807373047, |
| "logps/ref_chosen": -100.86032104492188, |
| "logps/ref_rejected": -109.44500732421875, |
| "logps/rejected": -145.6571502685547, |
| "loss": 1.0054, |
| "margin_dpo/margin_mean": 16.961894989013672, |
| "margin_dpo/margin_std": 21.036739349365234, |
| "step": 192 |
| }, |
| { |
| "epoch": 0.2834067547723935, |
| "fcm_dpo/beta": 0.0341544970870018, |
| "fcm_dpo/delta": 0.1115766242146492, |
| "fcm_dpo/margin": 14.43603515625, |
| "fcm_dpo/q_t": 0.3931388258934021, |
| "grad_norm": 14.872236251831055, |
| "learning_rate": 4.5180069639630236e-07, |
| "logits/chosen": -1.807457685470581, |
| "logits/rejected": -1.221724271774292, |
| "logps/chosen": -108.34063720703125, |
| "logps/ref_chosen": -92.93831634521484, |
| "logps/ref_rejected": -83.10917663574219, |
| "logps/rejected": -112.94754028320312, |
| "loss": 1.0871, |
| "margin_dpo/margin_mean": 14.43603515625, |
| "margin_dpo/margin_std": 22.31793975830078, |
| "step": 193 |
| }, |
| { |
| "epoch": 0.28487518355359764, |
| "fcm_dpo/beta": 0.0344829261302948, |
| "fcm_dpo/delta": 0.020925231277942657, |
| "fcm_dpo/margin": 16.809911727905273, |
| "fcm_dpo/q_t": 0.3748980760574341, |
| "grad_norm": 12.2451810836792, |
| "learning_rate": 4.510405240853854e-07, |
| "logits/chosen": -2.212796211242676, |
| "logits/rejected": -1.8392068147659302, |
| "logps/chosen": -79.75379180908203, |
| "logps/ref_chosen": -63.79157257080078, |
| "logps/ref_rejected": -68.72665405273438, |
| "logps/rejected": -101.498779296875, |
| "loss": 1.0019, |
| "margin_dpo/margin_mean": 16.809911727905273, |
| "margin_dpo/margin_std": 20.774627685546875, |
| "step": 194 |
| }, |
| { |
| "epoch": 0.28634361233480177, |
| "fcm_dpo/beta": 0.03373473137617111, |
| "fcm_dpo/delta": -0.14405208826065063, |
| "fcm_dpo/margin": 21.749523162841797, |
| "fcm_dpo/q_t": 0.3414178490638733, |
| "grad_norm": 12.763561248779297, |
| "learning_rate": 4.5027505416968985e-07, |
| "logits/chosen": -1.0410107374191284, |
| "logits/rejected": -0.9035718441009521, |
| "logps/chosen": -99.03205871582031, |
| "logps/ref_chosen": -83.88545989990234, |
| "logps/ref_rejected": -104.00125122070312, |
| "logps/rejected": -140.89736938476562, |
| "loss": 0.9105, |
| "margin_dpo/margin_mean": 21.749523162841797, |
| "margin_dpo/margin_std": 22.94123649597168, |
| "step": 195 |
| }, |
| { |
| "epoch": 0.2878120411160059, |
| "fcm_dpo/beta": 0.032888732850551605, |
| "fcm_dpo/delta": -0.09442660957574844, |
| "fcm_dpo/margin": 20.915809631347656, |
| "fcm_dpo/q_t": 0.34680312871932983, |
| "grad_norm": 11.398536682128906, |
| "learning_rate": 4.495043068200599e-07, |
| "logits/chosen": -1.0971794128417969, |
| "logits/rejected": -0.6393330693244934, |
| "logps/chosen": -106.94159698486328, |
| "logps/ref_chosen": -89.4013671875, |
| "logps/ref_rejected": -83.34735107421875, |
| "logps/rejected": -121.80339050292969, |
| "loss": 0.9011, |
| "margin_dpo/margin_mean": 20.915809631347656, |
| "margin_dpo/margin_std": 19.22466278076172, |
| "step": 196 |
| }, |
| { |
| "epoch": 0.28928046989721, |
| "fcm_dpo/beta": 0.03359599411487579, |
| "fcm_dpo/delta": 0.17620685696601868, |
| "fcm_dpo/margin": 12.843008041381836, |
| "fcm_dpo/q_t": 0.40246933698654175, |
| "grad_norm": 13.048999786376953, |
| "learning_rate": 4.4872830234640493e-07, |
| "logits/chosen": -1.3644063472747803, |
| "logits/rejected": -1.0448070764541626, |
| "logps/chosen": -105.15078735351562, |
| "logps/ref_chosen": -86.84070587158203, |
| "logps/ref_rejected": -82.79289245605469, |
| "logps/rejected": -113.94599914550781, |
| "loss": 1.0851, |
| "margin_dpo/margin_mean": 12.843008995056152, |
| "margin_dpo/margin_std": 18.110435485839844, |
| "step": 197 |
| }, |
| { |
| "epoch": 0.2907488986784141, |
| "fcm_dpo/beta": 0.03354540839791298, |
| "fcm_dpo/delta": -0.08072011172771454, |
| "fcm_dpo/margin": 20.13166046142578, |
| "fcm_dpo/q_t": 0.3533306121826172, |
| "grad_norm": 15.39021110534668, |
| "learning_rate": 4.479470611971645e-07, |
| "logits/chosen": -1.7479376792907715, |
| "logits/rejected": -1.5593535900115967, |
| "logps/chosen": -106.19425964355469, |
| "logps/ref_chosen": -85.04824829101562, |
| "logps/ref_rejected": -104.10365295410156, |
| "logps/rejected": -145.38133239746094, |
| "loss": 0.9229, |
| "margin_dpo/margin_mean": 20.13166046142578, |
| "margin_dpo/margin_std": 20.80800437927246, |
| "step": 198 |
| }, |
| { |
| "epoch": 0.2922173274596182, |
| "fcm_dpo/beta": 0.033529043197631836, |
| "fcm_dpo/delta": 0.020058371126651764, |
| "fcm_dpo/margin": 17.328767776489258, |
| "fcm_dpo/q_t": 0.373092383146286, |
| "grad_norm": 13.264050483703613, |
| "learning_rate": 4.471606039587695e-07, |
| "logits/chosen": -1.4798874855041504, |
| "logits/rejected": -1.2172993421554565, |
| "logps/chosen": -102.33380889892578, |
| "logps/ref_chosen": -81.78669738769531, |
| "logps/ref_rejected": -90.790771484375, |
| "logps/rejected": -128.66665649414062, |
| "loss": 0.992, |
| "margin_dpo/margin_mean": 17.328765869140625, |
| "margin_dpo/margin_std": 20.446475982666016, |
| "step": 199 |
| }, |
| { |
| "epoch": 0.2936857562408223, |
| "fcm_dpo/beta": 0.03291964530944824, |
| "fcm_dpo/delta": -0.0771205723285675, |
| "fcm_dpo/margin": 20.37543296813965, |
| "fcm_dpo/q_t": 0.35671910643577576, |
| "grad_norm": 12.404348373413086, |
| "learning_rate": 4.4636895135509966e-07, |
| "logits/chosen": -1.604175090789795, |
| "logits/rejected": -1.086242437362671, |
| "logps/chosen": -104.21473693847656, |
| "logps/ref_chosen": -84.26809692382812, |
| "logps/ref_rejected": -86.96786499023438, |
| "logps/rejected": -127.28994750976562, |
| "loss": 0.9603, |
| "margin_dpo/margin_mean": 20.37543296813965, |
| "margin_dpo/margin_std": 23.788284301757812, |
| "step": 200 |
| }, |
| { |
| "epoch": 0.2936857562408223, |
| "eval_fcm_dpo/beta": 0.0329931266605854, |
| "eval_logits/chosen": -1.439759373664856, |
| "eval_logits/rejected": -1.1262362003326416, |
| "eval_logps/chosen": -123.58830261230469, |
| "eval_logps/ref_chosen": -100.49356842041016, |
| "eval_logps/ref_rejected": -94.06775665283203, |
| "eval_logps/rejected": -132.22947692871094, |
| "eval_loss": 0.534487247467041, |
| "eval_margin_dpo/margin_mean": 15.06699275970459, |
| "eval_margin_dpo/margin_std": 21.761512756347656, |
| "eval_runtime": 43.741, |
| "eval_samples_per_second": 53.474, |
| "eval_steps_per_second": 1.692, |
| "step": 200 |
| }, |
| { |
| "epoch": 0.29515418502202645, |
| "fcm_dpo/beta": 0.03285796195268631, |
| "fcm_dpo/delta": -0.0432654544711113, |
| "fcm_dpo/margin": 19.492534637451172, |
| "fcm_dpo/q_t": 0.36308830976486206, |
| "grad_norm": 13.604676246643066, |
| "learning_rate": 4.455721242469372e-07, |
| "logits/chosen": -1.3040488958358765, |
| "logits/rejected": -0.9086974263191223, |
| "logps/chosen": -129.45172119140625, |
| "logps/ref_chosen": -107.77249145507812, |
| "logps/ref_rejected": -119.79248046875, |
| "logps/rejected": -160.9642333984375, |
| "loss": 0.9856, |
| "margin_dpo/margin_mean": 19.492534637451172, |
| "margin_dpo/margin_std": 24.657806396484375, |
| "step": 201 |
| }, |
| { |
| "epoch": 0.2966226138032305, |
| "fcm_dpo/beta": 0.03282172232866287, |
| "fcm_dpo/delta": 0.012272818945348263, |
| "fcm_dpo/margin": 17.926525115966797, |
| "fcm_dpo/q_t": 0.3727894723415375, |
| "grad_norm": 13.259603500366211, |
| "learning_rate": 4.4477014363141755e-07, |
| "logits/chosen": -1.6802592277526855, |
| "logits/rejected": -1.547353744506836, |
| "logps/chosen": -96.56066131591797, |
| "logps/ref_chosen": -75.97245788574219, |
| "logps/ref_rejected": -94.4599838256836, |
| "logps/rejected": -132.9747314453125, |
| "loss": 0.9903, |
| "margin_dpo/margin_mean": 17.926525115966797, |
| "margin_dpo/margin_std": 21.052555084228516, |
| "step": 202 |
| }, |
| { |
| "epoch": 0.29809104258443464, |
| "fcm_dpo/beta": 0.032457903027534485, |
| "fcm_dpo/delta": -0.07600473612546921, |
| "fcm_dpo/margin": 20.671689987182617, |
| "fcm_dpo/q_t": 0.3520776033401489, |
| "grad_norm": 11.885881423950195, |
| "learning_rate": 4.439630306414758e-07, |
| "logits/chosen": -1.69942307472229, |
| "logits/rejected": -1.2441532611846924, |
| "logps/chosen": -112.8557357788086, |
| "logps/ref_chosen": -94.96715545654297, |
| "logps/ref_rejected": -92.8876724243164, |
| "logps/rejected": -131.44793701171875, |
| "loss": 0.9224, |
| "margin_dpo/margin_mean": 20.671688079833984, |
| "margin_dpo/margin_std": 20.679279327392578, |
| "step": 203 |
| }, |
| { |
| "epoch": 0.29955947136563876, |
| "fcm_dpo/beta": 0.032535798847675323, |
| "fcm_dpo/delta": 0.02650514431297779, |
| "fcm_dpo/margin": 17.66338348388672, |
| "fcm_dpo/q_t": 0.3760603666305542, |
| "grad_norm": 14.426183700561523, |
| "learning_rate": 4.431508065452897e-07, |
| "logits/chosen": -1.7683607339859009, |
| "logits/rejected": -1.129054069519043, |
| "logps/chosen": -139.35574340820312, |
| "logps/ref_chosen": -116.35719299316406, |
| "logps/ref_rejected": -93.39759063720703, |
| "logps/rejected": -134.05953979492188, |
| "loss": 1.0127, |
| "margin_dpo/margin_mean": 17.66338348388672, |
| "margin_dpo/margin_std": 22.486337661743164, |
| "step": 204 |
| }, |
| { |
| "epoch": 0.3010279001468429, |
| "fcm_dpo/beta": 0.031746186316013336, |
| "fcm_dpo/delta": -0.1341342329978943, |
| "fcm_dpo/margin": 22.80514907836914, |
| "fcm_dpo/q_t": 0.3369899392127991, |
| "grad_norm": 11.1696195602417, |
| "learning_rate": 4.4233349274571974e-07, |
| "logits/chosen": -2.104504108428955, |
| "logits/rejected": -1.677614688873291, |
| "logps/chosen": -109.97764587402344, |
| "logps/ref_chosen": -88.85934448242188, |
| "logps/ref_rejected": -91.8544921875, |
| "logps/rejected": -135.77793884277344, |
| "loss": 0.8669, |
| "margin_dpo/margin_mean": 22.80514907836914, |
| "margin_dpo/margin_std": 18.854307174682617, |
| "step": 205 |
| }, |
| { |
| "epoch": 0.302496328928047, |
| "fcm_dpo/beta": 0.031177151948213577, |
| "fcm_dpo/delta": -0.0601668581366539, |
| "fcm_dpo/margin": 21.0255069732666, |
| "fcm_dpo/q_t": 0.3517179489135742, |
| "grad_norm": 14.36100959777832, |
| "learning_rate": 4.415111107797445e-07, |
| "logits/chosen": -1.566483497619629, |
| "logits/rejected": -1.3959765434265137, |
| "logps/chosen": -98.07730102539062, |
| "logps/ref_chosen": -76.54634857177734, |
| "logps/ref_rejected": -102.95314025878906, |
| "logps/rejected": -145.50961303710938, |
| "loss": 0.9067, |
| "margin_dpo/margin_mean": 21.0255069732666, |
| "margin_dpo/margin_std": 18.027606964111328, |
| "step": 206 |
| }, |
| { |
| "epoch": 0.3039647577092511, |
| "fcm_dpo/beta": 0.030731763690710068, |
| "fcm_dpo/delta": -0.11052694916725159, |
| "fcm_dpo/margin": 22.86851692199707, |
| "fcm_dpo/q_t": 0.35563924908638, |
| "grad_norm": 14.994292259216309, |
| "learning_rate": 4.4068368231789365e-07, |
| "logits/chosen": -1.886859655380249, |
| "logits/rejected": -1.45271635055542, |
| "logps/chosen": -109.18792724609375, |
| "logps/ref_chosen": -86.23164367675781, |
| "logps/ref_rejected": -90.65512084960938, |
| "logps/rejected": -136.47991943359375, |
| "loss": 0.9606, |
| "margin_dpo/margin_mean": 22.86851692199707, |
| "margin_dpo/margin_std": 28.827228546142578, |
| "step": 207 |
| }, |
| { |
| "epoch": 0.3054331864904552, |
| "fcm_dpo/beta": 0.03015347383916378, |
| "fcm_dpo/delta": -0.12842844426631927, |
| "fcm_dpo/margin": 23.85171890258789, |
| "fcm_dpo/q_t": 0.3396170735359192, |
| "grad_norm": 13.180238723754883, |
| "learning_rate": 4.398512291636768e-07, |
| "logits/chosen": -1.829242467880249, |
| "logits/rejected": -1.4700978994369507, |
| "logps/chosen": -117.53231811523438, |
| "logps/ref_chosen": -94.1595458984375, |
| "logps/ref_rejected": -100.96233367919922, |
| "logps/rejected": -148.18682861328125, |
| "loss": 0.8877, |
| "margin_dpo/margin_mean": 23.85171890258789, |
| "margin_dpo/margin_std": 21.651004791259766, |
| "step": 208 |
| }, |
| { |
| "epoch": 0.3069016152716593, |
| "fcm_dpo/beta": 0.029522810131311417, |
| "fcm_dpo/delta": -0.018623225390911102, |
| "fcm_dpo/margin": 20.891002655029297, |
| "fcm_dpo/q_t": 0.36556458473205566, |
| "grad_norm": 11.94274616241455, |
| "learning_rate": 4.3901377325300857e-07, |
| "logits/chosen": -1.8460993766784668, |
| "logits/rejected": -1.3643391132354736, |
| "logps/chosen": -104.04109954833984, |
| "logps/ref_chosen": -84.17056274414062, |
| "logps/ref_rejected": -87.61955261230469, |
| "logps/rejected": -128.38108825683594, |
| "loss": 0.9693, |
| "margin_dpo/margin_mean": 20.891002655029297, |
| "margin_dpo/margin_std": 23.366682052612305, |
| "step": 209 |
| }, |
| { |
| "epoch": 0.30837004405286345, |
| "fcm_dpo/beta": 0.029583394527435303, |
| "fcm_dpo/delta": 0.005795907229185104, |
| "fcm_dpo/margin": 20.090076446533203, |
| "fcm_dpo/q_t": 0.37067821621894836, |
| "grad_norm": 11.102954864501953, |
| "learning_rate": 4.381713366536311e-07, |
| "logits/chosen": -1.9762842655181885, |
| "logits/rejected": -1.5929546356201172, |
| "logps/chosen": -103.7457504272461, |
| "logps/ref_chosen": -81.17117309570312, |
| "logps/ref_rejected": -84.17478942871094, |
| "logps/rejected": -126.83943939208984, |
| "loss": 0.9765, |
| "margin_dpo/margin_mean": 20.090076446533203, |
| "margin_dpo/margin_std": 22.627273559570312, |
| "step": 210 |
| }, |
| { |
| "epoch": 0.30983847283406757, |
| "fcm_dpo/beta": 0.029787559062242508, |
| "fcm_dpo/delta": 0.0069667138159275055, |
| "fcm_dpo/margin": 19.919384002685547, |
| "fcm_dpo/q_t": 0.37133318185806274, |
| "grad_norm": 13.3599214553833, |
| "learning_rate": 4.373239415645323e-07, |
| "logits/chosen": -1.7666494846343994, |
| "logits/rejected": -1.1323686838150024, |
| "logps/chosen": -133.63140869140625, |
| "logps/ref_chosen": -108.71270751953125, |
| "logps/ref_rejected": -93.55564880371094, |
| "logps/rejected": -138.39373779296875, |
| "loss": 0.9858, |
| "margin_dpo/margin_mean": 19.919384002685547, |
| "margin_dpo/margin_std": 23.80620574951172, |
| "step": 211 |
| }, |
| { |
| "epoch": 0.31130690161527164, |
| "fcm_dpo/beta": 0.028646890074014664, |
| "fcm_dpo/delta": -0.22884619235992432, |
| "fcm_dpo/margin": 28.252504348754883, |
| "fcm_dpo/q_t": 0.32535678148269653, |
| "grad_norm": 11.055304527282715, |
| "learning_rate": 4.3647161031536086e-07, |
| "logits/chosen": -1.8360583782196045, |
| "logits/rejected": -1.422875165939331, |
| "logps/chosen": -114.99956512451172, |
| "logps/ref_chosen": -98.36195373535156, |
| "logps/ref_rejected": -109.88999938964844, |
| "logps/rejected": -154.78012084960938, |
| "loss": 0.8428, |
| "margin_dpo/margin_mean": 28.252504348754883, |
| "margin_dpo/margin_std": 25.65581703186035, |
| "step": 212 |
| }, |
| { |
| "epoch": 0.31277533039647576, |
| "fcm_dpo/beta": 0.028224751353263855, |
| "fcm_dpo/delta": -0.024367500096559525, |
| "fcm_dpo/margin": 22.059900283813477, |
| "fcm_dpo/q_t": 0.3628522753715515, |
| "grad_norm": 14.968526840209961, |
| "learning_rate": 4.3561436536583774e-07, |
| "logits/chosen": -1.8746771812438965, |
| "logits/rejected": -1.1175596714019775, |
| "logps/chosen": -128.48854064941406, |
| "logps/ref_chosen": -108.05531311035156, |
| "logps/ref_rejected": -100.14414978027344, |
| "logps/rejected": -142.63729858398438, |
| "loss": 0.9689, |
| "margin_dpo/margin_mean": 22.059898376464844, |
| "margin_dpo/margin_std": 25.182662963867188, |
| "step": 213 |
| }, |
| { |
| "epoch": 0.3142437591776799, |
| "fcm_dpo/beta": 0.02846229076385498, |
| "fcm_dpo/delta": 0.03237991780042648, |
| "fcm_dpo/margin": 20.002635955810547, |
| "fcm_dpo/q_t": 0.38030585646629333, |
| "grad_norm": 11.99617862701416, |
| "learning_rate": 4.3475222930516473e-07, |
| "logits/chosen": -1.8247478008270264, |
| "logits/rejected": -1.70417320728302, |
| "logps/chosen": -99.7642593383789, |
| "logps/ref_chosen": -77.80473327636719, |
| "logps/ref_rejected": -84.27578735351562, |
| "logps/rejected": -126.2379379272461, |
| "loss": 1.0348, |
| "margin_dpo/margin_mean": 20.002635955810547, |
| "margin_dpo/margin_std": 28.244205474853516, |
| "step": 214 |
| }, |
| { |
| "epoch": 0.315712187958884, |
| "fcm_dpo/beta": 0.027820151299238205, |
| "fcm_dpo/delta": -0.1480301022529602, |
| "fcm_dpo/margin": 26.492956161499023, |
| "fcm_dpo/q_t": 0.3393728733062744, |
| "grad_norm": 11.24252986907959, |
| "learning_rate": 4.3388522485142885e-07, |
| "logits/chosen": -1.7213172912597656, |
| "logits/rejected": -1.2412421703338623, |
| "logps/chosen": -103.63861083984375, |
| "logps/ref_chosen": -85.1138916015625, |
| "logps/ref_rejected": -96.86151123046875, |
| "logps/rejected": -141.87918090820312, |
| "loss": 0.8754, |
| "margin_dpo/margin_mean": 26.492958068847656, |
| "margin_dpo/margin_std": 24.533878326416016, |
| "step": 215 |
| }, |
| { |
| "epoch": 0.31718061674008813, |
| "fcm_dpo/beta": 0.027603790163993835, |
| "fcm_dpo/delta": -0.03746962174773216, |
| "fcm_dpo/margin": 22.996862411499023, |
| "fcm_dpo/q_t": 0.3604411780834198, |
| "grad_norm": 10.716496467590332, |
| "learning_rate": 4.330133748510036e-07, |
| "logits/chosen": -1.9421966075897217, |
| "logits/rejected": -1.5539422035217285, |
| "logps/chosen": -101.21246337890625, |
| "logps/ref_chosen": -80.59230041503906, |
| "logps/ref_rejected": -81.41983795166016, |
| "logps/rejected": -125.036865234375, |
| "loss": 0.9377, |
| "margin_dpo/margin_mean": 22.996862411499023, |
| "margin_dpo/margin_std": 23.105133056640625, |
| "step": 216 |
| }, |
| { |
| "epoch": 0.3186490455212922, |
| "fcm_dpo/beta": 0.027701357379555702, |
| "fcm_dpo/delta": 0.04847228527069092, |
| "fcm_dpo/margin": 19.994403839111328, |
| "fcm_dpo/q_t": 0.38077569007873535, |
| "grad_norm": 13.545554161071777, |
| "learning_rate": 4.3213670227794757e-07, |
| "logits/chosen": -1.772913932800293, |
| "logits/rejected": -1.4578328132629395, |
| "logps/chosen": -117.97203063964844, |
| "logps/ref_chosen": -93.47257995605469, |
| "logps/ref_rejected": -103.488525390625, |
| "logps/rejected": -147.9823760986328, |
| "loss": 1.0186, |
| "margin_dpo/margin_mean": 19.994403839111328, |
| "margin_dpo/margin_std": 25.860273361206055, |
| "step": 217 |
| }, |
| { |
| "epoch": 0.3201174743024963, |
| "fcm_dpo/beta": 0.02740955352783203, |
| "fcm_dpo/delta": -0.046444494277238846, |
| "fcm_dpo/margin": 23.46440887451172, |
| "fcm_dpo/q_t": 0.36053454875946045, |
| "grad_norm": 11.716761589050293, |
| "learning_rate": 4.3125523023339815e-07, |
| "logits/chosen": -1.467943787574768, |
| "logits/rejected": -1.0809111595153809, |
| "logps/chosen": -112.5135498046875, |
| "logps/ref_chosen": -89.05883026123047, |
| "logps/ref_rejected": -94.30680847167969, |
| "logps/rejected": -141.22592163085938, |
| "loss": 0.9696, |
| "margin_dpo/margin_mean": 23.46440887451172, |
| "margin_dpo/margin_std": 27.73025894165039, |
| "step": 218 |
| }, |
| { |
| "epoch": 0.32158590308370044, |
| "fcm_dpo/beta": 0.0276676956564188, |
| "fcm_dpo/delta": 0.009598739445209503, |
| "fcm_dpo/margin": 21.319698333740234, |
| "fcm_dpo/q_t": 0.37368935346603394, |
| "grad_norm": 14.510382652282715, |
| "learning_rate": 4.303689819449636e-07, |
| "logits/chosen": -1.6932134628295898, |
| "logits/rejected": -1.259705662727356, |
| "logps/chosen": -122.24336242675781, |
| "logps/ref_chosen": -101.00733947753906, |
| "logps/ref_rejected": -92.46794128417969, |
| "logps/rejected": -135.02365112304688, |
| "loss": 1.0173, |
| "margin_dpo/margin_mean": 21.319698333740234, |
| "margin_dpo/margin_std": 28.504629135131836, |
| "step": 219 |
| }, |
| { |
| "epoch": 0.32305433186490456, |
| "fcm_dpo/beta": 0.027722153812646866, |
| "fcm_dpo/delta": 0.056987300515174866, |
| "fcm_dpo/margin": 19.699235916137695, |
| "fcm_dpo/q_t": 0.3782733082771301, |
| "grad_norm": 12.883475303649902, |
| "learning_rate": 4.2947798076611047e-07, |
| "logits/chosen": -1.2188053131103516, |
| "logits/rejected": -0.8566290140151978, |
| "logps/chosen": -117.09075164794922, |
| "logps/ref_chosen": -95.53721618652344, |
| "logps/ref_rejected": -94.30703735351562, |
| "logps/rejected": -135.559814453125, |
| "loss": 0.9964, |
| "margin_dpo/margin_mean": 19.699234008789062, |
| "margin_dpo/margin_std": 21.820940017700195, |
| "step": 220 |
| }, |
| { |
| "epoch": 0.3245227606461087, |
| "fcm_dpo/beta": 0.02716447226703167, |
| "fcm_dpo/delta": -0.2148474156856537, |
| "fcm_dpo/margin": 29.37576675415039, |
| "fcm_dpo/q_t": 0.3273373246192932, |
| "grad_norm": 12.420080184936523, |
| "learning_rate": 4.285822501755485e-07, |
| "logits/chosen": -1.346145749092102, |
| "logits/rejected": -1.196437120437622, |
| "logps/chosen": -103.59893035888672, |
| "logps/ref_chosen": -82.84486389160156, |
| "logps/ref_rejected": -110.81179809570312, |
| "logps/rejected": -160.94163513183594, |
| "loss": 0.8462, |
| "margin_dpo/margin_mean": 29.375770568847656, |
| "margin_dpo/margin_std": 25.051647186279297, |
| "step": 221 |
| }, |
| { |
| "epoch": 0.32599118942731276, |
| "fcm_dpo/beta": 0.026609212160110474, |
| "fcm_dpo/delta": -0.045089878141880035, |
| "fcm_dpo/margin": 24.11734962463379, |
| "fcm_dpo/q_t": 0.36285531520843506, |
| "grad_norm": 14.774637222290039, |
| "learning_rate": 4.276818137766118e-07, |
| "logits/chosen": -1.9378188848495483, |
| "logits/rejected": -1.510035514831543, |
| "logps/chosen": -121.65461730957031, |
| "logps/ref_chosen": -95.14198303222656, |
| "logps/ref_rejected": -106.80441284179688, |
| "logps/rejected": -157.43441772460938, |
| "loss": 0.9999, |
| "margin_dpo/margin_mean": 24.117347717285156, |
| "margin_dpo/margin_std": 31.577239990234375, |
| "step": 222 |
| }, |
| { |
| "epoch": 0.3274596182085169, |
| "fcm_dpo/beta": 0.026801597326993942, |
| "fcm_dpo/delta": 0.11458225548267365, |
| "fcm_dpo/margin": 18.32587432861328, |
| "fcm_dpo/q_t": 0.3927801251411438, |
| "grad_norm": 13.29928970336914, |
| "learning_rate": 4.2677669529663686e-07, |
| "logits/chosen": -2.038062572479248, |
| "logits/rejected": -1.6366686820983887, |
| "logps/chosen": -110.8859634399414, |
| "logps/ref_chosen": -85.57511138916016, |
| "logps/ref_rejected": -86.45237731933594, |
| "logps/rejected": -130.089111328125, |
| "loss": 1.0622, |
| "margin_dpo/margin_mean": 18.32587432861328, |
| "margin_dpo/margin_std": 25.809633255004883, |
| "step": 223 |
| }, |
| { |
| "epoch": 0.328928046989721, |
| "fcm_dpo/beta": 0.026815548539161682, |
| "fcm_dpo/delta": -0.049758054316043854, |
| "fcm_dpo/margin": 24.111448287963867, |
| "fcm_dpo/q_t": 0.35970795154571533, |
| "grad_norm": 12.745670318603516, |
| "learning_rate": 4.2586691858633747e-07, |
| "logits/chosen": -1.7115750312805176, |
| "logits/rejected": -1.2611249685287476, |
| "logps/chosen": -106.57003784179688, |
| "logps/ref_chosen": -82.72380065917969, |
| "logps/ref_rejected": -82.59538269042969, |
| "logps/rejected": -130.55307006835938, |
| "loss": 0.9542, |
| "margin_dpo/margin_mean": 24.111448287963867, |
| "margin_dpo/margin_std": 27.108388900756836, |
| "step": 224 |
| }, |
| { |
| "epoch": 0.3303964757709251, |
| "fcm_dpo/beta": 0.026444047689437866, |
| "fcm_dpo/delta": -0.10222186148166656, |
| "fcm_dpo/margin": 26.288196563720703, |
| "fcm_dpo/q_t": 0.35237205028533936, |
| "grad_norm": 13.087396621704102, |
| "learning_rate": 4.249525076191759e-07, |
| "logits/chosen": -1.6562142372131348, |
| "logits/rejected": -1.1619675159454346, |
| "logps/chosen": -122.09429931640625, |
| "logps/ref_chosen": -95.67768859863281, |
| "logps/ref_rejected": -105.09687805175781, |
| "logps/rejected": -157.80169677734375, |
| "loss": 0.9433, |
| "margin_dpo/margin_mean": 26.288192749023438, |
| "margin_dpo/margin_std": 30.676586151123047, |
| "step": 225 |
| }, |
| { |
| "epoch": 0.33186490455212925, |
| "fcm_dpo/beta": 0.02590518817305565, |
| "fcm_dpo/delta": -0.03390205651521683, |
| "fcm_dpo/margin": 24.35424041748047, |
| "fcm_dpo/q_t": 0.3604390621185303, |
| "grad_norm": 10.917281150817871, |
| "learning_rate": 4.2403348649073167e-07, |
| "logits/chosen": -1.681097388267517, |
| "logits/rejected": -1.1455821990966797, |
| "logps/chosen": -115.38078308105469, |
| "logps/ref_chosen": -93.46092224121094, |
| "logps/ref_rejected": -86.7017593383789, |
| "logps/rejected": -132.97586059570312, |
| "loss": 0.9398, |
| "margin_dpo/margin_mean": 24.35424041748047, |
| "margin_dpo/margin_std": 24.179988861083984, |
| "step": 226 |
| }, |
| { |
| "epoch": 0.3333333333333333, |
| "fcm_dpo/beta": 0.02585548348724842, |
| "fcm_dpo/delta": -0.045401234179735184, |
| "fcm_dpo/margin": 24.849811553955078, |
| "fcm_dpo/q_t": 0.362407386302948, |
| "grad_norm": 12.172209739685059, |
| "learning_rate": 4.2310987941806615e-07, |
| "logits/chosen": -1.2314541339874268, |
| "logits/rejected": -0.8789317607879639, |
| "logps/chosen": -123.27052307128906, |
| "logps/ref_chosen": -92.81427001953125, |
| "logps/ref_rejected": -104.73693084716797, |
| "logps/rejected": -160.04299926757812, |
| "loss": 0.9737, |
| "margin_dpo/margin_mean": 24.849811553955078, |
| "margin_dpo/margin_std": 29.725021362304688, |
| "step": 227 |
| }, |
| { |
| "epoch": 0.33480176211453744, |
| "fcm_dpo/beta": 0.026136575266718864, |
| "fcm_dpo/delta": 0.09344188123941422, |
| "fcm_dpo/margin": 19.564260482788086, |
| "fcm_dpo/q_t": 0.3862323760986328, |
| "grad_norm": 12.771748542785645, |
| "learning_rate": 4.2218171073908463e-07, |
| "logits/chosen": -1.6004002094268799, |
| "logits/rejected": -1.1517033576965332, |
| "logps/chosen": -122.49995422363281, |
| "logps/ref_chosen": -94.03712463378906, |
| "logps/ref_rejected": -96.02151489257812, |
| "logps/rejected": -144.04861450195312, |
| "loss": 1.0429, |
| "margin_dpo/margin_mean": 19.56426239013672, |
| "margin_dpo/margin_std": 25.722578048706055, |
| "step": 228 |
| }, |
| { |
| "epoch": 0.33627019089574156, |
| "fcm_dpo/beta": 0.025405660271644592, |
| "fcm_dpo/delta": -0.2284693717956543, |
| "fcm_dpo/margin": 31.885055541992188, |
| "fcm_dpo/q_t": 0.3289991319179535, |
| "grad_norm": 11.33955192565918, |
| "learning_rate": 4.212490049118951e-07, |
| "logits/chosen": -1.9561712741851807, |
| "logits/rejected": -1.348691701889038, |
| "logps/chosen": -120.61212921142578, |
| "logps/ref_chosen": -95.57766723632812, |
| "logps/ref_rejected": -89.17379760742188, |
| "logps/rejected": -146.0933074951172, |
| "loss": 0.8603, |
| "margin_dpo/margin_mean": 31.885055541992188, |
| "margin_dpo/margin_std": 31.453580856323242, |
| "step": 229 |
| }, |
| { |
| "epoch": 0.3377386196769457, |
| "fcm_dpo/beta": 0.02440287545323372, |
| "fcm_dpo/delta": -0.1861468255519867, |
| "fcm_dpo/margin": 31.624526977539062, |
| "fcm_dpo/q_t": 0.3298434615135193, |
| "grad_norm": 12.601603507995605, |
| "learning_rate": 4.203117865141635e-07, |
| "logits/chosen": -1.638891339302063, |
| "logits/rejected": -1.555408239364624, |
| "logps/chosen": -90.70736694335938, |
| "logps/ref_chosen": -63.713626861572266, |
| "logps/ref_rejected": -91.90870666503906, |
| "logps/rejected": -150.5269775390625, |
| "loss": 0.8553, |
| "margin_dpo/margin_mean": 31.624528884887695, |
| "margin_dpo/margin_std": 27.797245025634766, |
| "step": 230 |
| }, |
| { |
| "epoch": 0.3392070484581498, |
| "fcm_dpo/beta": 0.023886041715741158, |
| "fcm_dpo/delta": -0.04298706725239754, |
| "fcm_dpo/margin": 26.76896858215332, |
| "fcm_dpo/q_t": 0.3564389944076538, |
| "grad_norm": 10.671087265014648, |
| "learning_rate": 4.1937008024246625e-07, |
| "logits/chosen": -1.6697638034820557, |
| "logits/rejected": -1.1297423839569092, |
| "logps/chosen": -125.79000091552734, |
| "logps/ref_chosen": -95.45567321777344, |
| "logps/ref_rejected": -80.95568084716797, |
| "logps/rejected": -138.05897521972656, |
| "loss": 0.9207, |
| "margin_dpo/margin_mean": 26.76896858215332, |
| "margin_dpo/margin_std": 24.022785186767578, |
| "step": 231 |
| }, |
| { |
| "epoch": 0.3406754772393539, |
| "fcm_dpo/beta": 0.02407444454729557, |
| "fcm_dpo/delta": 0.04270622134208679, |
| "fcm_dpo/margin": 23.248355865478516, |
| "fcm_dpo/q_t": 0.37612390518188477, |
| "grad_norm": 11.53981876373291, |
| "learning_rate": 4.1842391091163933e-07, |
| "logits/chosen": -1.4995675086975098, |
| "logits/rejected": -1.0280941724777222, |
| "logps/chosen": -128.28302001953125, |
| "logps/ref_chosen": -96.89726257324219, |
| "logps/ref_rejected": -89.76461791992188, |
| "logps/rejected": -144.39874267578125, |
| "loss": 0.9973, |
| "margin_dpo/margin_mean": 23.248355865478516, |
| "margin_dpo/margin_std": 26.718120574951172, |
| "step": 232 |
| }, |
| { |
| "epoch": 0.342143906020558, |
| "fcm_dpo/beta": 0.02357470616698265, |
| "fcm_dpo/delta": -0.1532088667154312, |
| "fcm_dpo/margin": 31.44921112060547, |
| "fcm_dpo/q_t": 0.3387308120727539, |
| "grad_norm": 11.674040794372559, |
| "learning_rate": 4.174733034541245e-07, |
| "logits/chosen": -1.721343994140625, |
| "logits/rejected": -1.4938092231750488, |
| "logps/chosen": -122.76866149902344, |
| "logps/ref_chosen": -89.05032348632812, |
| "logps/ref_rejected": -112.75917053222656, |
| "logps/rejected": -177.92672729492188, |
| "loss": 0.8881, |
| "margin_dpo/margin_mean": 31.44921112060547, |
| "margin_dpo/margin_std": 30.564104080200195, |
| "step": 233 |
| }, |
| { |
| "epoch": 0.3436123348017621, |
| "fcm_dpo/beta": 0.022600866854190826, |
| "fcm_dpo/delta": -0.17494013905525208, |
| "fcm_dpo/margin": 33.52787780761719, |
| "fcm_dpo/q_t": 0.3380278944969177, |
| "grad_norm": 11.083850860595703, |
| "learning_rate": 4.165182829193126e-07, |
| "logits/chosen": -1.2067891359329224, |
| "logits/rejected": -1.1254383325576782, |
| "logps/chosen": -106.61786651611328, |
| "logps/ref_chosen": -74.318359375, |
| "logps/ref_rejected": -106.38758850097656, |
| "logps/rejected": -172.2149658203125, |
| "loss": 0.8916, |
| "margin_dpo/margin_mean": 33.52787780761719, |
| "margin_dpo/margin_std": 33.51675796508789, |
| "step": 234 |
| }, |
| { |
| "epoch": 0.34508076358296624, |
| "fcm_dpo/beta": 0.022734005004167557, |
| "fcm_dpo/delta": 0.01751571148633957, |
| "fcm_dpo/margin": 25.663055419921875, |
| "fcm_dpo/q_t": 0.36923179030418396, |
| "grad_norm": 10.767046928405762, |
| "learning_rate": 4.1555887447288255e-07, |
| "logits/chosen": -1.3730831146240234, |
| "logits/rejected": -0.8100583553314209, |
| "logps/chosen": -136.84141540527344, |
| "logps/ref_chosen": -98.217041015625, |
| "logps/ref_rejected": -97.24677276611328, |
| "logps/rejected": -161.53419494628906, |
| "loss": 0.9624, |
| "margin_dpo/margin_mean": 25.663055419921875, |
| "margin_dpo/margin_std": 25.689247131347656, |
| "step": 235 |
| }, |
| { |
| "epoch": 0.3465491923641703, |
| "fcm_dpo/beta": 0.022359684109687805, |
| "fcm_dpo/delta": -0.12736767530441284, |
| "fcm_dpo/margin": 32.12920379638672, |
| "fcm_dpo/q_t": 0.35286009311676025, |
| "grad_norm": 11.318601608276367, |
| "learning_rate": 4.1459510339613946e-07, |
| "logits/chosen": -1.2687420845031738, |
| "logits/rejected": -1.2513408660888672, |
| "logps/chosen": -111.73262023925781, |
| "logps/ref_chosen": -78.83773040771484, |
| "logps/ref_rejected": -109.06343078613281, |
| "logps/rejected": -174.0875244140625, |
| "loss": 0.924, |
| "margin_dpo/margin_mean": 32.12920379638672, |
| "margin_dpo/margin_std": 37.443138122558594, |
| "step": 236 |
| }, |
| { |
| "epoch": 0.34801762114537443, |
| "fcm_dpo/beta": 0.02228570729494095, |
| "fcm_dpo/delta": 0.0468742772936821, |
| "fcm_dpo/margin": 24.937557220458984, |
| "fcm_dpo/q_t": 0.3862718939781189, |
| "grad_norm": 11.445574760437012, |
| "learning_rate": 4.136269950853473e-07, |
| "logits/chosen": -1.4098989963531494, |
| "logits/rejected": -1.0293853282928467, |
| "logps/chosen": -123.49139404296875, |
| "logps/ref_chosen": -85.21128845214844, |
| "logps/ref_rejected": -99.90999603271484, |
| "logps/rejected": -163.12767028808594, |
| "loss": 1.065, |
| "margin_dpo/margin_mean": 24.937557220458984, |
| "margin_dpo/margin_std": 39.00885009765625, |
| "step": 237 |
| }, |
| { |
| "epoch": 0.34948604992657856, |
| "fcm_dpo/beta": 0.022245164960622787, |
| "fcm_dpo/delta": -0.023596640676259995, |
| "fcm_dpo/margin": 27.963947296142578, |
| "fcm_dpo/q_t": 0.36633193492889404, |
| "grad_norm": 10.700308799743652, |
| "learning_rate": 4.126545750510605e-07, |
| "logits/chosen": -1.0491883754730225, |
| "logits/rejected": -0.8851519227027893, |
| "logps/chosen": -120.75160217285156, |
| "logps/ref_chosen": -78.73123168945312, |
| "logps/ref_rejected": -95.41840362548828, |
| "logps/rejected": -165.4027099609375, |
| "loss": 0.9704, |
| "margin_dpo/margin_mean": 27.963947296142578, |
| "margin_dpo/margin_std": 32.5536003112793, |
| "step": 238 |
| }, |
| { |
| "epoch": 0.3509544787077827, |
| "fcm_dpo/beta": 0.022143233567476273, |
| "fcm_dpo/delta": 0.018351107835769653, |
| "fcm_dpo/margin": 26.272315979003906, |
| "fcm_dpo/q_t": 0.37351781129837036, |
| "grad_norm": 11.168644905090332, |
| "learning_rate": 4.116778689174514e-07, |
| "logits/chosen": -1.7788722515106201, |
| "logits/rejected": -1.3021533489227295, |
| "logps/chosen": -131.14578247070312, |
| "logps/ref_chosen": -92.60093688964844, |
| "logps/ref_rejected": -100.51769256591797, |
| "logps/rejected": -165.3348388671875, |
| "loss": 0.9967, |
| "margin_dpo/margin_mean": 26.272315979003906, |
| "margin_dpo/margin_std": 30.892166137695312, |
| "step": 239 |
| }, |
| { |
| "epoch": 0.3524229074889868, |
| "fcm_dpo/beta": 0.022500187158584595, |
| "fcm_dpo/delta": 0.07234937697649002, |
| "fcm_dpo/margin": 23.62274932861328, |
| "fcm_dpo/q_t": 0.3824573755264282, |
| "grad_norm": 12.867546081542969, |
| "learning_rate": 4.106969024216348e-07, |
| "logits/chosen": -1.9507150650024414, |
| "logits/rejected": -1.6044831275939941, |
| "logps/chosen": -125.81748962402344, |
| "logps/ref_chosen": -86.15977478027344, |
| "logps/ref_rejected": -80.45567321777344, |
| "logps/rejected": -143.7361297607422, |
| "loss": 1.0213, |
| "margin_dpo/margin_mean": 23.62274932861328, |
| "margin_dpo/margin_std": 28.90222930908203, |
| "step": 240 |
| }, |
| { |
| "epoch": 0.35389133627019087, |
| "fcm_dpo/beta": 0.0217452310025692, |
| "fcm_dpo/delta": -0.3266553282737732, |
| "fcm_dpo/margin": 41.317291259765625, |
| "fcm_dpo/q_t": 0.30971965193748474, |
| "grad_norm": 10.297433853149414, |
| "learning_rate": 4.097117014129903e-07, |
| "logits/chosen": -1.5102795362472534, |
| "logits/rejected": -0.8343143463134766, |
| "logps/chosen": -136.01959228515625, |
| "logps/ref_chosen": -101.04594421386719, |
| "logps/ref_rejected": -94.04934692382812, |
| "logps/rejected": -170.34030151367188, |
| "loss": 0.8042, |
| "margin_dpo/margin_mean": 41.317291259765625, |
| "margin_dpo/margin_std": 36.07255554199219, |
| "step": 241 |
| }, |
| { |
| "epoch": 0.355359765051395, |
| "fcm_dpo/beta": 0.021258074790239334, |
| "fcm_dpo/delta": -0.021016836166381836, |
| "fcm_dpo/margin": 29.1357479095459, |
| "fcm_dpo/q_t": 0.36231163144111633, |
| "grad_norm": 11.23208999633789, |
| "learning_rate": 4.087222918524807e-07, |
| "logits/chosen": -1.8308205604553223, |
| "logits/rejected": -1.1828442811965942, |
| "logps/chosen": -138.07565307617188, |
| "logps/ref_chosen": -95.67266845703125, |
| "logps/ref_rejected": -90.65454864501953, |
| "logps/rejected": -162.19329833984375, |
| "loss": 0.9561, |
| "margin_dpo/margin_mean": 29.135746002197266, |
| "margin_dpo/margin_std": 30.477272033691406, |
| "step": 242 |
| }, |
| { |
| "epoch": 0.3568281938325991, |
| "fcm_dpo/beta": 0.021303560584783554, |
| "fcm_dpo/delta": -0.015803933143615723, |
| "fcm_dpo/margin": 28.77163314819336, |
| "fcm_dpo/q_t": 0.36690443754196167, |
| "grad_norm": 11.050640106201172, |
| "learning_rate": 4.07728699811968e-07, |
| "logits/chosen": -1.8111510276794434, |
| "logits/rejected": -1.0000836849212646, |
| "logps/chosen": -139.14649963378906, |
| "logps/ref_chosen": -98.03140258789062, |
| "logps/ref_rejected": -83.18806457519531, |
| "logps/rejected": -153.07479858398438, |
| "loss": 0.9904, |
| "margin_dpo/margin_mean": 28.77163314819336, |
| "margin_dpo/margin_std": 34.02631378173828, |
| "step": 243 |
| }, |
| { |
| "epoch": 0.35829662261380324, |
| "fcm_dpo/beta": 0.020892877131700516, |
| "fcm_dpo/delta": -0.03988625854253769, |
| "fcm_dpo/margin": 30.495805740356445, |
| "fcm_dpo/q_t": 0.36164575815200806, |
| "grad_norm": 11.020362854003906, |
| "learning_rate": 4.067309514735267e-07, |
| "logits/chosen": -2.001028060913086, |
| "logits/rejected": -1.656126856803894, |
| "logps/chosen": -131.77944946289062, |
| "logps/ref_chosen": -88.8939208984375, |
| "logps/ref_rejected": -102.57278442382812, |
| "logps/rejected": -175.95413208007812, |
| "loss": 0.9556, |
| "margin_dpo/margin_mean": 30.495803833007812, |
| "margin_dpo/margin_std": 33.84397888183594, |
| "step": 244 |
| }, |
| { |
| "epoch": 0.35976505139500736, |
| "fcm_dpo/beta": 0.02063831314444542, |
| "fcm_dpo/delta": -0.0902470201253891, |
| "fcm_dpo/margin": 33.148372650146484, |
| "fcm_dpo/q_t": 0.34877562522888184, |
| "grad_norm": 10.457184791564941, |
| "learning_rate": 4.057290731287531e-07, |
| "logits/chosen": -2.292057514190674, |
| "logits/rejected": -1.4950196743011475, |
| "logps/chosen": -140.51174926757812, |
| "logps/ref_chosen": -104.19400024414062, |
| "logps/ref_rejected": -92.65645599365234, |
| "logps/rejected": -162.12255859375, |
| "loss": 0.9075, |
| "margin_dpo/margin_mean": 33.14836883544922, |
| "margin_dpo/margin_std": 31.503116607666016, |
| "step": 245 |
| }, |
| { |
| "epoch": 0.36123348017621143, |
| "fcm_dpo/beta": 0.020679624751210213, |
| "fcm_dpo/delta": 0.07523977756500244, |
| "fcm_dpo/margin": 25.567752838134766, |
| "fcm_dpo/q_t": 0.3833276629447937, |
| "grad_norm": 11.005114555358887, |
| "learning_rate": 4.047230911780736e-07, |
| "logits/chosen": -1.9735589027404785, |
| "logits/rejected": -1.4642459154129028, |
| "logps/chosen": -149.89358520507812, |
| "logps/ref_chosen": -103.21904754638672, |
| "logps/ref_rejected": -90.99221801757812, |
| "logps/rejected": -163.2344970703125, |
| "loss": 1.021, |
| "margin_dpo/margin_mean": 25.567752838134766, |
| "margin_dpo/margin_std": 31.323787689208984, |
| "step": 246 |
| }, |
| { |
| "epoch": 0.36270190895741555, |
| "fcm_dpo/beta": 0.020045291632413864, |
| "fcm_dpo/delta": -0.22309784591197968, |
| "fcm_dpo/margin": 40.07710647583008, |
| "fcm_dpo/q_t": 0.328274130821228, |
| "grad_norm": 10.08717155456543, |
| "learning_rate": 4.0371303213004814e-07, |
| "logits/chosen": -2.199232578277588, |
| "logits/rejected": -1.9928696155548096, |
| "logps/chosen": -129.20089721679688, |
| "logps/ref_chosen": -86.99437713623047, |
| "logps/ref_rejected": -111.33802795410156, |
| "logps/rejected": -193.62164306640625, |
| "loss": 0.8532, |
| "margin_dpo/margin_mean": 40.07710647583008, |
| "margin_dpo/margin_std": 37.57669448852539, |
| "step": 247 |
| }, |
| { |
| "epoch": 0.3641703377386197, |
| "fcm_dpo/beta": 0.020135609433054924, |
| "fcm_dpo/delta": 0.0492338202893734, |
| "fcm_dpo/margin": 27.472003936767578, |
| "fcm_dpo/q_t": 0.3776289224624634, |
| "grad_norm": 11.691912651062012, |
| "learning_rate": 4.0269892260067197e-07, |
| "logits/chosen": -1.5299792289733887, |
| "logits/rejected": -1.482997179031372, |
| "logps/chosen": -123.07820129394531, |
| "logps/ref_chosen": -74.7855224609375, |
| "logps/ref_rejected": -98.27689361572266, |
| "logps/rejected": -174.0415802001953, |
| "loss": 0.9963, |
| "margin_dpo/margin_mean": 27.472003936767578, |
| "margin_dpo/margin_std": 31.208698272705078, |
| "step": 248 |
| }, |
| { |
| "epoch": 0.3656387665198238, |
| "fcm_dpo/beta": 0.02031414583325386, |
| "fcm_dpo/delta": 0.08933912217617035, |
| "fcm_dpo/margin": 25.36626434326172, |
| "fcm_dpo/q_t": 0.38820499181747437, |
| "grad_norm": 12.994771957397461, |
| "learning_rate": 4.0168078931267426e-07, |
| "logits/chosen": -2.2492823600769043, |
| "logits/rejected": -1.739271640777588, |
| "logps/chosen": -148.12538146972656, |
| "logps/ref_chosen": -95.70379638671875, |
| "logps/ref_rejected": -87.14646911621094, |
| "logps/rejected": -164.934326171875, |
| "loss": 1.0546, |
| "margin_dpo/margin_mean": 25.36626434326172, |
| "margin_dpo/margin_std": 35.47053527832031, |
| "step": 249 |
| }, |
| { |
| "epoch": 0.3671071953010279, |
| "fcm_dpo/beta": 0.020489081740379333, |
| "fcm_dpo/delta": 0.04790526628494263, |
| "fcm_dpo/margin": 27.043155670166016, |
| "fcm_dpo/q_t": 0.37720349431037903, |
| "grad_norm": 13.011922836303711, |
| "learning_rate": 4.006586590948141e-07, |
| "logits/chosen": -1.7810206413269043, |
| "logits/rejected": -0.8627009391784668, |
| "logps/chosen": -161.8379364013672, |
| "logps/ref_chosen": -114.05221557617188, |
| "logps/ref_rejected": -81.08767700195312, |
| "logps/rejected": -155.91656494140625, |
| "loss": 1.0209, |
| "margin_dpo/margin_mean": 27.043155670166016, |
| "margin_dpo/margin_std": 33.19232940673828, |
| "step": 250 |
| }, |
| { |
| "epoch": 0.368575624082232, |
| "fcm_dpo/beta": 0.020762145519256592, |
| "fcm_dpo/delta": 0.07583161443471909, |
| "fcm_dpo/margin": 25.410945892333984, |
| "fcm_dpo/q_t": 0.3874654173851013, |
| "grad_norm": 12.704300880432129, |
| "learning_rate": 3.9963255888117325e-07, |
| "logits/chosen": -1.9245502948760986, |
| "logits/rejected": -1.244246006011963, |
| "logps/chosen": -144.52914428710938, |
| "logps/ref_chosen": -97.71128845214844, |
| "logps/ref_rejected": -83.52742004394531, |
| "logps/rejected": -155.7562255859375, |
| "loss": 1.0538, |
| "margin_dpo/margin_mean": 25.410945892333984, |
| "margin_dpo/margin_std": 35.388427734375, |
| "step": 251 |
| }, |
| { |
| "epoch": 0.3700440528634361, |
| "fcm_dpo/beta": 0.02117466926574707, |
| "fcm_dpo/delta": 0.05208258330821991, |
| "fcm_dpo/margin": 26.01020050048828, |
| "fcm_dpo/q_t": 0.37829625606536865, |
| "grad_norm": 13.06750774383545, |
| "learning_rate": 3.9860251571044666e-07, |
| "logits/chosen": -1.8967103958129883, |
| "logits/rejected": -1.3722825050354004, |
| "logps/chosen": -155.25262451171875, |
| "logps/ref_chosen": -108.9861068725586, |
| "logps/ref_rejected": -91.56423950195312, |
| "logps/rejected": -163.84097290039062, |
| "loss": 1.0169, |
| "margin_dpo/margin_mean": 26.01020050048828, |
| "margin_dpo/margin_std": 32.340423583984375, |
| "step": 252 |
| }, |
| { |
| "epoch": 0.37151248164464024, |
| "fcm_dpo/beta": 0.021657634526491165, |
| "fcm_dpo/delta": 0.07324262708425522, |
| "fcm_dpo/margin": 24.389892578125, |
| "fcm_dpo/q_t": 0.3817564845085144, |
| "grad_norm": 12.639901161193848, |
| "learning_rate": 3.9756855672522986e-07, |
| "logits/chosen": -2.0871338844299316, |
| "logits/rejected": -1.880314588546753, |
| "logps/chosen": -150.90634155273438, |
| "logps/ref_chosen": -100.21630096435547, |
| "logps/ref_rejected": -105.67670440673828, |
| "logps/rejected": -180.7566375732422, |
| "loss": 1.0268, |
| "margin_dpo/margin_mean": 24.389892578125, |
| "margin_dpo/margin_std": 29.48230743408203, |
| "step": 253 |
| }, |
| { |
| "epoch": 0.37298091042584436, |
| "fcm_dpo/beta": 0.021658308804035187, |
| "fcm_dpo/delta": 0.0799470990896225, |
| "fcm_dpo/margin": 24.19228744506836, |
| "fcm_dpo/q_t": 0.38415348529815674, |
| "grad_norm": 12.239904403686523, |
| "learning_rate": 3.965307091713037e-07, |
| "logits/chosen": -1.8840199708938599, |
| "logits/rejected": -1.340732455253601, |
| "logps/chosen": -148.34524536132812, |
| "logps/ref_chosen": -98.73518371582031, |
| "logps/ref_rejected": -93.73825073242188, |
| "logps/rejected": -167.5406036376953, |
| "loss": 1.0444, |
| "margin_dpo/margin_mean": 24.19228744506836, |
| "margin_dpo/margin_std": 32.14631652832031, |
| "step": 254 |
| }, |
| { |
| "epoch": 0.3744493392070485, |
| "fcm_dpo/beta": 0.021404996514320374, |
| "fcm_dpo/delta": -0.153507798910141, |
| "fcm_dpo/margin": 34.67428207397461, |
| "fcm_dpo/q_t": 0.33902448415756226, |
| "grad_norm": 11.967893600463867, |
| "learning_rate": 3.954890003969163e-07, |
| "logits/chosen": -1.910501480102539, |
| "logits/rejected": -1.5700030326843262, |
| "logps/chosen": -141.0276641845703, |
| "logps/ref_chosen": -90.382568359375, |
| "logps/ref_rejected": -97.07624816894531, |
| "logps/rejected": -182.3956298828125, |
| "loss": 0.8959, |
| "margin_dpo/margin_mean": 34.674278259277344, |
| "margin_dpo/margin_std": 34.8538818359375, |
| "step": 255 |
| }, |
| { |
| "epoch": 0.37591776798825255, |
| "fcm_dpo/beta": 0.02097788080573082, |
| "fcm_dpo/delta": -0.10094420611858368, |
| "fcm_dpo/margin": 33.08401107788086, |
| "fcm_dpo/q_t": 0.3532879948616028, |
| "grad_norm": 12.47603702545166, |
| "learning_rate": 3.944434578520628e-07, |
| "logits/chosen": -1.874246597290039, |
| "logits/rejected": -1.5234124660491943, |
| "logps/chosen": -136.12049865722656, |
| "logps/ref_chosen": -88.7528076171875, |
| "logps/ref_rejected": -98.49383544921875, |
| "logps/rejected": -178.94552612304688, |
| "loss": 0.9704, |
| "margin_dpo/margin_mean": 33.08401107788086, |
| "margin_dpo/margin_std": 41.36225891113281, |
| "step": 256 |
| }, |
| { |
| "epoch": 0.37738619676945667, |
| "fcm_dpo/beta": 0.020358510315418243, |
| "fcm_dpo/delta": -0.06614407151937485, |
| "fcm_dpo/margin": 32.34702682495117, |
| "fcm_dpo/q_t": 0.35797828435897827, |
| "grad_norm": 11.72060775756836, |
| "learning_rate": 3.933941090877615e-07, |
| "logits/chosen": -1.9517452716827393, |
| "logits/rejected": -1.572911262512207, |
| "logps/chosen": -132.3583984375, |
| "logps/ref_chosen": -82.80352020263672, |
| "logps/ref_rejected": -85.8677978515625, |
| "logps/rejected": -167.7696990966797, |
| "loss": 0.9599, |
| "margin_dpo/margin_mean": 32.34702682495117, |
| "margin_dpo/margin_std": 36.041168212890625, |
| "step": 257 |
| }, |
| { |
| "epoch": 0.3788546255506608, |
| "fcm_dpo/beta": 0.02057628706097603, |
| "fcm_dpo/delta": -0.028431270271539688, |
| "fcm_dpo/margin": 30.42020034790039, |
| "fcm_dpo/q_t": 0.36126869916915894, |
| "grad_norm": 12.226306915283203, |
| "learning_rate": 3.923409817553284e-07, |
| "logits/chosen": -2.099985122680664, |
| "logits/rejected": -1.7437987327575684, |
| "logps/chosen": -140.68870544433594, |
| "logps/ref_chosen": -90.187744140625, |
| "logps/ref_rejected": -103.47068786621094, |
| "logps/rejected": -184.39186096191406, |
| "loss": 0.9628, |
| "margin_dpo/margin_mean": 30.42020034790039, |
| "margin_dpo/margin_std": 32.89296340942383, |
| "step": 258 |
| }, |
| { |
| "epoch": 0.3803230543318649, |
| "fcm_dpo/beta": 0.020515816286206245, |
| "fcm_dpo/delta": 0.08078932762145996, |
| "fcm_dpo/margin": 25.49618148803711, |
| "fcm_dpo/q_t": 0.3858429789543152, |
| "grad_norm": 12.687917709350586, |
| "learning_rate": 3.9128410360564793e-07, |
| "logits/chosen": -1.4180266857147217, |
| "logits/rejected": -1.1736819744110107, |
| "logps/chosen": -146.1452178955078, |
| "logps/ref_chosen": -90.77254486083984, |
| "logps/ref_rejected": -94.58816528320312, |
| "logps/rejected": -175.45701599121094, |
| "loss": 1.0682, |
| "margin_dpo/margin_mean": 25.49618148803711, |
| "margin_dpo/margin_std": 38.16563415527344, |
| "step": 259 |
| }, |
| { |
| "epoch": 0.38179148311306904, |
| "fcm_dpo/beta": 0.021066930145025253, |
| "fcm_dpo/delta": 0.11459525674581528, |
| "fcm_dpo/margin": 23.313533782958984, |
| "fcm_dpo/q_t": 0.39374488592147827, |
| "grad_norm": 14.11849594116211, |
| "learning_rate": 3.9022350248844246e-07, |
| "logits/chosen": -1.6266734600067139, |
| "logits/rejected": -1.6325747966766357, |
| "logps/chosen": -131.62429809570312, |
| "logps/ref_chosen": -75.59269714355469, |
| "logps/ref_rejected": -100.84554290771484, |
| "logps/rejected": -180.190673828125, |
| "loss": 1.113, |
| "margin_dpo/margin_mean": 23.313533782958984, |
| "margin_dpo/margin_std": 39.60096740722656, |
| "step": 260 |
| }, |
| { |
| "epoch": 0.3832599118942731, |
| "fcm_dpo/beta": 0.02059752494096756, |
| "fcm_dpo/delta": -0.1640065312385559, |
| "fcm_dpo/margin": 36.42674255371094, |
| "fcm_dpo/q_t": 0.33720719814300537, |
| "grad_norm": 11.14237117767334, |
| "learning_rate": 3.891592063515376e-07, |
| "logits/chosen": -1.8282723426818848, |
| "logits/rejected": -1.453067421913147, |
| "logps/chosen": -138.38714599609375, |
| "logps/ref_chosen": -94.25491333007812, |
| "logps/ref_rejected": -93.65699005126953, |
| "logps/rejected": -174.21595764160156, |
| "loss": 0.8778, |
| "margin_dpo/margin_mean": 36.42674255371094, |
| "margin_dpo/margin_std": 34.84246826171875, |
| "step": 261 |
| }, |
| { |
| "epoch": 0.38472834067547723, |
| "fcm_dpo/beta": 0.020204119384288788, |
| "fcm_dpo/delta": -0.09102602303028107, |
| "fcm_dpo/margin": 33.87533950805664, |
| "fcm_dpo/q_t": 0.3513264060020447, |
| "grad_norm": 11.49547290802002, |
| "learning_rate": 3.880912432401264e-07, |
| "logits/chosen": -1.7930582761764526, |
| "logits/rejected": -1.2549785375595093, |
| "logps/chosen": -133.25894165039062, |
| "logps/ref_chosen": -85.26730346679688, |
| "logps/ref_rejected": -90.82609558105469, |
| "logps/rejected": -172.69308471679688, |
| "loss": 0.9266, |
| "margin_dpo/margin_mean": 33.87533950805664, |
| "margin_dpo/margin_std": 35.84063720703125, |
| "step": 262 |
| }, |
| { |
| "epoch": 0.38619676945668135, |
| "fcm_dpo/beta": 0.019306186586618423, |
| "fcm_dpo/delta": -0.29907822608947754, |
| "fcm_dpo/margin": 45.224769592285156, |
| "fcm_dpo/q_t": 0.3136481046676636, |
| "grad_norm": 10.701422691345215, |
| "learning_rate": 3.870196412960302e-07, |
| "logits/chosen": -1.9696038961410522, |
| "logits/rejected": -1.4115229845046997, |
| "logps/chosen": -134.94964599609375, |
| "logps/ref_chosen": -95.75790405273438, |
| "logps/ref_rejected": -101.83377075195312, |
| "logps/rejected": -186.2502899169922, |
| "loss": 0.83, |
| "margin_dpo/margin_mean": 45.224769592285156, |
| "margin_dpo/margin_std": 42.148963928222656, |
| "step": 263 |
| }, |
| { |
| "epoch": 0.3876651982378855, |
| "fcm_dpo/beta": 0.01862514019012451, |
| "fcm_dpo/delta": -0.11325480043888092, |
| "fcm_dpo/margin": 37.84281921386719, |
| "fcm_dpo/q_t": 0.34645789861679077, |
| "grad_norm": 10.080230712890625, |
| "learning_rate": 3.8594442875695665e-07, |
| "logits/chosen": -2.0475120544433594, |
| "logits/rejected": -1.7664682865142822, |
| "logps/chosen": -134.43109130859375, |
| "logps/ref_chosen": -90.6226577758789, |
| "logps/ref_rejected": -100.32554626464844, |
| "logps/rejected": -181.976806640625, |
| "loss": 0.9163, |
| "margin_dpo/margin_mean": 37.84281921386719, |
| "margin_dpo/margin_std": 38.84701919555664, |
| "step": 264 |
| }, |
| { |
| "epoch": 0.3891336270190896, |
| "fcm_dpo/beta": 0.018290959298610687, |
| "fcm_dpo/delta": -0.06356701254844666, |
| "fcm_dpo/margin": 35.99824523925781, |
| "fcm_dpo/q_t": 0.3593207895755768, |
| "grad_norm": 10.072009086608887, |
| "learning_rate": 3.848656339557562e-07, |
| "logits/chosen": -1.925950050354004, |
| "logits/rejected": -1.5249955654144287, |
| "logps/chosen": -138.44578552246094, |
| "logps/ref_chosen": -92.37232971191406, |
| "logps/ref_rejected": -94.62757873535156, |
| "logps/rejected": -176.69927978515625, |
| "loss": 0.9603, |
| "margin_dpo/margin_mean": 35.99824905395508, |
| "margin_dpo/margin_std": 41.63231658935547, |
| "step": 265 |
| }, |
| { |
| "epoch": 0.39060205580029367, |
| "fcm_dpo/beta": 0.018374420702457428, |
| "fcm_dpo/delta": 0.031407758593559265, |
| "fcm_dpo/margin": 31.039928436279297, |
| "fcm_dpo/q_t": 0.3749583959579468, |
| "grad_norm": 12.884138107299805, |
| "learning_rate": 3.8378328531967507e-07, |
| "logits/chosen": -2.2668302059173584, |
| "logits/rejected": -1.4866418838500977, |
| "logps/chosen": -155.75039672851562, |
| "logps/ref_chosen": -102.20002746582031, |
| "logps/ref_rejected": -74.36642456054688, |
| "logps/rejected": -158.9567108154297, |
| "loss": 1.002, |
| "margin_dpo/margin_mean": 31.039928436279297, |
| "margin_dpo/margin_std": 37.078948974609375, |
| "step": 266 |
| }, |
| { |
| "epoch": 0.3920704845814978, |
| "fcm_dpo/beta": 0.018142802640795708, |
| "fcm_dpo/delta": -0.07580061256885529, |
| "fcm_dpo/margin": 36.915672302246094, |
| "fcm_dpo/q_t": 0.35151273012161255, |
| "grad_norm": 10.919096946716309, |
| "learning_rate": 3.8269741136960646e-07, |
| "logits/chosen": -1.7907602787017822, |
| "logits/rejected": -1.1931796073913574, |
| "logps/chosen": -150.03753662109375, |
| "logps/ref_chosen": -104.28599548339844, |
| "logps/ref_rejected": -95.98719024658203, |
| "logps/rejected": -178.6544189453125, |
| "loss": 0.9123, |
| "margin_dpo/margin_mean": 36.915672302246094, |
| "margin_dpo/margin_std": 34.50056076049805, |
| "step": 267 |
| }, |
| { |
| "epoch": 0.3935389133627019, |
| "fcm_dpo/beta": 0.017767982557415962, |
| "fcm_dpo/delta": -0.12636685371398926, |
| "fcm_dpo/margin": 40.33136749267578, |
| "fcm_dpo/q_t": 0.34127211570739746, |
| "grad_norm": 10.990910530090332, |
| "learning_rate": 3.8160804071933894e-07, |
| "logits/chosen": -1.6720902919769287, |
| "logits/rejected": -1.495853304862976, |
| "logps/chosen": -132.07077026367188, |
| "logps/ref_chosen": -86.69622039794922, |
| "logps/ref_rejected": -109.19184112548828, |
| "logps/rejected": -194.89776611328125, |
| "loss": 0.8877, |
| "margin_dpo/margin_mean": 40.33136749267578, |
| "margin_dpo/margin_std": 36.7901611328125, |
| "step": 268 |
| }, |
| { |
| "epoch": 0.39500734214390604, |
| "fcm_dpo/beta": 0.017438966780900955, |
| "fcm_dpo/delta": -0.1256072223186493, |
| "fcm_dpo/margin": 41.10211944580078, |
| "fcm_dpo/q_t": 0.3434303104877472, |
| "grad_norm": 10.632892608642578, |
| "learning_rate": 3.8051520207480204e-07, |
| "logits/chosen": -1.8700554370880127, |
| "logits/rejected": -1.408653974533081, |
| "logps/chosen": -152.26226806640625, |
| "logps/ref_chosen": -104.97181701660156, |
| "logps/ref_rejected": -112.47647094726562, |
| "logps/rejected": -200.86904907226562, |
| "loss": 0.8995, |
| "margin_dpo/margin_mean": 41.10211944580078, |
| "margin_dpo/margin_std": 40.0078125, |
| "step": 269 |
| }, |
| { |
| "epoch": 0.3964757709251101, |
| "fcm_dpo/beta": 0.017290018498897552, |
| "fcm_dpo/delta": -0.01898367889225483, |
| "fcm_dpo/margin": 35.722171783447266, |
| "fcm_dpo/q_t": 0.3673768937587738, |
| "grad_norm": 10.380319595336914, |
| "learning_rate": 3.794189242333106e-07, |
| "logits/chosen": -2.5234313011169434, |
| "logits/rejected": -2.1158437728881836, |
| "logps/chosen": -153.2642822265625, |
| "logps/ref_chosen": -101.07383728027344, |
| "logps/ref_rejected": -117.75289154052734, |
| "logps/rejected": -205.66552734375, |
| "loss": 1.0017, |
| "margin_dpo/margin_mean": 35.722171783447266, |
| "margin_dpo/margin_std": 45.860267639160156, |
| "step": 270 |
| }, |
| { |
| "epoch": 0.39794419970631423, |
| "fcm_dpo/beta": 0.0168063472956419, |
| "fcm_dpo/delta": -0.1483364999294281, |
| "fcm_dpo/margin": 43.856292724609375, |
| "fcm_dpo/q_t": 0.3364192843437195, |
| "grad_norm": 10.36974811553955, |
| "learning_rate": 3.7831923608280514e-07, |
| "logits/chosen": -1.4026761054992676, |
| "logits/rejected": -0.9475973844528198, |
| "logps/chosen": -140.18299865722656, |
| "logps/ref_chosen": -96.72459411621094, |
| "logps/ref_rejected": -98.5244140625, |
| "logps/rejected": -185.839111328125, |
| "loss": 0.9012, |
| "margin_dpo/margin_mean": 43.856292724609375, |
| "margin_dpo/margin_std": 43.59748840332031, |
| "step": 271 |
| }, |
| { |
| "epoch": 0.39941262848751835, |
| "fcm_dpo/beta": 0.016488002613186836, |
| "fcm_dpo/delta": -0.08753229677677155, |
| "fcm_dpo/margin": 41.335079193115234, |
| "fcm_dpo/q_t": 0.35309892892837524, |
| "grad_norm": 10.801867485046387, |
| "learning_rate": 3.772161666010912e-07, |
| "logits/chosen": -1.7714588642120361, |
| "logits/rejected": -1.6390788555145264, |
| "logps/chosen": -117.77945709228516, |
| "logps/ref_chosen": -80.97721862792969, |
| "logps/ref_rejected": -108.55535888671875, |
| "logps/rejected": -186.69268798828125, |
| "loss": 0.9354, |
| "margin_dpo/margin_mean": 41.3350830078125, |
| "margin_dpo/margin_std": 44.84138107299805, |
| "step": 272 |
| }, |
| { |
| "epoch": 0.4008810572687225, |
| "fcm_dpo/beta": 0.016001848503947258, |
| "fcm_dpo/delta": -0.10035118460655212, |
| "fcm_dpo/margin": 43.12675857543945, |
| "fcm_dpo/q_t": 0.350558340549469, |
| "grad_norm": 9.880205154418945, |
| "learning_rate": 3.761097448550755e-07, |
| "logits/chosen": -1.484649419784546, |
| "logits/rejected": -1.0760293006896973, |
| "logps/chosen": -142.65541076660156, |
| "logps/ref_chosen": -92.22460174560547, |
| "logps/ref_rejected": -97.3630599975586, |
| "logps/rejected": -190.9206085205078, |
| "loss": 0.9532, |
| "margin_dpo/margin_mean": 43.12675857543945, |
| "margin_dpo/margin_std": 49.11865234375, |
| "step": 273 |
| }, |
| { |
| "epoch": 0.4023494860499266, |
| "fcm_dpo/beta": 0.016166478395462036, |
| "fcm_dpo/delta": 0.05010783299803734, |
| "fcm_dpo/margin": 34.18390655517578, |
| "fcm_dpo/q_t": 0.37679600715637207, |
| "grad_norm": 12.417606353759766, |
| "learning_rate": 3.75e-07, |
| "logits/chosen": -1.9811569452285767, |
| "logits/rejected": -1.5822125673294067, |
| "logps/chosen": -133.99008178710938, |
| "logps/ref_chosen": -82.26089477539062, |
| "logps/ref_rejected": -83.87699890136719, |
| "logps/rejected": -169.79010009765625, |
| "loss": 0.9924, |
| "margin_dpo/margin_mean": 34.18390655517578, |
| "margin_dpo/margin_std": 37.70836639404297, |
| "step": 274 |
| }, |
| { |
| "epoch": 0.40381791483113066, |
| "fcm_dpo/beta": 0.016330569982528687, |
| "fcm_dpo/delta": -0.013110160827636719, |
| "fcm_dpo/margin": 37.44886016845703, |
| "fcm_dpo/q_t": 0.3667539358139038, |
| "grad_norm": 11.84593391418457, |
| "learning_rate": 3.738869612786737e-07, |
| "logits/chosen": -2.191549301147461, |
| "logits/rejected": -1.8682620525360107, |
| "logps/chosen": -123.36338806152344, |
| "logps/ref_chosen": -79.68696594238281, |
| "logps/ref_rejected": -98.7509765625, |
| "logps/rejected": -179.87625122070312, |
| "loss": 1.0023, |
| "margin_dpo/margin_mean": 37.44886016845703, |
| "margin_dpo/margin_std": 47.065589904785156, |
| "step": 275 |
| }, |
| { |
| "epoch": 0.4052863436123348, |
| "fcm_dpo/beta": 0.016055840998888016, |
| "fcm_dpo/delta": -0.008165836334228516, |
| "fcm_dpo/margin": 37.754920959472656, |
| "fcm_dpo/q_t": 0.3685857951641083, |
| "grad_norm": 12.060040473937988, |
| "learning_rate": 3.7277065802070204e-07, |
| "logits/chosen": -2.0615105628967285, |
| "logits/rejected": -1.5026049613952637, |
| "logps/chosen": -140.79737854003906, |
| "logps/ref_chosen": -86.53970336914062, |
| "logps/ref_rejected": -77.85394287109375, |
| "logps/rejected": -169.86654663085938, |
| "loss": 0.9909, |
| "margin_dpo/margin_mean": 37.754920959472656, |
| "margin_dpo/margin_std": 45.30341339111328, |
| "step": 276 |
| }, |
| { |
| "epoch": 0.4067547723935389, |
| "fcm_dpo/beta": 0.016346724703907967, |
| "fcm_dpo/delta": 0.08950374275445938, |
| "fcm_dpo/margin": 31.514963150024414, |
| "fcm_dpo/q_t": 0.38510024547576904, |
| "grad_norm": 13.390870094299316, |
| "learning_rate": 3.71651119641714e-07, |
| "logits/chosen": -2.061767101287842, |
| "logits/rejected": -1.646317481994629, |
| "logps/chosen": -141.05987548828125, |
| "logps/ref_chosen": -84.24411010742188, |
| "logps/ref_rejected": -98.83421325683594, |
| "logps/rejected": -187.1649169921875, |
| "loss": 1.0305, |
| "margin_dpo/margin_mean": 31.514963150024414, |
| "margin_dpo/margin_std": 38.913230895996094, |
| "step": 277 |
| }, |
| { |
| "epoch": 0.40822320117474303, |
| "fcm_dpo/beta": 0.0163891538977623, |
| "fcm_dpo/delta": -0.006686456501483917, |
| "fcm_dpo/margin": 36.96751403808594, |
| "fcm_dpo/q_t": 0.36568570137023926, |
| "grad_norm": 12.524548530578613, |
| "learning_rate": 3.705283756425872e-07, |
| "logits/chosen": -2.1564512252807617, |
| "logits/rejected": -1.948979139328003, |
| "logps/chosen": -136.97509765625, |
| "logps/ref_chosen": -82.431884765625, |
| "logps/ref_rejected": -97.85691833496094, |
| "logps/rejected": -189.3676300048828, |
| "loss": 0.9865, |
| "margin_dpo/margin_mean": 36.9675178527832, |
| "margin_dpo/margin_std": 43.24678039550781, |
| "step": 278 |
| }, |
| { |
| "epoch": 0.40969162995594716, |
| "fcm_dpo/beta": 0.01617228053510189, |
| "fcm_dpo/delta": -0.08998230844736099, |
| "fcm_dpo/margin": 42.24311447143555, |
| "fcm_dpo/q_t": 0.35100412368774414, |
| "grad_norm": 11.406697273254395, |
| "learning_rate": 3.6940245560867e-07, |
| "logits/chosen": -1.8248711824417114, |
| "logits/rejected": -1.607770562171936, |
| "logps/chosen": -143.13755798339844, |
| "logps/ref_chosen": -85.16799926757812, |
| "logps/ref_rejected": -94.12664794921875, |
| "logps/rejected": -194.33932495117188, |
| "loss": 0.9281, |
| "margin_dpo/margin_mean": 42.24311828613281, |
| "margin_dpo/margin_std": 44.3424072265625, |
| "step": 279 |
| }, |
| { |
| "epoch": 0.4111600587371512, |
| "fcm_dpo/beta": 0.01589279994368553, |
| "fcm_dpo/delta": -0.12768404185771942, |
| "fcm_dpo/margin": 45.2197265625, |
| "fcm_dpo/q_t": 0.34247255325317383, |
| "grad_norm": 11.279327392578125, |
| "learning_rate": 3.6827338920900253e-07, |
| "logits/chosen": -1.7638534307479858, |
| "logits/rejected": -1.555558204650879, |
| "logps/chosen": -143.5164337158203, |
| "logps/ref_chosen": -85.85641479492188, |
| "logps/ref_rejected": -104.11859130859375, |
| "logps/rejected": -206.9983367919922, |
| "loss": 0.8992, |
| "margin_dpo/margin_mean": 45.2197265625, |
| "margin_dpo/margin_std": 43.852996826171875, |
| "step": 280 |
| }, |
| { |
| "epoch": 0.41262848751835535, |
| "fcm_dpo/beta": 0.015449896454811096, |
| "fcm_dpo/delta": -0.1223934069275856, |
| "fcm_dpo/margin": 46.17839050292969, |
| "fcm_dpo/q_t": 0.3489765524864197, |
| "grad_norm": 9.723531723022461, |
| "learning_rate": 3.6714120619553435e-07, |
| "logits/chosen": -2.1463782787323, |
| "logits/rejected": -1.422343373298645, |
| "logps/chosen": -147.69259643554688, |
| "logps/ref_chosen": -86.55082702636719, |
| "logps/ref_rejected": -88.62866973876953, |
| "logps/rejected": -195.94882202148438, |
| "loss": 0.9317, |
| "margin_dpo/margin_mean": 46.17839050292969, |
| "margin_dpo/margin_std": 52.58583068847656, |
| "step": 281 |
| }, |
| { |
| "epoch": 0.41409691629955947, |
| "fcm_dpo/beta": 0.014763940125703812, |
| "fcm_dpo/delta": -0.25571346282958984, |
| "fcm_dpo/margin": 56.46161651611328, |
| "fcm_dpo/q_t": 0.3185679316520691, |
| "grad_norm": 9.334823608398438, |
| "learning_rate": 3.660059364023408e-07, |
| "logits/chosen": -1.9072599411010742, |
| "logits/rejected": -1.542052984237671, |
| "logps/chosen": -164.61256408691406, |
| "logps/ref_chosen": -105.10511016845703, |
| "logps/ref_rejected": -102.85336303710938, |
| "logps/rejected": -218.82244873046875, |
| "loss": 0.8212, |
| "margin_dpo/margin_mean": 56.46161651611328, |
| "margin_dpo/margin_std": 48.17137908935547, |
| "step": 282 |
| }, |
| { |
| "epoch": 0.4155653450807636, |
| "fcm_dpo/beta": 0.014660017564892769, |
| "fcm_dpo/delta": 0.026379965245723724, |
| "fcm_dpo/margin": 39.232139587402344, |
| "fcm_dpo/q_t": 0.3760044276714325, |
| "grad_norm": 12.626619338989258, |
| "learning_rate": 3.6486760974483685e-07, |
| "logits/chosen": -2.1855897903442383, |
| "logits/rejected": -1.8464176654815674, |
| "logps/chosen": -164.91322326660156, |
| "logps/ref_chosen": -95.05259704589844, |
| "logps/ref_rejected": -103.54454803466797, |
| "logps/rejected": -212.63731384277344, |
| "loss": 1.0276, |
| "margin_dpo/margin_mean": 39.232139587402344, |
| "margin_dpo/margin_std": 53.09544372558594, |
| "step": 283 |
| }, |
| { |
| "epoch": 0.4170337738619677, |
| "fcm_dpo/beta": 0.01422587689012289, |
| "fcm_dpo/delta": -0.17442496120929718, |
| "fcm_dpo/margin": 53.390506744384766, |
| "fcm_dpo/q_t": 0.33068281412124634, |
| "grad_norm": 9.979324340820312, |
| "learning_rate": 3.6372625621898863e-07, |
| "logits/chosen": -1.7961182594299316, |
| "logits/rejected": -1.4766089916229248, |
| "logps/chosen": -148.95257568359375, |
| "logps/ref_chosen": -87.66647338867188, |
| "logps/ref_rejected": -98.75103759765625, |
| "logps/rejected": -213.42764282226562, |
| "loss": 0.8488, |
| "margin_dpo/margin_mean": 53.3905029296875, |
| "margin_dpo/margin_std": 44.12660217285156, |
| "step": 284 |
| }, |
| { |
| "epoch": 0.4185022026431718, |
| "fcm_dpo/beta": 0.01436292938888073, |
| "fcm_dpo/delta": 0.06020905822515488, |
| "fcm_dpo/margin": 37.7760009765625, |
| "fcm_dpo/q_t": 0.3767179250717163, |
| "grad_norm": 14.075101852416992, |
| "learning_rate": 3.625819059005228e-07, |
| "logits/chosen": -2.3716185092926025, |
| "logits/rejected": -2.0925536155700684, |
| "logps/chosen": -164.59915161132812, |
| "logps/ref_chosen": -94.43303680419922, |
| "logps/ref_rejected": -104.07194519042969, |
| "logps/rejected": -212.01405334472656, |
| "loss": 0.9954, |
| "margin_dpo/margin_mean": 37.7760009765625, |
| "margin_dpo/margin_std": 40.709510803222656, |
| "step": 285 |
| }, |
| { |
| "epoch": 0.4199706314243759, |
| "fcm_dpo/beta": 0.014110833406448364, |
| "fcm_dpo/delta": -0.10124547779560089, |
| "fcm_dpo/margin": 49.17796325683594, |
| "fcm_dpo/q_t": 0.3466648757457733, |
| "grad_norm": 10.761466026306152, |
| "learning_rate": 3.614345889441346e-07, |
| "logits/chosen": -1.7981414794921875, |
| "logits/rejected": -1.32814621925354, |
| "logps/chosen": -178.29949951171875, |
| "logps/ref_chosen": -103.72039794921875, |
| "logps/ref_rejected": -96.25775909423828, |
| "logps/rejected": -220.01480102539062, |
| "loss": 0.9137, |
| "margin_dpo/margin_mean": 49.17796325683594, |
| "margin_dpo/margin_std": 48.17053985595703, |
| "step": 286 |
| }, |
| { |
| "epoch": 0.42143906020558003, |
| "fcm_dpo/beta": 0.014512348920106888, |
| "fcm_dpo/delta": 0.16535604000091553, |
| "fcm_dpo/margin": 30.320154190063477, |
| "fcm_dpo/q_t": 0.4061054587364197, |
| "grad_norm": 11.057209968566895, |
| "learning_rate": 3.6028433558269275e-07, |
| "logits/chosen": -2.0745434761047363, |
| "logits/rejected": -1.4789698123931885, |
| "logps/chosen": -181.6774444580078, |
| "logps/ref_chosen": -93.88988494873047, |
| "logps/ref_rejected": -83.33365631103516, |
| "logps/rejected": -201.44137573242188, |
| "loss": 1.1162, |
| "margin_dpo/margin_mean": 30.320154190063477, |
| "margin_dpo/margin_std": 49.62172317504883, |
| "step": 287 |
| }, |
| { |
| "epoch": 0.42290748898678415, |
| "fcm_dpo/beta": 0.014212418347597122, |
| "fcm_dpo/delta": -0.13853885233402252, |
| "fcm_dpo/margin": 51.22802734375, |
| "fcm_dpo/q_t": 0.34565305709838867, |
| "grad_norm": 10.420333862304688, |
| "learning_rate": 3.5913117612644327e-07, |
| "logits/chosen": -1.952336311340332, |
| "logits/rejected": -1.555095911026001, |
| "logps/chosen": -172.57025146484375, |
| "logps/ref_chosen": -88.15602111816406, |
| "logps/ref_rejected": -93.28195190429688, |
| "logps/rejected": -228.92422485351562, |
| "loss": 0.9128, |
| "margin_dpo/margin_mean": 51.22802734375, |
| "margin_dpo/margin_std": 54.86450958251953, |
| "step": 288 |
| }, |
| { |
| "epoch": 0.4243759177679883, |
| "fcm_dpo/beta": 0.013875823467969894, |
| "fcm_dpo/delta": -0.14725634455680847, |
| "fcm_dpo/margin": 53.087974548339844, |
| "fcm_dpo/q_t": 0.34943529963493347, |
| "grad_norm": 9.893397331237793, |
| "learning_rate": 3.5797514096221024e-07, |
| "logits/chosen": -1.9582253694534302, |
| "logits/rejected": -1.757028341293335, |
| "logps/chosen": -158.78115844726562, |
| "logps/ref_chosen": -75.39292907714844, |
| "logps/ref_rejected": -93.15428161621094, |
| "logps/rejected": -229.6304931640625, |
| "loss": 0.9302, |
| "margin_dpo/margin_mean": 53.087974548339844, |
| "margin_dpo/margin_std": 61.871124267578125, |
| "step": 289 |
| }, |
| { |
| "epoch": 0.42584434654919234, |
| "fcm_dpo/beta": 0.013547876849770546, |
| "fcm_dpo/delta": -0.14979226887226105, |
| "fcm_dpo/margin": 54.51033020019531, |
| "fcm_dpo/q_t": 0.3426121473312378, |
| "grad_norm": 10.341540336608887, |
| "learning_rate": 3.568162605525952e-07, |
| "logits/chosen": -1.157566785812378, |
| "logits/rejected": -1.1871843338012695, |
| "logps/chosen": -179.93722534179688, |
| "logps/ref_chosen": -88.04197692871094, |
| "logps/ref_rejected": -123.212158203125, |
| "logps/rejected": -269.61773681640625, |
| "loss": 0.9071, |
| "margin_dpo/margin_mean": 54.51033020019531, |
| "margin_dpo/margin_std": 57.139404296875, |
| "step": 290 |
| }, |
| { |
| "epoch": 0.42731277533039647, |
| "fcm_dpo/beta": 0.013345437124371529, |
| "fcm_dpo/delta": -0.0021981801837682724, |
| "fcm_dpo/margin": 45.109642028808594, |
| "fcm_dpo/q_t": 0.3656271994113922, |
| "grad_norm": 10.851905822753906, |
| "learning_rate": 3.5565456543517485e-07, |
| "logits/chosen": -2.3467676639556885, |
| "logits/rejected": -1.8936057090759277, |
| "logps/chosen": -181.8697967529297, |
| "logps/ref_chosen": -94.09524536132812, |
| "logps/ref_rejected": -96.05006408691406, |
| "logps/rejected": -228.93426513671875, |
| "loss": 0.9784, |
| "margin_dpo/margin_mean": 45.109642028808594, |
| "margin_dpo/margin_std": 51.156219482421875, |
| "step": 291 |
| }, |
| { |
| "epoch": 0.4287812041116006, |
| "fcm_dpo/beta": 0.013189633376896381, |
| "fcm_dpo/delta": -0.062153078615665436, |
| "fcm_dpo/margin": 49.88909912109375, |
| "fcm_dpo/q_t": 0.3522568643093109, |
| "grad_norm": 10.579051971435547, |
| "learning_rate": 3.5449008622169583e-07, |
| "logits/chosen": -2.1592330932617188, |
| "logits/rejected": -1.730700969696045, |
| "logps/chosen": -178.15887451171875, |
| "logps/ref_chosen": -88.25041198730469, |
| "logps/ref_rejected": -96.41764068603516, |
| "logps/rejected": -236.21517944335938, |
| "loss": 0.9094, |
| "margin_dpo/margin_mean": 49.889095306396484, |
| "margin_dpo/margin_std": 44.906211853027344, |
| "step": 292 |
| }, |
| { |
| "epoch": 0.4302496328928047, |
| "fcm_dpo/beta": 0.013408501632511616, |
| "fcm_dpo/delta": 0.0868854746222496, |
| "fcm_dpo/margin": 38.51031494140625, |
| "fcm_dpo/q_t": 0.3900435268878937, |
| "grad_norm": 9.791333198547363, |
| "learning_rate": 3.5332285359726846e-07, |
| "logits/chosen": -2.625321388244629, |
| "logits/rejected": -2.0894412994384766, |
| "logps/chosen": -180.33511352539062, |
| "logps/ref_chosen": -87.37654113769531, |
| "logps/ref_rejected": -85.75579833984375, |
| "logps/rejected": -217.22467041015625, |
| "loss": 1.0576, |
| "margin_dpo/margin_mean": 38.51031494140625, |
| "margin_dpo/margin_std": 54.91911315917969, |
| "step": 293 |
| }, |
| { |
| "epoch": 0.43171806167400884, |
| "fcm_dpo/beta": 0.013757754117250443, |
| "fcm_dpo/delta": 0.16101771593093872, |
| "fcm_dpo/margin": 32.374855041503906, |
| "fcm_dpo/q_t": 0.4055211544036865, |
| "grad_norm": 11.367213249206543, |
| "learning_rate": 3.5215289831955786e-07, |
| "logits/chosen": -1.6977550983428955, |
| "logits/rejected": -1.5189887285232544, |
| "logps/chosen": -170.7028045654297, |
| "logps/ref_chosen": -73.50794982910156, |
| "logps/ref_rejected": -88.08877563476562, |
| "logps/rejected": -217.65847778320312, |
| "loss": 1.1326, |
| "margin_dpo/margin_mean": 32.374855041503906, |
| "margin_dpo/margin_std": 57.856483459472656, |
| "step": 294 |
| }, |
| { |
| "epoch": 0.4331864904552129, |
| "fcm_dpo/beta": 0.013606134802103043, |
| "fcm_dpo/delta": -0.06477653235197067, |
| "fcm_dpo/margin": 48.46016311645508, |
| "fcm_dpo/q_t": 0.3579823672771454, |
| "grad_norm": 10.51223373413086, |
| "learning_rate": 3.509802512179737e-07, |
| "logits/chosen": -1.6513359546661377, |
| "logits/rejected": -1.5038331747055054, |
| "logps/chosen": -172.31985473632812, |
| "logps/ref_chosen": -77.76548767089844, |
| "logps/ref_rejected": -94.24726867675781, |
| "logps/rejected": -237.2617950439453, |
| "loss": 0.9532, |
| "margin_dpo/margin_mean": 48.46016311645508, |
| "margin_dpo/margin_std": 54.77019500732422, |
| "step": 295 |
| }, |
| { |
| "epoch": 0.434654919236417, |
| "fcm_dpo/beta": 0.01337181031703949, |
| "fcm_dpo/delta": -0.10661374777555466, |
| "fcm_dpo/margin": 52.22368240356445, |
| "fcm_dpo/q_t": 0.34752362966537476, |
| "grad_norm": 11.95097827911377, |
| "learning_rate": 3.498049431928577e-07, |
| "logits/chosen": -2.5504400730133057, |
| "logits/rejected": -2.0862655639648438, |
| "logps/chosen": -198.88778686523438, |
| "logps/ref_chosen": -97.85641479492188, |
| "logps/ref_rejected": -100.81631469726562, |
| "logps/rejected": -254.07135009765625, |
| "loss": 0.9174, |
| "margin_dpo/margin_mean": 52.22368240356445, |
| "margin_dpo/margin_std": 53.6197509765625, |
| "step": 296 |
| }, |
| { |
| "epoch": 0.43612334801762115, |
| "fcm_dpo/beta": 0.013420840725302696, |
| "fcm_dpo/delta": 0.018867414444684982, |
| "fcm_dpo/margin": 43.375144958496094, |
| "fcm_dpo/q_t": 0.37541282176971436, |
| "grad_norm": 13.033909797668457, |
| "learning_rate": 3.486270052146694e-07, |
| "logits/chosen": -1.2705745697021484, |
| "logits/rejected": -0.9568313360214233, |
| "logps/chosen": -192.59295654296875, |
| "logps/ref_chosen": -88.56583404541016, |
| "logps/ref_rejected": -101.55656433105469, |
| "logps/rejected": -248.9588165283203, |
| "loss": 1.0032, |
| "margin_dpo/margin_mean": 43.375144958496094, |
| "margin_dpo/margin_std": 54.68861389160156, |
| "step": 297 |
| }, |
| { |
| "epoch": 0.43759177679882527, |
| "fcm_dpo/beta": 0.013230125419795513, |
| "fcm_dpo/delta": -0.11564385145902634, |
| "fcm_dpo/margin": 53.48369216918945, |
| "fcm_dpo/q_t": 0.35127192735671997, |
| "grad_norm": 16.36045265197754, |
| "learning_rate": 3.474464683231698e-07, |
| "logits/chosen": -2.012399911880493, |
| "logits/rejected": -1.9581105709075928, |
| "logps/chosen": -186.6326904296875, |
| "logps/ref_chosen": -94.88043975830078, |
| "logps/ref_rejected": -122.31101989746094, |
| "logps/rejected": -267.54693603515625, |
| "loss": 0.9249, |
| "margin_dpo/margin_mean": 53.48369216918945, |
| "margin_dpo/margin_std": 59.249088287353516, |
| "step": 298 |
| }, |
| { |
| "epoch": 0.4390602055800294, |
| "fcm_dpo/beta": 0.012806476093828678, |
| "fcm_dpo/delta": -0.13323679566383362, |
| "fcm_dpo/margin": 56.45828628540039, |
| "fcm_dpo/q_t": 0.34347003698349, |
| "grad_norm": 10.563586235046387, |
| "learning_rate": 3.462633636266041e-07, |
| "logits/chosen": -0.9475471377372742, |
| "logits/rejected": -0.7138333320617676, |
| "logps/chosen": -175.18414306640625, |
| "logps/ref_chosen": -80.40835571289062, |
| "logps/ref_rejected": -89.53717041015625, |
| "logps/rejected": -240.771240234375, |
| "loss": 0.9033, |
| "margin_dpo/margin_mean": 56.458290100097656, |
| "margin_dpo/margin_std": 57.19697570800781, |
| "step": 299 |
| }, |
| { |
| "epoch": 0.44052863436123346, |
| "fcm_dpo/beta": 0.012716519646346569, |
| "fcm_dpo/delta": 0.005831874907016754, |
| "fcm_dpo/margin": 46.72724151611328, |
| "fcm_dpo/q_t": 0.37154459953308105, |
| "grad_norm": 11.477516174316406, |
| "learning_rate": 3.4507772230088147e-07, |
| "logits/chosen": -1.3495773077011108, |
| "logits/rejected": -1.1283777952194214, |
| "logps/chosen": -187.920654296875, |
| "logps/ref_chosen": -88.15890502929688, |
| "logps/ref_rejected": -100.93919372558594, |
| "logps/rejected": -247.4281768798828, |
| "loss": 1.0079, |
| "margin_dpo/margin_mean": 46.727237701416016, |
| "margin_dpo/margin_std": 59.988731384277344, |
| "step": 300 |
| }, |
| { |
| "epoch": 0.44052863436123346, |
| "eval_fcm_dpo/beta": 0.01277273241430521, |
| "eval_logits/chosen": -1.7159112691879272, |
| "eval_logits/rejected": -1.3590905666351318, |
| "eval_logps/chosen": -199.09510803222656, |
| "eval_logps/ref_chosen": -100.49356842041016, |
| "eval_logps/ref_rejected": -94.06775665283203, |
| "eval_logps/rejected": -235.35121154785156, |
| "eval_loss": 0.5174583792686462, |
| "eval_margin_dpo/margin_mean": 42.681915283203125, |
| "eval_margin_dpo/margin_std": 56.65989685058594, |
| "eval_runtime": 43.743, |
| "eval_samples_per_second": 53.471, |
| "eval_steps_per_second": 1.692, |
| "step": 300 |
| }, |
| { |
| "epoch": 0.4419970631424376, |
| "fcm_dpo/beta": 0.012361442670226097, |
| "fcm_dpo/delta": -0.16100239753723145, |
| "fcm_dpo/margin": 60.36054992675781, |
| "fcm_dpo/q_t": 0.3411176800727844, |
| "grad_norm": 10.854103088378906, |
| "learning_rate": 3.4388957558875316e-07, |
| "logits/chosen": -1.7507340908050537, |
| "logits/rejected": -1.315748691558838, |
| "logps/chosen": -178.2888946533203, |
| "logps/ref_chosen": -85.50491333007812, |
| "logps/ref_rejected": -98.91683959960938, |
| "logps/rejected": -252.06137084960938, |
| "loss": 0.9009, |
| "margin_dpo/margin_mean": 60.36054992675781, |
| "margin_dpo/margin_std": 62.385398864746094, |
| "step": 301 |
| }, |
| { |
| "epoch": 0.4434654919236417, |
| "fcm_dpo/beta": 0.012308208271861076, |
| "fcm_dpo/delta": -0.004877906292676926, |
| "fcm_dpo/margin": 49.0919189453125, |
| "fcm_dpo/q_t": 0.3680340647697449, |
| "grad_norm": 11.78134536743164, |
| "learning_rate": 3.426989547989902e-07, |
| "logits/chosen": -1.391321063041687, |
| "logits/rejected": -1.2118873596191406, |
| "logps/chosen": -172.2723388671875, |
| "logps/ref_chosen": -79.45040893554688, |
| "logps/ref_rejected": -103.0591049194336, |
| "logps/rejected": -244.97293090820312, |
| "loss": 0.9875, |
| "margin_dpo/margin_mean": 49.091915130615234, |
| "margin_dpo/margin_std": 58.554237365722656, |
| "step": 302 |
| }, |
| { |
| "epoch": 0.44493392070484583, |
| "fcm_dpo/beta": 0.01226123422384262, |
| "fcm_dpo/delta": -0.08874759823083878, |
| "fcm_dpo/margin": 55.66910934448242, |
| "fcm_dpo/q_t": 0.3502175211906433, |
| "grad_norm": 10.105868339538574, |
| "learning_rate": 3.4150589130555773e-07, |
| "logits/chosen": -1.8867647647857666, |
| "logits/rejected": -1.4471575021743774, |
| "logps/chosen": -179.00445556640625, |
| "logps/ref_chosen": -95.59829711914062, |
| "logps/ref_rejected": -92.0645751953125, |
| "logps/rejected": -231.1398468017578, |
| "loss": 0.9267, |
| "margin_dpo/margin_mean": 55.669105529785156, |
| "margin_dpo/margin_std": 56.76445007324219, |
| "step": 303 |
| }, |
| { |
| "epoch": 0.44640234948604995, |
| "fcm_dpo/beta": 0.011957298964262009, |
| "fcm_dpo/delta": -0.11238184571266174, |
| "fcm_dpo/margin": 58.92472839355469, |
| "fcm_dpo/q_t": 0.3453929126262665, |
| "grad_norm": 11.284574508666992, |
| "learning_rate": 3.403104165467883e-07, |
| "logits/chosen": -2.2299141883850098, |
| "logits/rejected": -1.8196072578430176, |
| "logps/chosen": -183.23292541503906, |
| "logps/ref_chosen": -99.4531478881836, |
| "logps/ref_rejected": -98.08226013183594, |
| "logps/rejected": -240.78677368164062, |
| "loss": 0.9148, |
| "margin_dpo/margin_mean": 58.92472839355469, |
| "margin_dpo/margin_std": 60.10872268676758, |
| "step": 304 |
| }, |
| { |
| "epoch": 0.447870778267254, |
| "fcm_dpo/beta": 0.011641704477369785, |
| "fcm_dpo/delta": -0.06990103423595428, |
| "fcm_dpo/margin": 56.961299896240234, |
| "fcm_dpo/q_t": 0.35451453924179077, |
| "grad_norm": 10.1016263961792, |
| "learning_rate": 3.391125620245535e-07, |
| "logits/chosen": -2.020512104034424, |
| "logits/rejected": -1.5305230617523193, |
| "logps/chosen": -190.84634399414062, |
| "logps/ref_chosen": -103.26339721679688, |
| "logps/ref_rejected": -97.82897186279297, |
| "logps/rejected": -242.3732147216797, |
| "loss": 0.9438, |
| "margin_dpo/margin_mean": 56.96129608154297, |
| "margin_dpo/margin_std": 59.46485137939453, |
| "step": 305 |
| }, |
| { |
| "epoch": 0.44933920704845814, |
| "fcm_dpo/beta": 0.011314811185002327, |
| "fcm_dpo/delta": -0.20089520514011383, |
| "fcm_dpo/margin": 69.29483032226562, |
| "fcm_dpo/q_t": 0.3232324719429016, |
| "grad_norm": 10.051947593688965, |
| "learning_rate": 3.3791235930343417e-07, |
| "logits/chosen": -2.1980226039886475, |
| "logits/rejected": -1.6485023498535156, |
| "logps/chosen": -177.880615234375, |
| "logps/ref_chosen": -100.3038330078125, |
| "logps/ref_rejected": -92.23086547851562, |
| "logps/rejected": -239.10247802734375, |
| "loss": 0.8261, |
| "margin_dpo/margin_mean": 69.29483032226562, |
| "margin_dpo/margin_std": 51.61286926269531, |
| "step": 306 |
| }, |
| { |
| "epoch": 0.45080763582966227, |
| "fcm_dpo/beta": 0.011268500238656998, |
| "fcm_dpo/delta": 0.04018982872366905, |
| "fcm_dpo/margin": 49.86820983886719, |
| "fcm_dpo/q_t": 0.37846463918685913, |
| "grad_norm": 10.908080101013184, |
| "learning_rate": 3.367098400098881e-07, |
| "logits/chosen": -2.024630308151245, |
| "logits/rejected": -1.6950145959854126, |
| "logps/chosen": -181.67286682128906, |
| "logps/ref_chosen": -96.64976501464844, |
| "logps/ref_rejected": -93.34860229492188, |
| "logps/rejected": -228.23992919921875, |
| "loss": 1.026, |
| "margin_dpo/margin_mean": 49.86820602416992, |
| "margin_dpo/margin_std": 66.2600326538086, |
| "step": 307 |
| }, |
| { |
| "epoch": 0.4522760646108664, |
| "fcm_dpo/beta": 0.011317985132336617, |
| "fcm_dpo/delta": 0.04552755504846573, |
| "fcm_dpo/margin": 49.12928009033203, |
| "fcm_dpo/q_t": 0.38196253776550293, |
| "grad_norm": 11.59121322631836, |
| "learning_rate": 3.355050358314172e-07, |
| "logits/chosen": -1.7888622283935547, |
| "logits/rejected": -1.5279297828674316, |
| "logps/chosen": -173.8710479736328, |
| "logps/ref_chosen": -85.72467041015625, |
| "logps/ref_rejected": -86.42317962646484, |
| "logps/rejected": -223.69882202148438, |
| "loss": 1.0387, |
| "margin_dpo/margin_mean": 49.12928009033203, |
| "margin_dpo/margin_std": 66.89312744140625, |
| "step": 308 |
| }, |
| { |
| "epoch": 0.45374449339207046, |
| "fcm_dpo/beta": 0.011532912030816078, |
| "fcm_dpo/delta": 0.044411551207304, |
| "fcm_dpo/margin": 48.37786102294922, |
| "fcm_dpo/q_t": 0.3778493404388428, |
| "grad_norm": 14.045127868652344, |
| "learning_rate": 3.3429797851573183e-07, |
| "logits/chosen": -1.8127508163452148, |
| "logits/rejected": -1.389906883239746, |
| "logps/chosen": -179.91854858398438, |
| "logps/ref_chosen": -83.15145874023438, |
| "logps/ref_rejected": -86.58602142333984, |
| "logps/rejected": -231.73095703125, |
| "loss": 1.0173, |
| "margin_dpo/margin_mean": 48.37785720825195, |
| "margin_dpo/margin_std": 60.719688415527344, |
| "step": 309 |
| }, |
| { |
| "epoch": 0.4552129221732746, |
| "fcm_dpo/beta": 0.011562330648303032, |
| "fcm_dpo/delta": 0.013482505455613136, |
| "fcm_dpo/margin": 50.79509735107422, |
| "fcm_dpo/q_t": 0.3710099160671234, |
| "grad_norm": 10.965914726257324, |
| "learning_rate": 3.3308869986991487e-07, |
| "logits/chosen": -2.2912349700927734, |
| "logits/rejected": -1.8876441717147827, |
| "logps/chosen": -175.0748291015625, |
| "logps/ref_chosen": -89.73799133300781, |
| "logps/ref_rejected": -87.07083129882812, |
| "logps/rejected": -223.2027587890625, |
| "loss": 0.9925, |
| "margin_dpo/margin_mean": 50.79509353637695, |
| "margin_dpo/margin_std": 60.45803451538086, |
| "step": 310 |
| }, |
| { |
| "epoch": 0.4566813509544787, |
| "fcm_dpo/beta": 0.011323259212076664, |
| "fcm_dpo/delta": -0.14433330297470093, |
| "fcm_dpo/margin": 64.79643249511719, |
| "fcm_dpo/q_t": 0.33913880586624146, |
| "grad_norm": 10.768723487854004, |
| "learning_rate": 3.3187723175958346e-07, |
| "logits/chosen": -1.8347514867782593, |
| "logits/rejected": -1.1266942024230957, |
| "logps/chosen": -192.47508239746094, |
| "logps/ref_chosen": -88.52644348144531, |
| "logps/ref_rejected": -82.26608276367188, |
| "logps/rejected": -251.0111541748047, |
| "loss": 0.875, |
| "margin_dpo/margin_mean": 64.79643249511719, |
| "margin_dpo/margin_std": 58.919742584228516, |
| "step": 311 |
| }, |
| { |
| "epoch": 0.4581497797356828, |
| "fcm_dpo/beta": 0.011255646124482155, |
| "fcm_dpo/delta": -0.024957779794931412, |
| "fcm_dpo/margin": 55.35394287109375, |
| "fcm_dpo/q_t": 0.36366888880729675, |
| "grad_norm": 10.373575210571289, |
| "learning_rate": 3.306636061080487e-07, |
| "logits/chosen": -1.5337181091308594, |
| "logits/rejected": -1.003326654434204, |
| "logps/chosen": -179.2374267578125, |
| "logps/ref_chosen": -80.40069580078125, |
| "logps/ref_rejected": -82.4471435546875, |
| "logps/rejected": -236.63783264160156, |
| "loss": 0.9707, |
| "margin_dpo/margin_mean": 55.35394287109375, |
| "margin_dpo/margin_std": 63.06084442138672, |
| "step": 312 |
| }, |
| { |
| "epoch": 0.45961820851688695, |
| "fcm_dpo/beta": 0.01121255662292242, |
| "fcm_dpo/delta": 0.0006225090473890305, |
| "fcm_dpo/margin": 53.44978332519531, |
| "fcm_dpo/q_t": 0.3703840374946594, |
| "grad_norm": 11.122648239135742, |
| "learning_rate": 3.2944785489547537e-07, |
| "logits/chosen": -2.4546828269958496, |
| "logits/rejected": -2.0480782985687256, |
| "logps/chosen": -176.66587829589844, |
| "logps/ref_chosen": -84.14190673828125, |
| "logps/ref_rejected": -93.42684173583984, |
| "logps/rejected": -239.40060424804688, |
| "loss": 1.008, |
| "margin_dpo/margin_mean": 53.44977569580078, |
| "margin_dpo/margin_std": 69.2515869140625, |
| "step": 313 |
| }, |
| { |
| "epoch": 0.461086637298091, |
| "fcm_dpo/beta": 0.011162392795085907, |
| "fcm_dpo/delta": 0.010066971182823181, |
| "fcm_dpo/margin": 52.88270568847656, |
| "fcm_dpo/q_t": 0.3719868063926697, |
| "grad_norm": 11.244734764099121, |
| "learning_rate": 3.2823001015803857e-07, |
| "logits/chosen": -1.9083623886108398, |
| "logits/rejected": -1.6891626119613647, |
| "logps/chosen": -181.61160278320312, |
| "logps/ref_chosen": -88.80972290039062, |
| "logps/ref_rejected": -104.15892028808594, |
| "logps/rejected": -249.843505859375, |
| "loss": 0.9914, |
| "margin_dpo/margin_mean": 52.88270568847656, |
| "margin_dpo/margin_std": 63.093101501464844, |
| "step": 314 |
| }, |
| { |
| "epoch": 0.46255506607929514, |
| "fcm_dpo/beta": 0.011188652366399765, |
| "fcm_dpo/delta": -0.006126678083091974, |
| "fcm_dpo/margin": 54.138145446777344, |
| "fcm_dpo/q_t": 0.37136292457580566, |
| "grad_norm": 12.068286895751953, |
| "learning_rate": 3.270101039870797e-07, |
| "logits/chosen": -1.4232127666473389, |
| "logits/rejected": -1.2410612106323242, |
| "logps/chosen": -171.7213134765625, |
| "logps/ref_chosen": -80.793701171875, |
| "logps/ref_rejected": -92.13711547851562, |
| "logps/rejected": -237.202880859375, |
| "loss": 1.0249, |
| "margin_dpo/margin_mean": 54.138145446777344, |
| "margin_dpo/margin_std": 74.93022155761719, |
| "step": 315 |
| }, |
| { |
| "epoch": 0.46402349486049926, |
| "fcm_dpo/beta": 0.011153796687722206, |
| "fcm_dpo/delta": -0.032783642411231995, |
| "fcm_dpo/margin": 56.54811096191406, |
| "fcm_dpo/q_t": 0.364329993724823, |
| "grad_norm": 10.284096717834473, |
| "learning_rate": 3.2578816852826086e-07, |
| "logits/chosen": -1.860671043395996, |
| "logits/rejected": -1.8614685535430908, |
| "logps/chosen": -179.6904296875, |
| "logps/ref_chosen": -84.18424987792969, |
| "logps/ref_rejected": -105.6908187866211, |
| "logps/rejected": -257.7451171875, |
| "loss": 0.9706, |
| "margin_dpo/margin_mean": 56.54811096191406, |
| "margin_dpo/margin_std": 67.02822875976562, |
| "step": 316 |
| }, |
| { |
| "epoch": 0.4654919236417034, |
| "fcm_dpo/beta": 0.010856937617063522, |
| "fcm_dpo/delta": -0.19055171310901642, |
| "fcm_dpo/margin": 71.48945617675781, |
| "fcm_dpo/q_t": 0.3373391628265381, |
| "grad_norm": 12.972799301147461, |
| "learning_rate": 3.2456423598071783e-07, |
| "logits/chosen": -1.9898185729980469, |
| "logits/rejected": -1.512327790260315, |
| "logps/chosen": -185.63662719726562, |
| "logps/ref_chosen": -87.87348937988281, |
| "logps/ref_rejected": -108.26519775390625, |
| "logps/rejected": -277.5177917480469, |
| "loss": 0.9042, |
| "margin_dpo/margin_mean": 71.48945617675781, |
| "margin_dpo/margin_std": 77.33946228027344, |
| "step": 317 |
| }, |
| { |
| "epoch": 0.4669603524229075, |
| "fcm_dpo/beta": 0.010556013323366642, |
| "fcm_dpo/delta": -0.08519938588142395, |
| "fcm_dpo/margin": 64.34666442871094, |
| "fcm_dpo/q_t": 0.351060688495636, |
| "grad_norm": 10.77450180053711, |
| "learning_rate": 3.233383385962115e-07, |
| "logits/chosen": -2.1584458351135254, |
| "logits/rejected": -1.4637277126312256, |
| "logps/chosen": -193.25021362304688, |
| "logps/ref_chosen": -98.20553588867188, |
| "logps/ref_rejected": -88.13629150390625, |
| "logps/rejected": -247.52764892578125, |
| "loss": 0.925, |
| "margin_dpo/margin_mean": 64.34667205810547, |
| "margin_dpo/margin_std": 66.24368286132812, |
| "step": 318 |
| }, |
| { |
| "epoch": 0.4684287812041116, |
| "fcm_dpo/beta": 0.010318400338292122, |
| "fcm_dpo/delta": -0.1146102324128151, |
| "fcm_dpo/margin": 68.42571258544922, |
| "fcm_dpo/q_t": 0.3458014726638794, |
| "grad_norm": 10.647491455078125, |
| "learning_rate": 3.2211050867827805e-07, |
| "logits/chosen": -1.9602619409561157, |
| "logits/rejected": -1.8778321743011475, |
| "logps/chosen": -164.64324951171875, |
| "logps/ref_chosen": -82.50337219238281, |
| "logps/ref_rejected": -118.65068054199219, |
| "logps/rejected": -269.21624755859375, |
| "loss": 0.9143, |
| "margin_dpo/margin_mean": 68.42571258544922, |
| "margin_dpo/margin_std": 70.17927551269531, |
| "step": 319 |
| }, |
| { |
| "epoch": 0.4698972099853157, |
| "fcm_dpo/beta": 0.010274864733219147, |
| "fcm_dpo/delta": 0.008025115355849266, |
| "fcm_dpo/margin": 57.651084899902344, |
| "fcm_dpo/q_t": 0.369371235370636, |
| "grad_norm": 10.72866439819336, |
| "learning_rate": 3.208807785813777e-07, |
| "logits/chosen": -2.084186553955078, |
| "logits/rejected": -1.8974592685699463, |
| "logps/chosen": -172.93923950195312, |
| "logps/ref_chosen": -87.13722229003906, |
| "logps/ref_rejected": -103.22412109375, |
| "logps/rejected": -246.67721557617188, |
| "loss": 0.974, |
| "margin_dpo/margin_mean": 57.651084899902344, |
| "margin_dpo/margin_std": 64.01679992675781, |
| "step": 320 |
| }, |
| { |
| "epoch": 0.4713656387665198, |
| "fcm_dpo/beta": 0.010156042873859406, |
| "fcm_dpo/delta": -0.06489168107509613, |
| "fcm_dpo/margin": 64.97313690185547, |
| "fcm_dpo/q_t": 0.3531816005706787, |
| "grad_norm": 10.770258903503418, |
| "learning_rate": 3.1964918071004217e-07, |
| "logits/chosen": -2.1117072105407715, |
| "logits/rejected": -1.7369625568389893, |
| "logps/chosen": -188.44654846191406, |
| "logps/ref_chosen": -93.10466766357422, |
| "logps/ref_rejected": -97.4681396484375, |
| "logps/rejected": -257.78314208984375, |
| "loss": 0.9341, |
| "margin_dpo/margin_mean": 64.97313690185547, |
| "margin_dpo/margin_std": 66.21844482421875, |
| "step": 321 |
| }, |
| { |
| "epoch": 0.47283406754772395, |
| "fcm_dpo/beta": 0.009905429556965828, |
| "fcm_dpo/delta": -0.12228944152593613, |
| "fcm_dpo/margin": 71.8099594116211, |
| "fcm_dpo/q_t": 0.3459140360355377, |
| "grad_norm": 13.35485553741455, |
| "learning_rate": 3.184157475180207e-07, |
| "logits/chosen": -1.7865667343139648, |
| "logits/rejected": -1.4356210231781006, |
| "logps/chosen": -182.97547912597656, |
| "logps/ref_chosen": -92.19291687011719, |
| "logps/ref_rejected": -100.89321899414062, |
| "logps/rejected": -263.4857177734375, |
| "loss": 0.9237, |
| "margin_dpo/margin_mean": 71.8099594116211, |
| "margin_dpo/margin_std": 75.52076721191406, |
| "step": 322 |
| }, |
| { |
| "epoch": 0.47430249632892807, |
| "fcm_dpo/beta": 0.009861964732408524, |
| "fcm_dpo/delta": -0.021682463586330414, |
| "fcm_dpo/margin": 62.875831604003906, |
| "fcm_dpo/q_t": 0.3650767207145691, |
| "grad_norm": 10.933591842651367, |
| "learning_rate": 3.171805115074251e-07, |
| "logits/chosen": -2.0441133975982666, |
| "logits/rejected": -1.6695351600646973, |
| "logps/chosen": -184.3953857421875, |
| "logps/ref_chosen": -80.27210235595703, |
| "logps/ref_rejected": -80.3033447265625, |
| "logps/rejected": -247.302490234375, |
| "loss": 0.9606, |
| "margin_dpo/margin_mean": 62.875831604003906, |
| "margin_dpo/margin_std": 68.70439147949219, |
| "step": 323 |
| }, |
| { |
| "epoch": 0.47577092511013214, |
| "fcm_dpo/beta": 0.009814316406846046, |
| "fcm_dpo/delta": -0.05098802596330643, |
| "fcm_dpo/margin": 65.9963150024414, |
| "fcm_dpo/q_t": 0.3542330861091614, |
| "grad_norm": 10.099251747131348, |
| "learning_rate": 3.1594350522787295e-07, |
| "logits/chosen": -1.5629843473434448, |
| "logits/rejected": -0.8973981142044067, |
| "logps/chosen": -200.52267456054688, |
| "logps/ref_chosen": -95.72935485839844, |
| "logps/ref_rejected": -94.16924285888672, |
| "logps/rejected": -264.9588623046875, |
| "loss": 0.9235, |
| "margin_dpo/margin_mean": 65.9963150024414, |
| "margin_dpo/margin_std": 62.014060974121094, |
| "step": 324 |
| }, |
| { |
| "epoch": 0.47723935389133626, |
| "fcm_dpo/beta": 0.009698811918497086, |
| "fcm_dpo/delta": -0.06353026628494263, |
| "fcm_dpo/margin": 67.9854965209961, |
| "fcm_dpo/q_t": 0.3524365723133087, |
| "grad_norm": 8.840824127197266, |
| "learning_rate": 3.147047612756302e-07, |
| "logits/chosen": -1.7049955129623413, |
| "logits/rejected": -0.977316677570343, |
| "logps/chosen": -196.92950439453125, |
| "logps/ref_chosen": -98.70686340332031, |
| "logps/ref_rejected": -76.40809631347656, |
| "logps/rejected": -242.61624145507812, |
| "loss": 0.9246, |
| "margin_dpo/margin_mean": 67.9854965209961, |
| "margin_dpo/margin_std": 65.83146667480469, |
| "step": 325 |
| }, |
| { |
| "epoch": 0.4787077826725404, |
| "fcm_dpo/beta": 0.009657101705670357, |
| "fcm_dpo/delta": 0.0018943261820822954, |
| "fcm_dpo/margin": 61.94574737548828, |
| "fcm_dpo/q_t": 0.36700719594955444, |
| "grad_norm": 10.065813064575195, |
| "learning_rate": 3.134643122927519e-07, |
| "logits/chosen": -2.4169249534606934, |
| "logits/rejected": -1.762148380279541, |
| "logps/chosen": -214.70034790039062, |
| "logps/ref_chosen": -104.66130828857422, |
| "logps/ref_rejected": -86.82537841796875, |
| "logps/rejected": -258.8101806640625, |
| "loss": 0.9749, |
| "margin_dpo/margin_mean": 61.94574737548828, |
| "margin_dpo/margin_std": 68.8028793334961, |
| "step": 326 |
| }, |
| { |
| "epoch": 0.4801762114537445, |
| "fcm_dpo/beta": 0.00965862162411213, |
| "fcm_dpo/delta": 0.015111330896615982, |
| "fcm_dpo/margin": 60.6379508972168, |
| "fcm_dpo/q_t": 0.3714819848537445, |
| "grad_norm": 10.074041366577148, |
| "learning_rate": 3.1222219096622264e-07, |
| "logits/chosen": -1.8287179470062256, |
| "logits/rejected": -1.238955020904541, |
| "logps/chosen": -213.91966247558594, |
| "logps/ref_chosen": -107.44023895263672, |
| "logps/ref_rejected": -117.05323028564453, |
| "logps/rejected": -284.17059326171875, |
| "loss": 0.9792, |
| "margin_dpo/margin_mean": 60.6379508972168, |
| "margin_dpo/margin_std": 67.89471435546875, |
| "step": 327 |
| }, |
| { |
| "epoch": 0.48164464023494863, |
| "fcm_dpo/beta": 0.0094821248203516, |
| "fcm_dpo/delta": -0.14909884333610535, |
| "fcm_dpo/margin": 77.84373474121094, |
| "fcm_dpo/q_t": 0.33735549449920654, |
| "grad_norm": 12.075971603393555, |
| "learning_rate": 3.1097843002709427e-07, |
| "logits/chosen": -1.6724544763565063, |
| "logits/rejected": -1.3806228637695312, |
| "logps/chosen": -192.94517517089844, |
| "logps/ref_chosen": -86.6106948852539, |
| "logps/ref_rejected": -97.40188598632812, |
| "logps/rejected": -281.5801086425781, |
| "loss": 0.8778, |
| "margin_dpo/margin_mean": 77.84373474121094, |
| "margin_dpo/margin_std": 71.33271789550781, |
| "step": 328 |
| }, |
| { |
| "epoch": 0.4831130690161527, |
| "fcm_dpo/beta": 0.0093916654586792, |
| "fcm_dpo/delta": 0.019437361508607864, |
| "fcm_dpo/margin": 61.90422058105469, |
| "fcm_dpo/q_t": 0.371404230594635, |
| "grad_norm": 11.608654975891113, |
| "learning_rate": 3.0973306224962437e-07, |
| "logits/chosen": -1.5943433046340942, |
| "logits/rejected": -1.0977287292480469, |
| "logps/chosen": -212.25148010253906, |
| "logps/ref_chosen": -100.61308288574219, |
| "logps/ref_rejected": -104.95370483398438, |
| "logps/rejected": -278.496337890625, |
| "loss": 0.9812, |
| "margin_dpo/margin_mean": 61.90422058105469, |
| "margin_dpo/margin_std": 68.27637481689453, |
| "step": 329 |
| }, |
| { |
| "epoch": 0.4845814977973568, |
| "fcm_dpo/beta": 0.00939023308455944, |
| "fcm_dpo/delta": -0.020257247611880302, |
| "fcm_dpo/margin": 65.90046691894531, |
| "fcm_dpo/q_t": 0.3579813539981842, |
| "grad_norm": 9.650596618652344, |
| "learning_rate": 3.084861204504122e-07, |
| "logits/chosen": -2.022364377975464, |
| "logits/rejected": -1.9466552734375, |
| "logps/chosen": -175.4729766845703, |
| "logps/ref_chosen": -76.95843505859375, |
| "logps/ref_rejected": -92.81494140625, |
| "logps/rejected": -257.22991943359375, |
| "loss": 0.9311, |
| "margin_dpo/margin_mean": 65.90046691894531, |
| "margin_dpo/margin_std": 57.93687438964844, |
| "step": 330 |
| }, |
| { |
| "epoch": 0.48604992657856094, |
| "fcm_dpo/beta": 0.009334873408079147, |
| "fcm_dpo/delta": -0.08815296739339828, |
| "fcm_dpo/margin": 73.05988311767578, |
| "fcm_dpo/q_t": 0.34697699546813965, |
| "grad_norm": 12.887969970703125, |
| "learning_rate": 3.072376374875335e-07, |
| "logits/chosen": -1.7896149158477783, |
| "logits/rejected": -1.4816310405731201, |
| "logps/chosen": -188.48745727539062, |
| "logps/ref_chosen": -83.05941772460938, |
| "logps/ref_rejected": -94.13113403320312, |
| "logps/rejected": -272.6190490722656, |
| "loss": 0.8953, |
| "margin_dpo/margin_mean": 73.05988311767578, |
| "margin_dpo/margin_std": 62.86804962158203, |
| "step": 331 |
| }, |
| { |
| "epoch": 0.48751835535976507, |
| "fcm_dpo/beta": 0.00939233973622322, |
| "fcm_dpo/delta": 0.1038806289434433, |
| "fcm_dpo/margin": 53.36603927612305, |
| "fcm_dpo/q_t": 0.3885546326637268, |
| "grad_norm": 9.494643211364746, |
| "learning_rate": 3.059876462596758e-07, |
| "logits/chosen": -2.1970367431640625, |
| "logits/rejected": -1.778074026107788, |
| "logps/chosen": -191.7408447265625, |
| "logps/ref_chosen": -81.93089294433594, |
| "logps/ref_rejected": -82.84590148925781, |
| "logps/rejected": -246.02191162109375, |
| "loss": 1.0397, |
| "margin_dpo/margin_mean": 53.36603546142578, |
| "margin_dpo/margin_std": 68.24900817871094, |
| "step": 332 |
| }, |
| { |
| "epoch": 0.4889867841409692, |
| "fcm_dpo/beta": 0.009271856397390366, |
| "fcm_dpo/delta": -0.06427404284477234, |
| "fcm_dpo/margin": 71.00698852539062, |
| "fcm_dpo/q_t": 0.35842740535736084, |
| "grad_norm": 10.024383544921875, |
| "learning_rate": 3.0473617970527015e-07, |
| "logits/chosen": -1.5461020469665527, |
| "logits/rejected": -1.212287425994873, |
| "logps/chosen": -216.07131958007812, |
| "logps/ref_chosen": -88.53257751464844, |
| "logps/ref_rejected": -102.46601867675781, |
| "logps/rejected": -301.01171875, |
| "loss": 0.968, |
| "margin_dpo/margin_mean": 71.00698852539062, |
| "margin_dpo/margin_std": 83.59139251708984, |
| "step": 333 |
| }, |
| { |
| "epoch": 0.49045521292217326, |
| "fcm_dpo/beta": 0.009264590218663216, |
| "fcm_dpo/delta": 0.004638519138097763, |
| "fcm_dpo/margin": 64.21815490722656, |
| "fcm_dpo/q_t": 0.3684922456741333, |
| "grad_norm": 11.035482406616211, |
| "learning_rate": 3.034832708016243e-07, |
| "logits/chosen": -1.366419792175293, |
| "logits/rejected": -1.0714776515960693, |
| "logps/chosen": -213.7954864501953, |
| "logps/ref_chosen": -97.75778198242188, |
| "logps/ref_rejected": -100.0058364868164, |
| "logps/rejected": -280.2616882324219, |
| "loss": 1.0037, |
| "margin_dpo/margin_mean": 64.21814727783203, |
| "margin_dpo/margin_std": 79.95184326171875, |
| "step": 334 |
| }, |
| { |
| "epoch": 0.4919236417033774, |
| "fcm_dpo/beta": 0.009400870651006699, |
| "fcm_dpo/delta": 0.05613856390118599, |
| "fcm_dpo/margin": 58.17888641357422, |
| "fcm_dpo/q_t": 0.38059741258621216, |
| "grad_norm": 11.443718910217285, |
| "learning_rate": 3.022289525640531e-07, |
| "logits/chosen": -2.1963934898376465, |
| "logits/rejected": -1.7753444910049438, |
| "logps/chosen": -212.1436767578125, |
| "logps/ref_chosen": -96.37603759765625, |
| "logps/ref_rejected": -94.92401123046875, |
| "logps/rejected": -268.87054443359375, |
| "loss": 1.0258, |
| "margin_dpo/margin_mean": 58.17888641357422, |
| "margin_dpo/margin_std": 75.49417114257812, |
| "step": 335 |
| }, |
| { |
| "epoch": 0.4933920704845815, |
| "fcm_dpo/beta": 0.009332764893770218, |
| "fcm_dpo/delta": -0.08782423287630081, |
| "fcm_dpo/margin": 73.06648254394531, |
| "fcm_dpo/q_t": 0.3505851626396179, |
| "grad_norm": 11.681970596313477, |
| "learning_rate": 3.009732580450086e-07, |
| "logits/chosen": -1.6232037544250488, |
| "logits/rejected": -1.4630122184753418, |
| "logps/chosen": -203.955078125, |
| "logps/ref_chosen": -88.52952575683594, |
| "logps/ref_rejected": -109.5557861328125, |
| "logps/rejected": -298.04779052734375, |
| "loss": 0.9236, |
| "margin_dpo/margin_mean": 73.06648254394531, |
| "margin_dpo/margin_std": 76.55876159667969, |
| "step": 336 |
| }, |
| { |
| "epoch": 0.4948604992657856, |
| "fcm_dpo/beta": 0.009262502193450928, |
| "fcm_dpo/delta": -0.02163381688296795, |
| "fcm_dpo/margin": 66.955322265625, |
| "fcm_dpo/q_t": 0.36714088916778564, |
| "grad_norm": 12.159340858459473, |
| "learning_rate": 2.9971622033320914e-07, |
| "logits/chosen": -2.4223062992095947, |
| "logits/rejected": -2.1396877765655518, |
| "logps/chosen": -213.5079345703125, |
| "logps/ref_chosen": -97.3123779296875, |
| "logps/ref_rejected": -110.13134765625, |
| "logps/rejected": -293.2822265625, |
| "loss": 0.9977, |
| "margin_dpo/margin_mean": 66.955322265625, |
| "margin_dpo/margin_std": 85.39122772216797, |
| "step": 337 |
| }, |
| { |
| "epoch": 0.49632892804698975, |
| "fcm_dpo/beta": 0.00921336468309164, |
| "fcm_dpo/delta": 0.004760187119245529, |
| "fcm_dpo/margin": 64.63055419921875, |
| "fcm_dpo/q_t": 0.36690112948417664, |
| "grad_norm": 10.842123985290527, |
| "learning_rate": 2.984578725527675e-07, |
| "logits/chosen": -1.7986483573913574, |
| "logits/rejected": -1.6366183757781982, |
| "logps/chosen": -202.72119140625, |
| "logps/ref_chosen": -83.63011169433594, |
| "logps/ref_rejected": -93.69235229492188, |
| "logps/rejected": -277.41400146484375, |
| "loss": 0.9782, |
| "margin_dpo/margin_mean": 64.63055419921875, |
| "margin_dpo/margin_std": 72.1498794555664, |
| "step": 338 |
| }, |
| { |
| "epoch": 0.4977973568281938, |
| "fcm_dpo/beta": 0.009260453283786774, |
| "fcm_dpo/delta": -0.030668731778860092, |
| "fcm_dpo/margin": 67.80081939697266, |
| "fcm_dpo/q_t": 0.3631947636604309, |
| "grad_norm": 10.40760612487793, |
| "learning_rate": 2.9719824786231796e-07, |
| "logits/chosen": -3.007730484008789, |
| "logits/rejected": -2.590148448944092, |
| "logps/chosen": -223.9995880126953, |
| "logps/ref_chosen": -99.85182189941406, |
| "logps/ref_rejected": -109.0743637084961, |
| "logps/rejected": -301.02294921875, |
| "loss": 0.9867, |
| "margin_dpo/margin_mean": 67.80081939697266, |
| "margin_dpo/margin_std": 81.80381774902344, |
| "step": 339 |
| }, |
| { |
| "epoch": 0.49926578560939794, |
| "fcm_dpo/beta": 0.009243492037057877, |
| "fcm_dpo/delta": 0.04011049121618271, |
| "fcm_dpo/margin": 60.805809020996094, |
| "fcm_dpo/q_t": 0.37543347477912903, |
| "grad_norm": 12.52755355834961, |
| "learning_rate": 2.959373794541426e-07, |
| "logits/chosen": -1.4719023704528809, |
| "logits/rejected": -0.9998151659965515, |
| "logps/chosen": -218.23899841308594, |
| "logps/ref_chosen": -84.97396850585938, |
| "logps/ref_rejected": -78.80033874511719, |
| "logps/rejected": -272.87115478515625, |
| "loss": 1.0069, |
| "margin_dpo/margin_mean": 60.805809020996094, |
| "margin_dpo/margin_std": 75.30453491210938, |
| "step": 340 |
| }, |
| { |
| "epoch": 0.5007342143906021, |
| "fcm_dpo/beta": 0.009315932169556618, |
| "fcm_dpo/delta": 0.06518840789794922, |
| "fcm_dpo/margin": 57.787986755371094, |
| "fcm_dpo/q_t": 0.383889377117157, |
| "grad_norm": 12.319915771484375, |
| "learning_rate": 2.946753005532965e-07, |
| "logits/chosen": -1.5798420906066895, |
| "logits/rejected": -1.3175228834152222, |
| "logps/chosen": -224.97544860839844, |
| "logps/ref_chosen": -85.26399230957031, |
| "logps/ref_rejected": -106.08595275878906, |
| "logps/rejected": -303.58538818359375, |
| "loss": 1.0502, |
| "margin_dpo/margin_mean": 57.787986755371094, |
| "margin_dpo/margin_std": 81.41255187988281, |
| "step": 341 |
| }, |
| { |
| "epoch": 0.5022026431718062, |
| "fcm_dpo/beta": 0.009050893597304821, |
| "fcm_dpo/delta": -0.1713675707578659, |
| "fcm_dpo/margin": 83.44084167480469, |
| "fcm_dpo/q_t": 0.33266481757164, |
| "grad_norm": 10.313692092895508, |
| "learning_rate": 2.934120444167326e-07, |
| "logits/chosen": -1.3843135833740234, |
| "logits/rejected": -0.633280336856842, |
| "logps/chosen": -212.62014770507812, |
| "logps/ref_chosen": -101.03860473632812, |
| "logps/ref_rejected": -98.89851379394531, |
| "logps/rejected": -293.9208984375, |
| "loss": 0.8903, |
| "margin_dpo/margin_mean": 83.44084167480469, |
| "margin_dpo/margin_std": 78.70523834228516, |
| "step": 342 |
| }, |
| { |
| "epoch": 0.5036710719530103, |
| "fcm_dpo/beta": 0.009080128744244576, |
| "fcm_dpo/delta": -0.036977700889110565, |
| "fcm_dpo/margin": 69.78305053710938, |
| "fcm_dpo/q_t": 0.3612483739852905, |
| "grad_norm": 12.353619575500488, |
| "learning_rate": 2.9214764433242476e-07, |
| "logits/chosen": -2.0943527221679688, |
| "logits/rejected": -1.9702728986740112, |
| "logps/chosen": -203.23928833007812, |
| "logps/ref_chosen": -83.74736022949219, |
| "logps/ref_rejected": -116.57908630371094, |
| "logps/rejected": -305.85406494140625, |
| "loss": 0.9632, |
| "margin_dpo/margin_mean": 69.78305053710938, |
| "margin_dpo/margin_std": 76.31233215332031, |
| "step": 343 |
| }, |
| { |
| "epoch": 0.5051395007342144, |
| "fcm_dpo/beta": 0.008997892960906029, |
| "fcm_dpo/delta": -0.030076302587985992, |
| "fcm_dpo/margin": 69.76876068115234, |
| "fcm_dpo/q_t": 0.3652355372905731, |
| "grad_norm": 10.976330757141113, |
| "learning_rate": 2.9088213361849126e-07, |
| "logits/chosen": -1.680814504623413, |
| "logits/rejected": -1.425823450088501, |
| "logps/chosen": -202.2406768798828, |
| "logps/ref_chosen": -78.25396728515625, |
| "logps/ref_rejected": -95.42870330810547, |
| "logps/rejected": -289.1842041015625, |
| "loss": 0.9842, |
| "margin_dpo/margin_mean": 69.76876068115234, |
| "margin_dpo/margin_std": 85.37368774414062, |
| "step": 344 |
| }, |
| { |
| "epoch": 0.5066079295154186, |
| "fcm_dpo/beta": 0.008805211633443832, |
| "fcm_dpo/delta": -0.08041529357433319, |
| "fcm_dpo/margin": 76.61846160888672, |
| "fcm_dpo/q_t": 0.3534577190876007, |
| "grad_norm": 11.980306625366211, |
| "learning_rate": 2.896155456223163e-07, |
| "logits/chosen": -1.5205774307250977, |
| "logits/rejected": -1.2206867933273315, |
| "logps/chosen": -221.85317993164062, |
| "logps/ref_chosen": -98.0794906616211, |
| "logps/ref_rejected": -104.03477478027344, |
| "logps/rejected": -304.42694091796875, |
| "loss": 0.9355, |
| "margin_dpo/margin_mean": 76.61846923828125, |
| "margin_dpo/margin_std": 82.15892028808594, |
| "step": 345 |
| }, |
| { |
| "epoch": 0.5080763582966226, |
| "fcm_dpo/beta": 0.008871208876371384, |
| "fcm_dpo/delta": 0.0582590326666832, |
| "fcm_dpo/margin": 61.423545837402344, |
| "fcm_dpo/q_t": 0.3825676441192627, |
| "grad_norm": 10.243391036987305, |
| "learning_rate": 2.883479137196714e-07, |
| "logits/chosen": -1.9583630561828613, |
| "logits/rejected": -1.651193618774414, |
| "logps/chosen": -219.33963012695312, |
| "logps/ref_chosen": -80.68348693847656, |
| "logps/ref_rejected": -85.89260864257812, |
| "logps/rejected": -285.9722900390625, |
| "loss": 1.0424, |
| "margin_dpo/margin_mean": 61.423545837402344, |
| "margin_dpo/margin_std": 86.02749633789062, |
| "step": 346 |
| }, |
| { |
| "epoch": 0.5095447870778267, |
| "fcm_dpo/beta": 0.009024476632475853, |
| "fcm_dpo/delta": 0.0737469345331192, |
| "fcm_dpo/margin": 58.69518280029297, |
| "fcm_dpo/q_t": 0.385664701461792, |
| "grad_norm": 11.432808876037598, |
| "learning_rate": 2.8707927131383614e-07, |
| "logits/chosen": -1.6128592491149902, |
| "logits/rejected": -1.2691833972930908, |
| "logps/chosen": -223.65142822265625, |
| "logps/ref_chosen": -93.67938232421875, |
| "logps/ref_rejected": -97.54251098632812, |
| "logps/rejected": -286.209716796875, |
| "loss": 1.047, |
| "margin_dpo/margin_mean": 58.69518280029297, |
| "margin_dpo/margin_std": 81.67510986328125, |
| "step": 347 |
| }, |
| { |
| "epoch": 0.5110132158590308, |
| "fcm_dpo/beta": 0.008929546922445297, |
| "fcm_dpo/delta": -0.0036687031388282776, |
| "fcm_dpo/margin": 67.33739471435547, |
| "fcm_dpo/q_t": 0.3671216368675232, |
| "grad_norm": 9.5287446975708, |
| "learning_rate": 2.858096518347179e-07, |
| "logits/chosen": -1.8974089622497559, |
| "logits/rejected": -1.6134772300720215, |
| "logps/chosen": -202.92189025878906, |
| "logps/ref_chosen": -90.25224304199219, |
| "logps/ref_rejected": -95.77027893066406, |
| "logps/rejected": -275.7773132324219, |
| "loss": 0.9868, |
| "margin_dpo/margin_mean": 67.33739471435547, |
| "margin_dpo/margin_std": 76.04698181152344, |
| "step": 348 |
| }, |
| { |
| "epoch": 0.5124816446402349, |
| "fcm_dpo/beta": 0.009099038317799568, |
| "fcm_dpo/delta": 0.019386742264032364, |
| "fcm_dpo/margin": 63.853363037109375, |
| "fcm_dpo/q_t": 0.3738081157207489, |
| "grad_norm": 11.725251197814941, |
| "learning_rate": 2.845390887379706e-07, |
| "logits/chosen": -1.996931552886963, |
| "logits/rejected": -1.7708909511566162, |
| "logps/chosen": -214.01986694335938, |
| "logps/ref_chosen": -92.05809020996094, |
| "logps/ref_rejected": -103.96527099609375, |
| "logps/rejected": -289.7804260253906, |
| "loss": 1.0031, |
| "margin_dpo/margin_mean": 63.853363037109375, |
| "margin_dpo/margin_std": 78.11573791503906, |
| "step": 349 |
| }, |
| { |
| "epoch": 0.5139500734214391, |
| "fcm_dpo/beta": 0.009226538240909576, |
| "fcm_dpo/delta": 0.1344294250011444, |
| "fcm_dpo/margin": 51.163047790527344, |
| "fcm_dpo/q_t": 0.39684417843818665, |
| "grad_norm": 12.850371360778809, |
| "learning_rate": 2.8326761550411346e-07, |
| "logits/chosen": -1.785254955291748, |
| "logits/rejected": -1.556738018989563, |
| "logps/chosen": -219.18960571289062, |
| "logps/ref_chosen": -87.05419921875, |
| "logps/ref_rejected": -95.72193145751953, |
| "logps/rejected": -279.0203857421875, |
| "loss": 1.0736, |
| "margin_dpo/margin_mean": 51.16304397583008, |
| "margin_dpo/margin_std": 73.25152587890625, |
| "step": 350 |
| }, |
| { |
| "epoch": 0.5154185022026432, |
| "fcm_dpo/beta": 0.009196523576974869, |
| "fcm_dpo/delta": -0.1071363314986229, |
| "fcm_dpo/margin": 76.08428192138672, |
| "fcm_dpo/q_t": 0.34812554717063904, |
| "grad_norm": 11.473617553710938, |
| "learning_rate": 2.819952656376487e-07, |
| "logits/chosen": -1.5400876998901367, |
| "logits/rejected": -1.2160134315490723, |
| "logps/chosen": -218.47886657714844, |
| "logps/ref_chosen": -89.16903686523438, |
| "logps/ref_rejected": -108.29801940917969, |
| "logps/rejected": -313.692138671875, |
| "loss": 0.9389, |
| "margin_dpo/margin_mean": 76.08428192138672, |
| "margin_dpo/margin_std": 84.34982299804688, |
| "step": 351 |
| }, |
| { |
| "epoch": 0.5168869309838473, |
| "fcm_dpo/beta": 0.009029903449118137, |
| "fcm_dpo/delta": -0.0580860860645771, |
| "fcm_dpo/margin": 72.45895385742188, |
| "fcm_dpo/q_t": 0.3607955574989319, |
| "grad_norm": 11.246063232421875, |
| "learning_rate": 2.8072207266617854e-07, |
| "logits/chosen": -1.9249012470245361, |
| "logits/rejected": -1.3872631788253784, |
| "logps/chosen": -215.51181030273438, |
| "logps/ref_chosen": -96.81147766113281, |
| "logps/ref_rejected": -84.40483093261719, |
| "logps/rejected": -275.5640869140625, |
| "loss": 0.9591, |
| "margin_dpo/margin_mean": 72.45895385742188, |
| "margin_dpo/margin_std": 83.17478942871094, |
| "step": 352 |
| }, |
| { |
| "epoch": 0.5183553597650514, |
| "fcm_dpo/beta": 0.009232399985194206, |
| "fcm_dpo/delta": 0.1866103559732437, |
| "fcm_dpo/margin": 45.66265869140625, |
| "fcm_dpo/q_t": 0.40671753883361816, |
| "grad_norm": 12.68930721282959, |
| "learning_rate": 2.794480701395219e-07, |
| "logits/chosen": -1.9192724227905273, |
| "logits/rejected": -1.527640700340271, |
| "logps/chosen": -221.19326782226562, |
| "logps/ref_chosen": -85.36248779296875, |
| "logps/ref_rejected": -87.36732482910156, |
| "logps/rejected": -268.86077880859375, |
| "loss": 1.1274, |
| "margin_dpo/margin_mean": 45.66265869140625, |
| "margin_dpo/margin_std": 76.734375, |
| "step": 353 |
| }, |
| { |
| "epoch": 0.5198237885462555, |
| "fcm_dpo/beta": 0.009293053299188614, |
| "fcm_dpo/delta": -0.0588521845638752, |
| "fcm_dpo/margin": 70.47891235351562, |
| "fcm_dpo/q_t": 0.35432109236717224, |
| "grad_norm": 10.813716888427734, |
| "learning_rate": 2.781732916288303e-07, |
| "logits/chosen": -1.8877973556518555, |
| "logits/rejected": -1.4194233417510986, |
| "logps/chosen": -215.95494079589844, |
| "logps/ref_chosen": -95.8011703491211, |
| "logps/ref_rejected": -94.86614227294922, |
| "logps/rejected": -285.49884033203125, |
| "loss": 0.9172, |
| "margin_dpo/margin_mean": 70.47891235351562, |
| "margin_dpo/margin_std": 65.81661987304688, |
| "step": 354 |
| }, |
| { |
| "epoch": 0.5212922173274597, |
| "fcm_dpo/beta": 0.009149078279733658, |
| "fcm_dpo/delta": -0.0197719968855381, |
| "fcm_dpo/margin": 67.51188659667969, |
| "fcm_dpo/q_t": 0.3642021119594574, |
| "grad_norm": 11.531266212463379, |
| "learning_rate": 2.7689777072570284e-07, |
| "logits/chosen": -2.4402689933776855, |
| "logits/rejected": -1.9639763832092285, |
| "logps/chosen": -205.29983520507812, |
| "logps/ref_chosen": -86.185546875, |
| "logps/ref_rejected": -88.4615707397461, |
| "logps/rejected": -275.0877685546875, |
| "loss": 0.9754, |
| "margin_dpo/margin_mean": 67.51187896728516, |
| "margin_dpo/margin_std": 75.14073181152344, |
| "step": 355 |
| }, |
| { |
| "epoch": 0.5227606461086637, |
| "fcm_dpo/beta": 0.009418745525181293, |
| "fcm_dpo/delta": 0.13494420051574707, |
| "fcm_dpo/margin": 49.970237731933594, |
| "fcm_dpo/q_t": 0.3972604274749756, |
| "grad_norm": 13.176185607910156, |
| "learning_rate": 2.7562154104130176e-07, |
| "logits/chosen": -1.8549089431762695, |
| "logits/rejected": -1.394954800605774, |
| "logps/chosen": -231.3189697265625, |
| "logps/ref_chosen": -90.818115234375, |
| "logps/ref_rejected": -82.1273193359375, |
| "logps/rejected": -272.5984191894531, |
| "loss": 1.1045, |
| "margin_dpo/margin_mean": 49.970237731933594, |
| "margin_dpo/margin_std": 81.81539916992188, |
| "step": 356 |
| }, |
| { |
| "epoch": 0.5242290748898678, |
| "fcm_dpo/beta": 0.009089802391827106, |
| "fcm_dpo/delta": -0.18602949380874634, |
| "fcm_dpo/margin": 84.42666625976562, |
| "fcm_dpo/q_t": 0.3301994502544403, |
| "grad_norm": 9.98563003540039, |
| "learning_rate": 2.7434463620546594e-07, |
| "logits/chosen": -2.099191188812256, |
| "logits/rejected": -1.7402398586273193, |
| "logps/chosen": -202.91748046875, |
| "logps/ref_chosen": -87.5047607421875, |
| "logps/ref_rejected": -92.24937438964844, |
| "logps/rejected": -292.0887756347656, |
| "loss": 0.861, |
| "margin_dpo/margin_mean": 84.42666625976562, |
| "margin_dpo/margin_std": 71.71357727050781, |
| "step": 357 |
| }, |
| { |
| "epoch": 0.5256975036710719, |
| "fcm_dpo/beta": 0.009189213626086712, |
| "fcm_dpo/delta": 0.059452179819345474, |
| "fcm_dpo/margin": 59.17223358154297, |
| "fcm_dpo/q_t": 0.3846597373485565, |
| "grad_norm": 11.328146934509277, |
| "learning_rate": 2.730670898658255e-07, |
| "logits/chosen": -1.47278892993927, |
| "logits/rejected": -0.9826061725616455, |
| "logps/chosen": -225.228271484375, |
| "logps/ref_chosen": -95.05546569824219, |
| "logps/ref_rejected": -96.47611999511719, |
| "logps/rejected": -285.8211669921875, |
| "loss": 1.0297, |
| "margin_dpo/margin_mean": 59.17223358154297, |
| "margin_dpo/margin_std": 79.3198471069336, |
| "step": 358 |
| }, |
| { |
| "epoch": 0.527165932452276, |
| "fcm_dpo/beta": 0.009268088266253471, |
| "fcm_dpo/delta": 0.09260153770446777, |
| "fcm_dpo/margin": 55.203125, |
| "fcm_dpo/q_t": 0.39258408546447754, |
| "grad_norm": 11.787094116210938, |
| "learning_rate": 2.717889356869146e-07, |
| "logits/chosen": -1.5292563438415527, |
| "logits/rejected": -0.9679604768753052, |
| "logps/chosen": -230.56655883789062, |
| "logps/ref_chosen": -87.37332916259766, |
| "logps/ref_rejected": -88.77188110351562, |
| "logps/rejected": -287.1682434082031, |
| "loss": 1.0731, |
| "margin_dpo/margin_mean": 55.203128814697266, |
| "margin_dpo/margin_std": 82.28927612304688, |
| "step": 359 |
| }, |
| { |
| "epoch": 0.5286343612334802, |
| "fcm_dpo/beta": 0.009283397346735, |
| "fcm_dpo/delta": -0.06985802948474884, |
| "fcm_dpo/margin": 71.65193176269531, |
| "fcm_dpo/q_t": 0.35196417570114136, |
| "grad_norm": 14.178664207458496, |
| "learning_rate": 2.7051020734928443e-07, |
| "logits/chosen": -1.6124732494354248, |
| "logits/rejected": -1.2661912441253662, |
| "logps/chosen": -209.75942993164062, |
| "logps/ref_chosen": -75.22344970703125, |
| "logps/ref_rejected": -76.09400177001953, |
| "logps/rejected": -282.28192138671875, |
| "loss": 0.9315, |
| "margin_dpo/margin_mean": 71.65193176269531, |
| "margin_dpo/margin_std": 73.52704620361328, |
| "step": 360 |
| }, |
| { |
| "epoch": 0.5301027900146843, |
| "fcm_dpo/beta": 0.009296550415456295, |
| "fcm_dpo/delta": 0.019596338272094727, |
| "fcm_dpo/margin": 62.54603958129883, |
| "fcm_dpo/q_t": 0.37713533639907837, |
| "grad_norm": 12.717004776000977, |
| "learning_rate": 2.6923093854861593e-07, |
| "logits/chosen": -1.715510606765747, |
| "logits/rejected": -1.4632185697555542, |
| "logps/chosen": -218.74740600585938, |
| "logps/ref_chosen": -81.95333099365234, |
| "logps/ref_rejected": -96.67919921875, |
| "logps/rejected": -296.0193176269531, |
| "loss": 1.0228, |
| "margin_dpo/margin_mean": 62.54604721069336, |
| "margin_dpo/margin_std": 84.71768188476562, |
| "step": 361 |
| }, |
| { |
| "epoch": 0.5315712187958884, |
| "fcm_dpo/beta": 0.009247587993741035, |
| "fcm_dpo/delta": -0.015775587409734726, |
| "fcm_dpo/margin": 66.46771240234375, |
| "fcm_dpo/q_t": 0.3636864423751831, |
| "grad_norm": 11.942319869995117, |
| "learning_rate": 2.679511629948319e-07, |
| "logits/chosen": -1.6847763061523438, |
| "logits/rejected": -1.467708706855774, |
| "logps/chosen": -230.143310546875, |
| "logps/ref_chosen": -89.33971405029297, |
| "logps/ref_rejected": -110.50196838378906, |
| "logps/rejected": -317.7733154296875, |
| "loss": 0.9741, |
| "margin_dpo/margin_mean": 66.46771240234375, |
| "margin_dpo/margin_std": 74.62181854248047, |
| "step": 362 |
| }, |
| { |
| "epoch": 0.5330396475770925, |
| "fcm_dpo/beta": 0.009058544412255287, |
| "fcm_dpo/delta": -0.18251146376132965, |
| "fcm_dpo/margin": 84.8752670288086, |
| "fcm_dpo/q_t": 0.33618301153182983, |
| "grad_norm": 12.65180778503418, |
| "learning_rate": 2.6667091441120816e-07, |
| "logits/chosen": -1.7909729480743408, |
| "logits/rejected": -1.3521623611450195, |
| "logps/chosen": -214.4105987548828, |
| "logps/ref_chosen": -80.77499389648438, |
| "logps/ref_rejected": -79.43267822265625, |
| "logps/rejected": -297.94354248046875, |
| "loss": 0.8884, |
| "margin_dpo/margin_mean": 84.87525939941406, |
| "margin_dpo/margin_std": 86.01889038085938, |
| "step": 363 |
| }, |
| { |
| "epoch": 0.5345080763582967, |
| "fcm_dpo/beta": 0.008983662351965904, |
| "fcm_dpo/delta": -0.018669523298740387, |
| "fcm_dpo/margin": 68.58975219726562, |
| "fcm_dpo/q_t": 0.36190158128738403, |
| "grad_norm": 13.727071762084961, |
| "learning_rate": 2.6539022653348575e-07, |
| "logits/chosen": -2.0300331115722656, |
| "logits/rejected": -1.877135992050171, |
| "logps/chosen": -209.77499389648438, |
| "logps/ref_chosen": -76.52960205078125, |
| "logps/ref_rejected": -97.56173706054688, |
| "logps/rejected": -299.39691162109375, |
| "loss": 0.9511, |
| "margin_dpo/margin_mean": 68.58975219726562, |
| "margin_dpo/margin_std": 66.92192840576172, |
| "step": 364 |
| }, |
| { |
| "epoch": 0.5359765051395007, |
| "fcm_dpo/beta": 0.008843779563903809, |
| "fcm_dpo/delta": -0.06726966798305511, |
| "fcm_dpo/margin": 74.93986511230469, |
| "fcm_dpo/q_t": 0.35855019092559814, |
| "grad_norm": 10.768917083740234, |
| "learning_rate": 2.641091331089811e-07, |
| "logits/chosen": -1.5261728763580322, |
| "logits/rejected": -1.3805526494979858, |
| "logps/chosen": -210.90951538085938, |
| "logps/ref_chosen": -80.54489135742188, |
| "logps/ref_rejected": -99.32316589355469, |
| "logps/rejected": -304.62762451171875, |
| "loss": 0.9621, |
| "margin_dpo/margin_mean": 74.93986511230469, |
| "margin_dpo/margin_std": 88.24283599853516, |
| "step": 365 |
| }, |
| { |
| "epoch": 0.5374449339207048, |
| "fcm_dpo/beta": 0.008807620033621788, |
| "fcm_dpo/delta": 0.024917863309383392, |
| "fcm_dpo/margin": 65.45736694335938, |
| "fcm_dpo/q_t": 0.37715762853622437, |
| "grad_norm": 10.28955364227295, |
| "learning_rate": 2.6282766789569736e-07, |
| "logits/chosen": -1.706976294517517, |
| "logits/rejected": -1.7277071475982666, |
| "logps/chosen": -201.939697265625, |
| "logps/ref_chosen": -72.23806762695312, |
| "logps/ref_rejected": -91.67182922363281, |
| "logps/rejected": -286.830810546875, |
| "loss": 0.9911, |
| "margin_dpo/margin_mean": 65.45736694335938, |
| "margin_dpo/margin_std": 78.65293884277344, |
| "step": 366 |
| }, |
| { |
| "epoch": 0.5389133627019089, |
| "fcm_dpo/beta": 0.008751322515308857, |
| "fcm_dpo/delta": -0.05687132105231285, |
| "fcm_dpo/margin": 74.63752746582031, |
| "fcm_dpo/q_t": 0.3563315272331238, |
| "grad_norm": 10.187382698059082, |
| "learning_rate": 2.615458646614349e-07, |
| "logits/chosen": -1.7047667503356934, |
| "logits/rejected": -1.3686751127243042, |
| "logps/chosen": -207.76351928710938, |
| "logps/ref_chosen": -85.42201232910156, |
| "logps/ref_rejected": -83.51779174804688, |
| "logps/rejected": -280.496826171875, |
| "loss": 0.9312, |
| "margin_dpo/margin_mean": 74.63752746582031, |
| "margin_dpo/margin_std": 76.12283325195312, |
| "step": 367 |
| }, |
| { |
| "epoch": 0.540381791483113, |
| "fcm_dpo/beta": 0.008638489991426468, |
| "fcm_dpo/delta": -0.10402532666921616, |
| "fcm_dpo/margin": 80.64979553222656, |
| "fcm_dpo/q_t": 0.34605395793914795, |
| "grad_norm": 10.933233261108398, |
| "learning_rate": 2.6026375718290083e-07, |
| "logits/chosen": -1.6838431358337402, |
| "logits/rejected": -1.6408352851867676, |
| "logps/chosen": -189.81283569335938, |
| "logps/ref_chosen": -73.84112548828125, |
| "logps/ref_rejected": -104.69528198242188, |
| "logps/rejected": -301.3167724609375, |
| "loss": 0.9216, |
| "margin_dpo/margin_mean": 80.64979553222656, |
| "margin_dpo/margin_std": 81.0087661743164, |
| "step": 368 |
| }, |
| { |
| "epoch": 0.5418502202643172, |
| "fcm_dpo/beta": 0.008491117507219315, |
| "fcm_dpo/delta": -0.05438908189535141, |
| "fcm_dpo/margin": 76.65316772460938, |
| "fcm_dpo/q_t": 0.35426294803619385, |
| "grad_norm": 11.064504623413086, |
| "learning_rate": 2.589813792448196e-07, |
| "logits/chosen": -1.6709437370300293, |
| "logits/rejected": -1.30281400680542, |
| "logps/chosen": -224.9449920654297, |
| "logps/ref_chosen": -98.46195983886719, |
| "logps/ref_rejected": -99.15672302246094, |
| "logps/rejected": -302.29290771484375, |
| "loss": 0.915, |
| "margin_dpo/margin_mean": 76.65316772460938, |
| "margin_dpo/margin_std": 69.63180541992188, |
| "step": 369 |
| }, |
| { |
| "epoch": 0.5433186490455213, |
| "fcm_dpo/beta": 0.008652754127979279, |
| "fcm_dpo/delta": 0.14852631092071533, |
| "fcm_dpo/margin": 52.930091857910156, |
| "fcm_dpo/q_t": 0.3984871506690979, |
| "grad_norm": 11.095954895019531, |
| "learning_rate": 2.5769876463904263e-07, |
| "logits/chosen": -1.8884553909301758, |
| "logits/rejected": -1.686102032661438, |
| "logps/chosen": -236.5087432861328, |
| "logps/ref_chosen": -96.77574157714844, |
| "logps/ref_rejected": -102.74411010742188, |
| "logps/rejected": -295.4071960449219, |
| "loss": 1.0924, |
| "margin_dpo/margin_mean": 52.930091857910156, |
| "margin_dpo/margin_std": 80.57627868652344, |
| "step": 370 |
| }, |
| { |
| "epoch": 0.5447870778267254, |
| "fcm_dpo/beta": 0.008863227441906929, |
| "fcm_dpo/delta": 0.09689568728208542, |
| "fcm_dpo/margin": 57.24983215332031, |
| "fcm_dpo/q_t": 0.38820239901542664, |
| "grad_norm": 12.403904914855957, |
| "learning_rate": 2.5641594716365744e-07, |
| "logits/chosen": -1.7923095226287842, |
| "logits/rejected": -1.426107406616211, |
| "logps/chosen": -235.0094451904297, |
| "logps/ref_chosen": -102.99020385742188, |
| "logps/ref_rejected": -105.35062408447266, |
| "logps/rejected": -294.61968994140625, |
| "loss": 1.0522, |
| "margin_dpo/margin_mean": 57.24983215332031, |
| "margin_dpo/margin_std": 77.89881134033203, |
| "step": 371 |
| }, |
| { |
| "epoch": 0.5462555066079295, |
| "fcm_dpo/beta": 0.00865775067359209, |
| "fcm_dpo/delta": -0.1739071160554886, |
| "fcm_dpo/margin": 87.87632751464844, |
| "fcm_dpo/q_t": 0.3345174789428711, |
| "grad_norm": 9.950636863708496, |
| "learning_rate": 2.551329606220976e-07, |
| "logits/chosen": -1.8596062660217285, |
| "logits/rejected": -1.2332671880722046, |
| "logps/chosen": -204.2379608154297, |
| "logps/ref_chosen": -91.02623748779297, |
| "logps/ref_rejected": -85.6102294921875, |
| "logps/rejected": -286.69830322265625, |
| "loss": 0.8762, |
| "margin_dpo/margin_mean": 87.87632751464844, |
| "margin_dpo/margin_std": 83.01412963867188, |
| "step": 372 |
| }, |
| { |
| "epoch": 0.5477239353891337, |
| "fcm_dpo/beta": 0.008587082847952843, |
| "fcm_dpo/delta": 0.02046929858624935, |
| "fcm_dpo/margin": 67.62651824951172, |
| "fcm_dpo/q_t": 0.3730928897857666, |
| "grad_norm": 13.208666801452637, |
| "learning_rate": 2.538498388222517e-07, |
| "logits/chosen": -2.059213161468506, |
| "logits/rejected": -1.5383615493774414, |
| "logps/chosen": -229.08303833007812, |
| "logps/ref_chosen": -100.07307434082031, |
| "logps/ref_rejected": -91.11634826660156, |
| "logps/rejected": -287.7528076171875, |
| "loss": 1.0004, |
| "margin_dpo/margin_mean": 67.62651062011719, |
| "margin_dpo/margin_std": 82.54972839355469, |
| "step": 373 |
| }, |
| { |
| "epoch": 0.5491923641703378, |
| "fcm_dpo/beta": 0.008557072840631008, |
| "fcm_dpo/delta": -0.01339799165725708, |
| "fcm_dpo/margin": 71.54954528808594, |
| "fcm_dpo/q_t": 0.36381733417510986, |
| "grad_norm": 10.388188362121582, |
| "learning_rate": 2.525666155755725e-07, |
| "logits/chosen": -2.204723834991455, |
| "logits/rejected": -1.814666986465454, |
| "logps/chosen": -211.86734008789062, |
| "logps/ref_chosen": -98.01234436035156, |
| "logps/ref_rejected": -100.75224304199219, |
| "logps/rejected": -286.1567687988281, |
| "loss": 0.9477, |
| "margin_dpo/margin_mean": 71.54954528808594, |
| "margin_dpo/margin_std": 71.1581802368164, |
| "step": 374 |
| }, |
| { |
| "epoch": 0.5506607929515418, |
| "fcm_dpo/beta": 0.00840807519853115, |
| "fcm_dpo/delta": -0.07696233689785004, |
| "fcm_dpo/margin": 79.625244140625, |
| "fcm_dpo/q_t": 0.3517950177192688, |
| "grad_norm": 10.864441871643066, |
| "learning_rate": 2.512833246961859e-07, |
| "logits/chosen": -1.7122812271118164, |
| "logits/rejected": -1.4867812395095825, |
| "logps/chosen": -202.69105529785156, |
| "logps/ref_chosen": -84.39627838134766, |
| "logps/ref_rejected": -95.7188491821289, |
| "logps/rejected": -293.63885498046875, |
| "loss": 0.9258, |
| "margin_dpo/margin_mean": 79.62525177001953, |
| "margin_dpo/margin_std": 77.73374938964844, |
| "step": 375 |
| }, |
| { |
| "epoch": 0.5521292217327459, |
| "fcm_dpo/beta": 0.00827401876449585, |
| "fcm_dpo/delta": -0.13464483618736267, |
| "fcm_dpo/margin": 87.5845718383789, |
| "fcm_dpo/q_t": 0.3414178490638733, |
| "grad_norm": 11.585356712341309, |
| "learning_rate": 2.5e-07, |
| "logits/chosen": -1.7216506004333496, |
| "logits/rejected": -1.4160125255584717, |
| "logps/chosen": -206.31048583984375, |
| "logps/ref_chosen": -95.73770141601562, |
| "logps/ref_rejected": -111.97331237792969, |
| "logps/rejected": -310.13067626953125, |
| "loss": 0.8974, |
| "margin_dpo/margin_mean": 87.5845718383789, |
| "margin_dpo/margin_std": 86.00065612792969, |
| "step": 376 |
| }, |
| { |
| "epoch": 0.55359765051395, |
| "fcm_dpo/beta": 0.008009790442883968, |
| "fcm_dpo/delta": -0.14207610487937927, |
| "fcm_dpo/margin": 91.17427062988281, |
| "fcm_dpo/q_t": 0.339458703994751, |
| "grad_norm": 10.219127655029297, |
| "learning_rate": 2.487166753038141e-07, |
| "logits/chosen": -1.5154132843017578, |
| "logits/rejected": -1.391338586807251, |
| "logps/chosen": -199.823486328125, |
| "logps/ref_chosen": -78.92694854736328, |
| "logps/ref_rejected": -105.89892578125, |
| "logps/rejected": -317.9697265625, |
| "loss": 0.8889, |
| "margin_dpo/margin_mean": 91.17427062988281, |
| "margin_dpo/margin_std": 85.8505859375, |
| "step": 377 |
| }, |
| { |
| "epoch": 0.5550660792951542, |
| "fcm_dpo/beta": 0.008050184696912766, |
| "fcm_dpo/delta": 0.0010249018669128418, |
| "fcm_dpo/margin": 74.31245422363281, |
| "fcm_dpo/q_t": 0.36496371030807495, |
| "grad_norm": 10.996868133544922, |
| "learning_rate": 2.4743338442442754e-07, |
| "logits/chosen": -2.0298149585723877, |
| "logits/rejected": -1.7492952346801758, |
| "logps/chosen": -192.58639526367188, |
| "logps/ref_chosen": -75.37240600585938, |
| "logps/ref_rejected": -93.70409393310547, |
| "logps/rejected": -285.2305603027344, |
| "loss": 0.9746, |
| "margin_dpo/margin_mean": 74.31245422363281, |
| "margin_dpo/margin_std": 80.56228637695312, |
| "step": 378 |
| }, |
| { |
| "epoch": 0.5565345080763583, |
| "fcm_dpo/beta": 0.008063051849603653, |
| "fcm_dpo/delta": 0.05794180929660797, |
| "fcm_dpo/margin": 67.62254333496094, |
| "fcm_dpo/q_t": 0.38331836462020874, |
| "grad_norm": 10.951027870178223, |
| "learning_rate": 2.461501611777483e-07, |
| "logits/chosen": -1.8049771785736084, |
| "logits/rejected": -1.708174467086792, |
| "logps/chosen": -221.08468627929688, |
| "logps/ref_chosen": -88.85568237304688, |
| "logps/ref_rejected": -121.38114166259766, |
| "logps/rejected": -321.2326965332031, |
| "loss": 1.0446, |
| "margin_dpo/margin_mean": 67.62254333496094, |
| "margin_dpo/margin_std": 94.80967712402344, |
| "step": 379 |
| }, |
| { |
| "epoch": 0.5580029368575624, |
| "fcm_dpo/beta": 0.008038859814405441, |
| "fcm_dpo/delta": -0.04042629525065422, |
| "fcm_dpo/margin": 79.3418197631836, |
| "fcm_dpo/q_t": 0.364249050617218, |
| "grad_norm": 13.981470108032227, |
| "learning_rate": 2.4486703937790243e-07, |
| "logits/chosen": -1.3323712348937988, |
| "logits/rejected": -1.299829125404358, |
| "logps/chosen": -227.69309997558594, |
| "logps/ref_chosen": -79.83321380615234, |
| "logps/ref_rejected": -110.7447509765625, |
| "logps/rejected": -337.94647216796875, |
| "loss": 0.9705, |
| "margin_dpo/margin_mean": 79.3418197631836, |
| "margin_dpo/margin_std": 95.55670928955078, |
| "step": 380 |
| }, |
| { |
| "epoch": 0.5594713656387665, |
| "fcm_dpo/beta": 0.00806824117898941, |
| "fcm_dpo/delta": 0.03291797637939453, |
| "fcm_dpo/margin": 70.51676940917969, |
| "fcm_dpo/q_t": 0.37344565987586975, |
| "grad_norm": 12.015775680541992, |
| "learning_rate": 2.435840528363426e-07, |
| "logits/chosen": -1.5715270042419434, |
| "logits/rejected": -1.0036929845809937, |
| "logps/chosen": -214.4466552734375, |
| "logps/ref_chosen": -86.56683349609375, |
| "logps/ref_rejected": -84.416748046875, |
| "logps/rejected": -282.8133544921875, |
| "loss": 0.9954, |
| "margin_dpo/margin_mean": 70.51676940917969, |
| "margin_dpo/margin_std": 82.17634582519531, |
| "step": 381 |
| }, |
| { |
| "epoch": 0.5609397944199707, |
| "fcm_dpo/beta": 0.007982393726706505, |
| "fcm_dpo/delta": -0.02618768811225891, |
| "fcm_dpo/margin": 78.04951477050781, |
| "fcm_dpo/q_t": 0.36266398429870605, |
| "grad_norm": 10.413619041442871, |
| "learning_rate": 2.4230123536095745e-07, |
| "logits/chosen": -1.9479053020477295, |
| "logits/rejected": -1.6788208484649658, |
| "logps/chosen": -233.09474182128906, |
| "logps/ref_chosen": -104.46180725097656, |
| "logps/ref_rejected": -114.60223388671875, |
| "logps/rejected": -321.28466796875, |
| "loss": 0.9563, |
| "margin_dpo/margin_mean": 78.04950714111328, |
| "margin_dpo/margin_std": 81.13053894042969, |
| "step": 382 |
| }, |
| { |
| "epoch": 0.5624082232011748, |
| "fcm_dpo/beta": 0.008084449917078018, |
| "fcm_dpo/delta": 0.05731251835823059, |
| "fcm_dpo/margin": 67.51455688476562, |
| "fcm_dpo/q_t": 0.3798357844352722, |
| "grad_norm": 11.326423645019531, |
| "learning_rate": 2.4101862075518037e-07, |
| "logits/chosen": -1.5943937301635742, |
| "logits/rejected": -1.4752601385116577, |
| "logps/chosen": -219.62017822265625, |
| "logps/ref_chosen": -82.11607360839844, |
| "logps/ref_rejected": -98.77716064453125, |
| "logps/rejected": -303.79583740234375, |
| "loss": 1.0111, |
| "margin_dpo/margin_mean": 67.51455688476562, |
| "margin_dpo/margin_std": 80.03602600097656, |
| "step": 383 |
| }, |
| { |
| "epoch": 0.5638766519823789, |
| "fcm_dpo/beta": 0.008265078067779541, |
| "fcm_dpo/delta": 0.04529654234647751, |
| "fcm_dpo/margin": 67.14823913574219, |
| "fcm_dpo/q_t": 0.37842482328414917, |
| "grad_norm": 13.126636505126953, |
| "learning_rate": 2.397362428170992e-07, |
| "logits/chosen": -1.7413570880889893, |
| "logits/rejected": -1.5120550394058228, |
| "logps/chosen": -228.9114990234375, |
| "logps/ref_chosen": -80.58525085449219, |
| "logps/ref_rejected": -91.44789123535156, |
| "logps/rejected": -306.92236328125, |
| "loss": 1.0505, |
| "margin_dpo/margin_mean": 67.14823913574219, |
| "margin_dpo/margin_std": 92.4796371459961, |
| "step": 384 |
| }, |
| { |
| "epoch": 0.5653450807635829, |
| "fcm_dpo/beta": 0.008029671385884285, |
| "fcm_dpo/delta": -0.09009828418493271, |
| "fcm_dpo/margin": 84.93191528320312, |
| "fcm_dpo/q_t": 0.35304510593414307, |
| "grad_norm": 12.234119415283203, |
| "learning_rate": 2.3845413533856514e-07, |
| "logits/chosen": -2.2260751724243164, |
| "logits/rejected": -1.6197378635406494, |
| "logps/chosen": -217.18515014648438, |
| "logps/ref_chosen": -92.76016235351562, |
| "logps/ref_rejected": -84.45997619628906, |
| "logps/rejected": -293.81689453125, |
| "loss": 0.9373, |
| "margin_dpo/margin_mean": 84.93191528320312, |
| "margin_dpo/margin_std": 90.8809814453125, |
| "step": 385 |
| }, |
| { |
| "epoch": 0.566813509544787, |
| "fcm_dpo/beta": 0.007991664111614227, |
| "fcm_dpo/delta": -0.05802081525325775, |
| "fcm_dpo/margin": 81.86547088623047, |
| "fcm_dpo/q_t": 0.35287386178970337, |
| "grad_norm": 9.855354309082031, |
| "learning_rate": 2.3717233210430254e-07, |
| "logits/chosen": -1.597925066947937, |
| "logits/rejected": -1.3119235038757324, |
| "logps/chosen": -215.71453857421875, |
| "logps/ref_chosen": -85.35896301269531, |
| "logps/ref_rejected": -96.99065399169922, |
| "logps/rejected": -309.211669921875, |
| "loss": 0.9444, |
| "margin_dpo/margin_mean": 81.86546325683594, |
| "margin_dpo/margin_std": 86.34203338623047, |
| "step": 386 |
| }, |
| { |
| "epoch": 0.5682819383259912, |
| "fcm_dpo/beta": 0.007988503202795982, |
| "fcm_dpo/delta": 0.00018469803035259247, |
| "fcm_dpo/margin": 75.06538391113281, |
| "fcm_dpo/q_t": 0.36637941002845764, |
| "grad_norm": 12.132233619689941, |
| "learning_rate": 2.3589086689101889e-07, |
| "logits/chosen": -2.5154049396514893, |
| "logits/rejected": -1.8449430465698242, |
| "logps/chosen": -253.63768005371094, |
| "logps/ref_chosen": -102.68814086914062, |
| "logps/ref_rejected": -99.42858123779297, |
| "logps/rejected": -325.4435119628906, |
| "loss": 0.9641, |
| "margin_dpo/margin_mean": 75.06538391113281, |
| "margin_dpo/margin_std": 79.062255859375, |
| "step": 387 |
| }, |
| { |
| "epoch": 0.5697503671071953, |
| "fcm_dpo/beta": 0.00789298489689827, |
| "fcm_dpo/delta": -0.05572465807199478, |
| "fcm_dpo/margin": 82.60919952392578, |
| "fcm_dpo/q_t": 0.35591650009155273, |
| "grad_norm": 10.612791061401367, |
| "learning_rate": 2.3460977346651428e-07, |
| "logits/chosen": -1.7140839099884033, |
| "logits/rejected": -1.5836071968078613, |
| "logps/chosen": -234.08363342285156, |
| "logps/ref_chosen": -83.58523559570312, |
| "logps/ref_rejected": -110.10037231445312, |
| "logps/rejected": -343.2079772949219, |
| "loss": 0.9393, |
| "margin_dpo/margin_mean": 82.60919189453125, |
| "margin_dpo/margin_std": 85.55877685546875, |
| "step": 388 |
| }, |
| { |
| "epoch": 0.5712187958883994, |
| "fcm_dpo/beta": 0.007903015241026878, |
| "fcm_dpo/delta": 0.023557592183351517, |
| "fcm_dpo/margin": 73.11224365234375, |
| "fcm_dpo/q_t": 0.3752698302268982, |
| "grad_norm": 12.45739459991455, |
| "learning_rate": 2.3332908558879177e-07, |
| "logits/chosen": -2.490940570831299, |
| "logits/rejected": -2.0545895099639893, |
| "logps/chosen": -244.65711975097656, |
| "logps/ref_chosen": -89.822265625, |
| "logps/ref_rejected": -84.85376739501953, |
| "logps/rejected": -312.80084228515625, |
| "loss": 1.0175, |
| "margin_dpo/margin_mean": 73.11224365234375, |
| "margin_dpo/margin_std": 96.38984680175781, |
| "step": 389 |
| }, |
| { |
| "epoch": 0.5726872246696035, |
| "fcm_dpo/beta": 0.007932371459901333, |
| "fcm_dpo/delta": 0.060371283441782, |
| "fcm_dpo/margin": 68.37907409667969, |
| "fcm_dpo/q_t": 0.3861868381500244, |
| "grad_norm": 10.967287063598633, |
| "learning_rate": 2.320488370051681e-07, |
| "logits/chosen": -1.6026420593261719, |
| "logits/rejected": -1.389652132987976, |
| "logps/chosen": -234.77609252929688, |
| "logps/ref_chosen": -77.53244018554688, |
| "logps/ref_rejected": -92.58392333984375, |
| "logps/rejected": -318.2066650390625, |
| "loss": 1.0473, |
| "margin_dpo/margin_mean": 68.37908172607422, |
| "margin_dpo/margin_std": 97.29962158203125, |
| "step": 390 |
| }, |
| { |
| "epoch": 0.5741556534508077, |
| "fcm_dpo/beta": 0.007975282147526741, |
| "fcm_dpo/delta": -0.00302848219871521, |
| "fcm_dpo/margin": 75.55020141601562, |
| "fcm_dpo/q_t": 0.3686513304710388, |
| "grad_norm": 11.11927604675293, |
| "learning_rate": 2.3076906145138405e-07, |
| "logits/chosen": -1.8575489521026611, |
| "logits/rejected": -1.4008793830871582, |
| "logps/chosen": -229.98764038085938, |
| "logps/ref_chosen": -88.2329330444336, |
| "logps/ref_rejected": -93.00756072998047, |
| "logps/rejected": -310.3124694824219, |
| "loss": 0.9856, |
| "margin_dpo/margin_mean": 75.55020141601562, |
| "margin_dpo/margin_std": 88.94224548339844, |
| "step": 391 |
| }, |
| { |
| "epoch": 0.5756240822320118, |
| "fcm_dpo/beta": 0.007913609966635704, |
| "fcm_dpo/delta": -0.104721799492836, |
| "fcm_dpo/margin": 88.1395263671875, |
| "fcm_dpo/q_t": 0.3488277792930603, |
| "grad_norm": 13.28966999053955, |
| "learning_rate": 2.294897926507156e-07, |
| "logits/chosen": -1.2604981660842896, |
| "logits/rejected": -1.1343798637390137, |
| "logps/chosen": -231.86595153808594, |
| "logps/ref_chosen": -87.74442291259766, |
| "logps/ref_rejected": -108.57071685791016, |
| "logps/rejected": -340.831787109375, |
| "loss": 0.9373, |
| "margin_dpo/margin_mean": 88.1395263671875, |
| "margin_dpo/margin_std": 97.56378173828125, |
| "step": 392 |
| }, |
| { |
| "epoch": 0.5770925110132159, |
| "fcm_dpo/beta": 0.0078022656962275505, |
| "fcm_dpo/delta": -0.027617137879133224, |
| "fcm_dpo/margin": 80.21553802490234, |
| "fcm_dpo/q_t": 0.3618813455104828, |
| "grad_norm": 10.270767211914062, |
| "learning_rate": 2.2821106431308543e-07, |
| "logits/chosen": -1.5781824588775635, |
| "logits/rejected": -1.2512006759643555, |
| "logps/chosen": -205.15457153320312, |
| "logps/ref_chosen": -76.56381225585938, |
| "logps/ref_rejected": -79.72618103027344, |
| "logps/rejected": -288.5325012207031, |
| "loss": 0.9551, |
| "margin_dpo/margin_mean": 80.21553802490234, |
| "margin_dpo/margin_std": 85.68644714355469, |
| "step": 393 |
| }, |
| { |
| "epoch": 0.57856093979442, |
| "fcm_dpo/beta": 0.007711788639426231, |
| "fcm_dpo/delta": -0.06800325214862823, |
| "fcm_dpo/margin": 86.02757263183594, |
| "fcm_dpo/q_t": 0.3508681356906891, |
| "grad_norm": 10.564422607421875, |
| "learning_rate": 2.2693291013417452e-07, |
| "logits/chosen": -1.5955944061279297, |
| "logits/rejected": -1.3630306720733643, |
| "logps/chosen": -219.39657592773438, |
| "logps/ref_chosen": -84.69866180419922, |
| "logps/ref_rejected": -97.457763671875, |
| "logps/rejected": -318.1832275390625, |
| "loss": 0.9127, |
| "margin_dpo/margin_mean": 86.02757263183594, |
| "margin_dpo/margin_std": 78.89590454101562, |
| "step": 394 |
| }, |
| { |
| "epoch": 0.580029368575624, |
| "fcm_dpo/beta": 0.00774542149156332, |
| "fcm_dpo/delta": 0.08366027474403381, |
| "fcm_dpo/margin": 67.20747375488281, |
| "fcm_dpo/q_t": 0.3928409814834595, |
| "grad_norm": 11.868879318237305, |
| "learning_rate": 2.2565536379453404e-07, |
| "logits/chosen": -2.6224658489227295, |
| "logits/rejected": -2.231447219848633, |
| "logps/chosen": -229.33065795898438, |
| "logps/ref_chosen": -88.88763427734375, |
| "logps/ref_rejected": -91.0455322265625, |
| "logps/rejected": -298.696044921875, |
| "loss": 1.1162, |
| "margin_dpo/margin_mean": 67.20747375488281, |
| "margin_dpo/margin_std": 117.15966796875, |
| "step": 395 |
| }, |
| { |
| "epoch": 0.5814977973568282, |
| "fcm_dpo/beta": 0.007993882521986961, |
| "fcm_dpo/delta": 0.14788684248924255, |
| "fcm_dpo/margin": 57.40891647338867, |
| "fcm_dpo/q_t": 0.3995135426521301, |
| "grad_norm": 15.381148338317871, |
| "learning_rate": 2.2437845895869825e-07, |
| "logits/chosen": -1.979184627532959, |
| "logits/rejected": -1.4956262111663818, |
| "logps/chosen": -252.1164093017578, |
| "logps/ref_chosen": -102.47132873535156, |
| "logps/ref_rejected": -93.23478698730469, |
| "logps/rejected": -300.28875732421875, |
| "loss": 1.0905, |
| "margin_dpo/margin_mean": 57.40891647338867, |
| "margin_dpo/margin_std": 87.18148803710938, |
| "step": 396 |
| }, |
| { |
| "epoch": 0.5829662261380323, |
| "fcm_dpo/beta": 0.0077405949123203754, |
| "fcm_dpo/delta": -0.2481895387172699, |
| "fcm_dpo/margin": 106.80502319335938, |
| "fcm_dpo/q_t": 0.3245747983455658, |
| "grad_norm": 11.806785583496094, |
| "learning_rate": 2.2310222927429716e-07, |
| "logits/chosen": -2.333134889602661, |
| "logits/rejected": -1.7398998737335205, |
| "logps/chosen": -234.05245971679688, |
| "logps/ref_chosen": -101.04441833496094, |
| "logps/ref_rejected": -108.8511734008789, |
| "logps/rejected": -348.66424560546875, |
| "loss": 0.8711, |
| "margin_dpo/margin_mean": 106.80502319335938, |
| "margin_dpo/margin_std": 107.98983001708984, |
| "step": 397 |
| }, |
| { |
| "epoch": 0.5844346549192364, |
| "fcm_dpo/beta": 0.00766055379062891, |
| "fcm_dpo/delta": 0.006077233701944351, |
| "fcm_dpo/margin": 77.572265625, |
| "fcm_dpo/q_t": 0.3723902702331543, |
| "grad_norm": 11.528839111328125, |
| "learning_rate": 2.2182670837116972e-07, |
| "logits/chosen": -2.045728921890259, |
| "logits/rejected": -1.755936861038208, |
| "logps/chosen": -235.32550048828125, |
| "logps/ref_chosen": -95.04922485351562, |
| "logps/ref_rejected": -114.00563049316406, |
| "logps/rejected": -331.85418701171875, |
| "loss": 1.0144, |
| "margin_dpo/margin_mean": 77.572265625, |
| "margin_dpo/margin_std": 103.19320678710938, |
| "step": 398 |
| }, |
| { |
| "epoch": 0.5859030837004405, |
| "fcm_dpo/beta": 0.0075286137871444225, |
| "fcm_dpo/delta": -0.15626700222492218, |
| "fcm_dpo/margin": 98.94232940673828, |
| "fcm_dpo/q_t": 0.3343579173088074, |
| "grad_norm": 10.617576599121094, |
| "learning_rate": 2.2055192986047804e-07, |
| "logits/chosen": -2.003676414489746, |
| "logits/rejected": -1.1966228485107422, |
| "logps/chosen": -222.55133056640625, |
| "logps/ref_chosen": -96.63854217529297, |
| "logps/ref_rejected": -83.53645324707031, |
| "logps/rejected": -308.3916015625, |
| "loss": 0.8622, |
| "margin_dpo/margin_mean": 98.94233703613281, |
| "margin_dpo/margin_std": 83.51806640625, |
| "step": 399 |
| }, |
| { |
| "epoch": 0.5873715124816447, |
| "fcm_dpo/beta": 0.007274920120835304, |
| "fcm_dpo/delta": -0.14528706669807434, |
| "fcm_dpo/margin": 100.96365356445312, |
| "fcm_dpo/q_t": 0.3401756286621094, |
| "grad_norm": 14.20820426940918, |
| "learning_rate": 2.192779273338215e-07, |
| "logits/chosen": -2.292538642883301, |
| "logits/rejected": -1.8154387474060059, |
| "logps/chosen": -234.29095458984375, |
| "logps/ref_chosen": -105.30314636230469, |
| "logps/ref_rejected": -108.6555404663086, |
| "logps/rejected": -338.60699462890625, |
| "loss": 0.9031, |
| "margin_dpo/margin_mean": 100.96365356445312, |
| "margin_dpo/margin_std": 101.42951965332031, |
| "step": 400 |
| }, |
| { |
| "epoch": 0.5873715124816447, |
| "eval_fcm_dpo/beta": 0.0072255851700901985, |
| "eval_logits/chosen": -1.9563562870025635, |
| "eval_logits/rejected": -1.5862782001495361, |
| "eval_logps/chosen": -255.01856994628906, |
| "eval_logps/ref_chosen": -100.49356842041016, |
| "eval_logps/ref_rejected": -94.06775665283203, |
| "eval_logps/rejected": -314.7674865722656, |
| "eval_loss": 0.5320852398872375, |
| "eval_margin_dpo/margin_mean": 66.17474365234375, |
| "eval_margin_dpo/margin_std": 89.82697296142578, |
| "eval_runtime": 43.7256, |
| "eval_samples_per_second": 53.493, |
| "eval_steps_per_second": 1.692, |
| "step": 400 |
| }, |
| { |
| "epoch": 0.5888399412628488, |
| "fcm_dpo/beta": 0.007348657585680485, |
| "fcm_dpo/delta": 0.13697752356529236, |
| "fcm_dpo/margin": 63.91277313232422, |
| "fcm_dpo/q_t": 0.40259644389152527, |
| "grad_norm": 14.34986686706543, |
| "learning_rate": 2.1800473436235136e-07, |
| "logits/chosen": -1.7043498754501343, |
| "logits/rejected": -1.4674514532089233, |
| "logps/chosen": -227.89923095703125, |
| "logps/ref_chosen": -84.28916931152344, |
| "logps/ref_rejected": -90.943115234375, |
| "logps/rejected": -298.4659423828125, |
| "loss": 1.1053, |
| "margin_dpo/margin_mean": 63.91277313232422, |
| "margin_dpo/margin_std": 107.23680114746094, |
| "step": 401 |
| }, |
| { |
| "epoch": 0.5903083700440529, |
| "fcm_dpo/beta": 0.007161559071391821, |
| "fcm_dpo/delta": -0.25808143615722656, |
| "fcm_dpo/margin": 116.8514404296875, |
| "fcm_dpo/q_t": 0.3132210671901703, |
| "grad_norm": 11.211475372314453, |
| "learning_rate": 2.1673238449588665e-07, |
| "logits/chosen": -1.7466460466384888, |
| "logits/rejected": -1.2662107944488525, |
| "logps/chosen": -213.04330444335938, |
| "logps/ref_chosen": -83.59312438964844, |
| "logps/ref_rejected": -87.81027221679688, |
| "logps/rejected": -334.11187744140625, |
| "loss": 0.8053, |
| "margin_dpo/margin_mean": 116.8514404296875, |
| "margin_dpo/margin_std": 88.61773681640625, |
| "step": 402 |
| }, |
| { |
| "epoch": 0.591776798825257, |
| "fcm_dpo/beta": 0.007018306292593479, |
| "fcm_dpo/delta": -0.011818572878837585, |
| "fcm_dpo/margin": 87.03970336914062, |
| "fcm_dpo/q_t": 0.3649277985095978, |
| "grad_norm": 10.342928886413574, |
| "learning_rate": 2.154609112620295e-07, |
| "logits/chosen": -2.358241081237793, |
| "logits/rejected": -1.9733853340148926, |
| "logps/chosen": -205.4605712890625, |
| "logps/ref_chosen": -73.75308227539062, |
| "logps/ref_rejected": -83.92012786865234, |
| "logps/rejected": -302.6673278808594, |
| "loss": 0.9675, |
| "margin_dpo/margin_mean": 87.03970336914062, |
| "margin_dpo/margin_std": 95.28384399414062, |
| "step": 403 |
| }, |
| { |
| "epoch": 0.593245227606461, |
| "fcm_dpo/beta": 0.007098081987351179, |
| "fcm_dpo/delta": 0.07683449983596802, |
| "fcm_dpo/margin": 74.28379821777344, |
| "fcm_dpo/q_t": 0.38582703471183777, |
| "grad_norm": 15.550983428955078, |
| "learning_rate": 2.1419034816528218e-07, |
| "logits/chosen": -2.0557281970977783, |
| "logits/rejected": -1.7230305671691895, |
| "logps/chosen": -237.1645050048828, |
| "logps/ref_chosen": -79.67617797851562, |
| "logps/ref_rejected": -84.280517578125, |
| "logps/rejected": -316.0526428222656, |
| "loss": 1.0557, |
| "margin_dpo/margin_mean": 74.28379821777344, |
| "margin_dpo/margin_std": 107.16883087158203, |
| "step": 404 |
| }, |
| { |
| "epoch": 0.5947136563876652, |
| "fcm_dpo/beta": 0.007242183201014996, |
| "fcm_dpo/delta": 0.046799372881650925, |
| "fcm_dpo/margin": 76.58912658691406, |
| "fcm_dpo/q_t": 0.3775564432144165, |
| "grad_norm": 9.922403335571289, |
| "learning_rate": 2.129207286861638e-07, |
| "logits/chosen": -1.4873124361038208, |
| "logits/rejected": -1.0152088403701782, |
| "logps/chosen": -255.55612182617188, |
| "logps/ref_chosen": -96.46195220947266, |
| "logps/ref_rejected": -92.87071228027344, |
| "logps/rejected": -328.55401611328125, |
| "loss": 0.9945, |
| "margin_dpo/margin_mean": 76.58912658691406, |
| "margin_dpo/margin_std": 85.40648651123047, |
| "step": 405 |
| }, |
| { |
| "epoch": 0.5961820851688693, |
| "fcm_dpo/beta": 0.007322330959141254, |
| "fcm_dpo/delta": 0.09602566063404083, |
| "fcm_dpo/margin": 69.48136138916016, |
| "fcm_dpo/q_t": 0.3899584114551544, |
| "grad_norm": 17.966711044311523, |
| "learning_rate": 2.1165208628032861e-07, |
| "logits/chosen": -2.0658977031707764, |
| "logits/rejected": -1.8964214324951172, |
| "logps/chosen": -228.33712768554688, |
| "logps/ref_chosen": -78.13396453857422, |
| "logps/ref_rejected": -98.28359985351562, |
| "logps/rejected": -317.9681396484375, |
| "loss": 1.0736, |
| "margin_dpo/margin_mean": 69.48136138916016, |
| "margin_dpo/margin_std": 104.44835662841797, |
| "step": 406 |
| }, |
| { |
| "epoch": 0.5976505139500734, |
| "fcm_dpo/beta": 0.007518821395933628, |
| "fcm_dpo/delta": 0.14913788437843323, |
| "fcm_dpo/margin": 60.868133544921875, |
| "fcm_dpo/q_t": 0.39967426657676697, |
| "grad_norm": 13.635825157165527, |
| "learning_rate": 2.1038445437768375e-07, |
| "logits/chosen": -2.2239134311676025, |
| "logits/rejected": -1.7089815139770508, |
| "logps/chosen": -243.0015869140625, |
| "logps/ref_chosen": -84.01283264160156, |
| "logps/ref_rejected": -82.78103637695312, |
| "logps/rejected": -302.6379089355469, |
| "loss": 1.0895, |
| "margin_dpo/margin_mean": 60.868133544921875, |
| "margin_dpo/margin_std": 93.1767578125, |
| "step": 407 |
| }, |
| { |
| "epoch": 0.5991189427312775, |
| "fcm_dpo/beta": 0.007688861805945635, |
| "fcm_dpo/delta": 0.11370319128036499, |
| "fcm_dpo/margin": 63.99238204956055, |
| "fcm_dpo/q_t": 0.39551639556884766, |
| "grad_norm": 13.247913360595703, |
| "learning_rate": 2.0911786638150872e-07, |
| "logits/chosen": -2.340407133102417, |
| "logits/rejected": -1.736910343170166, |
| "logps/chosen": -254.63917541503906, |
| "logps/ref_chosen": -104.46174621582031, |
| "logps/ref_rejected": -96.37217712402344, |
| "logps/rejected": -310.5419921875, |
| "loss": 1.0843, |
| "margin_dpo/margin_mean": 63.99237823486328, |
| "margin_dpo/margin_std": 99.77020263671875, |
| "step": 408 |
| }, |
| { |
| "epoch": 0.6005873715124816, |
| "fcm_dpo/beta": 0.007855923846364021, |
| "fcm_dpo/delta": 0.098321832716465, |
| "fcm_dpo/margin": 64.50284576416016, |
| "fcm_dpo/q_t": 0.3866105079650879, |
| "grad_norm": 11.660709381103516, |
| "learning_rate": 2.0785235566757517e-07, |
| "logits/chosen": -1.5960277318954468, |
| "logits/rejected": -1.1638531684875488, |
| "logps/chosen": -259.2583923339844, |
| "logps/ref_chosen": -97.66830444335938, |
| "logps/ref_rejected": -90.04583740234375, |
| "logps/rejected": -316.13873291015625, |
| "loss": 1.0741, |
| "margin_dpo/margin_mean": 64.50284576416016, |
| "margin_dpo/margin_std": 95.21458435058594, |
| "step": 409 |
| }, |
| { |
| "epoch": 0.6020558002936858, |
| "fcm_dpo/beta": 0.007921447977423668, |
| "fcm_dpo/delta": 0.01372436247766018, |
| "fcm_dpo/margin": 74.1120834350586, |
| "fcm_dpo/q_t": 0.3695540428161621, |
| "grad_norm": 11.11070442199707, |
| "learning_rate": 2.065879555832674e-07, |
| "logits/chosen": -1.7564365863800049, |
| "logits/rejected": -1.6230534315109253, |
| "logps/chosen": -235.60093688964844, |
| "logps/ref_chosen": -76.46923828125, |
| "logps/ref_rejected": -88.64064025878906, |
| "logps/rejected": -321.8843994140625, |
| "loss": 0.9988, |
| "margin_dpo/margin_mean": 74.11209106445312, |
| "margin_dpo/margin_std": 90.84548950195312, |
| "step": 410 |
| }, |
| { |
| "epoch": 0.6035242290748899, |
| "fcm_dpo/beta": 0.007924282923340797, |
| "fcm_dpo/delta": 0.025149427354335785, |
| "fcm_dpo/margin": 72.70220184326172, |
| "fcm_dpo/q_t": 0.37517866492271423, |
| "grad_norm": 11.753535270690918, |
| "learning_rate": 2.0532469944670343e-07, |
| "logits/chosen": -2.482978343963623, |
| "logits/rejected": -2.093186140060425, |
| "logps/chosen": -254.4764862060547, |
| "logps/ref_chosen": -87.16630554199219, |
| "logps/ref_rejected": -87.09603118896484, |
| "logps/rejected": -327.1083984375, |
| "loss": 0.9966, |
| "margin_dpo/margin_mean": 72.70220184326172, |
| "margin_dpo/margin_std": 86.63870239257812, |
| "step": 411 |
| }, |
| { |
| "epoch": 0.604992657856094, |
| "fcm_dpo/beta": 0.007970331236720085, |
| "fcm_dpo/delta": 0.03290639817714691, |
| "fcm_dpo/margin": 71.35936737060547, |
| "fcm_dpo/q_t": 0.38153523206710815, |
| "grad_norm": 12.409209251403809, |
| "learning_rate": 2.0406262054585738e-07, |
| "logits/chosen": -2.412627696990967, |
| "logits/rejected": -2.191441059112549, |
| "logps/chosen": -236.4732208251953, |
| "logps/ref_chosen": -78.94734191894531, |
| "logps/ref_rejected": -106.10554504394531, |
| "logps/rejected": -334.99078369140625, |
| "loss": 1.0581, |
| "margin_dpo/margin_mean": 71.35936737060547, |
| "margin_dpo/margin_std": 107.4913330078125, |
| "step": 412 |
| }, |
| { |
| "epoch": 0.6064610866372981, |
| "fcm_dpo/beta": 0.0079441387206316, |
| "fcm_dpo/delta": -0.033406391739845276, |
| "fcm_dpo/margin": 79.40829467773438, |
| "fcm_dpo/q_t": 0.3635585904121399, |
| "grad_norm": 11.994926452636719, |
| "learning_rate": 2.0280175213768205e-07, |
| "logits/chosen": -1.2549076080322266, |
| "logits/rejected": -0.9982696771621704, |
| "logps/chosen": -265.3006591796875, |
| "logps/ref_chosen": -95.69470977783203, |
| "logps/ref_rejected": -107.96085357666016, |
| "logps/rejected": -356.97509765625, |
| "loss": 0.9942, |
| "margin_dpo/margin_mean": 79.40829467773438, |
| "margin_dpo/margin_std": 102.14022827148438, |
| "step": 413 |
| }, |
| { |
| "epoch": 0.6079295154185022, |
| "fcm_dpo/beta": 0.007994389161467552, |
| "fcm_dpo/delta": 0.021429970860481262, |
| "fcm_dpo/margin": 72.52461242675781, |
| "fcm_dpo/q_t": 0.37816792726516724, |
| "grad_norm": 16.40660858154297, |
| "learning_rate": 2.0154212744723247e-07, |
| "logits/chosen": -1.648315668106079, |
| "logits/rejected": -1.2972140312194824, |
| "logps/chosen": -245.08993530273438, |
| "logps/ref_chosen": -88.27667236328125, |
| "logps/ref_rejected": -92.87004089355469, |
| "logps/rejected": -322.2078857421875, |
| "loss": 1.0534, |
| "margin_dpo/margin_mean": 72.52461242675781, |
| "margin_dpo/margin_std": 109.23208618164062, |
| "step": 414 |
| }, |
| { |
| "epoch": 0.6093979441997063, |
| "fcm_dpo/beta": 0.008126772940158844, |
| "fcm_dpo/delta": 0.09716188907623291, |
| "fcm_dpo/margin": 62.45792007446289, |
| "fcm_dpo/q_t": 0.39152491092681885, |
| "grad_norm": 13.653444290161133, |
| "learning_rate": 2.002837796667909e-07, |
| "logits/chosen": -1.602043867111206, |
| "logits/rejected": -1.2246946096420288, |
| "logps/chosen": -278.5191955566406, |
| "logps/ref_chosen": -108.91590118408203, |
| "logps/ref_rejected": -107.47135925292969, |
| "logps/rejected": -339.5325622558594, |
| "loss": 1.0767, |
| "margin_dpo/margin_mean": 62.45792007446289, |
| "margin_dpo/margin_std": 95.18862915039062, |
| "step": 415 |
| }, |
| { |
| "epoch": 0.6108663729809104, |
| "fcm_dpo/beta": 0.008111116476356983, |
| "fcm_dpo/delta": -0.1124371886253357, |
| "fcm_dpo/margin": 86.70777893066406, |
| "fcm_dpo/q_t": 0.3507331311702728, |
| "grad_norm": 11.193100929260254, |
| "learning_rate": 1.990267419549914e-07, |
| "logits/chosen": -1.8388745784759521, |
| "logits/rejected": -1.336613655090332, |
| "logps/chosen": -259.5759582519531, |
| "logps/ref_chosen": -93.39888000488281, |
| "logps/ref_rejected": -97.6729736328125, |
| "logps/rejected": -350.5578308105469, |
| "loss": 0.9505, |
| "margin_dpo/margin_mean": 86.70777893066406, |
| "margin_dpo/margin_std": 98.13165283203125, |
| "step": 416 |
| }, |
| { |
| "epoch": 0.6123348017621145, |
| "fcm_dpo/beta": 0.007943946868181229, |
| "fcm_dpo/delta": -0.04033021628856659, |
| "fcm_dpo/margin": 80.280517578125, |
| "fcm_dpo/q_t": 0.36008456349372864, |
| "grad_norm": 11.344613075256348, |
| "learning_rate": 1.9777104743594686e-07, |
| "logits/chosen": -2.1126794815063477, |
| "logits/rejected": -1.1838053464889526, |
| "logps/chosen": -239.34365844726562, |
| "logps/ref_chosen": -83.53533172607422, |
| "logps/ref_rejected": -74.44184875488281, |
| "logps/rejected": -310.53070068359375, |
| "loss": 0.9601, |
| "margin_dpo/margin_mean": 80.280517578125, |
| "margin_dpo/margin_std": 89.32632446289062, |
| "step": 417 |
| }, |
| { |
| "epoch": 0.6138032305433186, |
| "fcm_dpo/beta": 0.007776966318488121, |
| "fcm_dpo/delta": -0.20287159085273743, |
| "fcm_dpo/margin": 101.0872802734375, |
| "fcm_dpo/q_t": 0.3297736644744873, |
| "grad_norm": 14.166253089904785, |
| "learning_rate": 1.965167291983757e-07, |
| "logits/chosen": -2.374035358428955, |
| "logits/rejected": -1.742133378982544, |
| "logps/chosen": -255.0244598388672, |
| "logps/ref_chosen": -108.22152709960938, |
| "logps/ref_rejected": -111.8646469116211, |
| "logps/rejected": -359.7548828125, |
| "loss": 0.8604, |
| "margin_dpo/margin_mean": 101.0872802734375, |
| "margin_dpo/margin_std": 87.68946838378906, |
| "step": 418 |
| }, |
| { |
| "epoch": 0.6152716593245228, |
| "fcm_dpo/beta": 0.007567869499325752, |
| "fcm_dpo/delta": -0.06575581431388855, |
| "fcm_dpo/margin": 87.35041809082031, |
| "fcm_dpo/q_t": 0.3550986051559448, |
| "grad_norm": 11.697413444519043, |
| "learning_rate": 1.9526382029472988e-07, |
| "logits/chosen": -1.4137977361679077, |
| "logits/rejected": -0.960525631904602, |
| "logps/chosen": -251.655517578125, |
| "logps/ref_chosen": -97.18328094482422, |
| "logps/ref_rejected": -98.18531799316406, |
| "logps/rejected": -340.0079345703125, |
| "loss": 0.9383, |
| "margin_dpo/margin_mean": 87.35041809082031, |
| "margin_dpo/margin_std": 90.81410217285156, |
| "step": 419 |
| }, |
| { |
| "epoch": 0.6167400881057269, |
| "fcm_dpo/beta": 0.007550552487373352, |
| "fcm_dpo/delta": 0.013763319700956345, |
| "fcm_dpo/margin": 77.66873931884766, |
| "fcm_dpo/q_t": 0.37317919731140137, |
| "grad_norm": 16.0394344329834, |
| "learning_rate": 1.9401235374032425e-07, |
| "logits/chosen": -2.1781749725341797, |
| "logits/rejected": -1.0760204792022705, |
| "logps/chosen": -272.736572265625, |
| "logps/ref_chosen": -114.30848693847656, |
| "logps/ref_rejected": -75.68355560302734, |
| "logps/rejected": -311.7803955078125, |
| "loss": 1.0063, |
| "margin_dpo/margin_mean": 77.66874694824219, |
| "margin_dpo/margin_std": 97.64486694335938, |
| "step": 420 |
| }, |
| { |
| "epoch": 0.618208516886931, |
| "fcm_dpo/beta": 0.007453433237969875, |
| "fcm_dpo/delta": -0.04194428399205208, |
| "fcm_dpo/margin": 85.74388885498047, |
| "fcm_dpo/q_t": 0.35998445749282837, |
| "grad_norm": 10.710517883300781, |
| "learning_rate": 1.9276236251246653e-07, |
| "logits/chosen": -1.8413752317428589, |
| "logits/rejected": -1.4720008373260498, |
| "logps/chosen": -239.76809692382812, |
| "logps/ref_chosen": -85.87985229492188, |
| "logps/ref_rejected": -96.33648681640625, |
| "logps/rejected": -335.9686279296875, |
| "loss": 0.9597, |
| "margin_dpo/margin_mean": 85.743896484375, |
| "margin_dpo/margin_std": 95.09271240234375, |
| "step": 421 |
| }, |
| { |
| "epoch": 0.6196769456681351, |
| "fcm_dpo/beta": 0.007458934560418129, |
| "fcm_dpo/delta": -0.008748026564717293, |
| "fcm_dpo/margin": 81.53353881835938, |
| "fcm_dpo/q_t": 0.3671715557575226, |
| "grad_norm": 12.771909713745117, |
| "learning_rate": 1.9151387954958792e-07, |
| "logits/chosen": -2.399049758911133, |
| "logits/rejected": -2.0303146839141846, |
| "logps/chosen": -251.56912231445312, |
| "logps/ref_chosen": -100.48060607910156, |
| "logps/ref_rejected": -94.40821838378906, |
| "logps/rejected": -327.0303039550781, |
| "loss": 0.9712, |
| "margin_dpo/margin_mean": 81.53353881835938, |
| "margin_dpo/margin_std": 92.20245361328125, |
| "step": 422 |
| }, |
| { |
| "epoch": 0.6211453744493393, |
| "fcm_dpo/beta": 0.00747031532227993, |
| "fcm_dpo/delta": -0.004844721406698227, |
| "fcm_dpo/margin": 80.87257385253906, |
| "fcm_dpo/q_t": 0.3732362687587738, |
| "grad_norm": 12.241605758666992, |
| "learning_rate": 1.902669377503756e-07, |
| "logits/chosen": -1.9506278038024902, |
| "logits/rejected": -1.833554744720459, |
| "logps/chosen": -219.573486328125, |
| "logps/ref_chosen": -78.44993591308594, |
| "logps/ref_rejected": -92.04652404785156, |
| "logps/rejected": -314.04266357421875, |
| "loss": 1.0157, |
| "margin_dpo/margin_mean": 80.8725814819336, |
| "margin_dpo/margin_std": 108.63528442382812, |
| "step": 423 |
| }, |
| { |
| "epoch": 0.6226138032305433, |
| "fcm_dpo/beta": 0.007418894208967686, |
| "fcm_dpo/delta": -0.012280648574233055, |
| "fcm_dpo/margin": 82.428466796875, |
| "fcm_dpo/q_t": 0.3642910420894623, |
| "grad_norm": 10.28632640838623, |
| "learning_rate": 1.890215699729057e-07, |
| "logits/chosen": -1.6936588287353516, |
| "logits/rejected": -1.089149832725525, |
| "logps/chosen": -245.31788635253906, |
| "logps/ref_chosen": -87.64236450195312, |
| "logps/ref_rejected": -72.36566162109375, |
| "logps/rejected": -312.46966552734375, |
| "loss": 0.973, |
| "margin_dpo/margin_mean": 82.428466796875, |
| "margin_dpo/margin_std": 93.11031341552734, |
| "step": 424 |
| }, |
| { |
| "epoch": 0.6240822320117474, |
| "fcm_dpo/beta": 0.007599799428135157, |
| "fcm_dpo/delta": 0.1809379905462265, |
| "fcm_dpo/margin": 56.19786071777344, |
| "fcm_dpo/q_t": 0.41294217109680176, |
| "grad_norm": 13.262221336364746, |
| "learning_rate": 1.8777780903377732e-07, |
| "logits/chosen": -1.6487901210784912, |
| "logits/rejected": -1.5741477012634277, |
| "logps/chosen": -241.32943725585938, |
| "logps/ref_chosen": -78.51979064941406, |
| "logps/ref_rejected": -102.74864196777344, |
| "logps/rejected": -321.75616455078125, |
| "loss": 1.1604, |
| "margin_dpo/margin_mean": 56.19786071777344, |
| "margin_dpo/margin_std": 110.29562377929688, |
| "step": 425 |
| }, |
| { |
| "epoch": 0.6255506607929515, |
| "fcm_dpo/beta": 0.007676470093429089, |
| "fcm_dpo/delta": -0.026276282966136932, |
| "fcm_dpo/margin": 81.36367797851562, |
| "fcm_dpo/q_t": 0.36279982328414917, |
| "grad_norm": 14.7294282913208, |
| "learning_rate": 1.8653568770724803e-07, |
| "logits/chosen": -1.8185904026031494, |
| "logits/rejected": -1.1083682775497437, |
| "logps/chosen": -247.37643432617188, |
| "logps/ref_chosen": -108.50582885742188, |
| "logps/ref_rejected": -88.300048828125, |
| "logps/rejected": -308.53436279296875, |
| "loss": 0.9699, |
| "margin_dpo/margin_mean": 81.36367797851562, |
| "margin_dpo/margin_std": 92.48495483398438, |
| "step": 426 |
| }, |
| { |
| "epoch": 0.6270190895741556, |
| "fcm_dpo/beta": 0.007682330906391144, |
| "fcm_dpo/delta": 0.018056850880384445, |
| "fcm_dpo/margin": 75.87571716308594, |
| "fcm_dpo/q_t": 0.3702627420425415, |
| "grad_norm": 10.52995491027832, |
| "learning_rate": 1.8529523872436977e-07, |
| "logits/chosen": -2.527616262435913, |
| "logits/rejected": -1.911233901977539, |
| "logps/chosen": -213.69146728515625, |
| "logps/ref_chosen": -99.12046813964844, |
| "logps/ref_rejected": -85.724609375, |
| "logps/rejected": -276.17132568359375, |
| "loss": 0.9796, |
| "margin_dpo/margin_mean": 75.87571716308594, |
| "margin_dpo/margin_std": 83.9215316772461, |
| "step": 427 |
| }, |
| { |
| "epoch": 0.6284875183553598, |
| "fcm_dpo/beta": 0.007750866934657097, |
| "fcm_dpo/delta": 0.1085626631975174, |
| "fcm_dpo/margin": 64.08061981201172, |
| "fcm_dpo/q_t": 0.3931909203529358, |
| "grad_norm": 11.369186401367188, |
| "learning_rate": 1.8405649477212697e-07, |
| "logits/chosen": -1.6850427389144897, |
| "logits/rejected": -1.4209712743759155, |
| "logps/chosen": -267.068115234375, |
| "logps/ref_chosen": -105.96925354003906, |
| "logps/ref_rejected": -109.10218048095703, |
| "logps/rejected": -334.28167724609375, |
| "loss": 1.0875, |
| "margin_dpo/margin_mean": 64.08061981201172, |
| "margin_dpo/margin_std": 99.4907455444336, |
| "step": 428 |
| }, |
| { |
| "epoch": 0.6299559471365639, |
| "fcm_dpo/beta": 0.0078573664650321, |
| "fcm_dpo/delta": -0.057729579508304596, |
| "fcm_dpo/margin": 83.04490661621094, |
| "fcm_dpo/q_t": 0.3629707396030426, |
| "grad_norm": 11.991065979003906, |
| "learning_rate": 1.828194884925749e-07, |
| "logits/chosen": -2.019653797149658, |
| "logits/rejected": -1.3182734251022339, |
| "logps/chosen": -259.6905517578125, |
| "logps/ref_chosen": -113.54487609863281, |
| "logps/ref_rejected": -98.24201965332031, |
| "logps/rejected": -327.4326171875, |
| "loss": 0.9858, |
| "margin_dpo/margin_mean": 83.04490661621094, |
| "margin_dpo/margin_std": 102.54899597167969, |
| "step": 429 |
| }, |
| { |
| "epoch": 0.631424375917768, |
| "fcm_dpo/beta": 0.00791855901479721, |
| "fcm_dpo/delta": 0.11582081019878387, |
| "fcm_dpo/margin": 61.77325439453125, |
| "fcm_dpo/q_t": 0.39517396688461304, |
| "grad_norm": 11.80427074432373, |
| "learning_rate": 1.8158425248197928e-07, |
| "logits/chosen": -1.3399078845977783, |
| "logits/rejected": -1.298473596572876, |
| "logps/chosen": -235.0428009033203, |
| "logps/ref_chosen": -91.31936645507812, |
| "logps/ref_rejected": -110.1096420288086, |
| "logps/rejected": -315.6063232421875, |
| "loss": 1.0881, |
| "margin_dpo/margin_mean": 61.773258209228516, |
| "margin_dpo/margin_std": 96.4741439819336, |
| "step": 430 |
| }, |
| { |
| "epoch": 0.6328928046989721, |
| "fcm_dpo/beta": 0.007927455008029938, |
| "fcm_dpo/delta": 0.0042446693405508995, |
| "fcm_dpo/margin": 75.17669677734375, |
| "fcm_dpo/q_t": 0.371150940656662, |
| "grad_norm": 12.48121166229248, |
| "learning_rate": 1.8035081928995788e-07, |
| "logits/chosen": -1.9271609783172607, |
| "logits/rejected": -1.5382022857666016, |
| "logps/chosen": -235.30166625976562, |
| "logps/ref_chosen": -93.18122100830078, |
| "logps/ref_rejected": -98.13226318359375, |
| "logps/rejected": -315.4294128417969, |
| "loss": 1.0076, |
| "margin_dpo/margin_mean": 75.17669677734375, |
| "margin_dpo/margin_std": 96.68666076660156, |
| "step": 431 |
| }, |
| { |
| "epoch": 0.6343612334801763, |
| "fcm_dpo/beta": 0.007823891006410122, |
| "fcm_dpo/delta": -0.13029845058918, |
| "fcm_dpo/margin": 92.16277313232422, |
| "fcm_dpo/q_t": 0.34236711263656616, |
| "grad_norm": 10.779977798461914, |
| "learning_rate": 1.791192214186223e-07, |
| "logits/chosen": -1.6222333908081055, |
| "logits/rejected": -1.0763202905654907, |
| "logps/chosen": -229.423828125, |
| "logps/ref_chosen": -104.43478393554688, |
| "logps/ref_rejected": -105.08955383300781, |
| "logps/rejected": -322.24139404296875, |
| "loss": 0.9015, |
| "margin_dpo/margin_mean": 92.16278076171875, |
| "margin_dpo/margin_std": 89.09420776367188, |
| "step": 432 |
| }, |
| { |
| "epoch": 0.6358296622613803, |
| "fcm_dpo/beta": 0.007831670343875885, |
| "fcm_dpo/delta": 0.033429112285375595, |
| "fcm_dpo/margin": 72.43510437011719, |
| "fcm_dpo/q_t": 0.3780878186225891, |
| "grad_norm": 11.7706880569458, |
| "learning_rate": 1.7788949132172193e-07, |
| "logits/chosen": -1.7953574657440186, |
| "logits/rejected": -1.4741289615631104, |
| "logps/chosen": -237.90676879882812, |
| "logps/ref_chosen": -89.84322357177734, |
| "logps/ref_rejected": -101.73345947265625, |
| "logps/rejected": -322.23211669921875, |
| "loss": 1.0302, |
| "margin_dpo/margin_mean": 72.43511199951172, |
| "margin_dpo/margin_std": 97.10591125488281, |
| "step": 433 |
| }, |
| { |
| "epoch": 0.6372980910425844, |
| "fcm_dpo/beta": 0.007778637111186981, |
| "fcm_dpo/delta": -0.019643208011984825, |
| "fcm_dpo/margin": 79.50172424316406, |
| "fcm_dpo/q_t": 0.37190669775009155, |
| "grad_norm": 11.87337875366211, |
| "learning_rate": 1.7666166140378853e-07, |
| "logits/chosen": -2.532245635986328, |
| "logits/rejected": -1.888089656829834, |
| "logps/chosen": -229.92483520507812, |
| "logps/ref_chosen": -97.6925277709961, |
| "logps/ref_rejected": -84.09129333496094, |
| "logps/rejected": -295.8253173828125, |
| "loss": 1.0079, |
| "margin_dpo/margin_mean": 79.50172424316406, |
| "margin_dpo/margin_std": 108.16815185546875, |
| "step": 434 |
| }, |
| { |
| "epoch": 0.6387665198237885, |
| "fcm_dpo/beta": 0.007717899512499571, |
| "fcm_dpo/delta": -0.0468808189034462, |
| "fcm_dpo/margin": 83.42731475830078, |
| "fcm_dpo/q_t": 0.36141249537467957, |
| "grad_norm": 12.820802688598633, |
| "learning_rate": 1.7543576401928218e-07, |
| "logits/chosen": -2.351439952850342, |
| "logits/rejected": -1.8964674472808838, |
| "logps/chosen": -214.12286376953125, |
| "logps/ref_chosen": -86.17192077636719, |
| "logps/ref_rejected": -93.751708984375, |
| "logps/rejected": -305.12994384765625, |
| "loss": 0.9617, |
| "margin_dpo/margin_mean": 83.42730712890625, |
| "margin_dpo/margin_std": 96.24213409423828, |
| "step": 435 |
| }, |
| { |
| "epoch": 0.6402349486049926, |
| "fcm_dpo/beta": 0.00758952833712101, |
| "fcm_dpo/delta": -0.08244708925485611, |
| "fcm_dpo/margin": 89.17771911621094, |
| "fcm_dpo/q_t": 0.3484029769897461, |
| "grad_norm": 11.64830493927002, |
| "learning_rate": 1.742118314717391e-07, |
| "logits/chosen": -1.9208111763000488, |
| "logits/rejected": -1.0720462799072266, |
| "logps/chosen": -211.42588806152344, |
| "logps/ref_chosen": -105.78710174560547, |
| "logps/ref_rejected": -88.62471008300781, |
| "logps/rejected": -283.44122314453125, |
| "loss": 0.9044, |
| "margin_dpo/margin_mean": 89.17771911621094, |
| "margin_dpo/margin_std": 80.94717407226562, |
| "step": 436 |
| }, |
| { |
| "epoch": 0.6417033773861968, |
| "fcm_dpo/beta": 0.007655961439013481, |
| "fcm_dpo/delta": 0.07603086531162262, |
| "fcm_dpo/margin": 68.94998168945312, |
| "fcm_dpo/q_t": 0.38024958968162537, |
| "grad_norm": 11.85803508758545, |
| "learning_rate": 1.7298989601292036e-07, |
| "logits/chosen": -1.9982421398162842, |
| "logits/rejected": -1.3477120399475098, |
| "logps/chosen": -220.46435546875, |
| "logps/ref_chosen": -96.06204223632812, |
| "logps/ref_rejected": -89.01220703125, |
| "logps/rejected": -282.364501953125, |
| "loss": 1.0144, |
| "margin_dpo/margin_mean": 68.94998168945312, |
| "margin_dpo/margin_std": 80.42849731445312, |
| "step": 437 |
| }, |
| { |
| "epoch": 0.6431718061674009, |
| "fcm_dpo/beta": 0.00769776338711381, |
| "fcm_dpo/delta": 0.05381428450345993, |
| "fcm_dpo/margin": 71.3000717163086, |
| "fcm_dpo/q_t": 0.3784894645214081, |
| "grad_norm": 12.725931167602539, |
| "learning_rate": 1.7176998984196144e-07, |
| "logits/chosen": -2.1903576850891113, |
| "logits/rejected": -1.3835668563842773, |
| "logps/chosen": -227.7496337890625, |
| "logps/ref_chosen": -101.85537719726562, |
| "logps/ref_rejected": -89.44766235351562, |
| "logps/rejected": -286.6419677734375, |
| "loss": 1.0157, |
| "margin_dpo/margin_mean": 71.3000717163086, |
| "margin_dpo/margin_std": 86.569091796875, |
| "step": 438 |
| }, |
| { |
| "epoch": 0.644640234948605, |
| "fcm_dpo/beta": 0.007735989056527615, |
| "fcm_dpo/delta": 0.037646569311618805, |
| "fcm_dpo/margin": 72.81299591064453, |
| "fcm_dpo/q_t": 0.3774023652076721, |
| "grad_norm": 13.056401252746582, |
| "learning_rate": 1.7055214510452458e-07, |
| "logits/chosen": -1.570444941520691, |
| "logits/rejected": -1.3217337131500244, |
| "logps/chosen": -219.38943481445312, |
| "logps/ref_chosen": -81.75563049316406, |
| "logps/ref_rejected": -90.58636474609375, |
| "logps/rejected": -301.03314208984375, |
| "loss": 1.0128, |
| "margin_dpo/margin_mean": 72.81299591064453, |
| "margin_dpo/margin_std": 88.81855773925781, |
| "step": 439 |
| }, |
| { |
| "epoch": 0.6461086637298091, |
| "fcm_dpo/beta": 0.007678865920752287, |
| "fcm_dpo/delta": -0.07341113686561584, |
| "fcm_dpo/margin": 86.8636703491211, |
| "fcm_dpo/q_t": 0.35673287510871887, |
| "grad_norm": 10.554864883422852, |
| "learning_rate": 1.6933639389195134e-07, |
| "logits/chosen": -2.4091434478759766, |
| "logits/rejected": -1.9146919250488281, |
| "logps/chosen": -229.0126190185547, |
| "logps/ref_chosen": -105.64108276367188, |
| "logps/ref_rejected": -103.4010009765625, |
| "logps/rejected": -313.63623046875, |
| "loss": 0.9517, |
| "margin_dpo/margin_mean": 86.86366271972656, |
| "margin_dpo/margin_std": 97.84516143798828, |
| "step": 440 |
| }, |
| { |
| "epoch": 0.6475770925110133, |
| "fcm_dpo/beta": 0.007652001921087503, |
| "fcm_dpo/delta": -0.02822795696556568, |
| "fcm_dpo/margin": 81.84053039550781, |
| "fcm_dpo/q_t": 0.36398565769195557, |
| "grad_norm": 11.060223579406738, |
| "learning_rate": 1.681227682404166e-07, |
| "logits/chosen": -2.021010398864746, |
| "logits/rejected": -1.6652534008026123, |
| "logps/chosen": -243.90728759765625, |
| "logps/ref_chosen": -91.529541015625, |
| "logps/ref_rejected": -103.619384765625, |
| "logps/rejected": -337.8376770019531, |
| "loss": 0.9485, |
| "margin_dpo/margin_mean": 81.84053039550781, |
| "margin_dpo/margin_std": 87.4005126953125, |
| "step": 441 |
| }, |
| { |
| "epoch": 0.6490455212922174, |
| "fcm_dpo/beta": 0.007508729584515095, |
| "fcm_dpo/delta": -0.13639724254608154, |
| "fcm_dpo/margin": 96.75260162353516, |
| "fcm_dpo/q_t": 0.33930784463882446, |
| "grad_norm": 11.458242416381836, |
| "learning_rate": 1.669113001300851e-07, |
| "logits/chosen": -1.8102037906646729, |
| "logits/rejected": -1.3520574569702148, |
| "logps/chosen": -236.66015625, |
| "logps/ref_chosen": -84.77755737304688, |
| "logps/ref_rejected": -83.82415771484375, |
| "logps/rejected": -332.4593505859375, |
| "loss": 0.8827, |
| "margin_dpo/margin_mean": 96.75260162353516, |
| "margin_dpo/margin_std": 88.31156158447266, |
| "step": 442 |
| }, |
| { |
| "epoch": 0.6505139500734214, |
| "fcm_dpo/beta": 0.007541971281170845, |
| "fcm_dpo/delta": 0.12367384880781174, |
| "fcm_dpo/margin": 63.93056869506836, |
| "fcm_dpo/q_t": 0.3926345407962799, |
| "grad_norm": 11.647642135620117, |
| "learning_rate": 1.6570202148426815e-07, |
| "logits/chosen": -2.023066997528076, |
| "logits/rejected": -1.4911339282989502, |
| "logps/chosen": -249.26025390625, |
| "logps/ref_chosen": -102.64927673339844, |
| "logps/ref_rejected": -93.03807067871094, |
| "logps/rejected": -303.5796203613281, |
| "loss": 1.057, |
| "margin_dpo/margin_mean": 63.930564880371094, |
| "margin_dpo/margin_std": 82.83998107910156, |
| "step": 443 |
| }, |
| { |
| "epoch": 0.6519823788546255, |
| "fcm_dpo/beta": 0.0077362204901874065, |
| "fcm_dpo/delta": 0.07435087114572525, |
| "fcm_dpo/margin": 68.44133758544922, |
| "fcm_dpo/q_t": 0.3861270546913147, |
| "grad_norm": 12.801765441894531, |
| "learning_rate": 1.6449496416858282e-07, |
| "logits/chosen": -1.6765544414520264, |
| "logits/rejected": -1.5086448192596436, |
| "logps/chosen": -242.2406463623047, |
| "logps/ref_chosen": -87.91971588134766, |
| "logps/ref_rejected": -103.32345581054688, |
| "logps/rejected": -326.08575439453125, |
| "loss": 1.0539, |
| "margin_dpo/margin_mean": 68.44134521484375, |
| "margin_dpo/margin_std": 97.85394287109375, |
| "step": 444 |
| }, |
| { |
| "epoch": 0.6534508076358296, |
| "fcm_dpo/beta": 0.007698110304772854, |
| "fcm_dpo/delta": -0.021563369780778885, |
| "fcm_dpo/margin": 80.50859832763672, |
| "fcm_dpo/q_t": 0.3627839982509613, |
| "grad_norm": 11.485919952392578, |
| "learning_rate": 1.6329015999011182e-07, |
| "logits/chosen": -1.9858006238937378, |
| "logits/rejected": -1.5744715929031372, |
| "logps/chosen": -236.77423095703125, |
| "logps/ref_chosen": -101.40086364746094, |
| "logps/ref_rejected": -99.03790283203125, |
| "logps/rejected": -314.91986083984375, |
| "loss": 0.9579, |
| "margin_dpo/margin_mean": 80.50859832763672, |
| "margin_dpo/margin_std": 85.82842254638672, |
| "step": 445 |
| }, |
| { |
| "epoch": 0.6549192364170338, |
| "fcm_dpo/beta": 0.007764734327793121, |
| "fcm_dpo/delta": 0.028329739347100258, |
| "fcm_dpo/margin": 73.82595825195312, |
| "fcm_dpo/q_t": 0.3735341429710388, |
| "grad_norm": 14.396162986755371, |
| "learning_rate": 1.6208764069656578e-07, |
| "logits/chosen": -1.9688878059387207, |
| "logits/rejected": -1.821672797203064, |
| "logps/chosen": -217.6196746826172, |
| "logps/ref_chosen": -87.4223403930664, |
| "logps/ref_rejected": -106.70075988769531, |
| "logps/rejected": -310.72406005859375, |
| "loss": 0.9884, |
| "margin_dpo/margin_mean": 73.82595825195312, |
| "margin_dpo/margin_std": 83.8984603881836, |
| "step": 446 |
| }, |
| { |
| "epoch": 0.6563876651982379, |
| "fcm_dpo/beta": 0.007610865402966738, |
| "fcm_dpo/delta": -0.12789002060890198, |
| "fcm_dpo/margin": 94.40399169921875, |
| "fcm_dpo/q_t": 0.34479495882987976, |
| "grad_norm": 11.825496673583984, |
| "learning_rate": 1.608874379754465e-07, |
| "logits/chosen": -2.1955559253692627, |
| "logits/rejected": -2.148773670196533, |
| "logps/chosen": -214.28981018066406, |
| "logps/ref_chosen": -83.6152572631836, |
| "logps/ref_rejected": -104.91239929199219, |
| "logps/rejected": -329.990966796875, |
| "loss": 0.9191, |
| "margin_dpo/margin_mean": 94.40399169921875, |
| "margin_dpo/margin_std": 100.11077880859375, |
| "step": 447 |
| }, |
| { |
| "epoch": 0.657856093979442, |
| "fcm_dpo/beta": 0.007517419755458832, |
| "fcm_dpo/delta": -0.09112964570522308, |
| "fcm_dpo/margin": 91.07563781738281, |
| "fcm_dpo/q_t": 0.35592204332351685, |
| "grad_norm": 12.4849271774292, |
| "learning_rate": 1.5968958345321177e-07, |
| "logits/chosen": -1.9084062576293945, |
| "logits/rejected": -1.6441185474395752, |
| "logps/chosen": -232.83050537109375, |
| "logps/ref_chosen": -92.5757827758789, |
| "logps/ref_rejected": -107.68977355957031, |
| "logps/rejected": -339.0201110839844, |
| "loss": 0.9746, |
| "margin_dpo/margin_mean": 91.07563781738281, |
| "margin_dpo/margin_std": 112.18666076660156, |
| "step": 448 |
| }, |
| { |
| "epoch": 0.6593245227606461, |
| "fcm_dpo/beta": 0.007335230708122253, |
| "fcm_dpo/delta": -0.09747156500816345, |
| "fcm_dpo/margin": 94.17698669433594, |
| "fcm_dpo/q_t": 0.34920936822891235, |
| "grad_norm": 10.423608779907227, |
| "learning_rate": 1.584941086944423e-07, |
| "logits/chosen": -1.7010339498519897, |
| "logits/rejected": -1.2772597074508667, |
| "logps/chosen": -240.6586151123047, |
| "logps/ref_chosen": -102.39893341064453, |
| "logps/ref_rejected": -95.14886474609375, |
| "logps/rejected": -327.58551025390625, |
| "loss": 0.9312, |
| "margin_dpo/margin_mean": 94.17698669433594, |
| "margin_dpo/margin_std": 100.80087280273438, |
| "step": 449 |
| }, |
| { |
| "epoch": 0.6607929515418502, |
| "fcm_dpo/beta": 0.007234559860080481, |
| "fcm_dpo/delta": -0.046480074524879456, |
| "fcm_dpo/margin": 88.93949127197266, |
| "fcm_dpo/q_t": 0.35815033316612244, |
| "grad_norm": 11.052569389343262, |
| "learning_rate": 1.573010452010098e-07, |
| "logits/chosen": -2.46598482131958, |
| "logits/rejected": -2.271839141845703, |
| "logps/chosen": -210.14553833007812, |
| "logps/ref_chosen": -86.99285888671875, |
| "logps/ref_rejected": -108.53203582763672, |
| "logps/rejected": -320.62420654296875, |
| "loss": 0.9593, |
| "margin_dpo/margin_mean": 88.93949890136719, |
| "margin_dpo/margin_std": 99.60723876953125, |
| "step": 450 |
| }, |
| { |
| "epoch": 0.6622613803230544, |
| "fcm_dpo/beta": 0.007310773245990276, |
| "fcm_dpo/delta": 0.062415413558483124, |
| "fcm_dpo/margin": 73.95240020751953, |
| "fcm_dpo/q_t": 0.3855394721031189, |
| "grad_norm": 19.515153884887695, |
| "learning_rate": 1.5611042441124687e-07, |
| "logits/chosen": -2.4347171783447266, |
| "logits/rejected": -1.9352984428405762, |
| "logps/chosen": -226.25865173339844, |
| "logps/ref_chosen": -86.811279296875, |
| "logps/ref_rejected": -79.8555908203125, |
| "logps/rejected": -293.25537109375, |
| "loss": 1.0629, |
| "margin_dpo/margin_mean": 73.952392578125, |
| "margin_dpo/margin_std": 110.59036254882812, |
| "step": 451 |
| }, |
| { |
| "epoch": 0.6637298091042585, |
| "fcm_dpo/beta": 0.007255923468619585, |
| "fcm_dpo/delta": 0.012697272002696991, |
| "fcm_dpo/margin": 80.79637145996094, |
| "fcm_dpo/q_t": 0.36846262216567993, |
| "grad_norm": 10.28548812866211, |
| "learning_rate": 1.549222776991186e-07, |
| "logits/chosen": -1.8156119585037231, |
| "logits/rejected": -1.7515356540679932, |
| "logps/chosen": -201.019287109375, |
| "logps/ref_chosen": -79.379638671875, |
| "logps/ref_rejected": -103.71539306640625, |
| "logps/rejected": -306.15142822265625, |
| "loss": 0.9648, |
| "margin_dpo/margin_mean": 80.79637145996094, |
| "margin_dpo/margin_std": 77.09156799316406, |
| "step": 452 |
| }, |
| { |
| "epoch": 0.6651982378854625, |
| "fcm_dpo/beta": 0.007229423150420189, |
| "fcm_dpo/delta": -0.10436852276325226, |
| "fcm_dpo/margin": 96.43700408935547, |
| "fcm_dpo/q_t": 0.34717243909835815, |
| "grad_norm": 14.366576194763184, |
| "learning_rate": 1.5373663637339584e-07, |
| "logits/chosen": -2.0531060695648193, |
| "logits/rejected": -1.5909814834594727, |
| "logps/chosen": -234.8137664794922, |
| "logps/ref_chosen": -87.6951904296875, |
| "logps/ref_rejected": -90.0582275390625, |
| "logps/rejected": -333.6138000488281, |
| "loss": 0.9015, |
| "margin_dpo/margin_mean": 96.43701171875, |
| "margin_dpo/margin_std": 90.89103698730469, |
| "step": 453 |
| }, |
| { |
| "epoch": 0.6666666666666666, |
| "fcm_dpo/beta": 0.007256688550114632, |
| "fcm_dpo/delta": 0.03500938415527344, |
| "fcm_dpo/margin": 78.03349304199219, |
| "fcm_dpo/q_t": 0.3794366121292114, |
| "grad_norm": 12.678661346435547, |
| "learning_rate": 1.5255353167683017e-07, |
| "logits/chosen": -1.8184224367141724, |
| "logits/rejected": -1.3561699390411377, |
| "logps/chosen": -246.05104064941406, |
| "logps/ref_chosen": -89.56623840332031, |
| "logps/ref_rejected": -92.92105102539062, |
| "logps/rejected": -327.4393310546875, |
| "loss": 1.0581, |
| "margin_dpo/margin_mean": 78.03349304199219, |
| "margin_dpo/margin_std": 115.84363555908203, |
| "step": 454 |
| }, |
| { |
| "epoch": 0.6681350954478708, |
| "fcm_dpo/beta": 0.0073183816857635975, |
| "fcm_dpo/delta": 0.04829338192939758, |
| "fcm_dpo/margin": 75.66259765625, |
| "fcm_dpo/q_t": 0.37961339950561523, |
| "grad_norm": 11.24074935913086, |
| "learning_rate": 1.5137299478533064e-07, |
| "logits/chosen": -2.125541925430298, |
| "logits/rejected": -2.0756983757019043, |
| "logps/chosen": -222.00599670410156, |
| "logps/ref_chosen": -77.62998962402344, |
| "logps/ref_rejected": -118.97795104980469, |
| "logps/rejected": -339.0165710449219, |
| "loss": 1.0307, |
| "margin_dpo/margin_mean": 75.66259765625, |
| "margin_dpo/margin_std": 100.04164123535156, |
| "step": 455 |
| }, |
| { |
| "epoch": 0.6696035242290749, |
| "fcm_dpo/beta": 0.00738774798810482, |
| "fcm_dpo/delta": 0.056851405650377274, |
| "fcm_dpo/margin": 73.88893127441406, |
| "fcm_dpo/q_t": 0.37978994846343994, |
| "grad_norm": 13.592082023620605, |
| "learning_rate": 1.5019505680714232e-07, |
| "logits/chosen": -1.9894077777862549, |
| "logits/rejected": -1.8558869361877441, |
| "logps/chosen": -236.3739013671875, |
| "logps/ref_chosen": -89.61686706542969, |
| "logps/ref_rejected": -109.5597152709961, |
| "logps/rejected": -330.2056884765625, |
| "loss": 1.0154, |
| "margin_dpo/margin_mean": 73.88893127441406, |
| "margin_dpo/margin_std": 90.75523376464844, |
| "step": 456 |
| }, |
| { |
| "epoch": 0.671071953010279, |
| "fcm_dpo/beta": 0.007179590407758951, |
| "fcm_dpo/delta": -0.12014248967170715, |
| "fcm_dpo/margin": 98.66180419921875, |
| "fcm_dpo/q_t": 0.33989059925079346, |
| "grad_norm": 12.305672645568848, |
| "learning_rate": 1.4901974878202627e-07, |
| "logits/chosen": -2.244877338409424, |
| "logits/rejected": -1.9393136501312256, |
| "logps/chosen": -209.8995819091797, |
| "logps/ref_chosen": -87.32168579101562, |
| "logps/ref_rejected": -90.76660919189453, |
| "logps/rejected": -312.0063171386719, |
| "loss": 0.8772, |
| "margin_dpo/margin_mean": 98.66180419921875, |
| "margin_dpo/margin_std": 78.503662109375, |
| "step": 457 |
| }, |
| { |
| "epoch": 0.6725403817914831, |
| "fcm_dpo/beta": 0.00729702552780509, |
| "fcm_dpo/delta": 0.06375245004892349, |
| "fcm_dpo/margin": 73.92202758789062, |
| "fcm_dpo/q_t": 0.38597458600997925, |
| "grad_norm": 12.738073348999023, |
| "learning_rate": 1.4784710168044212e-07, |
| "logits/chosen": -1.7707667350769043, |
| "logits/rejected": -1.5331363677978516, |
| "logps/chosen": -233.474365234375, |
| "logps/ref_chosen": -93.52044677734375, |
| "logps/ref_rejected": -103.36898803710938, |
| "logps/rejected": -317.24493408203125, |
| "loss": 1.0455, |
| "margin_dpo/margin_mean": 73.92202758789062, |
| "margin_dpo/margin_std": 105.11477661132812, |
| "step": 458 |
| }, |
| { |
| "epoch": 0.6740088105726872, |
| "fcm_dpo/beta": 0.007197215687483549, |
| "fcm_dpo/delta": -0.05717407539486885, |
| "fcm_dpo/margin": 90.64775085449219, |
| "fcm_dpo/q_t": 0.35962265729904175, |
| "grad_norm": 9.387142181396484, |
| "learning_rate": 1.466771464027316e-07, |
| "logits/chosen": -1.5463656187057495, |
| "logits/rejected": -1.285736322402954, |
| "logps/chosen": -208.82803344726562, |
| "logps/ref_chosen": -75.68820190429688, |
| "logps/ref_rejected": -92.17048645019531, |
| "logps/rejected": -315.95806884765625, |
| "loss": 0.9552, |
| "margin_dpo/margin_mean": 90.64775085449219, |
| "margin_dpo/margin_std": 101.39945983886719, |
| "step": 459 |
| }, |
| { |
| "epoch": 0.6754772393538914, |
| "fcm_dpo/beta": 0.007193270605057478, |
| "fcm_dpo/delta": 0.00503632053732872, |
| "fcm_dpo/margin": 82.70344543457031, |
| "fcm_dpo/q_t": 0.37113451957702637, |
| "grad_norm": 13.572725296020508, |
| "learning_rate": 1.4550991377830423e-07, |
| "logits/chosen": -1.9946367740631104, |
| "logits/rejected": -2.009538412094116, |
| "logps/chosen": -219.1749267578125, |
| "logps/ref_chosen": -81.11788940429688, |
| "logps/ref_rejected": -110.31238555908203, |
| "logps/rejected": -331.0728759765625, |
| "loss": 0.9981, |
| "margin_dpo/margin_mean": 82.70344543457031, |
| "margin_dpo/margin_std": 101.98342895507812, |
| "step": 460 |
| }, |
| { |
| "epoch": 0.6769456681350955, |
| "fcm_dpo/beta": 0.007337551563978195, |
| "fcm_dpo/delta": 0.12816324830055237, |
| "fcm_dpo/margin": 65.16386413574219, |
| "fcm_dpo/q_t": 0.3958069682121277, |
| "grad_norm": 10.882566452026367, |
| "learning_rate": 1.4434543456482518e-07, |
| "logits/chosen": -1.553586483001709, |
| "logits/rejected": -1.35811448097229, |
| "logps/chosen": -250.552734375, |
| "logps/ref_chosen": -81.58352661132812, |
| "logps/ref_rejected": -93.87710571289062, |
| "logps/rejected": -328.01019287109375, |
| "loss": 1.0685, |
| "margin_dpo/margin_mean": 65.16386413574219, |
| "margin_dpo/margin_std": 90.7364501953125, |
| "step": 461 |
| }, |
| { |
| "epoch": 0.6784140969162996, |
| "fcm_dpo/beta": 0.007375137880444527, |
| "fcm_dpo/delta": -0.05297492444515228, |
| "fcm_dpo/margin": 88.07096862792969, |
| "fcm_dpo/q_t": 0.3572496771812439, |
| "grad_norm": 10.971365928649902, |
| "learning_rate": 1.4318373944740484e-07, |
| "logits/chosen": -2.2870004177093506, |
| "logits/rejected": -1.787416696548462, |
| "logps/chosen": -221.18679809570312, |
| "logps/ref_chosen": -94.19854736328125, |
| "logps/ref_rejected": -85.63162994384766, |
| "logps/rejected": -300.69085693359375, |
| "loss": 0.9407, |
| "margin_dpo/margin_mean": 88.07096862792969, |
| "margin_dpo/margin_std": 91.79960632324219, |
| "step": 462 |
| }, |
| { |
| "epoch": 0.6798825256975036, |
| "fcm_dpo/beta": 0.007232788018882275, |
| "fcm_dpo/delta": -0.1597738415002823, |
| "fcm_dpo/margin": 103.35470581054688, |
| "fcm_dpo/q_t": 0.33559200167655945, |
| "grad_norm": 14.430492401123047, |
| "learning_rate": 1.4202485903778976e-07, |
| "logits/chosen": -1.9360418319702148, |
| "logits/rejected": -1.5115666389465332, |
| "logps/chosen": -210.00100708007812, |
| "logps/ref_chosen": -85.92474365234375, |
| "logps/ref_rejected": -96.90184020996094, |
| "logps/rejected": -324.33282470703125, |
| "loss": 0.8987, |
| "margin_dpo/margin_mean": 103.35470581054688, |
| "margin_dpo/margin_std": 98.86663818359375, |
| "step": 463 |
| }, |
| { |
| "epoch": 0.6813509544787077, |
| "fcm_dpo/beta": 0.007100261747837067, |
| "fcm_dpo/delta": 0.008855840191245079, |
| "fcm_dpo/margin": 83.32122802734375, |
| "fcm_dpo/q_t": 0.37016385793685913, |
| "grad_norm": 14.36965274810791, |
| "learning_rate": 1.4086882387355658e-07, |
| "logits/chosen": -2.0437088012695312, |
| "logits/rejected": -2.1165390014648438, |
| "logps/chosen": -232.46543884277344, |
| "logps/ref_chosen": -79.68919372558594, |
| "logps/ref_rejected": -107.29232025146484, |
| "logps/rejected": -343.3898010253906, |
| "loss": 0.9944, |
| "margin_dpo/margin_mean": 83.32122802734375, |
| "margin_dpo/margin_std": 101.45448303222656, |
| "step": 464 |
| }, |
| { |
| "epoch": 0.6828193832599119, |
| "fcm_dpo/beta": 0.006998377852141857, |
| "fcm_dpo/delta": -0.09421905875205994, |
| "fcm_dpo/margin": 98.20915222167969, |
| "fcm_dpo/q_t": 0.3482474386692047, |
| "grad_norm": 12.179363250732422, |
| "learning_rate": 1.3971566441730714e-07, |
| "logits/chosen": -1.6612213850021362, |
| "logits/rejected": -1.3575751781463623, |
| "logps/chosen": -235.50955200195312, |
| "logps/ref_chosen": -91.8602294921875, |
| "logps/ref_rejected": -118.71001434326172, |
| "logps/rejected": -360.5684814453125, |
| "loss": 0.9084, |
| "margin_dpo/margin_mean": 98.20915222167969, |
| "margin_dpo/margin_std": 94.13998413085938, |
| "step": 465 |
| }, |
| { |
| "epoch": 0.684287812041116, |
| "fcm_dpo/beta": 0.006858200766146183, |
| "fcm_dpo/delta": -0.10749548673629761, |
| "fcm_dpo/margin": 102.01173400878906, |
| "fcm_dpo/q_t": 0.3455418348312378, |
| "grad_norm": 8.609355926513672, |
| "learning_rate": 1.3856541105586545e-07, |
| "logits/chosen": -1.733262300491333, |
| "logits/rejected": -1.417356252670288, |
| "logps/chosen": -213.08628845214844, |
| "logps/ref_chosen": -84.70140838623047, |
| "logps/ref_rejected": -96.05084228515625, |
| "logps/rejected": -326.44744873046875, |
| "loss": 0.9046, |
| "margin_dpo/margin_mean": 102.01173400878906, |
| "margin_dpo/margin_std": 98.38445281982422, |
| "step": 466 |
| }, |
| { |
| "epoch": 0.6857562408223201, |
| "fcm_dpo/beta": 0.006808539852499962, |
| "fcm_dpo/delta": 0.01279231533408165, |
| "fcm_dpo/margin": 86.26240539550781, |
| "fcm_dpo/q_t": 0.3721468448638916, |
| "grad_norm": 13.353436470031738, |
| "learning_rate": 1.3741809409947729e-07, |
| "logits/chosen": -2.1667706966400146, |
| "logits/rejected": -1.576673984527588, |
| "logps/chosen": -269.39898681640625, |
| "logps/ref_chosen": -109.29832458496094, |
| "logps/ref_rejected": -108.8436508178711, |
| "logps/rejected": -355.20672607421875, |
| "loss": 1.0157, |
| "margin_dpo/margin_mean": 86.26240539550781, |
| "margin_dpo/margin_std": 111.6978759765625, |
| "step": 467 |
| }, |
| { |
| "epoch": 0.6872246696035242, |
| "fcm_dpo/beta": 0.006919668987393379, |
| "fcm_dpo/delta": 0.07496071606874466, |
| "fcm_dpo/margin": 76.45877075195312, |
| "fcm_dpo/q_t": 0.38796383142471313, |
| "grad_norm": 11.61962604522705, |
| "learning_rate": 1.362737437810114e-07, |
| "logits/chosen": -1.8730924129486084, |
| "logits/rejected": -1.6779046058654785, |
| "logps/chosen": -226.63394165039062, |
| "logps/ref_chosen": -98.32164764404297, |
| "logps/ref_rejected": -106.68048095703125, |
| "logps/rejected": -311.4515380859375, |
| "loss": 1.0509, |
| "margin_dpo/margin_mean": 76.45877075195312, |
| "margin_dpo/margin_std": 109.24627685546875, |
| "step": 468 |
| }, |
| { |
| "epoch": 0.6886930983847284, |
| "fcm_dpo/beta": 0.006858726963400841, |
| "fcm_dpo/delta": -0.05379898101091385, |
| "fcm_dpo/margin": 94.6535873413086, |
| "fcm_dpo/q_t": 0.35775983333587646, |
| "grad_norm": 13.751016616821289, |
| "learning_rate": 1.351323902551631e-07, |
| "logits/chosen": -2.398892879486084, |
| "logits/rejected": -1.9867908954620361, |
| "logps/chosen": -252.68528747558594, |
| "logps/ref_chosen": -96.76420593261719, |
| "logps/ref_rejected": -109.59500885009766, |
| "logps/rejected": -360.169677734375, |
| "loss": 0.9539, |
| "margin_dpo/margin_mean": 94.6535873413086, |
| "margin_dpo/margin_std": 103.79216003417969, |
| "step": 469 |
| }, |
| { |
| "epoch": 0.6901615271659325, |
| "fcm_dpo/beta": 0.00688820518553257, |
| "fcm_dpo/delta": 0.0024176451843231916, |
| "fcm_dpo/margin": 86.77494812011719, |
| "fcm_dpo/q_t": 0.3669547140598297, |
| "grad_norm": 10.803339004516602, |
| "learning_rate": 1.339940635976592e-07, |
| "logits/chosen": -1.6873784065246582, |
| "logits/rejected": -1.2658147811889648, |
| "logps/chosen": -228.20758056640625, |
| "logps/ref_chosen": -83.49665832519531, |
| "logps/ref_rejected": -88.48578643798828, |
| "logps/rejected": -319.9716796875, |
| "loss": 0.962, |
| "margin_dpo/margin_mean": 86.77494812011719, |
| "margin_dpo/margin_std": 90.58299255371094, |
| "step": 470 |
| }, |
| { |
| "epoch": 0.6916299559471366, |
| "fcm_dpo/beta": 0.00690112542361021, |
| "fcm_dpo/delta": 0.02034003473818302, |
| "fcm_dpo/margin": 84.1663589477539, |
| "fcm_dpo/q_t": 0.36976000666618347, |
| "grad_norm": 11.01422119140625, |
| "learning_rate": 1.3285879380446563e-07, |
| "logits/chosen": -1.8118414878845215, |
| "logits/rejected": -1.4235670566558838, |
| "logps/chosen": -255.23658752441406, |
| "logps/ref_chosen": -88.47430419921875, |
| "logps/ref_rejected": -90.48171997070312, |
| "logps/rejected": -341.41033935546875, |
| "loss": 0.973, |
| "margin_dpo/margin_mean": 84.16635131835938, |
| "margin_dpo/margin_std": 89.08399963378906, |
| "step": 471 |
| }, |
| { |
| "epoch": 0.6930983847283406, |
| "fcm_dpo/beta": 0.006775230169296265, |
| "fcm_dpo/delta": -0.1039826050400734, |
| "fcm_dpo/margin": 102.69633483886719, |
| "fcm_dpo/q_t": 0.3456118404865265, |
| "grad_norm": 9.285819053649902, |
| "learning_rate": 1.317266107909975e-07, |
| "logits/chosen": -2.095188617706299, |
| "logits/rejected": -1.393007516860962, |
| "logps/chosen": -276.9514465332031, |
| "logps/ref_chosen": -125.23370361328125, |
| "logps/ref_rejected": -121.05349731445312, |
| "logps/rejected": -375.46759033203125, |
| "loss": 0.8906, |
| "margin_dpo/margin_mean": 102.69633483886719, |
| "margin_dpo/margin_std": 90.67658233642578, |
| "step": 472 |
| }, |
| { |
| "epoch": 0.6945668135095447, |
| "fcm_dpo/beta": 0.0069533102214336395, |
| "fcm_dpo/delta": 0.16166609525680542, |
| "fcm_dpo/margin": 64.00907897949219, |
| "fcm_dpo/q_t": 0.4053763747215271, |
| "grad_norm": 12.549628257751465, |
| "learning_rate": 1.3059754439133002e-07, |
| "logits/chosen": -1.8435192108154297, |
| "logits/rejected": -1.3511978387832642, |
| "logps/chosen": -268.58892822265625, |
| "logps/ref_chosen": -95.61137390136719, |
| "logps/ref_rejected": -88.15115356445312, |
| "logps/rejected": -325.1377868652344, |
| "loss": 1.1087, |
| "margin_dpo/margin_mean": 64.00907897949219, |
| "margin_dpo/margin_std": 105.01445770263672, |
| "step": 473 |
| }, |
| { |
| "epoch": 0.6960352422907489, |
| "fcm_dpo/beta": 0.007215796038508415, |
| "fcm_dpo/delta": 0.14754922688007355, |
| "fcm_dpo/margin": 63.232147216796875, |
| "fcm_dpo/q_t": 0.4007920026779175, |
| "grad_norm": 14.762158393859863, |
| "learning_rate": 1.2947162435741277e-07, |
| "logits/chosen": -1.478015422821045, |
| "logits/rejected": -1.284541368484497, |
| "logps/chosen": -245.74856567382812, |
| "logps/ref_chosen": -81.47975158691406, |
| "logps/ref_rejected": -96.46562957763672, |
| "logps/rejected": -323.9665832519531, |
| "loss": 1.112, |
| "margin_dpo/margin_mean": 63.23215103149414, |
| "margin_dpo/margin_std": 100.12031555175781, |
| "step": 474 |
| }, |
| { |
| "epoch": 0.697503671071953, |
| "fcm_dpo/beta": 0.007110551930963993, |
| "fcm_dpo/delta": -0.09296688437461853, |
| "fcm_dpo/margin": 96.56515502929688, |
| "fcm_dpo/q_t": 0.35164913535118103, |
| "grad_norm": 10.715070724487305, |
| "learning_rate": 1.2834888035828596e-07, |
| "logits/chosen": -2.3761649131774902, |
| "logits/rejected": -2.3209524154663086, |
| "logps/chosen": -197.6427001953125, |
| "logps/ref_chosen": -74.19598388671875, |
| "logps/ref_rejected": -94.69242095947266, |
| "logps/rejected": -314.70428466796875, |
| "loss": 0.9244, |
| "margin_dpo/margin_mean": 96.56515502929688, |
| "margin_dpo/margin_std": 101.15115356445312, |
| "step": 475 |
| }, |
| { |
| "epoch": 0.6989720998531571, |
| "fcm_dpo/beta": 0.007045960985124111, |
| "fcm_dpo/delta": 0.005972862243652344, |
| "fcm_dpo/margin": 84.30451965332031, |
| "fcm_dpo/q_t": 0.3728053569793701, |
| "grad_norm": 14.778757095336914, |
| "learning_rate": 1.2722934197929802e-07, |
| "logits/chosen": -2.1326091289520264, |
| "logits/rejected": -1.7579033374786377, |
| "logps/chosen": -218.2198944091797, |
| "logps/ref_chosen": -71.97109985351562, |
| "logps/ref_rejected": -80.26224517822266, |
| "logps/rejected": -310.8155517578125, |
| "loss": 0.9929, |
| "margin_dpo/margin_mean": 84.30451965332031, |
| "margin_dpo/margin_std": 102.45613098144531, |
| "step": 476 |
| }, |
| { |
| "epoch": 0.7004405286343612, |
| "fcm_dpo/beta": 0.007064092438668013, |
| "fcm_dpo/delta": -0.026253901422023773, |
| "fcm_dpo/margin": 88.41944885253906, |
| "fcm_dpo/q_t": 0.3624919652938843, |
| "grad_norm": 11.6608304977417, |
| "learning_rate": 1.2611303872132631e-07, |
| "logits/chosen": -2.2218666076660156, |
| "logits/rejected": -1.326930284500122, |
| "logps/chosen": -249.8019561767578, |
| "logps/ref_chosen": -105.00555419921875, |
| "logps/ref_rejected": -81.87843322753906, |
| "logps/rejected": -315.09429931640625, |
| "loss": 0.9793, |
| "margin_dpo/margin_mean": 88.41944885253906, |
| "margin_dpo/margin_std": 104.81526947021484, |
| "step": 477 |
| }, |
| { |
| "epoch": 0.7019089574155654, |
| "fcm_dpo/beta": 0.007006206549704075, |
| "fcm_dpo/delta": -0.0406154990196228, |
| "fcm_dpo/margin": 91.0680923461914, |
| "fcm_dpo/q_t": 0.3596052825450897, |
| "grad_norm": 10.416852951049805, |
| "learning_rate": 1.2500000000000005e-07, |
| "logits/chosen": -1.7539746761322021, |
| "logits/rejected": -1.58416748046875, |
| "logps/chosen": -215.23123168945312, |
| "logps/ref_chosen": -76.7882080078125, |
| "logps/ref_rejected": -90.43994140625, |
| "logps/rejected": -319.9510498046875, |
| "loss": 0.9565, |
| "margin_dpo/margin_mean": 91.06809997558594, |
| "margin_dpo/margin_std": 100.75262451171875, |
| "step": 478 |
| }, |
| { |
| "epoch": 0.7033773861967695, |
| "fcm_dpo/beta": 0.007025442086160183, |
| "fcm_dpo/delta": 0.00436464324593544, |
| "fcm_dpo/margin": 84.7488784790039, |
| "fcm_dpo/q_t": 0.36682647466659546, |
| "grad_norm": 13.093260765075684, |
| "learning_rate": 1.2389025514492456e-07, |
| "logits/chosen": -1.6674690246582031, |
| "logits/rejected": -1.5295199155807495, |
| "logps/chosen": -226.80955505371094, |
| "logps/ref_chosen": -81.3623046875, |
| "logps/ref_rejected": -101.09114074707031, |
| "logps/rejected": -331.2872619628906, |
| "loss": 0.9753, |
| "margin_dpo/margin_mean": 84.7488784790039, |
| "margin_dpo/margin_std": 92.19709014892578, |
| "step": 479 |
| }, |
| { |
| "epoch": 0.7048458149779736, |
| "fcm_dpo/beta": 0.007094934582710266, |
| "fcm_dpo/delta": 0.09958771616220474, |
| "fcm_dpo/margin": 71.2434310913086, |
| "fcm_dpo/q_t": 0.3897866904735565, |
| "grad_norm": 13.344147682189941, |
| "learning_rate": 1.227838333989088e-07, |
| "logits/chosen": -1.7132782936096191, |
| "logits/rejected": -1.0284159183502197, |
| "logps/chosen": -262.8040771484375, |
| "logps/ref_chosen": -96.7739028930664, |
| "logps/ref_rejected": -86.40473937988281, |
| "logps/rejected": -323.6783447265625, |
| "loss": 1.0558, |
| "margin_dpo/margin_mean": 71.2434310913086, |
| "margin_dpo/margin_std": 99.02870178222656, |
| "step": 480 |
| }, |
| { |
| "epoch": 0.7063142437591777, |
| "fcm_dpo/beta": 0.007131170481443405, |
| "fcm_dpo/delta": 0.0027018017135560513, |
| "fcm_dpo/margin": 83.7805404663086, |
| "fcm_dpo/q_t": 0.3683137893676758, |
| "grad_norm": 14.994175910949707, |
| "learning_rate": 1.2168076391719489e-07, |
| "logits/chosen": -2.007399559020996, |
| "logits/rejected": -1.5706861019134521, |
| "logps/chosen": -255.43331909179688, |
| "logps/ref_chosen": -91.670166015625, |
| "logps/ref_rejected": -98.69490051269531, |
| "logps/rejected": -346.2386169433594, |
| "loss": 0.9915, |
| "margin_dpo/margin_mean": 83.7805404663086, |
| "margin_dpo/margin_std": 100.96444702148438, |
| "step": 481 |
| }, |
| { |
| "epoch": 0.7077826725403817, |
| "fcm_dpo/beta": 0.00711690541356802, |
| "fcm_dpo/delta": 0.00016288645565509796, |
| "fcm_dpo/margin": 84.26334381103516, |
| "fcm_dpo/q_t": 0.3666335344314575, |
| "grad_norm": 14.931113243103027, |
| "learning_rate": 1.2058107576668938e-07, |
| "logits/chosen": -1.6981770992279053, |
| "logits/rejected": -1.342320442199707, |
| "logps/chosen": -254.57460021972656, |
| "logps/ref_chosen": -98.52011108398438, |
| "logps/ref_rejected": -94.82945251464844, |
| "logps/rejected": -335.14727783203125, |
| "loss": 0.9789, |
| "margin_dpo/margin_mean": 84.26334381103516, |
| "margin_dpo/margin_std": 95.54747009277344, |
| "step": 482 |
| }, |
| { |
| "epoch": 0.7092511013215859, |
| "fcm_dpo/beta": 0.007158739026635885, |
| "fcm_dpo/delta": -0.03249619901180267, |
| "fcm_dpo/margin": 87.9349365234375, |
| "fcm_dpo/q_t": 0.3601437211036682, |
| "grad_norm": 9.465551376342773, |
| "learning_rate": 1.194847979251979e-07, |
| "logits/chosen": -1.9062191247940063, |
| "logits/rejected": -1.2519001960754395, |
| "logps/chosen": -269.0156555175781, |
| "logps/ref_chosen": -107.11860656738281, |
| "logps/ref_rejected": -101.11499786376953, |
| "logps/rejected": -350.9469909667969, |
| "loss": 0.9553, |
| "margin_dpo/margin_mean": 87.9349365234375, |
| "margin_dpo/margin_std": 92.60682678222656, |
| "step": 483 |
| }, |
| { |
| "epoch": 0.71071953010279, |
| "fcm_dpo/beta": 0.007042921148240566, |
| "fcm_dpo/delta": -0.07625816017389297, |
| "fcm_dpo/margin": 95.28274536132812, |
| "fcm_dpo/q_t": 0.3532974123954773, |
| "grad_norm": 10.351212501525879, |
| "learning_rate": 1.1839195928066101e-07, |
| "logits/chosen": -2.5401973724365234, |
| "logits/rejected": -2.068960189819336, |
| "logps/chosen": -238.0645751953125, |
| "logps/ref_chosen": -86.97991943359375, |
| "logps/ref_rejected": -90.72367095947266, |
| "logps/rejected": -337.091064453125, |
| "loss": 0.9457, |
| "margin_dpo/margin_mean": 95.28274536132812, |
| "margin_dpo/margin_std": 104.58644104003906, |
| "step": 484 |
| }, |
| { |
| "epoch": 0.7121879588839941, |
| "fcm_dpo/beta": 0.006840351969003677, |
| "fcm_dpo/delta": -0.14380024373531342, |
| "fcm_dpo/margin": 107.2056884765625, |
| "fcm_dpo/q_t": 0.3391224145889282, |
| "grad_norm": 10.769954681396484, |
| "learning_rate": 1.1730258863039347e-07, |
| "logits/chosen": -1.80538809299469, |
| "logits/rejected": -1.378275990486145, |
| "logps/chosen": -233.96734619140625, |
| "logps/ref_chosen": -94.05874633789062, |
| "logps/ref_rejected": -108.56298828125, |
| "logps/rejected": -355.67724609375, |
| "loss": 0.881, |
| "margin_dpo/margin_mean": 107.2056884765625, |
| "margin_dpo/margin_std": 98.861328125, |
| "step": 485 |
| }, |
| { |
| "epoch": 0.7136563876651982, |
| "fcm_dpo/beta": 0.006765207275748253, |
| "fcm_dpo/delta": -0.04138827323913574, |
| "fcm_dpo/margin": 94.40589904785156, |
| "fcm_dpo/q_t": 0.3596769869327545, |
| "grad_norm": 12.390087127685547, |
| "learning_rate": 1.1621671468032493e-07, |
| "logits/chosen": -1.9631407260894775, |
| "logits/rejected": -1.2379772663116455, |
| "logps/chosen": -255.2804718017578, |
| "logps/ref_chosen": -93.74588012695312, |
| "logps/ref_rejected": -98.07064819335938, |
| "logps/rejected": -354.0111389160156, |
| "loss": 0.9607, |
| "margin_dpo/margin_mean": 94.40589904785156, |
| "margin_dpo/margin_std": 104.61402893066406, |
| "step": 486 |
| }, |
| { |
| "epoch": 0.7151248164464024, |
| "fcm_dpo/beta": 0.006735553964972496, |
| "fcm_dpo/delta": 0.016399431973695755, |
| "fcm_dpo/margin": 86.7838134765625, |
| "fcm_dpo/q_t": 0.3709105849266052, |
| "grad_norm": 11.133604049682617, |
| "learning_rate": 1.1513436604424378e-07, |
| "logits/chosen": -1.875716209411621, |
| "logits/rejected": -1.5496511459350586, |
| "logps/chosen": -243.07168579101562, |
| "logps/ref_chosen": -88.0335693359375, |
| "logps/ref_rejected": -98.47209930419922, |
| "logps/rejected": -340.2940673828125, |
| "loss": 0.9787, |
| "margin_dpo/margin_mean": 86.78382110595703, |
| "margin_dpo/margin_std": 96.44297790527344, |
| "step": 487 |
| }, |
| { |
| "epoch": 0.7165932452276065, |
| "fcm_dpo/beta": 0.006739957258105278, |
| "fcm_dpo/delta": -0.01644238643348217, |
| "fcm_dpo/margin": 91.31090545654297, |
| "fcm_dpo/q_t": 0.3608691692352295, |
| "grad_norm": 12.61165714263916, |
| "learning_rate": 1.1405557124304335e-07, |
| "logits/chosen": -1.6958119869232178, |
| "logits/rejected": -1.4399720430374146, |
| "logps/chosen": -229.36561584472656, |
| "logps/ref_chosen": -84.78964233398438, |
| "logps/ref_rejected": -90.2734603881836, |
| "logps/rejected": -326.16033935546875, |
| "loss": 0.9391, |
| "margin_dpo/margin_mean": 91.3109130859375, |
| "margin_dpo/margin_std": 84.56864929199219, |
| "step": 488 |
| }, |
| { |
| "epoch": 0.7180616740088106, |
| "fcm_dpo/beta": 0.006757946684956551, |
| "fcm_dpo/delta": 0.025971170514822006, |
| "fcm_dpo/margin": 85.16073608398438, |
| "fcm_dpo/q_t": 0.3735937476158142, |
| "grad_norm": 11.42379093170166, |
| "learning_rate": 1.1298035870396985e-07, |
| "logits/chosen": -1.6805338859558105, |
| "logits/rejected": -1.1919324398040771, |
| "logps/chosen": -250.7588653564453, |
| "logps/ref_chosen": -90.46929931640625, |
| "logps/ref_rejected": -86.39761352539062, |
| "logps/rejected": -331.847900390625, |
| "loss": 0.9907, |
| "margin_dpo/margin_mean": 85.16073608398438, |
| "margin_dpo/margin_std": 98.77789306640625, |
| "step": 489 |
| }, |
| { |
| "epoch": 0.7195301027900147, |
| "fcm_dpo/beta": 0.006898378022015095, |
| "fcm_dpo/delta": 0.06882701814174652, |
| "fcm_dpo/margin": 77.1955795288086, |
| "fcm_dpo/q_t": 0.38805437088012695, |
| "grad_norm": 12.629081726074219, |
| "learning_rate": 1.1190875675987355e-07, |
| "logits/chosen": -2.1147122383117676, |
| "logits/rejected": -2.184906482696533, |
| "logps/chosen": -248.49630737304688, |
| "logps/ref_chosen": -85.32012939453125, |
| "logps/ref_rejected": -115.99385070800781, |
| "logps/rejected": -356.3656311035156, |
| "loss": 1.091, |
| "margin_dpo/margin_mean": 77.1955795288086, |
| "margin_dpo/margin_std": 123.41070556640625, |
| "step": 490 |
| }, |
| { |
| "epoch": 0.7209985315712188, |
| "fcm_dpo/beta": 0.006943402346223593, |
| "fcm_dpo/delta": 0.0842859223484993, |
| "fcm_dpo/margin": 74.90750122070312, |
| "fcm_dpo/q_t": 0.38507401943206787, |
| "grad_norm": 11.258004188537598, |
| "learning_rate": 1.1084079364846241e-07, |
| "logits/chosen": -1.9617629051208496, |
| "logits/rejected": -1.4733606576919556, |
| "logps/chosen": -249.405517578125, |
| "logps/ref_chosen": -86.14352416992188, |
| "logps/ref_rejected": -80.67945861816406, |
| "logps/rejected": -318.8489685058594, |
| "loss": 1.0404, |
| "margin_dpo/margin_mean": 74.9074935913086, |
| "margin_dpo/margin_std": 99.26988220214844, |
| "step": 491 |
| }, |
| { |
| "epoch": 0.7224669603524229, |
| "fcm_dpo/beta": 0.0071650012396276, |
| "fcm_dpo/delta": 0.1844560205936432, |
| "fcm_dpo/margin": 59.10064697265625, |
| "fcm_dpo/q_t": 0.40985268354415894, |
| "grad_norm": 16.328256607055664, |
| "learning_rate": 1.097764975115576e-07, |
| "logits/chosen": -2.2341504096984863, |
| "logits/rejected": -1.856196641921997, |
| "logps/chosen": -232.42469787597656, |
| "logps/ref_chosen": -81.10757446289062, |
| "logps/ref_rejected": -80.75199890136719, |
| "logps/rejected": -291.1697692871094, |
| "loss": 1.1711, |
| "margin_dpo/margin_mean": 59.10065460205078, |
| "margin_dpo/margin_std": 119.12300872802734, |
| "step": 492 |
| }, |
| { |
| "epoch": 0.723935389133627, |
| "fcm_dpo/beta": 0.007225475274026394, |
| "fcm_dpo/delta": 0.03164070099592209, |
| "fcm_dpo/margin": 78.82869720458984, |
| "fcm_dpo/q_t": 0.37277752161026, |
| "grad_norm": 15.412538528442383, |
| "learning_rate": 1.0871589639435203e-07, |
| "logits/chosen": -2.5492968559265137, |
| "logits/rejected": -1.8315892219543457, |
| "logps/chosen": -262.19671630859375, |
| "logps/ref_chosen": -112.20733642578125, |
| "logps/ref_rejected": -93.60719299316406, |
| "logps/rejected": -322.42523193359375, |
| "loss": 0.9933, |
| "margin_dpo/margin_mean": 78.82870483398438, |
| "margin_dpo/margin_std": 88.707275390625, |
| "step": 493 |
| }, |
| { |
| "epoch": 0.7254038179148311, |
| "fcm_dpo/beta": 0.007072390988469124, |
| "fcm_dpo/delta": -0.1455509066581726, |
| "fcm_dpo/margin": 103.59664916992188, |
| "fcm_dpo/q_t": 0.3390905261039734, |
| "grad_norm": 12.173338890075684, |
| "learning_rate": 1.0765901824467166e-07, |
| "logits/chosen": -1.4517457485198975, |
| "logits/rejected": -1.3680830001831055, |
| "logps/chosen": -218.484375, |
| "logps/ref_chosen": -73.11489868164062, |
| "logps/ref_rejected": -92.16300201416016, |
| "logps/rejected": -341.1291198730469, |
| "loss": 0.8945, |
| "margin_dpo/margin_mean": 103.59664916992188, |
| "margin_dpo/margin_std": 99.80902862548828, |
| "step": 494 |
| }, |
| { |
| "epoch": 0.7268722466960352, |
| "fcm_dpo/beta": 0.0070437416434288025, |
| "fcm_dpo/delta": -0.02011135406792164, |
| "fcm_dpo/margin": 87.85247802734375, |
| "fcm_dpo/q_t": 0.36548200249671936, |
| "grad_norm": 12.353002548217773, |
| "learning_rate": 1.0660589091223854e-07, |
| "logits/chosen": -2.263014316558838, |
| "logits/rejected": -1.8048030138015747, |
| "logps/chosen": -248.50387573242188, |
| "logps/ref_chosen": -99.52031707763672, |
| "logps/ref_rejected": -97.93089294433594, |
| "logps/rejected": -334.76690673828125, |
| "loss": 0.9874, |
| "margin_dpo/margin_mean": 87.85247802734375, |
| "margin_dpo/margin_std": 108.39643859863281, |
| "step": 495 |
| }, |
| { |
| "epoch": 0.7283406754772394, |
| "fcm_dpo/beta": 0.007146789226680994, |
| "fcm_dpo/delta": 0.0695204809308052, |
| "fcm_dpo/margin": 74.6668701171875, |
| "fcm_dpo/q_t": 0.3810140788555145, |
| "grad_norm": 11.570090293884277, |
| "learning_rate": 1.0555654214793722e-07, |
| "logits/chosen": -2.214664936065674, |
| "logits/rejected": -1.5754952430725098, |
| "logps/chosen": -272.01220703125, |
| "logps/ref_chosen": -107.85675048828125, |
| "logps/ref_rejected": -92.77056121826172, |
| "logps/rejected": -331.5928955078125, |
| "loss": 1.0234, |
| "margin_dpo/margin_mean": 74.6668701171875, |
| "margin_dpo/margin_std": 91.9163818359375, |
| "step": 496 |
| }, |
| { |
| "epoch": 0.7298091042584435, |
| "fcm_dpo/beta": 0.007371243089437485, |
| "fcm_dpo/delta": 0.22922512888908386, |
| "fcm_dpo/margin": 51.54722595214844, |
| "fcm_dpo/q_t": 0.4190720319747925, |
| "grad_norm": 11.735376358032227, |
| "learning_rate": 1.0451099960308374e-07, |
| "logits/chosen": -1.9056165218353271, |
| "logits/rejected": -1.4678566455841064, |
| "logps/chosen": -271.3009033203125, |
| "logps/ref_chosen": -92.08322143554688, |
| "logps/ref_rejected": -81.79503631591797, |
| "logps/rejected": -312.5599365234375, |
| "loss": 1.1688, |
| "margin_dpo/margin_mean": 51.54722213745117, |
| "margin_dpo/margin_std": 99.53231048583984, |
| "step": 497 |
| }, |
| { |
| "epoch": 0.7312775330396476, |
| "fcm_dpo/beta": 0.007443058304488659, |
| "fcm_dpo/delta": -0.029292959719896317, |
| "fcm_dpo/margin": 84.29811096191406, |
| "fcm_dpo/q_t": 0.3616979122161865, |
| "grad_norm": 13.830591201782227, |
| "learning_rate": 1.0346929082869641e-07, |
| "logits/chosen": -1.9414262771606445, |
| "logits/rejected": -1.4592478275299072, |
| "logps/chosen": -244.99317932128906, |
| "logps/ref_chosen": -98.19437408447266, |
| "logps/ref_rejected": -90.68745422363281, |
| "logps/rejected": -321.7843933105469, |
| "loss": 0.9608, |
| "margin_dpo/margin_mean": 84.29811096191406, |
| "margin_dpo/margin_std": 93.23235321044922, |
| "step": 498 |
| }, |
| { |
| "epoch": 0.7327459618208517, |
| "fcm_dpo/beta": 0.0074178483337163925, |
| "fcm_dpo/delta": -0.029103467240929604, |
| "fcm_dpo/margin": 84.56077575683594, |
| "fcm_dpo/q_t": 0.36542099714279175, |
| "grad_norm": 15.131891250610352, |
| "learning_rate": 1.0243144327477013e-07, |
| "logits/chosen": -1.9305214881896973, |
| "logits/rejected": -1.8768131732940674, |
| "logps/chosen": -239.75906372070312, |
| "logps/ref_chosen": -81.0399169921875, |
| "logps/ref_rejected": -106.92170715332031, |
| "logps/rejected": -350.2016296386719, |
| "loss": 0.992, |
| "margin_dpo/margin_mean": 84.56077575683594, |
| "margin_dpo/margin_std": 106.96080017089844, |
| "step": 499 |
| }, |
| { |
| "epoch": 0.7342143906020558, |
| "fcm_dpo/beta": 0.007414363324642181, |
| "fcm_dpo/delta": 0.03089684061706066, |
| "fcm_dpo/margin": 76.994140625, |
| "fcm_dpo/q_t": 0.381319522857666, |
| "grad_norm": 14.555001258850098, |
| "learning_rate": 1.0139748428955333e-07, |
| "logits/chosen": -1.7026747465133667, |
| "logits/rejected": -1.5297117233276367, |
| "logps/chosen": -260.76824951171875, |
| "logps/ref_chosen": -89.248046875, |
| "logps/ref_rejected": -100.41021728515625, |
| "logps/rejected": -348.9245300292969, |
| "loss": 1.0372, |
| "margin_dpo/margin_mean": 76.994140625, |
| "margin_dpo/margin_std": 109.05560302734375, |
| "step": 500 |
| }, |
| { |
| "epoch": 0.7342143906020558, |
| "eval_fcm_dpo/beta": 0.0074382987804710865, |
| "eval_logits/chosen": -1.973371148109436, |
| "eval_logits/rejected": -1.5978766679763794, |
| "eval_logps/chosen": -264.56976318359375, |
| "eval_logps/ref_chosen": -100.49356842041016, |
| "eval_logps/ref_rejected": -94.06775665283203, |
| "eval_logps/rejected": -330.363525390625, |
| "eval_loss": 0.5229191184043884, |
| "eval_margin_dpo/margin_mean": 72.21954345703125, |
| "eval_margin_dpo/margin_std": 99.1610107421875, |
| "eval_runtime": 43.7228, |
| "eval_samples_per_second": 53.496, |
| "eval_steps_per_second": 1.692, |
| "step": 500 |
| }, |
| { |
| "epoch": 0.73568281938326, |
| "fcm_dpo/beta": 0.007509919814765453, |
| "fcm_dpo/delta": -0.011263996362686157, |
| "fcm_dpo/margin": 80.9957046508789, |
| "fcm_dpo/q_t": 0.3736144006252289, |
| "grad_norm": 14.008468627929688, |
| "learning_rate": 1.0036744111882672e-07, |
| "logits/chosen": -1.6845519542694092, |
| "logits/rejected": -1.2528064250946045, |
| "logps/chosen": -241.5672607421875, |
| "logps/ref_chosen": -90.596923828125, |
| "logps/ref_rejected": -85.2972640991211, |
| "logps/rejected": -317.2633361816406, |
| "loss": 1.0472, |
| "margin_dpo/margin_mean": 80.99571228027344, |
| "margin_dpo/margin_std": 117.20719909667969, |
| "step": 501 |
| }, |
| { |
| "epoch": 0.737151248164464, |
| "fcm_dpo/beta": 0.007532845251262188, |
| "fcm_dpo/delta": 0.05232547968626022, |
| "fcm_dpo/margin": 72.90962982177734, |
| "fcm_dpo/q_t": 0.38146716356277466, |
| "grad_norm": 12.48260498046875, |
| "learning_rate": 9.934134090518592e-08, |
| "logits/chosen": -1.9113637208938599, |
| "logits/rejected": -1.1727447509765625, |
| "logps/chosen": -257.06707763671875, |
| "logps/ref_chosen": -103.47073364257812, |
| "logps/ref_rejected": -89.04415893554688, |
| "logps/rejected": -315.55010986328125, |
| "loss": 1.0651, |
| "margin_dpo/margin_mean": 72.90962982177734, |
| "margin_dpo/margin_std": 107.84392547607422, |
| "step": 502 |
| }, |
| { |
| "epoch": 0.7386196769456681, |
| "fcm_dpo/beta": 0.007508490234613419, |
| "fcm_dpo/delta": -0.042639248073101044, |
| "fcm_dpo/margin": 85.12161254882812, |
| "fcm_dpo/q_t": 0.36175140738487244, |
| "grad_norm": 11.972295761108398, |
| "learning_rate": 9.831921068732571e-08, |
| "logits/chosen": -1.7123284339904785, |
| "logits/rejected": -1.2041406631469727, |
| "logps/chosen": -243.31890869140625, |
| "logps/ref_chosen": -93.72998809814453, |
| "logps/ref_rejected": -87.60896301269531, |
| "logps/rejected": -322.3194885253906, |
| "loss": 0.9774, |
| "margin_dpo/margin_mean": 85.12161254882812, |
| "margin_dpo/margin_std": 100.05804443359375, |
| "step": 503 |
| }, |
| { |
| "epoch": 0.7400881057268722, |
| "fcm_dpo/beta": 0.007257069461047649, |
| "fcm_dpo/delta": -0.17726479470729828, |
| "fcm_dpo/margin": 105.27082824707031, |
| "fcm_dpo/q_t": 0.3317434787750244, |
| "grad_norm": 15.06264877319336, |
| "learning_rate": 9.730107739932805e-08, |
| "logits/chosen": -2.0711169242858887, |
| "logits/rejected": -1.7322639226913452, |
| "logps/chosen": -242.6743927001953, |
| "logps/ref_chosen": -92.2918701171875, |
| "logps/ref_rejected": -109.03765106201172, |
| "logps/rejected": -364.6910095214844, |
| "loss": 0.8702, |
| "margin_dpo/margin_mean": 105.27082824707031, |
| "margin_dpo/margin_std": 96.85734558105469, |
| "step": 504 |
| }, |
| { |
| "epoch": 0.7415565345080763, |
| "fcm_dpo/beta": 0.007209095172584057, |
| "fcm_dpo/delta": 0.027257125824689865, |
| "fcm_dpo/margin": 79.66241455078125, |
| "fcm_dpo/q_t": 0.3737871050834656, |
| "grad_norm": 11.139007568359375, |
| "learning_rate": 9.628696786995188e-08, |
| "logits/chosen": -2.458643674850464, |
| "logits/rejected": -1.9072757959365845, |
| "logps/chosen": -274.4703063964844, |
| "logps/ref_chosen": -106.27693176269531, |
| "logps/ref_rejected": -94.20611572265625, |
| "logps/rejected": -342.0618896484375, |
| "loss": 0.9951, |
| "margin_dpo/margin_mean": 79.66241455078125, |
| "margin_dpo/margin_std": 93.06168365478516, |
| "step": 505 |
| }, |
| { |
| "epoch": 0.7430249632892805, |
| "fcm_dpo/beta": 0.007173755206167698, |
| "fcm_dpo/delta": -0.01147904247045517, |
| "fcm_dpo/margin": 85.08673095703125, |
| "fcm_dpo/q_t": 0.3622966408729553, |
| "grad_norm": 11.042329788208008, |
| "learning_rate": 9.527690882192635e-08, |
| "logits/chosen": -2.1348423957824707, |
| "logits/rejected": -1.7820146083831787, |
| "logps/chosen": -245.61795043945312, |
| "logps/ref_chosen": -81.44086456298828, |
| "logps/ref_rejected": -85.66439819335938, |
| "logps/rejected": -334.92822265625, |
| "loss": 0.9651, |
| "margin_dpo/margin_mean": 85.08673095703125, |
| "margin_dpo/margin_std": 90.8485107421875, |
| "step": 506 |
| }, |
| { |
| "epoch": 0.7444933920704846, |
| "fcm_dpo/beta": 0.007161298766732216, |
| "fcm_dpo/delta": -0.0466289147734642, |
| "fcm_dpo/margin": 89.87873077392578, |
| "fcm_dpo/q_t": 0.3620367646217346, |
| "grad_norm": 11.227087020874023, |
| "learning_rate": 9.427092687124691e-08, |
| "logits/chosen": -2.110520362854004, |
| "logits/rejected": -1.6617255210876465, |
| "logps/chosen": -269.304931640625, |
| "logps/ref_chosen": -99.42694854736328, |
| "logps/ref_rejected": -102.25296020507812, |
| "logps/rejected": -362.00970458984375, |
| "loss": 0.9832, |
| "margin_dpo/margin_mean": 89.87872314453125, |
| "margin_dpo/margin_std": 110.67439270019531, |
| "step": 507 |
| }, |
| { |
| "epoch": 0.7459618208516887, |
| "fcm_dpo/beta": 0.007206980139017105, |
| "fcm_dpo/delta": 0.07583308964967728, |
| "fcm_dpo/margin": 73.293212890625, |
| "fcm_dpo/q_t": 0.3894771933555603, |
| "grad_norm": 15.699549674987793, |
| "learning_rate": 9.326904852647344e-08, |
| "logits/chosen": -2.079139232635498, |
| "logits/rejected": -1.599036455154419, |
| "logps/chosen": -264.4894714355469, |
| "logps/ref_chosen": -97.9156494140625, |
| "logps/ref_rejected": -103.79782104492188, |
| "logps/rejected": -343.66485595703125, |
| "loss": 1.0835, |
| "margin_dpo/margin_mean": 73.293212890625, |
| "margin_dpo/margin_std": 114.59175872802734, |
| "step": 508 |
| }, |
| { |
| "epoch": 0.7474302496328928, |
| "fcm_dpo/beta": 0.007344736717641354, |
| "fcm_dpo/delta": 0.07915890216827393, |
| "fcm_dpo/margin": 71.44049072265625, |
| "fcm_dpo/q_t": 0.3829602003097534, |
| "grad_norm": 17.265066146850586, |
| "learning_rate": 9.227130018803195e-08, |
| "logits/chosen": -2.11910080909729, |
| "logits/rejected": -1.4653677940368652, |
| "logps/chosen": -257.30712890625, |
| "logps/ref_chosen": -100.72984313964844, |
| "logps/ref_rejected": -86.943359375, |
| "logps/rejected": -314.961181640625, |
| "loss": 1.0586, |
| "margin_dpo/margin_mean": 71.44050598144531, |
| "margin_dpo/margin_std": 101.78379821777344, |
| "step": 509 |
| }, |
| { |
| "epoch": 0.748898678414097, |
| "fcm_dpo/beta": 0.0073591480031609535, |
| "fcm_dpo/delta": -0.015946993604302406, |
| "fcm_dpo/margin": 83.5523452758789, |
| "fcm_dpo/q_t": 0.36579430103302, |
| "grad_norm": 14.097626686096191, |
| "learning_rate": 9.127770814751932e-08, |
| "logits/chosen": -1.7049915790557861, |
| "logits/rejected": -1.5141171216964722, |
| "logps/chosen": -272.86090087890625, |
| "logps/ref_chosen": -89.16938781738281, |
| "logps/ref_rejected": -107.91939544677734, |
| "logps/rejected": -375.1632385253906, |
| "loss": 0.9857, |
| "margin_dpo/margin_mean": 83.55235290527344, |
| "margin_dpo/margin_std": 98.11227416992188, |
| "step": 510 |
| }, |
| { |
| "epoch": 0.750367107195301, |
| "fcm_dpo/beta": 0.007278499659150839, |
| "fcm_dpo/delta": -0.07066594809293747, |
| "fcm_dpo/margin": 91.5009765625, |
| "fcm_dpo/q_t": 0.3567890226840973, |
| "grad_norm": 12.136595726013184, |
| "learning_rate": 9.028829858700973e-08, |
| "logits/chosen": -2.2216238975524902, |
| "logits/rejected": -1.6600124835968018, |
| "logps/chosen": -255.50558471679688, |
| "logps/ref_chosen": -89.74979400634766, |
| "logps/ref_rejected": -97.73127746582031, |
| "logps/rejected": -354.988037109375, |
| "loss": 0.9465, |
| "margin_dpo/margin_mean": 91.5009765625, |
| "margin_dpo/margin_std": 102.25430297851562, |
| "step": 511 |
| }, |
| { |
| "epoch": 0.7518355359765051, |
| "fcm_dpo/beta": 0.0072515252977609634, |
| "fcm_dpo/delta": -0.02720758691430092, |
| "fcm_dpo/margin": 86.15798950195312, |
| "fcm_dpo/q_t": 0.3647354543209076, |
| "grad_norm": 16.666259765625, |
| "learning_rate": 8.930309757836516e-08, |
| "logits/chosen": -1.8731253147125244, |
| "logits/rejected": -1.4305176734924316, |
| "logps/chosen": -244.8845977783203, |
| "logps/ref_chosen": -85.73515319824219, |
| "logps/ref_rejected": -87.6248779296875, |
| "logps/rejected": -332.93231201171875, |
| "loss": 0.9965, |
| "margin_dpo/margin_mean": 86.1579818725586, |
| "margin_dpo/margin_std": 108.06047058105469, |
| "step": 512 |
| }, |
| { |
| "epoch": 0.7533039647577092, |
| "fcm_dpo/beta": 0.007211950141936541, |
| "fcm_dpo/delta": 0.06090360879898071, |
| "fcm_dpo/margin": 75.15151977539062, |
| "fcm_dpo/q_t": 0.37852686643600464, |
| "grad_norm": 12.995072364807129, |
| "learning_rate": 8.832213108254863e-08, |
| "logits/chosen": -2.311227798461914, |
| "logits/rejected": -1.6142702102661133, |
| "logps/chosen": -264.4798583984375, |
| "logps/ref_chosen": -96.19050598144531, |
| "logps/ref_rejected": -82.91773986816406, |
| "logps/rejected": -326.3586120605469, |
| "loss": 1.0292, |
| "margin_dpo/margin_mean": 75.15151977539062, |
| "margin_dpo/margin_std": 95.23574829101562, |
| "step": 513 |
| }, |
| { |
| "epoch": 0.7547723935389133, |
| "fcm_dpo/beta": 0.007290856912732124, |
| "fcm_dpo/delta": 0.017535630613565445, |
| "fcm_dpo/margin": 80.029296875, |
| "fcm_dpo/q_t": 0.37469518184661865, |
| "grad_norm": 14.053443908691406, |
| "learning_rate": 8.734542494893954e-08, |
| "logits/chosen": -1.6758110523223877, |
| "logits/rejected": -1.0973663330078125, |
| "logps/chosen": -259.1905822753906, |
| "logps/ref_chosen": -97.82363891601562, |
| "logps/ref_rejected": -85.36982727050781, |
| "logps/rejected": -326.76605224609375, |
| "loss": 1.0028, |
| "margin_dpo/margin_mean": 80.029296875, |
| "margin_dpo/margin_std": 99.76383972167969, |
| "step": 514 |
| }, |
| { |
| "epoch": 0.7562408223201175, |
| "fcm_dpo/beta": 0.00737146707251668, |
| "fcm_dpo/delta": 0.06299495697021484, |
| "fcm_dpo/margin": 73.31365203857422, |
| "fcm_dpo/q_t": 0.381864994764328, |
| "grad_norm": 17.193218231201172, |
| "learning_rate": 8.637300491465272e-08, |
| "logits/chosen": -1.7850375175476074, |
| "logits/rejected": -1.5526132583618164, |
| "logps/chosen": -240.5751953125, |
| "logps/ref_chosen": -81.93355560302734, |
| "logps/ref_rejected": -93.57656860351562, |
| "logps/rejected": -325.5318603515625, |
| "loss": 1.0369, |
| "margin_dpo/margin_mean": 73.31365203857422, |
| "margin_dpo/margin_std": 97.8846206665039, |
| "step": 515 |
| }, |
| { |
| "epoch": 0.7577092511013216, |
| "fcm_dpo/beta": 0.00745687261223793, |
| "fcm_dpo/delta": 0.0405082106590271, |
| "fcm_dpo/margin": 75.3163833618164, |
| "fcm_dpo/q_t": 0.3833583891391754, |
| "grad_norm": 13.243304252624512, |
| "learning_rate": 8.540489660386064e-08, |
| "logits/chosen": -2.4003708362579346, |
| "logits/rejected": -2.211030960083008, |
| "logps/chosen": -257.3096923828125, |
| "logps/ref_chosen": -97.09166717529297, |
| "logps/ref_rejected": -118.55667114257812, |
| "logps/rejected": -354.0910949707031, |
| "loss": 1.0819, |
| "margin_dpo/margin_mean": 75.3163833618164, |
| "margin_dpo/margin_std": 121.37615966796875, |
| "step": 516 |
| }, |
| { |
| "epoch": 0.7591776798825257, |
| "fcm_dpo/beta": 0.007305598817765713, |
| "fcm_dpo/delta": -0.07763826102018356, |
| "fcm_dpo/margin": 91.7294921875, |
| "fcm_dpo/q_t": 0.3567124009132385, |
| "grad_norm": 12.040420532226562, |
| "learning_rate": 8.444112552711752e-08, |
| "logits/chosen": -1.9732433557510376, |
| "logits/rejected": -1.5516693592071533, |
| "logps/chosen": -264.7029113769531, |
| "logps/ref_chosen": -93.18656921386719, |
| "logps/ref_rejected": -95.5927963256836, |
| "logps/rejected": -358.838623046875, |
| "loss": 0.9666, |
| "margin_dpo/margin_mean": 91.7294921875, |
| "margin_dpo/margin_std": 106.93476104736328, |
| "step": 517 |
| }, |
| { |
| "epoch": 0.7606461086637298, |
| "fcm_dpo/beta": 0.007446288131177425, |
| "fcm_dpo/delta": 0.09759702533483505, |
| "fcm_dpo/margin": 68.14192199707031, |
| "fcm_dpo/q_t": 0.3923153877258301, |
| "grad_norm": 15.57699966430664, |
| "learning_rate": 8.348171708068747e-08, |
| "logits/chosen": -2.0085532665252686, |
| "logits/rejected": -1.6788051128387451, |
| "logps/chosen": -266.467041015625, |
| "logps/ref_chosen": -87.0213623046875, |
| "logps/ref_rejected": -91.24456024169922, |
| "logps/rejected": -338.8321533203125, |
| "loss": 1.0852, |
| "margin_dpo/margin_mean": 68.14192199707031, |
| "margin_dpo/margin_std": 108.119873046875, |
| "step": 518 |
| }, |
| { |
| "epoch": 0.762114537444934, |
| "fcm_dpo/beta": 0.007489525713026524, |
| "fcm_dpo/delta": 0.032412007451057434, |
| "fcm_dpo/margin": 75.99348449707031, |
| "fcm_dpo/q_t": 0.37510281801223755, |
| "grad_norm": 12.590337753295898, |
| "learning_rate": 8.25266965458755e-08, |
| "logits/chosen": -1.9630098342895508, |
| "logits/rejected": -1.5859870910644531, |
| "logps/chosen": -266.3951416015625, |
| "logps/ref_chosen": -100.68755340576172, |
| "logps/ref_rejected": -109.03042602539062, |
| "logps/rejected": -350.73150634765625, |
| "loss": 0.9882, |
| "margin_dpo/margin_mean": 75.99348449707031, |
| "margin_dpo/margin_std": 85.33174133300781, |
| "step": 519 |
| }, |
| { |
| "epoch": 0.7635829662261381, |
| "fcm_dpo/beta": 0.007443825248628855, |
| "fcm_dpo/delta": -0.06601864099502563, |
| "fcm_dpo/margin": 88.84642028808594, |
| "fcm_dpo/q_t": 0.3531188368797302, |
| "grad_norm": 11.373416900634766, |
| "learning_rate": 8.15760890883607e-08, |
| "logits/chosen": -1.54036545753479, |
| "logits/rejected": -1.0939218997955322, |
| "logps/chosen": -250.6941375732422, |
| "logps/ref_chosen": -98.77762603759766, |
| "logps/ref_rejected": -106.9457015991211, |
| "logps/rejected": -347.7086486816406, |
| "loss": 0.9295, |
| "margin_dpo/margin_mean": 88.84642028808594, |
| "margin_dpo/margin_std": 88.97753143310547, |
| "step": 520 |
| }, |
| { |
| "epoch": 0.7650513950073421, |
| "fcm_dpo/beta": 0.0074075451120734215, |
| "fcm_dpo/delta": -0.06705917418003082, |
| "fcm_dpo/margin": 89.41288757324219, |
| "fcm_dpo/q_t": 0.35395270586013794, |
| "grad_norm": 12.826030731201172, |
| "learning_rate": 8.062991975753378e-08, |
| "logits/chosen": -2.1257705688476562, |
| "logits/rejected": -1.7991995811462402, |
| "logps/chosen": -244.10650634765625, |
| "logps/ref_chosen": -86.35108947753906, |
| "logps/ref_rejected": -90.39317321777344, |
| "logps/rejected": -337.56146240234375, |
| "loss": 0.9291, |
| "margin_dpo/margin_mean": 89.41287994384766, |
| "margin_dpo/margin_std": 89.05442810058594, |
| "step": 521 |
| }, |
| { |
| "epoch": 0.7665198237885462, |
| "fcm_dpo/beta": 0.007335927337408066, |
| "fcm_dpo/delta": -0.02064390294253826, |
| "fcm_dpo/margin": 84.41744232177734, |
| "fcm_dpo/q_t": 0.36679452657699585, |
| "grad_norm": 12.274141311645508, |
| "learning_rate": 7.968821348583643e-08, |
| "logits/chosen": -1.7425016164779663, |
| "logits/rejected": -1.3717331886291504, |
| "logps/chosen": -236.16275024414062, |
| "logps/ref_chosen": -76.008056640625, |
| "logps/ref_rejected": -74.44125366210938, |
| "logps/rejected": -319.0133972167969, |
| "loss": 0.9823, |
| "margin_dpo/margin_mean": 84.41744995117188, |
| "margin_dpo/margin_std": 101.350341796875, |
| "step": 522 |
| }, |
| { |
| "epoch": 0.7679882525697503, |
| "fcm_dpo/beta": 0.007474465761333704, |
| "fcm_dpo/delta": 0.16874706745147705, |
| "fcm_dpo/margin": 58.72737121582031, |
| "fcm_dpo/q_t": 0.40729784965515137, |
| "grad_norm": 14.717813491821289, |
| "learning_rate": 7.875099508810484e-08, |
| "logits/chosen": -1.9797134399414062, |
| "logits/rejected": -1.2677710056304932, |
| "logps/chosen": -276.0887451171875, |
| "logps/ref_chosen": -96.49627685546875, |
| "logps/ref_rejected": -91.0152816772461, |
| "logps/rejected": -329.3351135253906, |
| "loss": 1.1446, |
| "margin_dpo/margin_mean": 58.72737503051758, |
| "margin_dpo/margin_std": 108.98553466796875, |
| "step": 523 |
| }, |
| { |
| "epoch": 0.7694566813509545, |
| "fcm_dpo/beta": 0.007531831972301006, |
| "fcm_dpo/delta": -0.0187044907361269, |
| "fcm_dpo/margin": 81.98780822753906, |
| "fcm_dpo/q_t": 0.36835557222366333, |
| "grad_norm": 16.366233825683594, |
| "learning_rate": 7.781828926091535e-08, |
| "logits/chosen": -2.228161334991455, |
| "logits/rejected": -1.5479540824890137, |
| "logps/chosen": -268.5673828125, |
| "logps/ref_chosen": -103.40550994873047, |
| "logps/ref_rejected": -87.91015625, |
| "logps/rejected": -335.059814453125, |
| "loss": 0.9845, |
| "margin_dpo/margin_mean": 81.98780822753906, |
| "margin_dpo/margin_std": 100.14673614501953, |
| "step": 524 |
| }, |
| { |
| "epoch": 0.7709251101321586, |
| "fcm_dpo/beta": 0.007332179695367813, |
| "fcm_dpo/delta": -0.08620290458202362, |
| "fcm_dpo/margin": 92.11900329589844, |
| "fcm_dpo/q_t": 0.3595668077468872, |
| "grad_norm": 12.162001609802246, |
| "learning_rate": 7.689012058193384e-08, |
| "logits/chosen": -1.6582214832305908, |
| "logits/rejected": -1.572461724281311, |
| "logps/chosen": -239.363525390625, |
| "logps/ref_chosen": -84.20648193359375, |
| "logps/ref_rejected": -106.26158905029297, |
| "logps/rejected": -353.53765869140625, |
| "loss": 0.9656, |
| "margin_dpo/margin_mean": 92.11900329589844, |
| "margin_dpo/margin_std": 107.05292510986328, |
| "step": 525 |
| }, |
| { |
| "epoch": 0.7723935389133627, |
| "fcm_dpo/beta": 0.007367183454334736, |
| "fcm_dpo/delta": -0.03810001537203789, |
| "fcm_dpo/margin": 86.28730773925781, |
| "fcm_dpo/q_t": 0.36337071657180786, |
| "grad_norm": 10.43152141571045, |
| "learning_rate": 7.596651350926836e-08, |
| "logits/chosen": -1.8134328126907349, |
| "logits/rejected": -1.1642718315124512, |
| "logps/chosen": -264.8533935546875, |
| "logps/ref_chosen": -102.85411071777344, |
| "logps/ref_rejected": -93.65145874023438, |
| "logps/rejected": -341.9380187988281, |
| "loss": 0.9824, |
| "margin_dpo/margin_mean": 86.28730773925781, |
| "margin_dpo/margin_std": 105.58222961425781, |
| "step": 526 |
| }, |
| { |
| "epoch": 0.7738619676945668, |
| "fcm_dpo/beta": 0.007302050478756428, |
| "fcm_dpo/delta": -0.04052596166729927, |
| "fcm_dpo/margin": 87.3643798828125, |
| "fcm_dpo/q_t": 0.3573962152004242, |
| "grad_norm": 12.692599296569824, |
| "learning_rate": 7.504749238082414e-08, |
| "logits/chosen": -3.0793709754943848, |
| "logits/rejected": -2.2557575702667236, |
| "logps/chosen": -257.8551025390625, |
| "logps/ref_chosen": -96.76543426513672, |
| "logps/ref_rejected": -85.39518737792969, |
| "logps/rejected": -333.8492431640625, |
| "loss": 0.9283, |
| "margin_dpo/margin_mean": 87.36438751220703, |
| "margin_dpo/margin_std": 82.9266357421875, |
| "step": 527 |
| }, |
| { |
| "epoch": 0.775330396475771, |
| "fcm_dpo/beta": 0.0072532519698143005, |
| "fcm_dpo/delta": -0.04537132754921913, |
| "fcm_dpo/margin": 88.57562255859375, |
| "fcm_dpo/q_t": 0.3604862689971924, |
| "grad_norm": 12.81235122680664, |
| "learning_rate": 7.413308141366254e-08, |
| "logits/chosen": -2.018289089202881, |
| "logits/rejected": -1.5019323825836182, |
| "logps/chosen": -244.18003845214844, |
| "logps/ref_chosen": -94.1649169921875, |
| "logps/ref_rejected": -100.372314453125, |
| "logps/rejected": -338.96307373046875, |
| "loss": 0.9672, |
| "margin_dpo/margin_mean": 88.57562255859375, |
| "margin_dpo/margin_std": 102.981201171875, |
| "step": 528 |
| }, |
| { |
| "epoch": 0.7767988252569751, |
| "fcm_dpo/beta": 0.007266518659889698, |
| "fcm_dpo/delta": 0.034581415355205536, |
| "fcm_dpo/margin": 78.07312774658203, |
| "fcm_dpo/q_t": 0.3804980516433716, |
| "grad_norm": 12.89798641204834, |
| "learning_rate": 7.322330470336313e-08, |
| "logits/chosen": -1.6607023477554321, |
| "logits/rejected": -1.4771153926849365, |
| "logps/chosen": -255.24560546875, |
| "logps/ref_chosen": -80.77680206298828, |
| "logps/ref_rejected": -97.191162109375, |
| "logps/rejected": -349.73309326171875, |
| "loss": 1.0404, |
| "margin_dpo/margin_mean": 78.0731201171875, |
| "margin_dpo/margin_std": 111.84912109375, |
| "step": 529 |
| }, |
| { |
| "epoch": 0.7782672540381792, |
| "fcm_dpo/beta": 0.007262610364705324, |
| "fcm_dpo/delta": 0.023171119391918182, |
| "fcm_dpo/margin": 79.56475830078125, |
| "fcm_dpo/q_t": 0.37278905510902405, |
| "grad_norm": 11.019624710083008, |
| "learning_rate": 7.231818622338822e-08, |
| "logits/chosen": -1.5359987020492554, |
| "logits/rejected": -1.2066240310668945, |
| "logps/chosen": -239.765869140625, |
| "logps/ref_chosen": -79.7757339477539, |
| "logps/ref_rejected": -92.05390167236328, |
| "logps/rejected": -331.6087646484375, |
| "loss": 0.977, |
| "margin_dpo/margin_mean": 79.56475830078125, |
| "margin_dpo/margin_std": 86.05628967285156, |
| "step": 530 |
| }, |
| { |
| "epoch": 0.7797356828193832, |
| "fcm_dpo/beta": 0.007175838574767113, |
| "fcm_dpo/delta": -0.11706428229808807, |
| "fcm_dpo/margin": 98.76272583007812, |
| "fcm_dpo/q_t": 0.3418416380882263, |
| "grad_norm": 11.516433715820312, |
| "learning_rate": 7.141774982445147e-08, |
| "logits/chosen": -1.8055527210235596, |
| "logits/rejected": -1.1960854530334473, |
| "logps/chosen": -235.0717315673828, |
| "logps/ref_chosen": -79.31919860839844, |
| "logps/ref_rejected": -78.10832214355469, |
| "logps/rejected": -332.62359619140625, |
| "loss": 0.8993, |
| "margin_dpo/margin_mean": 98.76272583007812, |
| "margin_dpo/margin_std": 93.84896850585938, |
| "step": 531 |
| }, |
| { |
| "epoch": 0.7812041116005873, |
| "fcm_dpo/beta": 0.006985160522162914, |
| "fcm_dpo/delta": -0.16617754101753235, |
| "fcm_dpo/margin": 107.93487548828125, |
| "fcm_dpo/q_t": 0.334781676530838, |
| "grad_norm": 11.529740333557129, |
| "learning_rate": 7.052201923388953e-08, |
| "logits/chosen": -1.7615710496902466, |
| "logits/rejected": -1.1453497409820557, |
| "logps/chosen": -266.70928955078125, |
| "logps/ref_chosen": -101.63691711425781, |
| "logps/ref_rejected": -93.34539031982422, |
| "logps/rejected": -366.3526611328125, |
| "loss": 0.8779, |
| "margin_dpo/margin_mean": 107.93487548828125, |
| "margin_dpo/margin_std": 99.59811401367188, |
| "step": 532 |
| }, |
| { |
| "epoch": 0.7826725403817915, |
| "fcm_dpo/beta": 0.007072297390550375, |
| "fcm_dpo/delta": 0.15511363744735718, |
| "fcm_dpo/margin": 63.86847686767578, |
| "fcm_dpo/q_t": 0.4039548635482788, |
| "grad_norm": 12.298042297363281, |
| "learning_rate": 6.963101805503646e-08, |
| "logits/chosen": -2.4961955547332764, |
| "logits/rejected": -1.911888837814331, |
| "logps/chosen": -251.64068603515625, |
| "logps/ref_chosen": -90.74664306640625, |
| "logps/ref_rejected": -82.88848876953125, |
| "logps/rejected": -307.6510009765625, |
| "loss": 1.1014, |
| "margin_dpo/margin_mean": 63.86847686767578, |
| "margin_dpo/margin_std": 102.30418395996094, |
| "step": 533 |
| }, |
| { |
| "epoch": 0.7841409691629956, |
| "fcm_dpo/beta": 0.007072807289659977, |
| "fcm_dpo/delta": -0.053309373557567596, |
| "fcm_dpo/margin": 91.88386535644531, |
| "fcm_dpo/q_t": 0.3540825843811035, |
| "grad_norm": 10.521297454833984, |
| "learning_rate": 6.874476976660184e-08, |
| "logits/chosen": -2.0540428161621094, |
| "logits/rejected": -1.6685407161712646, |
| "logps/chosen": -231.44000244140625, |
| "logps/ref_chosen": -86.6308364868164, |
| "logps/ref_rejected": -84.87196350097656, |
| "logps/rejected": -321.56500244140625, |
| "loss": 0.9216, |
| "margin_dpo/margin_mean": 91.88386535644531, |
| "margin_dpo/margin_std": 85.76850128173828, |
| "step": 534 |
| }, |
| { |
| "epoch": 0.7856093979441997, |
| "fcm_dpo/beta": 0.0070062074810266495, |
| "fcm_dpo/delta": -0.07865796238183975, |
| "fcm_dpo/margin": 96.07039642333984, |
| "fcm_dpo/q_t": 0.3533538579940796, |
| "grad_norm": 10.109777450561523, |
| "learning_rate": 6.786329772205246e-08, |
| "logits/chosen": -1.8793187141418457, |
| "logits/rejected": -1.6919231414794922, |
| "logps/chosen": -236.0832061767578, |
| "logps/ref_chosen": -90.34539031982422, |
| "logps/ref_rejected": -100.7779769897461, |
| "logps/rejected": -342.586181640625, |
| "loss": 0.939, |
| "margin_dpo/margin_mean": 96.07039642333984, |
| "margin_dpo/margin_std": 102.51802062988281, |
| "step": 535 |
| }, |
| { |
| "epoch": 0.7870778267254038, |
| "fcm_dpo/beta": 0.006879711989313364, |
| "fcm_dpo/delta": -0.037070631980895996, |
| "fcm_dpo/margin": 92.2296371459961, |
| "fcm_dpo/q_t": 0.3588649034500122, |
| "grad_norm": 10.414005279541016, |
| "learning_rate": 6.698662514899638e-08, |
| "logits/chosen": -1.927714228630066, |
| "logits/rejected": -1.678464651107788, |
| "logps/chosen": -227.02041625976562, |
| "logps/ref_chosen": -78.95956420898438, |
| "logps/ref_rejected": -93.08779907226562, |
| "logps/rejected": -333.3782958984375, |
| "loss": 0.9498, |
| "margin_dpo/margin_mean": 92.22964477539062, |
| "margin_dpo/margin_std": 97.07234954833984, |
| "step": 536 |
| }, |
| { |
| "epoch": 0.788546255506608, |
| "fcm_dpo/beta": 0.006824993994086981, |
| "fcm_dpo/delta": -0.11446214467287064, |
| "fcm_dpo/margin": 103.43373107910156, |
| "fcm_dpo/q_t": 0.34388312697410583, |
| "grad_norm": 9.952900886535645, |
| "learning_rate": 6.611477514857114e-08, |
| "logits/chosen": -1.647277593612671, |
| "logits/rejected": -0.9549525380134583, |
| "logps/chosen": -214.62451171875, |
| "logps/ref_chosen": -86.70939636230469, |
| "logps/ref_rejected": -78.90766906738281, |
| "logps/rejected": -310.25653076171875, |
| "loss": 0.8996, |
| "margin_dpo/margin_mean": 103.43373107910156, |
| "margin_dpo/margin_std": 94.62214660644531, |
| "step": 537 |
| }, |
| { |
| "epoch": 0.7900146842878121, |
| "fcm_dpo/beta": 0.006767315790057182, |
| "fcm_dpo/delta": 0.02974122203886509, |
| "fcm_dpo/margin": 84.50604248046875, |
| "fcm_dpo/q_t": 0.37143373489379883, |
| "grad_norm": 11.350445747375488, |
| "learning_rate": 6.524777069483525e-08, |
| "logits/chosen": -1.9193155765533447, |
| "logits/rejected": -1.3912053108215332, |
| "logps/chosen": -269.3667297363281, |
| "logps/ref_chosen": -98.44214630126953, |
| "logps/ref_rejected": -91.09465026855469, |
| "logps/rejected": -346.5252990722656, |
| "loss": 0.9744, |
| "margin_dpo/margin_mean": 84.50605010986328, |
| "margin_dpo/margin_std": 87.87483215332031, |
| "step": 538 |
| }, |
| { |
| "epoch": 0.7914831130690162, |
| "fcm_dpo/beta": 0.006792771629989147, |
| "fcm_dpo/delta": 0.016240041702985764, |
| "fcm_dpo/margin": 86.07090759277344, |
| "fcm_dpo/q_t": 0.3723304867744446, |
| "grad_norm": 10.20103645324707, |
| "learning_rate": 6.438563463416221e-08, |
| "logits/chosen": -2.425452709197998, |
| "logits/rejected": -1.947152018547058, |
| "logps/chosen": -242.98489379882812, |
| "logps/ref_chosen": -90.93211364746094, |
| "logps/ref_rejected": -97.44546508789062, |
| "logps/rejected": -335.56915283203125, |
| "loss": 1.005, |
| "margin_dpo/margin_mean": 86.07090759277344, |
| "margin_dpo/margin_std": 107.96048736572266, |
| "step": 539 |
| }, |
| { |
| "epoch": 0.7929515418502202, |
| "fcm_dpo/beta": 0.006723560392856598, |
| "fcm_dpo/delta": -0.10000798851251602, |
| "fcm_dpo/margin": 103.08011627197266, |
| "fcm_dpo/q_t": 0.3510266840457916, |
| "grad_norm": 12.132619857788086, |
| "learning_rate": 6.352838968463919e-08, |
| "logits/chosen": -2.134742498397827, |
| "logits/rejected": -1.8956007957458496, |
| "logps/chosen": -265.84393310546875, |
| "logps/ref_chosen": -107.86990356445312, |
| "logps/ref_rejected": -121.39350128173828, |
| "logps/rejected": -382.4476318359375, |
| "loss": 0.9372, |
| "margin_dpo/margin_mean": 103.08011627197266, |
| "margin_dpo/margin_std": 114.15684509277344, |
| "step": 540 |
| }, |
| { |
| "epoch": 0.7944199706314243, |
| "fcm_dpo/beta": 0.00676216883584857, |
| "fcm_dpo/delta": 0.15339398384094238, |
| "fcm_dpo/margin": 67.08287048339844, |
| "fcm_dpo/q_t": 0.4000394940376282, |
| "grad_norm": 11.626500129699707, |
| "learning_rate": 6.267605843546767e-08, |
| "logits/chosen": -2.422119140625, |
| "logits/rejected": -1.958017349243164, |
| "logps/chosen": -283.84246826171875, |
| "logps/ref_chosen": -106.56748962402344, |
| "logps/ref_rejected": -109.06633758544922, |
| "logps/rejected": -353.4242248535156, |
| "loss": 1.0758, |
| "margin_dpo/margin_mean": 67.08287048339844, |
| "margin_dpo/margin_std": 90.2316665649414, |
| "step": 541 |
| }, |
| { |
| "epoch": 0.7958883994126285, |
| "fcm_dpo/beta": 0.006774522364139557, |
| "fcm_dpo/delta": -0.09346829354763031, |
| "fcm_dpo/margin": 101.42529296875, |
| "fcm_dpo/q_t": 0.3474395275115967, |
| "grad_norm": 11.156708717346191, |
| "learning_rate": 6.182866334636888e-08, |
| "logits/chosen": -2.3203277587890625, |
| "logits/rejected": -2.1964328289031982, |
| "logps/chosen": -237.198486328125, |
| "logps/ref_chosen": -87.36929321289062, |
| "logps/ref_rejected": -102.82034301757812, |
| "logps/rejected": -354.0748291015625, |
| "loss": 0.9173, |
| "margin_dpo/margin_mean": 101.42529296875, |
| "margin_dpo/margin_std": 100.42752838134766, |
| "step": 542 |
| }, |
| { |
| "epoch": 0.7973568281938326, |
| "fcm_dpo/beta": 0.006940866820514202, |
| "fcm_dpo/delta": 0.23537762463092804, |
| "fcm_dpo/margin": 53.89453887939453, |
| "fcm_dpo/q_t": 0.422155499458313, |
| "grad_norm": 14.22153377532959, |
| "learning_rate": 6.098622674699147e-08, |
| "logits/chosen": -1.7961983680725098, |
| "logits/rejected": -1.828367829322815, |
| "logps/chosen": -261.8563232421875, |
| "logps/ref_chosen": -87.152587890625, |
| "logps/ref_rejected": -111.53172302246094, |
| "logps/rejected": -340.1300048828125, |
| "loss": 1.1992, |
| "margin_dpo/margin_mean": 53.89453887939453, |
| "margin_dpo/margin_std": 116.86991882324219, |
| "step": 543 |
| }, |
| { |
| "epoch": 0.7988252569750367, |
| "fcm_dpo/beta": 0.007195335812866688, |
| "fcm_dpo/delta": 0.13186031579971313, |
| "fcm_dpo/margin": 65.94219207763672, |
| "fcm_dpo/q_t": 0.3974402844905853, |
| "grad_norm": 13.812807083129883, |
| "learning_rate": 6.01487708363232e-08, |
| "logits/chosen": -2.0716145038604736, |
| "logits/rejected": -1.9726595878601074, |
| "logps/chosen": -265.1961669921875, |
| "logps/ref_chosen": -86.14691162109375, |
| "logps/ref_rejected": -106.3939208984375, |
| "logps/rejected": -351.3853759765625, |
| "loss": 1.0679, |
| "margin_dpo/margin_mean": 65.94218444824219, |
| "margin_dpo/margin_std": 94.23214721679688, |
| "step": 544 |
| }, |
| { |
| "epoch": 0.8002936857562408, |
| "fcm_dpo/beta": 0.007269757799804211, |
| "fcm_dpo/delta": 0.005659213289618492, |
| "fcm_dpo/margin": 81.78932189941406, |
| "fcm_dpo/q_t": 0.37121522426605225, |
| "grad_norm": 12.46208667755127, |
| "learning_rate": 5.9316317682106294e-08, |
| "logits/chosen": -1.712825059890747, |
| "logits/rejected": -1.2983132600784302, |
| "logps/chosen": -258.543701171875, |
| "logps/ref_chosen": -96.42424011230469, |
| "logps/ref_rejected": -100.73405456542969, |
| "logps/rejected": -344.642822265625, |
| "loss": 1.0252, |
| "margin_dpo/margin_mean": 81.7893295288086, |
| "margin_dpo/margin_std": 111.28877258300781, |
| "step": 545 |
| }, |
| { |
| "epoch": 0.801762114537445, |
| "fcm_dpo/beta": 0.007279456127434969, |
| "fcm_dpo/delta": 0.041319385170936584, |
| "fcm_dpo/margin": 77.05686950683594, |
| "fcm_dpo/q_t": 0.37605637311935425, |
| "grad_norm": 12.261453628540039, |
| "learning_rate": 5.848888922025552e-08, |
| "logits/chosen": -1.5237065553665161, |
| "logits/rejected": -1.1611510515213013, |
| "logps/chosen": -252.166259765625, |
| "logps/ref_chosen": -86.02112579345703, |
| "logps/ref_rejected": -87.37263488769531, |
| "logps/rejected": -330.5746154785156, |
| "loss": 1.0274, |
| "margin_dpo/margin_mean": 77.05686950683594, |
| "margin_dpo/margin_std": 102.12185668945312, |
| "step": 546 |
| }, |
| { |
| "epoch": 0.8032305433186491, |
| "fcm_dpo/beta": 0.007295276969671249, |
| "fcm_dpo/delta": -0.05277668312191963, |
| "fcm_dpo/margin": 88.98194885253906, |
| "fcm_dpo/q_t": 0.3542296886444092, |
| "grad_norm": 11.10513973236084, |
| "learning_rate": 5.7666507254280265e-08, |
| "logits/chosen": -1.8882625102996826, |
| "logits/rejected": -1.4319205284118652, |
| "logps/chosen": -260.6878662109375, |
| "logps/ref_chosen": -100.6510009765625, |
| "logps/ref_rejected": -97.9117431640625, |
| "logps/rejected": -346.9305419921875, |
| "loss": 0.9216, |
| "margin_dpo/margin_mean": 88.98194885253906, |
| "margin_dpo/margin_std": 82.17151641845703, |
| "step": 547 |
| }, |
| { |
| "epoch": 0.8046989720998532, |
| "fcm_dpo/beta": 0.00735054537653923, |
| "fcm_dpo/delta": 0.093236543238163, |
| "fcm_dpo/margin": 69.57292175292969, |
| "fcm_dpo/q_t": 0.39055800437927246, |
| "grad_norm": 13.908289909362793, |
| "learning_rate": 5.684919345471029e-08, |
| "logits/chosen": -2.291468858718872, |
| "logits/rejected": -1.938688039779663, |
| "logps/chosen": -255.26414489746094, |
| "logps/ref_chosen": -96.03010559082031, |
| "logps/ref_rejected": -98.41322326660156, |
| "logps/rejected": -327.2201843261719, |
| "loss": 1.1036, |
| "margin_dpo/margin_mean": 69.57292938232422, |
| "margin_dpo/margin_std": 116.0625, |
| "step": 548 |
| }, |
| { |
| "epoch": 0.8061674008810573, |
| "fcm_dpo/beta": 0.007444350048899651, |
| "fcm_dpo/delta": 0.020497113466262817, |
| "fcm_dpo/margin": 77.89366149902344, |
| "fcm_dpo/q_t": 0.3747206926345825, |
| "grad_norm": 14.169367790222168, |
| "learning_rate": 5.603696935852426e-08, |
| "logits/chosen": -1.9140429496765137, |
| "logits/rejected": -1.494286060333252, |
| "logps/chosen": -231.77978515625, |
| "logps/ref_chosen": -83.45095825195312, |
| "logps/ref_rejected": -81.38023376464844, |
| "logps/rejected": -307.60272216796875, |
| "loss": 1.0385, |
| "margin_dpo/margin_mean": 77.89366149902344, |
| "margin_dpo/margin_std": 108.47966003417969, |
| "step": 549 |
| }, |
| { |
| "epoch": 0.8076358296622613, |
| "fcm_dpo/beta": 0.007483618333935738, |
| "fcm_dpo/delta": 0.04548526555299759, |
| "fcm_dpo/margin": 74.3836898803711, |
| "fcm_dpo/q_t": 0.3776836395263672, |
| "grad_norm": 12.159344673156738, |
| "learning_rate": 5.5229856368582376e-08, |
| "logits/chosen": -1.6402368545532227, |
| "logits/rejected": -1.4186840057373047, |
| "logps/chosen": -255.8831329345703, |
| "logps/ref_chosen": -89.59434509277344, |
| "logps/ref_rejected": -101.31549835205078, |
| "logps/rejected": -341.98797607421875, |
| "loss": 1.0083, |
| "margin_dpo/margin_mean": 74.38369750976562, |
| "margin_dpo/margin_std": 88.22019958496094, |
| "step": 550 |
| }, |
| { |
| "epoch": 0.8091042584434655, |
| "fcm_dpo/beta": 0.007342686876654625, |
| "fcm_dpo/delta": -0.06708859652280807, |
| "fcm_dpo/margin": 90.0142593383789, |
| "fcm_dpo/q_t": 0.3585403561592102, |
| "grad_norm": 14.69068717956543, |
| "learning_rate": 5.4427875753062734e-08, |
| "logits/chosen": -1.7167030572891235, |
| "logits/rejected": -1.5703403949737549, |
| "logps/chosen": -241.60235595703125, |
| "logps/ref_chosen": -95.08216094970703, |
| "logps/ref_rejected": -117.7192611694336, |
| "logps/rejected": -354.25372314453125, |
| "loss": 0.942, |
| "margin_dpo/margin_mean": 90.0142593383789, |
| "margin_dpo/margin_std": 97.10916137695312, |
| "step": 551 |
| }, |
| { |
| "epoch": 0.8105726872246696, |
| "fcm_dpo/beta": 0.0069796196185052395, |
| "fcm_dpo/delta": -0.29933837056159973, |
| "fcm_dpo/margin": 124.37358093261719, |
| "fcm_dpo/q_t": 0.31151920557022095, |
| "grad_norm": 10.166218757629395, |
| "learning_rate": 5.363104864490034e-08, |
| "logits/chosen": -2.0635464191436768, |
| "logits/rejected": -1.6969006061553955, |
| "logps/chosen": -239.97976684570312, |
| "logps/ref_chosen": -101.40918731689453, |
| "logps/ref_rejected": -110.83843994140625, |
| "logps/rejected": -373.7825927734375, |
| "loss": 0.8214, |
| "margin_dpo/margin_mean": 124.37358093261719, |
| "margin_dpo/margin_std": 107.26200866699219, |
| "step": 552 |
| }, |
| { |
| "epoch": 0.8120411160058737, |
| "fcm_dpo/beta": 0.007029990665614605, |
| "fcm_dpo/delta": 0.06483317166566849, |
| "fcm_dpo/margin": 76.58863830566406, |
| "fcm_dpo/q_t": 0.3810678720474243, |
| "grad_norm": 11.677241325378418, |
| "learning_rate": 5.2839396041230415e-08, |
| "logits/chosen": -1.5280718803405762, |
| "logits/rejected": -1.200793981552124, |
| "logps/chosen": -251.30337524414062, |
| "logps/ref_chosen": -97.96388244628906, |
| "logps/ref_rejected": -103.57719421386719, |
| "logps/rejected": -333.50531005859375, |
| "loss": 1.0384, |
| "margin_dpo/margin_mean": 76.58863830566406, |
| "margin_dpo/margin_std": 102.57685089111328, |
| "step": 553 |
| }, |
| { |
| "epoch": 0.8135095447870778, |
| "fcm_dpo/beta": 0.007085290737450123, |
| "fcm_dpo/delta": 0.005978398025035858, |
| "fcm_dpo/margin": 83.78956604003906, |
| "fcm_dpo/q_t": 0.37405747175216675, |
| "grad_norm": 11.701247215270996, |
| "learning_rate": 5.205293880283551e-08, |
| "logits/chosen": -1.9315733909606934, |
| "logits/rejected": -1.1649553775787354, |
| "logps/chosen": -265.73052978515625, |
| "logps/ref_chosen": -103.68680572509766, |
| "logps/ref_rejected": -92.99210357666016, |
| "logps/rejected": -338.82537841796875, |
| "loss": 1.0497, |
| "margin_dpo/margin_mean": 83.78956604003906, |
| "margin_dpo/margin_std": 123.03929138183594, |
| "step": 554 |
| }, |
| { |
| "epoch": 0.8149779735682819, |
| "fcm_dpo/beta": 0.007088308688253164, |
| "fcm_dpo/delta": 0.04228687286376953, |
| "fcm_dpo/margin": 79.01618957519531, |
| "fcm_dpo/q_t": 0.38176795840263367, |
| "grad_norm": 12.377523422241211, |
| "learning_rate": 5.127169765359515e-08, |
| "logits/chosen": -1.7843549251556396, |
| "logits/rejected": -1.688063144683838, |
| "logps/chosen": -255.608154296875, |
| "logps/ref_chosen": -86.20362854003906, |
| "logps/ref_rejected": -114.54719543457031, |
| "logps/rejected": -362.9678955078125, |
| "loss": 1.0493, |
| "margin_dpo/margin_mean": 79.01618957519531, |
| "margin_dpo/margin_std": 114.8555908203125, |
| "step": 555 |
| }, |
| { |
| "epoch": 0.8164464023494861, |
| "fcm_dpo/beta": 0.007074237335473299, |
| "fcm_dpo/delta": -0.01300017535686493, |
| "fcm_dpo/margin": 86.51844024658203, |
| "fcm_dpo/q_t": 0.36618727445602417, |
| "grad_norm": 12.069602012634277, |
| "learning_rate": 5.049569317994012e-08, |
| "logits/chosen": -1.5851771831512451, |
| "logits/rejected": -1.2715528011322021, |
| "logps/chosen": -238.14895629882812, |
| "logps/ref_chosen": -88.40348815917969, |
| "logps/ref_rejected": -106.63911437988281, |
| "logps/rejected": -342.90301513671875, |
| "loss": 0.9853, |
| "margin_dpo/margin_mean": 86.5184326171875, |
| "margin_dpo/margin_std": 104.031494140625, |
| "step": 556 |
| }, |
| { |
| "epoch": 0.8179148311306902, |
| "fcm_dpo/beta": 0.006955698132514954, |
| "fcm_dpo/delta": -0.08430520445108414, |
| "fcm_dpo/margin": 97.38102722167969, |
| "fcm_dpo/q_t": 0.3552389144897461, |
| "grad_norm": 10.077157974243164, |
| "learning_rate": 4.9724945830310144e-08, |
| "logits/chosen": -1.8743904829025269, |
| "logits/rejected": -1.6990039348602295, |
| "logps/chosen": -259.66729736328125, |
| "logps/ref_chosen": -98.83523559570312, |
| "logps/ref_rejected": -115.64888763427734, |
| "logps/rejected": -373.8619689941406, |
| "loss": 0.9496, |
| "margin_dpo/margin_mean": 97.38102722167969, |
| "margin_dpo/margin_std": 110.20993041992188, |
| "step": 557 |
| }, |
| { |
| "epoch": 0.8193832599118943, |
| "fcm_dpo/beta": 0.006852718070149422, |
| "fcm_dpo/delta": -0.09594091773033142, |
| "fcm_dpo/margin": 100.5401840209961, |
| "fcm_dpo/q_t": 0.346677303314209, |
| "grad_norm": 12.043631553649902, |
| "learning_rate": 4.8959475914614554e-08, |
| "logits/chosen": -2.3513083457946777, |
| "logits/rejected": -1.7356767654418945, |
| "logps/chosen": -267.2415771484375, |
| "logps/ref_chosen": -106.01183319091797, |
| "logps/ref_rejected": -108.39131927490234, |
| "logps/rejected": -370.1612548828125, |
| "loss": 0.9025, |
| "margin_dpo/margin_mean": 100.5401840209961, |
| "margin_dpo/margin_std": 92.63386535644531, |
| "step": 558 |
| }, |
| { |
| "epoch": 0.8208516886930984, |
| "fcm_dpo/beta": 0.006862045265734196, |
| "fcm_dpo/delta": 0.020558176562190056, |
| "fcm_dpo/margin": 84.61210632324219, |
| "fcm_dpo/q_t": 0.3711848556995392, |
| "grad_norm": 13.065723419189453, |
| "learning_rate": 4.8199303603697614e-08, |
| "logits/chosen": -2.5563955307006836, |
| "logits/rejected": -2.0908899307250977, |
| "logps/chosen": -271.6568908691406, |
| "logps/ref_chosen": -100.42474365234375, |
| "logps/ref_rejected": -100.41270446777344, |
| "logps/rejected": -356.2569580078125, |
| "loss": 1.0026, |
| "margin_dpo/margin_mean": 84.61210632324219, |
| "margin_dpo/margin_std": 103.49603271484375, |
| "step": 559 |
| }, |
| { |
| "epoch": 0.8223201174743024, |
| "fcm_dpo/beta": 0.006905478425323963, |
| "fcm_dpo/delta": 0.04596704989671707, |
| "fcm_dpo/margin": 80.60478973388672, |
| "fcm_dpo/q_t": 0.37765079736709595, |
| "grad_norm": 13.970322608947754, |
| "learning_rate": 4.7444448928806615e-08, |
| "logits/chosen": -1.1586146354675293, |
| "logits/rejected": -0.6924932599067688, |
| "logps/chosen": -254.54705810546875, |
| "logps/ref_chosen": -87.63551330566406, |
| "logps/ref_rejected": -87.37985229492188, |
| "logps/rejected": -334.89617919921875, |
| "loss": 1.0214, |
| "margin_dpo/margin_mean": 80.60478210449219, |
| "margin_dpo/margin_std": 103.96781921386719, |
| "step": 560 |
| }, |
| { |
| "epoch": 0.8237885462555066, |
| "fcm_dpo/beta": 0.0070221684873104095, |
| "fcm_dpo/delta": 0.06730255484580994, |
| "fcm_dpo/margin": 76.30671691894531, |
| "fcm_dpo/q_t": 0.38102492690086365, |
| "grad_norm": 10.298686981201172, |
| "learning_rate": 4.669493178106432e-08, |
| "logits/chosen": -1.9619371891021729, |
| "logits/rejected": -1.9289535284042358, |
| "logps/chosen": -249.20779418945312, |
| "logps/ref_chosen": -74.88140106201172, |
| "logps/ref_rejected": -106.04796600341797, |
| "logps/rejected": -356.68109130859375, |
| "loss": 1.0044, |
| "margin_dpo/margin_mean": 76.30671691894531, |
| "margin_dpo/margin_std": 86.78927612304688, |
| "step": 561 |
| }, |
| { |
| "epoch": 0.8252569750367107, |
| "fcm_dpo/beta": 0.0070929634384810925, |
| "fcm_dpo/delta": 0.0866638720035553, |
| "fcm_dpo/margin": 73.01620483398438, |
| "fcm_dpo/q_t": 0.38909482955932617, |
| "grad_norm": 15.064519882202148, |
| "learning_rate": 4.5950771910944596e-08, |
| "logits/chosen": -2.21871280670166, |
| "logits/rejected": -1.770498514175415, |
| "logps/chosen": -259.5347900390625, |
| "logps/ref_chosen": -100.66322326660156, |
| "logps/ref_rejected": -102.24087524414062, |
| "logps/rejected": -334.1286315917969, |
| "loss": 1.0786, |
| "margin_dpo/margin_mean": 73.01620483398438, |
| "margin_dpo/margin_std": 114.18824768066406, |
| "step": 562 |
| }, |
| { |
| "epoch": 0.8267254038179148, |
| "fcm_dpo/beta": 0.007309372071176767, |
| "fcm_dpo/delta": 0.12443551421165466, |
| "fcm_dpo/margin": 65.76155090332031, |
| "fcm_dpo/q_t": 0.3946084678173065, |
| "grad_norm": 12.58653736114502, |
| "learning_rate": 4.521198892775202e-08, |
| "logits/chosen": -1.7142508029937744, |
| "logits/rejected": -1.3739852905273438, |
| "logps/chosen": -265.404296875, |
| "logps/ref_chosen": -96.25921630859375, |
| "logps/ref_rejected": -101.20503997802734, |
| "logps/rejected": -336.1116943359375, |
| "loss": 1.0736, |
| "margin_dpo/margin_mean": 65.76155090332031, |
| "margin_dpo/margin_std": 95.51123046875, |
| "step": 563 |
| }, |
| { |
| "epoch": 0.8281938325991189, |
| "fcm_dpo/beta": 0.007428737357258797, |
| "fcm_dpo/delta": 0.06579644232988358, |
| "fcm_dpo/margin": 72.30021667480469, |
| "fcm_dpo/q_t": 0.3801688849925995, |
| "grad_norm": 13.077413558959961, |
| "learning_rate": 4.447860229910544e-08, |
| "logits/chosen": -2.345478057861328, |
| "logits/rejected": -1.7775757312774658, |
| "logps/chosen": -277.65167236328125, |
| "logps/ref_chosen": -112.96040344238281, |
| "logps/ref_rejected": -99.52803802490234, |
| "logps/rejected": -336.51953125, |
| "loss": 1.0098, |
| "margin_dpo/margin_mean": 72.30021667480469, |
| "margin_dpo/margin_std": 83.55976867675781, |
| "step": 564 |
| }, |
| { |
| "epoch": 0.8296622613803231, |
| "fcm_dpo/beta": 0.007322209887206554, |
| "fcm_dpo/delta": -0.07626369595527649, |
| "fcm_dpo/margin": 91.62553405761719, |
| "fcm_dpo/q_t": 0.3552612066268921, |
| "grad_norm": 11.750452995300293, |
| "learning_rate": 4.375063135042445e-08, |
| "logits/chosen": -1.7991347312927246, |
| "logits/rejected": -1.296679139137268, |
| "logps/chosen": -253.31402587890625, |
| "logps/ref_chosen": -95.36558532714844, |
| "logps/ref_rejected": -92.21737670898438, |
| "logps/rejected": -341.7913513183594, |
| "loss": 0.9521, |
| "margin_dpo/margin_mean": 91.62553405761719, |
| "margin_dpo/margin_std": 103.9658203125, |
| "step": 565 |
| }, |
| { |
| "epoch": 0.8311306901615272, |
| "fcm_dpo/beta": 0.007290005683898926, |
| "fcm_dpo/delta": -0.03795613348484039, |
| "fcm_dpo/margin": 87.17108154296875, |
| "fcm_dpo/q_t": 0.36021772027015686, |
| "grad_norm": 10.284100532531738, |
| "learning_rate": 4.3028095264420525e-08, |
| "logits/chosen": -2.10237455368042, |
| "logits/rejected": -1.6776249408721924, |
| "logps/chosen": -264.3515625, |
| "logps/ref_chosen": -98.97320556640625, |
| "logps/ref_rejected": -110.82090759277344, |
| "logps/rejected": -363.3703308105469, |
| "loss": 0.9705, |
| "margin_dpo/margin_mean": 87.17108154296875, |
| "margin_dpo/margin_std": 100.97877502441406, |
| "step": 566 |
| }, |
| { |
| "epoch": 0.8325991189427313, |
| "fcm_dpo/beta": 0.007268569432199001, |
| "fcm_dpo/delta": 0.050699688494205475, |
| "fcm_dpo/margin": 75.91093444824219, |
| "fcm_dpo/q_t": 0.3769448399543762, |
| "grad_norm": 13.014694213867188, |
| "learning_rate": 4.231101308059165e-08, |
| "logits/chosen": -2.2701256275177, |
| "logits/rejected": -1.6002724170684814, |
| "logps/chosen": -259.28155517578125, |
| "logps/ref_chosen": -92.7035903930664, |
| "logps/ref_rejected": -91.22431945800781, |
| "logps/rejected": -333.7132568359375, |
| "loss": 1.0209, |
| "margin_dpo/margin_mean": 75.91093444824219, |
| "margin_dpo/margin_std": 94.99116516113281, |
| "step": 567 |
| }, |
| { |
| "epoch": 0.8340675477239354, |
| "fcm_dpo/beta": 0.007200797088444233, |
| "fcm_dpo/delta": -0.09029556810855865, |
| "fcm_dpo/margin": 94.91342163085938, |
| "fcm_dpo/q_t": 0.35067832469940186, |
| "grad_norm": 10.619403839111328, |
| "learning_rate": 4.1599403694720145e-08, |
| "logits/chosen": -1.4319794178009033, |
| "logits/rejected": -1.2937116622924805, |
| "logps/chosen": -230.70367431640625, |
| "logps/ref_chosen": -73.13226318359375, |
| "logps/ref_rejected": -94.95265197753906, |
| "logps/rejected": -347.4375, |
| "loss": 0.9189, |
| "margin_dpo/margin_mean": 94.91342163085938, |
| "margin_dpo/margin_std": 95.35987091064453, |
| "step": 568 |
| }, |
| { |
| "epoch": 0.8355359765051396, |
| "fcm_dpo/beta": 0.007249847054481506, |
| "fcm_dpo/delta": 0.02976367063820362, |
| "fcm_dpo/margin": 78.85527801513672, |
| "fcm_dpo/q_t": 0.37523531913757324, |
| "grad_norm": 17.318199157714844, |
| "learning_rate": 4.089328585837512e-08, |
| "logits/chosen": -2.0498688220977783, |
| "logits/rejected": -1.6809823513031006, |
| "logps/chosen": -256.4047546386719, |
| "logps/ref_chosen": -89.21708679199219, |
| "logps/ref_rejected": -86.41105651855469, |
| "logps/rejected": -332.4539794921875, |
| "loss": 1.0482, |
| "margin_dpo/margin_mean": 78.85527801513672, |
| "margin_dpo/margin_std": 112.83515930175781, |
| "step": 569 |
| }, |
| { |
| "epoch": 0.8370044052863436, |
| "fcm_dpo/beta": 0.00708900997415185, |
| "fcm_dpo/delta": -0.13322904706001282, |
| "fcm_dpo/margin": 102.04464721679688, |
| "fcm_dpo/q_t": 0.3389042615890503, |
| "grad_norm": 11.147782325744629, |
| "learning_rate": 4.019267817841834e-08, |
| "logits/chosen": -2.2344322204589844, |
| "logits/rejected": -1.5383317470550537, |
| "logps/chosen": -243.58880615234375, |
| "logps/ref_chosen": -94.76277160644531, |
| "logps/ref_rejected": -88.846923828125, |
| "logps/rejected": -339.71759033203125, |
| "loss": 0.8871, |
| "margin_dpo/margin_mean": 102.04464721679688, |
| "margin_dpo/margin_std": 94.04344177246094, |
| "step": 570 |
| }, |
| { |
| "epoch": 0.8384728340675477, |
| "fcm_dpo/beta": 0.00701398029923439, |
| "fcm_dpo/delta": -0.053493283689022064, |
| "fcm_dpo/margin": 92.67195129394531, |
| "fcm_dpo/q_t": 0.3594912588596344, |
| "grad_norm": 14.784183502197266, |
| "learning_rate": 3.9497599116513705e-08, |
| "logits/chosen": -1.6442519426345825, |
| "logits/rejected": -1.4821221828460693, |
| "logps/chosen": -234.88565063476562, |
| "logps/ref_chosen": -80.8212890625, |
| "logps/ref_rejected": -97.01528930664062, |
| "logps/rejected": -343.7516174316406, |
| "loss": 0.9712, |
| "margin_dpo/margin_mean": 92.67195129394531, |
| "margin_dpo/margin_std": 109.70027160644531, |
| "step": 571 |
| }, |
| { |
| "epoch": 0.8399412628487518, |
| "fcm_dpo/beta": 0.00692150229588151, |
| "fcm_dpo/delta": -0.035205624997615814, |
| "fcm_dpo/margin": 91.42375183105469, |
| "fcm_dpo/q_t": 0.36460578441619873, |
| "grad_norm": 11.84706974029541, |
| "learning_rate": 3.880806698864086e-08, |
| "logits/chosen": -1.739823341369629, |
| "logits/rejected": -1.5698119401931763, |
| "logps/chosen": -235.00161743164062, |
| "logps/ref_chosen": -77.37992858886719, |
| "logps/ref_rejected": -90.56597900390625, |
| "logps/rejected": -339.6114196777344, |
| "loss": 0.9675, |
| "margin_dpo/margin_mean": 91.42375183105469, |
| "margin_dpo/margin_std": 106.2401123046875, |
| "step": 572 |
| }, |
| { |
| "epoch": 0.8414096916299559, |
| "fcm_dpo/beta": 0.006989034358412027, |
| "fcm_dpo/delta": -0.0046057626605033875, |
| "fcm_dpo/margin": 86.20860290527344, |
| "fcm_dpo/q_t": 0.3665117621421814, |
| "grad_norm": 10.864632606506348, |
| "learning_rate": 3.812409996461275e-08, |
| "logits/chosen": -2.1315410137176514, |
| "logits/rejected": -1.9030604362487793, |
| "logps/chosen": -236.8482208251953, |
| "logps/ref_chosen": -83.7060775756836, |
| "logps/ref_rejected": -90.69746398925781, |
| "logps/rejected": -330.0482177734375, |
| "loss": 0.9925, |
| "margin_dpo/margin_mean": 86.20860290527344, |
| "margin_dpo/margin_std": 100.88923645019531, |
| "step": 573 |
| }, |
| { |
| "epoch": 0.8428781204111601, |
| "fcm_dpo/beta": 0.006914999336004257, |
| "fcm_dpo/delta": -0.02837441675364971, |
| "fcm_dpo/margin": 90.59143829345703, |
| "fcm_dpo/q_t": 0.36350974440574646, |
| "grad_norm": 13.734319686889648, |
| "learning_rate": 3.74457160675965e-08, |
| "logits/chosen": -2.0994553565979004, |
| "logits/rejected": -1.7679946422576904, |
| "logps/chosen": -246.83309936523438, |
| "logps/ref_chosen": -85.12948608398438, |
| "logps/ref_rejected": -99.05403137207031, |
| "logps/rejected": -351.3490905761719, |
| "loss": 0.9786, |
| "margin_dpo/margin_mean": 90.59143829345703, |
| "margin_dpo/margin_std": 107.298828125, |
| "step": 574 |
| }, |
| { |
| "epoch": 0.8443465491923642, |
| "fcm_dpo/beta": 0.006864962168037891, |
| "fcm_dpo/delta": -0.005945529788732529, |
| "fcm_dpo/margin": 88.20824432373047, |
| "fcm_dpo/q_t": 0.3688689172267914, |
| "grad_norm": 15.33983039855957, |
| "learning_rate": 3.677293317363864e-08, |
| "logits/chosen": -1.398423433303833, |
| "logits/rejected": -0.9362001419067383, |
| "logps/chosen": -272.59100341796875, |
| "logps/ref_chosen": -105.77084350585938, |
| "logps/ref_rejected": -100.94944763183594, |
| "logps/rejected": -355.9778747558594, |
| "loss": 1.0027, |
| "margin_dpo/margin_mean": 88.20824432373047, |
| "margin_dpo/margin_std": 113.10353088378906, |
| "step": 575 |
| }, |
| { |
| "epoch": 0.8458149779735683, |
| "fcm_dpo/beta": 0.006903272587805986, |
| "fcm_dpo/delta": 0.013031164184212685, |
| "fcm_dpo/margin": 85.11408996582031, |
| "fcm_dpo/q_t": 0.37200257182121277, |
| "grad_norm": 12.17516803741455, |
| "learning_rate": 3.6105769011194224e-08, |
| "logits/chosen": -2.0569982528686523, |
| "logits/rejected": -1.9845412969589233, |
| "logps/chosen": -248.39340209960938, |
| "logps/ref_chosen": -81.20452880859375, |
| "logps/ref_rejected": -105.40940856933594, |
| "logps/rejected": -357.7123718261719, |
| "loss": 1.0148, |
| "margin_dpo/margin_mean": 85.11408996582031, |
| "margin_dpo/margin_std": 110.67799377441406, |
| "step": 576 |
| }, |
| { |
| "epoch": 0.8472834067547724, |
| "fcm_dpo/beta": 0.00673964899033308, |
| "fcm_dpo/delta": -0.11396480351686478, |
| "fcm_dpo/margin": 104.58753204345703, |
| "fcm_dpo/q_t": 0.34595972299575806, |
| "grad_norm": 9.555338859558105, |
| "learning_rate": 3.5444241160659304e-08, |
| "logits/chosen": -1.9512568712234497, |
| "logits/rejected": -1.2471516132354736, |
| "logps/chosen": -249.70469665527344, |
| "logps/ref_chosen": -101.46419525146484, |
| "logps/ref_rejected": -89.32637023925781, |
| "logps/rejected": -342.1544189453125, |
| "loss": 0.9266, |
| "margin_dpo/margin_mean": 104.58753204345703, |
| "margin_dpo/margin_std": 111.21625518798828, |
| "step": 577 |
| }, |
| { |
| "epoch": 0.8487518355359766, |
| "fcm_dpo/beta": 0.006825780496001244, |
| "fcm_dpo/delta": 0.08758461475372314, |
| "fcm_dpo/margin": 75.72772979736328, |
| "fcm_dpo/q_t": 0.38606128096580505, |
| "grad_norm": 16.20516586303711, |
| "learning_rate": 3.478836705390808e-08, |
| "logits/chosen": -1.3706796169281006, |
| "logits/rejected": -1.131410837173462, |
| "logps/chosen": -253.5721435546875, |
| "logps/ref_chosen": -77.29241943359375, |
| "logps/ref_rejected": -88.24701690673828, |
| "logps/rejected": -340.25445556640625, |
| "loss": 1.0516, |
| "margin_dpo/margin_mean": 75.72772979736328, |
| "margin_dpo/margin_std": 105.21282196044922, |
| "step": 578 |
| }, |
| { |
| "epoch": 0.8502202643171806, |
| "fcm_dpo/beta": 0.006886948365718126, |
| "fcm_dpo/delta": 0.06752566248178482, |
| "fcm_dpo/margin": 77.81400299072266, |
| "fcm_dpo/q_t": 0.3831830620765686, |
| "grad_norm": 11.248205184936523, |
| "learning_rate": 3.41381639738331e-08, |
| "logits/chosen": -1.7072319984436035, |
| "logits/rejected": -1.3880934715270996, |
| "logps/chosen": -245.95306396484375, |
| "logps/ref_chosen": -87.34305572509766, |
| "logps/ref_rejected": -102.12100982666016, |
| "logps/rejected": -338.5450439453125, |
| "loss": 1.027, |
| "margin_dpo/margin_mean": 77.81400299072266, |
| "margin_dpo/margin_std": 96.24092102050781, |
| "step": 579 |
| }, |
| { |
| "epoch": 0.8516886930983847, |
| "fcm_dpo/beta": 0.006863095331937075, |
| "fcm_dpo/delta": -0.040312498807907104, |
| "fcm_dpo/margin": 92.7848129272461, |
| "fcm_dpo/q_t": 0.3631054759025574, |
| "grad_norm": 11.509198188781738, |
| "learning_rate": 3.349364905389032e-08, |
| "logits/chosen": -1.6127009391784668, |
| "logits/rejected": -1.3059601783752441, |
| "logps/chosen": -242.31298828125, |
| "logps/ref_chosen": -88.92879486083984, |
| "logps/ref_rejected": -88.69661712646484, |
| "logps/rejected": -334.8656311035156, |
| "loss": 0.9711, |
| "margin_dpo/margin_mean": 92.7848129272461, |
| "margin_dpo/margin_std": 108.14720153808594, |
| "step": 580 |
| }, |
| { |
| "epoch": 0.8531571218795888, |
| "fcm_dpo/beta": 0.006930704694241285, |
| "fcm_dpo/delta": 0.026802916079759598, |
| "fcm_dpo/margin": 82.91061401367188, |
| "fcm_dpo/q_t": 0.3752003312110901, |
| "grad_norm": 13.487149238586426, |
| "learning_rate": 3.285483927764726e-08, |
| "logits/chosen": -2.016202449798584, |
| "logits/rejected": -1.6698317527770996, |
| "logps/chosen": -254.0625762939453, |
| "logps/ref_chosen": -92.22323608398438, |
| "logps/ref_rejected": -99.38943481445312, |
| "logps/rejected": -344.139404296875, |
| "loss": 1.0399, |
| "margin_dpo/margin_mean": 82.91061401367188, |
| "margin_dpo/margin_std": 116.20519256591797, |
| "step": 581 |
| }, |
| { |
| "epoch": 0.8546255506607929, |
| "fcm_dpo/beta": 0.007061149924993515, |
| "fcm_dpo/delta": 0.04960351437330246, |
| "fcm_dpo/margin": 77.88203430175781, |
| "fcm_dpo/q_t": 0.3824002146720886, |
| "grad_norm": 11.220653533935547, |
| "learning_rate": 3.222175147833556e-08, |
| "logits/chosen": -1.9758110046386719, |
| "logits/rejected": -1.9616727828979492, |
| "logps/chosen": -229.19464111328125, |
| "logps/ref_chosen": -87.33561706542969, |
| "logps/ref_rejected": -115.1544189453125, |
| "logps/rejected": -334.8954772949219, |
| "loss": 1.039, |
| "margin_dpo/margin_mean": 77.88203430175781, |
| "margin_dpo/margin_std": 104.54690551757812, |
| "step": 582 |
| }, |
| { |
| "epoch": 0.856093979441997, |
| "fcm_dpo/beta": 0.006990609224885702, |
| "fcm_dpo/delta": 0.02330578863620758, |
| "fcm_dpo/margin": 82.64878845214844, |
| "fcm_dpo/q_t": 0.37635916471481323, |
| "grad_norm": 12.386331558227539, |
| "learning_rate": 3.159440233840763e-08, |
| "logits/chosen": -1.5934879779815674, |
| "logits/rejected": -1.364257574081421, |
| "logps/chosen": -257.1393737792969, |
| "logps/ref_chosen": -83.71707153320312, |
| "logps/ref_rejected": -94.58465576171875, |
| "logps/rejected": -350.65576171875, |
| "loss": 1.0101, |
| "margin_dpo/margin_mean": 82.64878845214844, |
| "margin_dpo/margin_std": 105.64147186279297, |
| "step": 583 |
| }, |
| { |
| "epoch": 0.8575624082232012, |
| "fcm_dpo/beta": 0.007066408172249794, |
| "fcm_dpo/delta": -0.031313344836235046, |
| "fcm_dpo/margin": 88.83890533447266, |
| "fcm_dpo/q_t": 0.3632817268371582, |
| "grad_norm": 10.694197654724121, |
| "learning_rate": 3.0972808389096635e-08, |
| "logits/chosen": -2.124901294708252, |
| "logits/rejected": -1.6727513074874878, |
| "logps/chosen": -246.2718963623047, |
| "logps/ref_chosen": -94.25955963134766, |
| "logps/ref_rejected": -104.48269653320312, |
| "logps/rejected": -345.33392333984375, |
| "loss": 0.9905, |
| "margin_dpo/margin_mean": 88.83891296386719, |
| "margin_dpo/margin_std": 106.91671752929688, |
| "step": 584 |
| }, |
| { |
| "epoch": 0.8590308370044053, |
| "fcm_dpo/beta": 0.00690193846821785, |
| "fcm_dpo/delta": -0.11694584041833878, |
| "fcm_dpo/margin": 102.68148040771484, |
| "fcm_dpo/q_t": 0.350594162940979, |
| "grad_norm": 11.292356491088867, |
| "learning_rate": 3.035698600998121e-08, |
| "logits/chosen": -1.7038193941116333, |
| "logits/rejected": -1.4119065999984741, |
| "logps/chosen": -247.07659912109375, |
| "logps/ref_chosen": -85.63676452636719, |
| "logps/ref_rejected": -90.28547668457031, |
| "logps/rejected": -354.40679931640625, |
| "loss": 0.955, |
| "margin_dpo/margin_mean": 102.68147277832031, |
| "margin_dpo/margin_std": 123.2242431640625, |
| "step": 585 |
| }, |
| { |
| "epoch": 0.8604992657856094, |
| "fcm_dpo/beta": 0.006955640856176615, |
| "fcm_dpo/delta": 0.1244577169418335, |
| "fcm_dpo/margin": 69.21916198730469, |
| "fcm_dpo/q_t": 0.3929838240146637, |
| "grad_norm": 13.513263702392578, |
| "learning_rate": 2.974695142855388e-08, |
| "logits/chosen": -1.7758688926696777, |
| "logits/rejected": -1.507314682006836, |
| "logps/chosen": -263.46990966796875, |
| "logps/ref_chosen": -83.68114471435547, |
| "logps/ref_rejected": -98.35916137695312, |
| "logps/rejected": -347.3670654296875, |
| "loss": 1.0502, |
| "margin_dpo/margin_mean": 69.21916198730469, |
| "margin_dpo/margin_std": 89.19205474853516, |
| "step": 586 |
| }, |
| { |
| "epoch": 0.8619676945668135, |
| "fcm_dpo/beta": 0.0070916032418608665, |
| "fcm_dpo/delta": 0.06968455016613007, |
| "fcm_dpo/margin": 75.24751281738281, |
| "fcm_dpo/q_t": 0.3832997679710388, |
| "grad_norm": 11.80888557434082, |
| "learning_rate": 2.9142720719793122e-08, |
| "logits/chosen": -2.346747636795044, |
| "logits/rejected": -2.054532527923584, |
| "logps/chosen": -222.94000244140625, |
| "logps/ref_chosen": -77.57750701904297, |
| "logps/ref_rejected": -89.32032775878906, |
| "logps/rejected": -309.93035888671875, |
| "loss": 1.0274, |
| "margin_dpo/margin_mean": 75.24751281738281, |
| "margin_dpo/margin_std": 96.26628875732422, |
| "step": 587 |
| }, |
| { |
| "epoch": 0.8634361233480177, |
| "fcm_dpo/beta": 0.00717865489423275, |
| "fcm_dpo/delta": 0.10119879245758057, |
| "fcm_dpo/margin": 70.20915222167969, |
| "fcm_dpo/q_t": 0.39323359727859497, |
| "grad_norm": 14.625384330749512, |
| "learning_rate": 2.8544309805740018e-08, |
| "logits/chosen": -1.971228003501892, |
| "logits/rejected": -1.9192923307418823, |
| "logps/chosen": -241.52024841308594, |
| "logps/ref_chosen": -83.80426025390625, |
| "logps/ref_rejected": -113.11943054199219, |
| "logps/rejected": -341.0445861816406, |
| "loss": 1.0898, |
| "margin_dpo/margin_mean": 70.20915222167969, |
| "margin_dpo/margin_std": 111.68272399902344, |
| "step": 588 |
| }, |
| { |
| "epoch": 0.8649045521292217, |
| "fcm_dpo/beta": 0.007164877839386463, |
| "fcm_dpo/delta": -0.06353701651096344, |
| "fcm_dpo/margin": 92.01499938964844, |
| "fcm_dpo/q_t": 0.35298681259155273, |
| "grad_norm": 12.12359619140625, |
| "learning_rate": 2.7951734455078786e-08, |
| "logits/chosen": -1.600557565689087, |
| "logits/rejected": -1.1946024894714355, |
| "logps/chosen": -266.53143310546875, |
| "logps/ref_chosen": -96.46501922607422, |
| "logps/ref_rejected": -116.03719329833984, |
| "logps/rejected": -378.11859130859375, |
| "loss": 0.9228, |
| "margin_dpo/margin_mean": 92.01498413085938, |
| "margin_dpo/margin_std": 88.87408447265625, |
| "step": 589 |
| }, |
| { |
| "epoch": 0.8663729809104258, |
| "fcm_dpo/beta": 0.007136746309697628, |
| "fcm_dpo/delta": -0.05460678040981293, |
| "fcm_dpo/margin": 91.158203125, |
| "fcm_dpo/q_t": 0.3565479516983032, |
| "grad_norm": 13.865069389343262, |
| "learning_rate": 2.736501028272095e-08, |
| "logits/chosen": -1.906883716583252, |
| "logits/rejected": -1.7435002326965332, |
| "logps/chosen": -242.1221160888672, |
| "logps/ref_chosen": -88.4278564453125, |
| "logps/ref_rejected": -110.19876861572266, |
| "logps/rejected": -355.0512390136719, |
| "loss": 0.9316, |
| "margin_dpo/margin_mean": 91.158203125, |
| "margin_dpo/margin_std": 90.19902038574219, |
| "step": 590 |
| }, |
| { |
| "epoch": 0.8678414096916299, |
| "fcm_dpo/beta": 0.00695719663053751, |
| "fcm_dpo/delta": -0.08778176456689835, |
| "fcm_dpo/margin": 97.93551635742188, |
| "fcm_dpo/q_t": 0.3495037257671356, |
| "grad_norm": 8.954543113708496, |
| "learning_rate": 2.678415274939408e-08, |
| "logits/chosen": -2.069058418273926, |
| "logits/rejected": -1.4569966793060303, |
| "logps/chosen": -236.16778564453125, |
| "logps/ref_chosen": -87.84968566894531, |
| "logps/ref_rejected": -89.57919311523438, |
| "logps/rejected": -335.83282470703125, |
| "loss": 0.9107, |
| "margin_dpo/margin_mean": 97.93551635742188, |
| "margin_dpo/margin_std": 93.53607177734375, |
| "step": 591 |
| }, |
| { |
| "epoch": 0.869309838472834, |
| "fcm_dpo/beta": 0.007020566612482071, |
| "fcm_dpo/delta": 0.06493528187274933, |
| "fcm_dpo/margin": 76.70272827148438, |
| "fcm_dpo/q_t": 0.3827671408653259, |
| "grad_norm": 13.41107177734375, |
| "learning_rate": 2.6209177161234442e-08, |
| "logits/chosen": -1.726281762123108, |
| "logits/rejected": -1.3931875228881836, |
| "logps/chosen": -250.2864990234375, |
| "logps/ref_chosen": -74.97573852539062, |
| "logps/ref_rejected": -81.55262756347656, |
| "logps/rejected": -333.5661315917969, |
| "loss": 1.0491, |
| "margin_dpo/margin_mean": 76.70272827148438, |
| "margin_dpo/margin_std": 108.41436004638672, |
| "step": 592 |
| }, |
| { |
| "epoch": 0.8707782672540382, |
| "fcm_dpo/beta": 0.007145709358155727, |
| "fcm_dpo/delta": 0.1387026607990265, |
| "fcm_dpo/margin": 65.48255157470703, |
| "fcm_dpo/q_t": 0.399899959564209, |
| "grad_norm": 17.311737060546875, |
| "learning_rate": 2.564009866938349e-08, |
| "logits/chosen": -0.939947247505188, |
| "logits/rejected": -0.6396098136901855, |
| "logps/chosen": -234.7220458984375, |
| "logps/ref_chosen": -72.23902893066406, |
| "logps/ref_rejected": -70.09652709960938, |
| "logps/rejected": -298.0621337890625, |
| "loss": 1.099, |
| "margin_dpo/margin_mean": 65.48255157470703, |
| "margin_dpo/margin_std": 103.57083129882812, |
| "step": 593 |
| }, |
| { |
| "epoch": 0.8722466960352423, |
| "fcm_dpo/beta": 0.0071470364928245544, |
| "fcm_dpo/delta": -0.11245208233594894, |
| "fcm_dpo/margin": 98.58993530273438, |
| "fcm_dpo/q_t": 0.3443548083305359, |
| "grad_norm": 10.641993522644043, |
| "learning_rate": 2.5076932269588708e-08, |
| "logits/chosen": -1.9720404148101807, |
| "logits/rejected": -1.4504088163375854, |
| "logps/chosen": -236.50711059570312, |
| "logps/ref_chosen": -90.43771362304688, |
| "logps/ref_rejected": -92.267578125, |
| "logps/rejected": -336.92694091796875, |
| "loss": 0.8928, |
| "margin_dpo/margin_mean": 98.58993530273438, |
| "margin_dpo/margin_std": 89.36225891113281, |
| "step": 594 |
| }, |
| { |
| "epoch": 0.8737151248164464, |
| "fcm_dpo/beta": 0.007122883573174477, |
| "fcm_dpo/delta": 0.020702892914414406, |
| "fcm_dpo/margin": 81.45152282714844, |
| "fcm_dpo/q_t": 0.3758925795555115, |
| "grad_norm": 12.234829902648926, |
| "learning_rate": 2.451969280180849e-08, |
| "logits/chosen": -1.6966605186462402, |
| "logits/rejected": -1.2726609706878662, |
| "logps/chosen": -236.58111572265625, |
| "logps/ref_chosen": -81.51480102539062, |
| "logps/ref_rejected": -86.52047729492188, |
| "logps/rejected": -323.038330078125, |
| "loss": 1.0138, |
| "margin_dpo/margin_mean": 81.45152282714844, |
| "margin_dpo/margin_std": 105.23150634765625, |
| "step": 595 |
| }, |
| { |
| "epoch": 0.8751835535976505, |
| "fcm_dpo/beta": 0.00720050185918808, |
| "fcm_dpo/delta": 0.08340231329202652, |
| "fcm_dpo/margin": 72.33407592773438, |
| "fcm_dpo/q_t": 0.3886866271495819, |
| "grad_norm": 15.935720443725586, |
| "learning_rate": 2.396839494982103e-08, |
| "logits/chosen": -1.7379255294799805, |
| "logits/rejected": -1.1650152206420898, |
| "logps/chosen": -248.61962890625, |
| "logps/ref_chosen": -88.20805358886719, |
| "logps/ref_rejected": -86.09375, |
| "logps/rejected": -318.83941650390625, |
| "loss": 1.0738, |
| "margin_dpo/margin_mean": 72.33407592773438, |
| "margin_dpo/margin_std": 109.93353271484375, |
| "step": 596 |
| }, |
| { |
| "epoch": 0.8766519823788547, |
| "fcm_dpo/beta": 0.007105865050107241, |
| "fcm_dpo/delta": -0.09072263538837433, |
| "fcm_dpo/margin": 96.26971435546875, |
| "fcm_dpo/q_t": 0.3528348505496979, |
| "grad_norm": 10.61738109588623, |
| "learning_rate": 2.3423053240837514e-08, |
| "logits/chosen": -1.3981832265853882, |
| "logits/rejected": -1.2777040004730225, |
| "logps/chosen": -240.2840576171875, |
| "logps/ref_chosen": -83.58998107910156, |
| "logps/ref_rejected": -106.94439697265625, |
| "logps/rejected": -359.908203125, |
| "loss": 0.9408, |
| "margin_dpo/margin_mean": 96.26971435546875, |
| "margin_dpo/margin_std": 107.11369323730469, |
| "step": 597 |
| }, |
| { |
| "epoch": 0.8781204111600588, |
| "fcm_dpo/beta": 0.007089652121067047, |
| "fcm_dpo/delta": 0.03473873436450958, |
| "fcm_dpo/margin": 79.942138671875, |
| "fcm_dpo/q_t": 0.37478604912757874, |
| "grad_norm": 11.30370807647705, |
| "learning_rate": 2.2883682045119062e-08, |
| "logits/chosen": -2.0580103397369385, |
| "logits/rejected": -1.748920202255249, |
| "logps/chosen": -239.97232055664062, |
| "logps/ref_chosen": -90.18262481689453, |
| "logps/ref_rejected": -94.79856872558594, |
| "logps/rejected": -324.5303955078125, |
| "loss": 1.021, |
| "margin_dpo/margin_mean": 79.942138671875, |
| "margin_dpo/margin_std": 102.13404846191406, |
| "step": 598 |
| }, |
| { |
| "epoch": 0.8795888399412628, |
| "fcm_dpo/beta": 0.007190364878624678, |
| "fcm_dpo/delta": 0.09109681099653244, |
| "fcm_dpo/margin": 71.37487030029297, |
| "fcm_dpo/q_t": 0.38869598507881165, |
| "grad_norm": 12.38364028930664, |
| "learning_rate": 2.2350295575598367e-08, |
| "logits/chosen": -1.799156665802002, |
| "logits/rejected": -1.6733660697937012, |
| "logps/chosen": -232.6363067626953, |
| "logps/ref_chosen": -77.84226989746094, |
| "logps/ref_rejected": -89.1976318359375, |
| "logps/rejected": -315.36651611328125, |
| "loss": 1.0679, |
| "margin_dpo/margin_mean": 71.37487030029297, |
| "margin_dpo/margin_std": 102.9161148071289, |
| "step": 599 |
| }, |
| { |
| "epoch": 0.8810572687224669, |
| "fcm_dpo/beta": 0.007273955270648003, |
| "fcm_dpo/delta": -0.006405937019735575, |
| "fcm_dpo/margin": 83.31228637695312, |
| "fcm_dpo/q_t": 0.3687729835510254, |
| "grad_norm": 10.347834587097168, |
| "learning_rate": 2.1822907887504932e-08, |
| "logits/chosen": -2.4206528663635254, |
| "logits/rejected": -2.0758280754089355, |
| "logps/chosen": -257.964111328125, |
| "logps/ref_chosen": -92.18781280517578, |
| "logps/ref_rejected": -92.95429992675781, |
| "logps/rejected": -342.0428771972656, |
| "loss": 0.9868, |
| "margin_dpo/margin_mean": 83.31228637695312, |
| "margin_dpo/margin_std": 99.32923889160156, |
| "step": 600 |
| }, |
| { |
| "epoch": 0.8810572687224669, |
| "eval_fcm_dpo/beta": 0.0072656371630728245, |
| "eval_logits/chosen": -1.840718150138855, |
| "eval_logits/rejected": -1.4554615020751953, |
| "eval_logps/chosen": -265.1147155761719, |
| "eval_logps/ref_chosen": -100.49356842041016, |
| "eval_logps/ref_rejected": -94.06775665283203, |
| "eval_logps/rejected": -332.2911376953125, |
| "eval_loss": 0.5223754644393921, |
| "eval_margin_dpo/margin_mean": 73.60225677490234, |
| "eval_margin_dpo/margin_std": 100.21179962158203, |
| "eval_runtime": 43.7553, |
| "eval_samples_per_second": 53.456, |
| "eval_steps_per_second": 1.691, |
| "step": 600 |
| }, |
| { |
| "epoch": 0.882525697503671, |
| "fcm_dpo/beta": 0.00720963254570961, |
| "fcm_dpo/delta": -0.023707086220383644, |
| "fcm_dpo/margin": 86.24479675292969, |
| "fcm_dpo/q_t": 0.3629060983657837, |
| "grad_norm": 11.4024019241333, |
| "learning_rate": 2.1301532877994742e-08, |
| "logits/chosen": -1.7290210723876953, |
| "logits/rejected": -1.4024295806884766, |
| "logps/chosen": -259.41070556640625, |
| "logps/ref_chosen": -89.21614074707031, |
| "logps/ref_rejected": -100.17054748535156, |
| "logps/rejected": -356.60992431640625, |
| "loss": 0.9523, |
| "margin_dpo/margin_mean": 86.24479675292969, |
| "margin_dpo/margin_std": 90.01136016845703, |
| "step": 601 |
| }, |
| { |
| "epoch": 0.8839941262848752, |
| "fcm_dpo/beta": 0.007033023051917553, |
| "fcm_dpo/delta": -0.185970276594162, |
| "fcm_dpo/margin": 109.70604705810547, |
| "fcm_dpo/q_t": 0.3343247175216675, |
| "grad_norm": 10.687939643859863, |
| "learning_rate": 2.0786184285784298e-08, |
| "logits/chosen": -2.0846588611602783, |
| "logits/rejected": -1.826683759689331, |
| "logps/chosen": -214.779052734375, |
| "logps/ref_chosen": -80.05760192871094, |
| "logps/ref_rejected": -93.197509765625, |
| "logps/rejected": -337.625, |
| "loss": 0.8887, |
| "margin_dpo/margin_mean": 109.70604705810547, |
| "margin_dpo/margin_std": 112.28135681152344, |
| "step": 602 |
| }, |
| { |
| "epoch": 0.8854625550660793, |
| "fcm_dpo/beta": 0.006929402239620686, |
| "fcm_dpo/delta": -0.0485604926943779, |
| "fcm_dpo/margin": 93.14605712890625, |
| "fcm_dpo/q_t": 0.3590378165245056, |
| "grad_norm": 10.695706367492676, |
| "learning_rate": 2.0276875690788204e-08, |
| "logits/chosen": -1.9837377071380615, |
| "logits/rejected": -1.3861087560653687, |
| "logps/chosen": -247.59893798828125, |
| "logps/ref_chosen": -102.30958557128906, |
| "logps/ref_rejected": -108.06884765625, |
| "logps/rejected": -346.5042724609375, |
| "loss": 0.9534, |
| "margin_dpo/margin_mean": 93.14605712890625, |
| "margin_dpo/margin_std": 104.23910522460938, |
| "step": 603 |
| }, |
| { |
| "epoch": 0.8869309838472834, |
| "fcm_dpo/beta": 0.006826011463999748, |
| "fcm_dpo/delta": -0.04624610394239426, |
| "fcm_dpo/margin": 94.17044067382812, |
| "fcm_dpo/q_t": 0.35456353425979614, |
| "grad_norm": 10.141494750976562, |
| "learning_rate": 1.977362051376158e-08, |
| "logits/chosen": -1.782772183418274, |
| "logits/rejected": -1.626922845840454, |
| "logps/chosen": -228.49160766601562, |
| "logps/ref_chosen": -78.17408752441406, |
| "logps/ref_rejected": -99.49610900878906, |
| "logps/rejected": -343.98406982421875, |
| "loss": 0.9365, |
| "margin_dpo/margin_mean": 94.17044067382812, |
| "margin_dpo/margin_std": 92.8728256225586, |
| "step": 604 |
| }, |
| { |
| "epoch": 0.8883994126284875, |
| "fcm_dpo/beta": 0.006894152611494064, |
| "fcm_dpo/delta": 0.06398770958185196, |
| "fcm_dpo/margin": 78.25582122802734, |
| "fcm_dpo/q_t": 0.38114598393440247, |
| "grad_norm": 10.904273986816406, |
| "learning_rate": 1.9276432015946446e-08, |
| "logits/chosen": -1.7464425563812256, |
| "logits/rejected": -1.2376749515533447, |
| "logps/chosen": -258.21099853515625, |
| "logps/ref_chosen": -94.77332305908203, |
| "logps/ref_rejected": -107.30490112304688, |
| "logps/rejected": -348.9984130859375, |
| "loss": 1.0158, |
| "margin_dpo/margin_mean": 78.25582885742188, |
| "margin_dpo/margin_std": 95.33082580566406, |
| "step": 605 |
| }, |
| { |
| "epoch": 0.8898678414096917, |
| "fcm_dpo/beta": 0.007000097073614597, |
| "fcm_dpo/delta": 0.02021496742963791, |
| "fcm_dpo/margin": 82.80071258544922, |
| "fcm_dpo/q_t": 0.3707614541053772, |
| "grad_norm": 11.527751922607422, |
| "learning_rate": 1.8785323298722093e-08, |
| "logits/chosen": -1.9277280569076538, |
| "logits/rejected": -1.524000883102417, |
| "logps/chosen": -245.60330200195312, |
| "logps/ref_chosen": -87.7533950805664, |
| "logps/ref_rejected": -104.2422103881836, |
| "logps/rejected": -344.892822265625, |
| "loss": 0.9999, |
| "margin_dpo/margin_mean": 82.80071258544922, |
| "margin_dpo/margin_std": 97.99324035644531, |
| "step": 606 |
| }, |
| { |
| "epoch": 0.8913362701908958, |
| "fcm_dpo/beta": 0.0069069629535079, |
| "fcm_dpo/delta": -0.04535501077771187, |
| "fcm_dpo/margin": 93.01445007324219, |
| "fcm_dpo/q_t": 0.36220258474349976, |
| "grad_norm": 14.510759353637695, |
| "learning_rate": 1.8300307303259904e-08, |
| "logits/chosen": -1.7667630910873413, |
| "logits/rejected": -1.2318663597106934, |
| "logps/chosen": -243.1822509765625, |
| "logps/ref_chosen": -88.32904052734375, |
| "logps/ref_rejected": -86.76810455322266, |
| "logps/rejected": -334.6357421875, |
| "loss": 1.0038, |
| "margin_dpo/margin_mean": 93.01445007324219, |
| "margin_dpo/margin_std": 122.94918823242188, |
| "step": 607 |
| }, |
| { |
| "epoch": 0.8928046989720999, |
| "fcm_dpo/beta": 0.006772700231522322, |
| "fcm_dpo/delta": -0.05694134533405304, |
| "fcm_dpo/margin": 96.16629028320312, |
| "fcm_dpo/q_t": 0.35457879304885864, |
| "grad_norm": 12.842971801757812, |
| "learning_rate": 1.7821396810182437e-08, |
| "logits/chosen": -1.7402276992797852, |
| "logits/rejected": -1.4407744407653809, |
| "logps/chosen": -226.5384521484375, |
| "logps/ref_chosen": -85.76937866210938, |
| "logps/ref_rejected": -100.23281860351562, |
| "logps/rejected": -337.1681823730469, |
| "loss": 0.9581, |
| "margin_dpo/margin_mean": 96.16629028320312, |
| "margin_dpo/margin_std": 104.13595581054688, |
| "step": 608 |
| }, |
| { |
| "epoch": 0.8942731277533039, |
| "fcm_dpo/beta": 0.006742427125573158, |
| "fcm_dpo/delta": -0.037574149668216705, |
| "fcm_dpo/margin": 94.12151336669922, |
| "fcm_dpo/q_t": 0.36158236861228943, |
| "grad_norm": 13.74590015411377, |
| "learning_rate": 1.7348604439226617e-08, |
| "logits/chosen": -2.4941606521606445, |
| "logits/rejected": -2.0870494842529297, |
| "logps/chosen": -246.841796875, |
| "logps/ref_chosen": -92.9665756225586, |
| "logps/ref_rejected": -95.91818237304688, |
| "logps/rejected": -343.9149169921875, |
| "loss": 0.9629, |
| "margin_dpo/margin_mean": 94.12152099609375, |
| "margin_dpo/margin_std": 102.90473937988281, |
| "step": 609 |
| }, |
| { |
| "epoch": 0.895741556534508, |
| "fcm_dpo/beta": 0.00668362807482481, |
| "fcm_dpo/delta": -0.060604121536016464, |
| "fcm_dpo/margin": 98.20524597167969, |
| "fcm_dpo/q_t": 0.3524959683418274, |
| "grad_norm": 10.179794311523438, |
| "learning_rate": 1.6881942648911074e-08, |
| "logits/chosen": -1.8149501085281372, |
| "logits/rejected": -1.2743051052093506, |
| "logps/chosen": -239.31069946289062, |
| "logps/ref_chosen": -94.70028686523438, |
| "logps/ref_rejected": -89.68739318847656, |
| "logps/rejected": -332.5030517578125, |
| "loss": 0.934, |
| "margin_dpo/margin_mean": 98.20524597167969, |
| "margin_dpo/margin_std": 99.45082092285156, |
| "step": 610 |
| }, |
| { |
| "epoch": 0.8972099853157122, |
| "fcm_dpo/beta": 0.00667162798345089, |
| "fcm_dpo/delta": -0.047587163746356964, |
| "fcm_dpo/margin": 96.54525756835938, |
| "fcm_dpo/q_t": 0.35953009128570557, |
| "grad_norm": 12.610694885253906, |
| "learning_rate": 1.6421423736208e-08, |
| "logits/chosen": -2.17702317237854, |
| "logits/rejected": -1.8332421779632568, |
| "logps/chosen": -237.6696319580078, |
| "logps/ref_chosen": -86.78334045410156, |
| "logps/ref_rejected": -89.84307861328125, |
| "logps/rejected": -337.274658203125, |
| "loss": 0.9441, |
| "margin_dpo/margin_mean": 96.54525756835938, |
| "margin_dpo/margin_std": 100.653076171875, |
| "step": 611 |
| }, |
| { |
| "epoch": 0.8986784140969163, |
| "fcm_dpo/beta": 0.006550832651555538, |
| "fcm_dpo/delta": -0.049320608377456665, |
| "fcm_dpo/margin": 98.5962142944336, |
| "fcm_dpo/q_t": 0.35573434829711914, |
| "grad_norm": 10.573701858520508, |
| "learning_rate": 1.5967059836219042e-08, |
| "logits/chosen": -2.050720453262329, |
| "logits/rejected": -1.197265625, |
| "logps/chosen": -265.29229736328125, |
| "logps/ref_chosen": -101.02015686035156, |
| "logps/ref_rejected": -93.78302764892578, |
| "logps/rejected": -356.6513671875, |
| "loss": 0.9267, |
| "margin_dpo/margin_mean": 98.5962142944336, |
| "margin_dpo/margin_std": 94.86685180664062, |
| "step": 612 |
| }, |
| { |
| "epoch": 0.9001468428781204, |
| "fcm_dpo/beta": 0.00644554290920496, |
| "fcm_dpo/delta": -0.09590654820203781, |
| "fcm_dpo/margin": 106.90154266357422, |
| "fcm_dpo/q_t": 0.34533023834228516, |
| "grad_norm": 10.872828483581543, |
| "learning_rate": 1.551886292185553e-08, |
| "logits/chosen": -1.9289722442626953, |
| "logits/rejected": -1.728402853012085, |
| "logps/chosen": -224.5867919921875, |
| "logps/ref_chosen": -88.98861694335938, |
| "logps/ref_rejected": -109.99551391601562, |
| "logps/rejected": -352.4952392578125, |
| "loss": 0.8885, |
| "margin_dpo/margin_mean": 106.90153503417969, |
| "margin_dpo/margin_std": 91.88870239257812, |
| "step": 613 |
| }, |
| { |
| "epoch": 0.9016152716593245, |
| "fcm_dpo/beta": 0.0064603416249156, |
| "fcm_dpo/delta": 0.04875949025154114, |
| "fcm_dpo/margin": 85.74708557128906, |
| "fcm_dpo/q_t": 0.37675923109054565, |
| "grad_norm": 12.27489948272705, |
| "learning_rate": 1.507684480352292e-08, |
| "logits/chosen": -1.6303455829620361, |
| "logits/rejected": -1.6167235374450684, |
| "logps/chosen": -247.53366088867188, |
| "logps/ref_chosen": -80.20005798339844, |
| "logps/ref_rejected": -109.86239624023438, |
| "logps/rejected": -362.94305419921875, |
| "loss": 1.0172, |
| "margin_dpo/margin_mean": 85.74708557128906, |
| "margin_dpo/margin_std": 105.97724151611328, |
| "step": 614 |
| }, |
| { |
| "epoch": 0.9030837004405287, |
| "fcm_dpo/beta": 0.0066406624391674995, |
| "fcm_dpo/delta": 0.09489738941192627, |
| "fcm_dpo/margin": 76.43423461914062, |
| "fcm_dpo/q_t": 0.38481491804122925, |
| "grad_norm": 11.700821876525879, |
| "learning_rate": 1.4641017128809801e-08, |
| "logits/chosen": -1.9045593738555908, |
| "logits/rejected": -1.5077332258224487, |
| "logps/chosen": -256.5118408203125, |
| "logps/ref_chosen": -100.43525695800781, |
| "logps/ref_rejected": -101.1800537109375, |
| "logps/rejected": -333.6908874511719, |
| "loss": 1.0219, |
| "margin_dpo/margin_mean": 76.43423461914062, |
| "margin_dpo/margin_std": 84.58399963378906, |
| "step": 615 |
| }, |
| { |
| "epoch": 0.9045521292217328, |
| "fcm_dpo/beta": 0.006577151361852884, |
| "fcm_dpo/delta": -0.0695214495062828, |
| "fcm_dpo/margin": 101.07377624511719, |
| "fcm_dpo/q_t": 0.35348814725875854, |
| "grad_norm": 10.295190811157227, |
| "learning_rate": 1.4211391382180637e-08, |
| "logits/chosen": -1.8900903463363647, |
| "logits/rejected": -1.4016035795211792, |
| "logps/chosen": -250.1123504638672, |
| "logps/ref_chosen": -92.49292755126953, |
| "logps/ref_rejected": -82.06065368652344, |
| "logps/rejected": -340.7538757324219, |
| "loss": 0.937, |
| "margin_dpo/margin_mean": 101.07378387451172, |
| "margin_dpo/margin_std": 106.29083251953125, |
| "step": 616 |
| }, |
| { |
| "epoch": 0.9060205580029369, |
| "fcm_dpo/beta": 0.006596277467906475, |
| "fcm_dpo/delta": 0.07905448973178864, |
| "fcm_dpo/margin": 79.61018371582031, |
| "fcm_dpo/q_t": 0.3824957013130188, |
| "grad_norm": 10.861001014709473, |
| "learning_rate": 1.378797888467345e-08, |
| "logits/chosen": -1.6198773384094238, |
| "logits/rejected": -0.9793082475662231, |
| "logps/chosen": -255.09112548828125, |
| "logps/ref_chosen": -91.09699249267578, |
| "logps/ref_rejected": -70.41004943847656, |
| "logps/rejected": -314.01434326171875, |
| "loss": 1.0275, |
| "margin_dpo/margin_mean": 79.61019897460938, |
| "margin_dpo/margin_std": 98.98773193359375, |
| "step": 617 |
| }, |
| { |
| "epoch": 0.9074889867841409, |
| "fcm_dpo/beta": 0.006635970436036587, |
| "fcm_dpo/delta": 0.026904039084911346, |
| "fcm_dpo/margin": 86.58537292480469, |
| "fcm_dpo/q_t": 0.37111544609069824, |
| "grad_norm": 11.58753776550293, |
| "learning_rate": 1.3370790793601371e-08, |
| "logits/chosen": -1.829303503036499, |
| "logits/rejected": -1.4636094570159912, |
| "logps/chosen": -266.91119384765625, |
| "logps/ref_chosen": -102.02059936523438, |
| "logps/ref_rejected": -99.80119323730469, |
| "logps/rejected": -351.27716064453125, |
| "loss": 1.0048, |
| "margin_dpo/margin_mean": 86.58537292480469, |
| "margin_dpo/margin_std": 104.85649108886719, |
| "step": 618 |
| }, |
| { |
| "epoch": 0.908957415565345, |
| "fcm_dpo/beta": 0.006731549743562937, |
| "fcm_dpo/delta": 0.07085485756397247, |
| "fcm_dpo/margin": 79.16950988769531, |
| "fcm_dpo/q_t": 0.38630783557891846, |
| "grad_norm": 11.098875999450684, |
| "learning_rate": 1.2959838102258535e-08, |
| "logits/chosen": -1.452412724494934, |
| "logits/rejected": -1.0741287469863892, |
| "logps/chosen": -248.23838806152344, |
| "logps/ref_chosen": -89.74136352539062, |
| "logps/ref_rejected": -99.90138244628906, |
| "logps/rejected": -337.56793212890625, |
| "loss": 1.0501, |
| "margin_dpo/margin_mean": 79.16950988769531, |
| "margin_dpo/margin_std": 113.035400390625, |
| "step": 619 |
| }, |
| { |
| "epoch": 0.9104258443465492, |
| "fcm_dpo/beta": 0.006684327032417059, |
| "fcm_dpo/delta": 0.012951068580150604, |
| "fcm_dpo/margin": 87.47955322265625, |
| "fcm_dpo/q_t": 0.3729515075683594, |
| "grad_norm": 11.075997352600098, |
| "learning_rate": 1.2555131639630567e-08, |
| "logits/chosen": -2.05159068107605, |
| "logits/rejected": -1.514744758605957, |
| "logps/chosen": -238.4269256591797, |
| "logps/ref_chosen": -85.12431335449219, |
| "logps/ref_rejected": -85.41253662109375, |
| "logps/rejected": -326.1947021484375, |
| "loss": 1.0182, |
| "margin_dpo/margin_mean": 87.47955322265625, |
| "margin_dpo/margin_std": 107.8419189453125, |
| "step": 620 |
| }, |
| { |
| "epoch": 0.9118942731277533, |
| "fcm_dpo/beta": 0.006873922888189554, |
| "fcm_dpo/delta": 0.017858199775218964, |
| "fcm_dpo/margin": 84.41682434082031, |
| "fcm_dpo/q_t": 0.3753046989440918, |
| "grad_norm": 11.407662391662598, |
| "learning_rate": 1.2156682070109086e-08, |
| "logits/chosen": -1.8650214672088623, |
| "logits/rejected": -1.598212480545044, |
| "logps/chosen": -245.00628662109375, |
| "logps/ref_chosen": -89.24842071533203, |
| "logps/ref_rejected": -95.46463775634766, |
| "logps/rejected": -335.6393127441406, |
| "loss": 1.0371, |
| "margin_dpo/margin_mean": 84.41682434082031, |
| "margin_dpo/margin_std": 113.9649658203125, |
| "step": 621 |
| }, |
| { |
| "epoch": 0.9133627019089574, |
| "fcm_dpo/beta": 0.0067746308632195, |
| "fcm_dpo/delta": -0.006477778777480125, |
| "fcm_dpo/margin": 89.44147491455078, |
| "fcm_dpo/q_t": 0.3655218780040741, |
| "grad_norm": 12.676264762878418, |
| "learning_rate": 1.1764499893210878e-08, |
| "logits/chosen": -1.4255738258361816, |
| "logits/rejected": -0.7507696747779846, |
| "logps/chosen": -255.17373657226562, |
| "logps/ref_chosen": -99.79412841796875, |
| "logps/ref_rejected": -90.82821655273438, |
| "logps/rejected": -335.6492919921875, |
| "loss": 0.9635, |
| "margin_dpo/margin_mean": 89.44148254394531, |
| "margin_dpo/margin_std": 94.65943908691406, |
| "step": 622 |
| }, |
| { |
| "epoch": 0.9148311306901615, |
| "fcm_dpo/beta": 0.006937139667570591, |
| "fcm_dpo/delta": 0.16734254360198975, |
| "fcm_dpo/margin": 63.47694396972656, |
| "fcm_dpo/q_t": 0.40689754486083984, |
| "grad_norm": 13.347342491149902, |
| "learning_rate": 1.1378595443300998e-08, |
| "logits/chosen": -2.32456111907959, |
| "logits/rejected": -1.8910653591156006, |
| "logps/chosen": -251.2813262939453, |
| "logps/ref_chosen": -90.45555114746094, |
| "logps/ref_rejected": -91.32276153564453, |
| "logps/rejected": -315.62548828125, |
| "loss": 1.138, |
| "margin_dpo/margin_mean": 63.47693634033203, |
| "margin_dpo/margin_std": 114.36991882324219, |
| "step": 623 |
| }, |
| { |
| "epoch": 0.9162995594713657, |
| "fcm_dpo/beta": 0.006922868080437183, |
| "fcm_dpo/delta": -0.09390503168106079, |
| "fcm_dpo/margin": 99.29826354980469, |
| "fcm_dpo/q_t": 0.35036206245422363, |
| "grad_norm": 12.173335075378418, |
| "learning_rate": 1.0998978889320582e-08, |
| "logits/chosen": -2.5920774936676025, |
| "logits/rejected": -1.668988585472107, |
| "logps/chosen": -272.736083984375, |
| "logps/ref_chosen": -109.87522888183594, |
| "logps/ref_rejected": -104.77320861816406, |
| "logps/rejected": -366.93231201171875, |
| "loss": 0.9295, |
| "margin_dpo/margin_mean": 99.29826354980469, |
| "margin_dpo/margin_std": 105.40048217773438, |
| "step": 624 |
| }, |
| { |
| "epoch": 0.9177679882525698, |
| "fcm_dpo/beta": 0.006994827184826136, |
| "fcm_dpo/delta": 0.08261488378047943, |
| "fcm_dpo/margin": 74.53196716308594, |
| "fcm_dpo/q_t": 0.38579094409942627, |
| "grad_norm": 10.665149688720703, |
| "learning_rate": 1.0625660234518913e-08, |
| "logits/chosen": -1.9561140537261963, |
| "logits/rejected": -1.590733289718628, |
| "logps/chosen": -259.02740478515625, |
| "logps/ref_chosen": -87.16815948486328, |
| "logps/ref_rejected": -91.86148071289062, |
| "logps/rejected": -338.252685546875, |
| "loss": 1.0348, |
| "margin_dpo/margin_mean": 74.53196716308594, |
| "margin_dpo/margin_std": 97.0251693725586, |
| "step": 625 |
| }, |
| { |
| "epoch": 0.9192364170337739, |
| "fcm_dpo/beta": 0.007120197638869286, |
| "fcm_dpo/delta": 0.08808340132236481, |
| "fcm_dpo/margin": 72.4712142944336, |
| "fcm_dpo/q_t": 0.39089781045913696, |
| "grad_norm": 13.34603214263916, |
| "learning_rate": 1.0258649316189721e-08, |
| "logits/chosen": -1.5797321796417236, |
| "logits/rejected": -1.1557903289794922, |
| "logps/chosen": -267.87542724609375, |
| "logps/ref_chosen": -104.22421264648438, |
| "logps/ref_rejected": -104.1774673461914, |
| "logps/rejected": -340.29986572265625, |
| "loss": 1.0775, |
| "margin_dpo/margin_mean": 72.47120666503906, |
| "margin_dpo/margin_std": 112.5821533203125, |
| "step": 626 |
| }, |
| { |
| "epoch": 0.920704845814978, |
| "fcm_dpo/beta": 0.007080005016177893, |
| "fcm_dpo/delta": -0.04163035377860069, |
| "fcm_dpo/margin": 90.24308776855469, |
| "fcm_dpo/q_t": 0.36456674337387085, |
| "grad_norm": 9.200769424438477, |
| "learning_rate": 9.897955805412e-09, |
| "logits/chosen": -1.5551083087921143, |
| "logits/rejected": -1.6972510814666748, |
| "logps/chosen": -221.11859130859375, |
| "logps/ref_chosen": -74.93461608886719, |
| "logps/ref_rejected": -112.57289123535156, |
| "logps/rejected": -348.9999694824219, |
| "loss": 0.9833, |
| "margin_dpo/margin_mean": 90.24308776855469, |
| "margin_dpo/margin_std": 113.31100463867188, |
| "step": 627 |
| }, |
| { |
| "epoch": 0.922173274596182, |
| "fcm_dpo/beta": 0.006989369168877602, |
| "fcm_dpo/delta": -0.0777251347899437, |
| "fcm_dpo/margin": 96.21630859375, |
| "fcm_dpo/q_t": 0.3529139757156372, |
| "grad_norm": 12.8143892288208, |
| "learning_rate": 9.543589206795238e-09, |
| "logits/chosen": -2.0264172554016113, |
| "logits/rejected": -1.7620849609375, |
| "logps/chosen": -246.24087524414062, |
| "logps/ref_chosen": -93.69107818603516, |
| "logps/ref_rejected": -107.34395599365234, |
| "logps/rejected": -356.11004638671875, |
| "loss": 0.9296, |
| "margin_dpo/margin_mean": 96.21630096435547, |
| "margin_dpo/margin_std": 98.98152923583984, |
| "step": 628 |
| }, |
| { |
| "epoch": 0.9236417033773862, |
| "fcm_dpo/beta": 0.0069330655969679356, |
| "fcm_dpo/delta": -0.0012096241116523743, |
| "fcm_dpo/margin": 86.66081237792969, |
| "fcm_dpo/q_t": 0.36651545763015747, |
| "grad_norm": 11.387245178222656, |
| "learning_rate": 9.19555885822887e-09, |
| "logits/chosen": -2.008543014526367, |
| "logits/rejected": -1.5725985765457153, |
| "logps/chosen": -266.6309814453125, |
| "logps/ref_chosen": -103.23037719726562, |
| "logps/ref_rejected": -97.16841888427734, |
| "logps/rejected": -347.2298278808594, |
| "loss": 0.9856, |
| "margin_dpo/margin_mean": 86.66081237792969, |
| "margin_dpo/margin_std": 100.66366577148438, |
| "step": 629 |
| }, |
| { |
| "epoch": 0.9251101321585903, |
| "fcm_dpo/beta": 0.007189781405031681, |
| "fcm_dpo/delta": 0.1778494417667389, |
| "fcm_dpo/margin": 59.536293029785156, |
| "fcm_dpo/q_t": 0.40600132942199707, |
| "grad_norm": 15.609967231750488, |
| "learning_rate": 8.85387393063622e-09, |
| "logits/chosen": -2.0184245109558105, |
| "logits/rejected": -1.4799931049346924, |
| "logps/chosen": -244.27438354492188, |
| "logps/ref_chosen": -93.89755249023438, |
| "logps/ref_rejected": -89.3743896484375, |
| "logps/rejected": -299.28753662109375, |
| "loss": 1.1223, |
| "margin_dpo/margin_mean": 59.536293029785156, |
| "margin_dpo/margin_std": 98.0714111328125, |
| "step": 630 |
| }, |
| { |
| "epoch": 0.9265785609397944, |
| "fcm_dpo/beta": 0.007201473228633404, |
| "fcm_dpo/delta": -0.001188160851597786, |
| "fcm_dpo/margin": 83.4664535522461, |
| "fcm_dpo/q_t": 0.36686837673187256, |
| "grad_norm": 11.061304092407227, |
| "learning_rate": 8.518543427732949e-09, |
| "logits/chosen": -2.5078396797180176, |
| "logits/rejected": -2.078655958175659, |
| "logps/chosen": -235.43612670898438, |
| "logps/ref_chosen": -87.77082061767578, |
| "logps/ref_rejected": -88.68241882324219, |
| "logps/rejected": -319.81414794921875, |
| "loss": 0.9793, |
| "margin_dpo/margin_mean": 83.4664535522461, |
| "margin_dpo/margin_std": 95.37765502929688, |
| "step": 631 |
| }, |
| { |
| "epoch": 0.9280469897209985, |
| "fcm_dpo/beta": 0.007080842275172472, |
| "fcm_dpo/delta": -0.06554602086544037, |
| "fcm_dpo/margin": 93.11445617675781, |
| "fcm_dpo/q_t": 0.35878896713256836, |
| "grad_norm": 12.773686408996582, |
| "learning_rate": 8.189576185789637e-09, |
| "logits/chosen": -1.9766130447387695, |
| "logits/rejected": -1.5455785989761353, |
| "logps/chosen": -233.4774169921875, |
| "logps/ref_chosen": -88.62652587890625, |
| "logps/ref_rejected": -91.45091247558594, |
| "logps/rejected": -329.416259765625, |
| "loss": 0.9759, |
| "margin_dpo/margin_mean": 93.11445617675781, |
| "margin_dpo/margin_std": 112.72152709960938, |
| "step": 632 |
| }, |
| { |
| "epoch": 0.9295154185022027, |
| "fcm_dpo/beta": 0.007372128311544657, |
| "fcm_dpo/delta": 0.18217022716999054, |
| "fcm_dpo/margin": 57.35297393798828, |
| "fcm_dpo/q_t": 0.41445010900497437, |
| "grad_norm": 14.506038665771484, |
| "learning_rate": 7.866980873399015e-09, |
| "logits/chosen": -2.240860939025879, |
| "logits/rejected": -2.0498335361480713, |
| "logps/chosen": -260.7783203125, |
| "logps/ref_chosen": -81.37442016601562, |
| "logps/ref_rejected": -98.62571716308594, |
| "logps/rejected": -335.3825988769531, |
| "loss": 1.1966, |
| "margin_dpo/margin_mean": 57.35297393798828, |
| "margin_dpo/margin_std": 124.81423950195312, |
| "step": 633 |
| }, |
| { |
| "epoch": 0.9309838472834068, |
| "fcm_dpo/beta": 0.007527489215135574, |
| "fcm_dpo/delta": 0.04844316840171814, |
| "fcm_dpo/margin": 73.15084838867188, |
| "fcm_dpo/q_t": 0.3781312108039856, |
| "grad_norm": 13.961141586303711, |
| "learning_rate": 7.550765991247654e-09, |
| "logits/chosen": -1.4511160850524902, |
| "logits/rejected": -1.2699205875396729, |
| "logps/chosen": -263.0788879394531, |
| "logps/ref_chosen": -96.12284851074219, |
| "logps/ref_rejected": -112.84780883789062, |
| "logps/rejected": -352.9547119140625, |
| "loss": 1.0141, |
| "margin_dpo/margin_mean": 73.15084838867188, |
| "margin_dpo/margin_std": 84.39659881591797, |
| "step": 634 |
| }, |
| { |
| "epoch": 0.9324522760646109, |
| "fcm_dpo/beta": 0.007684573996812105, |
| "fcm_dpo/delta": 0.16829924285411835, |
| "fcm_dpo/margin": 56.93890380859375, |
| "fcm_dpo/q_t": 0.40733087062835693, |
| "grad_norm": 16.105562210083008, |
| "learning_rate": 7.240939871891699e-09, |
| "logits/chosen": -1.7153419256210327, |
| "logits/rejected": -1.1771185398101807, |
| "logps/chosen": -269.976806640625, |
| "logps/ref_chosen": -98.68411254882812, |
| "logps/ref_rejected": -89.89920043945312, |
| "logps/rejected": -318.13079833984375, |
| "loss": 1.1449, |
| "margin_dpo/margin_mean": 56.93890380859375, |
| "margin_dpo/margin_std": 105.87216186523438, |
| "step": 635 |
| }, |
| { |
| "epoch": 0.933920704845815, |
| "fcm_dpo/beta": 0.007606941275298595, |
| "fcm_dpo/delta": -0.09959021955728531, |
| "fcm_dpo/margin": 91.07327270507812, |
| "fcm_dpo/q_t": 0.35048407316207886, |
| "grad_norm": 12.00423526763916, |
| "learning_rate": 6.937510679537628e-09, |
| "logits/chosen": -1.446263074874878, |
| "logits/rejected": -0.9623331427574158, |
| "logps/chosen": -246.98480224609375, |
| "logps/ref_chosen": -90.41796112060547, |
| "logps/ref_rejected": -87.70687866210938, |
| "logps/rejected": -335.34698486328125, |
| "loss": 0.9282, |
| "margin_dpo/margin_mean": 91.07327270507812, |
| "margin_dpo/margin_std": 97.26042175292969, |
| "step": 636 |
| }, |
| { |
| "epoch": 0.9353891336270191, |
| "fcm_dpo/beta": 0.0074293059296905994, |
| "fcm_dpo/delta": -0.1272699534893036, |
| "fcm_dpo/margin": 96.68272399902344, |
| "fcm_dpo/q_t": 0.34441307187080383, |
| "grad_norm": 10.729310989379883, |
| "learning_rate": 6.640486409826785e-09, |
| "logits/chosen": -2.0841870307922363, |
| "logits/rejected": -1.981583595275879, |
| "logps/chosen": -240.16139221191406, |
| "logps/ref_chosen": -82.44971466064453, |
| "logps/ref_rejected": -104.02860260009766, |
| "logps/rejected": -358.4230041503906, |
| "loss": 0.9195, |
| "margin_dpo/margin_mean": 96.68272399902344, |
| "margin_dpo/margin_std": 101.56159973144531, |
| "step": 637 |
| }, |
| { |
| "epoch": 0.9368575624082232, |
| "fcm_dpo/beta": 0.007293237838894129, |
| "fcm_dpo/delta": -0.11687527596950531, |
| "fcm_dpo/margin": 97.12511444091797, |
| "fcm_dpo/q_t": 0.3418697118759155, |
| "grad_norm": 10.414422988891602, |
| "learning_rate": 6.349874889624962e-09, |
| "logits/chosen": -1.5123820304870605, |
| "logits/rejected": -0.8182043433189392, |
| "logps/chosen": -238.3617401123047, |
| "logps/ref_chosen": -91.92498779296875, |
| "logps/ref_rejected": -86.28703308105469, |
| "logps/rejected": -329.8489074707031, |
| "loss": 0.8832, |
| "margin_dpo/margin_mean": 97.12511444091797, |
| "margin_dpo/margin_std": 81.02644348144531, |
| "step": 638 |
| }, |
| { |
| "epoch": 0.9383259911894273, |
| "fcm_dpo/beta": 0.007484002038836479, |
| "fcm_dpo/delta": 0.23280690610408783, |
| "fcm_dpo/margin": 50.03688430786133, |
| "fcm_dpo/q_t": 0.41923975944519043, |
| "grad_norm": 14.05245304107666, |
| "learning_rate": 6.065683776815933e-09, |
| "logits/chosen": -1.3220126628875732, |
| "logits/rejected": -0.5478463768959045, |
| "logps/chosen": -296.0784912109375, |
| "logps/ref_chosen": -104.52755737304688, |
| "logps/ref_rejected": -81.4803466796875, |
| "logps/rejected": -323.06817626953125, |
| "loss": 1.2022, |
| "margin_dpo/margin_mean": 50.036888122558594, |
| "margin_dpo/margin_std": 110.06419372558594, |
| "step": 639 |
| }, |
| { |
| "epoch": 0.9397944199706314, |
| "fcm_dpo/beta": 0.00756090460345149, |
| "fcm_dpo/delta": -0.06717022508382797, |
| "fcm_dpo/margin": 87.34074401855469, |
| "fcm_dpo/q_t": 0.3554985821247101, |
| "grad_norm": 12.982050895690918, |
| "learning_rate": 5.7879205600998296e-09, |
| "logits/chosen": -1.7683886289596558, |
| "logits/rejected": -1.2369413375854492, |
| "logps/chosen": -261.49896240234375, |
| "logps/ref_chosen": -97.88526916503906, |
| "logps/ref_rejected": -112.70502471923828, |
| "logps/rejected": -363.65948486328125, |
| "loss": 0.9568, |
| "margin_dpo/margin_mean": 87.34074401855469, |
| "margin_dpo/margin_std": 92.46745300292969, |
| "step": 640 |
| }, |
| { |
| "epoch": 0.9412628487518355, |
| "fcm_dpo/beta": 0.007357077207416296, |
| "fcm_dpo/delta": -0.08074723184108734, |
| "fcm_dpo/margin": 91.79606628417969, |
| "fcm_dpo/q_t": 0.3599125146865845, |
| "grad_norm": 10.860076904296875, |
| "learning_rate": 5.516592558795746e-09, |
| "logits/chosen": -2.443859338760376, |
| "logits/rejected": -1.9180916547775269, |
| "logps/chosen": -245.45460510253906, |
| "logps/ref_chosen": -96.4456787109375, |
| "logps/ref_rejected": -95.13568878173828, |
| "logps/rejected": -335.940673828125, |
| "loss": 0.9735, |
| "margin_dpo/margin_mean": 91.79606628417969, |
| "margin_dpo/margin_std": 117.18472290039062, |
| "step": 641 |
| }, |
| { |
| "epoch": 0.9427312775330396, |
| "fcm_dpo/beta": 0.007323989644646645, |
| "fcm_dpo/delta": -0.019782835617661476, |
| "fcm_dpo/margin": 84.43093872070312, |
| "fcm_dpo/q_t": 0.3701496422290802, |
| "grad_norm": 12.883811950683594, |
| "learning_rate": 5.251706922648868e-09, |
| "logits/chosen": -1.569532036781311, |
| "logits/rejected": -1.235536813735962, |
| "logps/chosen": -269.09429931640625, |
| "logps/ref_chosen": -100.75984954833984, |
| "logps/ref_rejected": -114.7076416015625, |
| "logps/rejected": -367.4729919433594, |
| "loss": 1.0067, |
| "margin_dpo/margin_mean": 84.43093872070312, |
| "margin_dpo/margin_std": 113.8532485961914, |
| "step": 642 |
| }, |
| { |
| "epoch": 0.9441997063142438, |
| "fcm_dpo/beta": 0.007224624045193195, |
| "fcm_dpo/delta": 0.009218744933605194, |
| "fcm_dpo/margin": 81.56900787353516, |
| "fcm_dpo/q_t": 0.37049052119255066, |
| "grad_norm": 13.749687194824219, |
| "learning_rate": 4.993270631642038e-09, |
| "logits/chosen": -2.2235236167907715, |
| "logits/rejected": -1.8363418579101562, |
| "logps/chosen": -241.62869262695312, |
| "logps/ref_chosen": -84.74365997314453, |
| "logps/ref_rejected": -94.31842041015625, |
| "logps/rejected": -332.7724609375, |
| "loss": 1.0051, |
| "margin_dpo/margin_mean": 81.56900787353516, |
| "margin_dpo/margin_std": 98.04161071777344, |
| "step": 643 |
| }, |
| { |
| "epoch": 0.9456681350954479, |
| "fcm_dpo/beta": 0.0074260118417441845, |
| "fcm_dpo/delta": 0.10681871324777603, |
| "fcm_dpo/margin": 67.11778259277344, |
| "fcm_dpo/q_t": 0.38967859745025635, |
| "grad_norm": 14.644425392150879, |
| "learning_rate": 4.741290495811873e-09, |
| "logits/chosen": -1.818040370941162, |
| "logits/rejected": -1.5412225723266602, |
| "logps/chosen": -246.3582763671875, |
| "logps/ref_chosen": -85.32274627685547, |
| "logps/ref_rejected": -94.60861206054688, |
| "logps/rejected": -322.76190185546875, |
| "loss": 1.0502, |
| "margin_dpo/margin_mean": 67.11778259277344, |
| "margin_dpo/margin_std": 89.98631286621094, |
| "step": 644 |
| }, |
| { |
| "epoch": 0.947136563876652, |
| "fcm_dpo/beta": 0.007773152552545071, |
| "fcm_dpo/delta": 0.2508290708065033, |
| "fcm_dpo/margin": 45.93043518066406, |
| "fcm_dpo/q_t": 0.425441175699234, |
| "grad_norm": 14.240182876586914, |
| "learning_rate": 4.495773155069299e-09, |
| "logits/chosen": -1.446195363998413, |
| "logits/rejected": -1.2669878005981445, |
| "logps/chosen": -257.103515625, |
| "logps/ref_chosen": -82.59024047851562, |
| "logps/ref_rejected": -103.00375366210938, |
| "logps/rejected": -323.4474792480469, |
| "loss": 1.2211, |
| "margin_dpo/margin_mean": 45.93043518066406, |
| "margin_dpo/margin_std": 107.12353515625, |
| "step": 645 |
| }, |
| { |
| "epoch": 0.9486049926578561, |
| "fcm_dpo/beta": 0.007703070994466543, |
| "fcm_dpo/delta": -0.08048854768276215, |
| "fcm_dpo/margin": 87.47425842285156, |
| "fcm_dpo/q_t": 0.3519340753555298, |
| "grad_norm": 11.884034156799316, |
| "learning_rate": 4.256725079024553e-09, |
| "logits/chosen": -1.8835480213165283, |
| "logits/rejected": -1.2794466018676758, |
| "logps/chosen": -241.96484375, |
| "logps/ref_chosen": -91.1920394897461, |
| "logps/ref_rejected": -83.77833557128906, |
| "logps/rejected": -322.025390625, |
| "loss": 0.9277, |
| "margin_dpo/margin_mean": 87.47425842285156, |
| "margin_dpo/margin_std": 88.36001586914062, |
| "step": 646 |
| }, |
| { |
| "epoch": 0.9500734214390602, |
| "fcm_dpo/beta": 0.0077326856553554535, |
| "fcm_dpo/delta": 0.018357396125793457, |
| "fcm_dpo/margin": 75.35665130615234, |
| "fcm_dpo/q_t": 0.37991610169410706, |
| "grad_norm": 12.295222282409668, |
| "learning_rate": 4.024152566816791e-09, |
| "logits/chosen": -1.3169585466384888, |
| "logits/rejected": -1.1735953092575073, |
| "logps/chosen": -254.99407958984375, |
| "logps/ref_chosen": -88.8444595336914, |
| "logps/ref_rejected": -99.49833679199219, |
| "logps/rejected": -341.00457763671875, |
| "loss": 1.0523, |
| "margin_dpo/margin_mean": 75.35665130615234, |
| "margin_dpo/margin_std": 113.26918029785156, |
| "step": 647 |
| }, |
| { |
| "epoch": 0.9515418502202643, |
| "fcm_dpo/beta": 0.007531121838837862, |
| "fcm_dpo/delta": -0.2151474952697754, |
| "fcm_dpo/margin": 106.015625, |
| "fcm_dpo/q_t": 0.3290584087371826, |
| "grad_norm": 12.194585800170898, |
| "learning_rate": 3.798061746947995e-09, |
| "logits/chosen": -1.9958875179290771, |
| "logits/rejected": -1.9187660217285156, |
| "logps/chosen": -240.71719360351562, |
| "logps/ref_chosen": -87.84810638427734, |
| "logps/ref_rejected": -104.67005920410156, |
| "logps/rejected": -363.5547790527344, |
| "loss": 0.8804, |
| "margin_dpo/margin_mean": 106.015625, |
| "margin_dpo/margin_std": 108.19938659667969, |
| "step": 648 |
| }, |
| { |
| "epoch": 0.9530102790014684, |
| "fcm_dpo/beta": 0.007324899546802044, |
| "fcm_dpo/delta": -0.042867887765169144, |
| "fcm_dpo/margin": 87.23211669921875, |
| "fcm_dpo/q_t": 0.35840868949890137, |
| "grad_norm": 13.264395713806152, |
| "learning_rate": 3.5784585771215235e-09, |
| "logits/chosen": -2.3595848083496094, |
| "logits/rejected": -1.9544248580932617, |
| "logps/chosen": -229.560302734375, |
| "logps/ref_chosen": -89.6925048828125, |
| "logps/ref_rejected": -88.70658111572266, |
| "logps/rejected": -315.8064880371094, |
| "loss": 0.9392, |
| "margin_dpo/margin_mean": 87.23211669921875, |
| "margin_dpo/margin_std": 85.69928741455078, |
| "step": 649 |
| }, |
| { |
| "epoch": 0.9544787077826725, |
| "fcm_dpo/beta": 0.007353082299232483, |
| "fcm_dpo/delta": -0.015879113227128983, |
| "fcm_dpo/margin": 83.62142181396484, |
| "fcm_dpo/q_t": 0.3669193983078003, |
| "grad_norm": 12.675481796264648, |
| "learning_rate": 3.3653488440851253e-09, |
| "logits/chosen": -1.3255770206451416, |
| "logits/rejected": -1.12962007522583, |
| "logps/chosen": -255.35781860351562, |
| "logps/ref_chosen": -89.93060302734375, |
| "logps/ref_rejected": -102.61282348632812, |
| "logps/rejected": -351.66143798828125, |
| "loss": 1.0064, |
| "margin_dpo/margin_mean": 83.62141418457031, |
| "margin_dpo/margin_std": 110.35063934326172, |
| "step": 650 |
| }, |
| { |
| "epoch": 0.9559471365638766, |
| "fcm_dpo/beta": 0.007182122208178043, |
| "fcm_dpo/delta": -0.1339079886674881, |
| "fcm_dpo/margin": 100.80975341796875, |
| "fcm_dpo/q_t": 0.3390667736530304, |
| "grad_norm": 11.261962890625, |
| "learning_rate": 3.158738163478475e-09, |
| "logits/chosen": -2.088024854660034, |
| "logits/rejected": -1.914245367050171, |
| "logps/chosen": -218.3494873046875, |
| "logps/ref_chosen": -79.18731689453125, |
| "logps/ref_rejected": -105.93333435058594, |
| "logps/rejected": -345.9052734375, |
| "loss": 0.8862, |
| "margin_dpo/margin_mean": 100.80976104736328, |
| "margin_dpo/margin_std": 92.29314422607422, |
| "step": 651 |
| }, |
| { |
| "epoch": 0.9574155653450808, |
| "fcm_dpo/beta": 0.007138779386878014, |
| "fcm_dpo/delta": -0.021067023277282715, |
| "fcm_dpo/margin": 86.80287170410156, |
| "fcm_dpo/q_t": 0.36111682653427124, |
| "grad_norm": 10.429312705993652, |
| "learning_rate": 2.9586319796851555e-09, |
| "logits/chosen": -2.4728636741638184, |
| "logits/rejected": -2.168954849243164, |
| "logps/chosen": -246.78160095214844, |
| "logps/ref_chosen": -101.79022216796875, |
| "logps/ref_rejected": -116.3245849609375, |
| "logps/rejected": -348.11883544921875, |
| "loss": 0.9352, |
| "margin_dpo/margin_mean": 86.80288696289062, |
| "margin_dpo/margin_std": 81.80032348632812, |
| "step": 652 |
| }, |
| { |
| "epoch": 0.9588839941262849, |
| "fcm_dpo/beta": 0.007149814628064632, |
| "fcm_dpo/delta": 0.018516050651669502, |
| "fcm_dpo/margin": 81.45347595214844, |
| "fcm_dpo/q_t": 0.36881303787231445, |
| "grad_norm": 12.834737777709961, |
| "learning_rate": 2.7650355656892166e-09, |
| "logits/chosen": -2.072822093963623, |
| "logits/rejected": -1.819368600845337, |
| "logps/chosen": -257.77667236328125, |
| "logps/ref_chosen": -93.35359191894531, |
| "logps/ref_rejected": -109.12324523925781, |
| "logps/rejected": -354.99981689453125, |
| "loss": 0.9894, |
| "margin_dpo/margin_mean": 81.45347595214844, |
| "margin_dpo/margin_std": 93.0560073852539, |
| "step": 653 |
| }, |
| { |
| "epoch": 0.960352422907489, |
| "fcm_dpo/beta": 0.007231119088828564, |
| "fcm_dpo/delta": 0.08581791818141937, |
| "fcm_dpo/margin": 71.71923828125, |
| "fcm_dpo/q_t": 0.3922192454338074, |
| "grad_norm": 15.544170379638672, |
| "learning_rate": 2.577954022936174e-09, |
| "logits/chosen": -2.3582053184509277, |
| "logits/rejected": -2.17484712600708, |
| "logps/chosen": -254.76153564453125, |
| "logps/ref_chosen": -89.11553955078125, |
| "logps/ref_rejected": -104.91995239257812, |
| "logps/rejected": -342.28515625, |
| "loss": 1.0873, |
| "margin_dpo/margin_mean": 71.71923828125, |
| "margin_dpo/margin_std": 116.78224182128906, |
| "step": 654 |
| }, |
| { |
| "epoch": 0.9618208516886931, |
| "fcm_dpo/beta": 0.007248979061841965, |
| "fcm_dpo/delta": -0.02873397059738636, |
| "fcm_dpo/margin": 86.4774169921875, |
| "fcm_dpo/q_t": 0.362434983253479, |
| "grad_norm": 11.312616348266602, |
| "learning_rate": 2.397392281198729e-09, |
| "logits/chosen": -2.129537582397461, |
| "logits/rejected": -2.125460147857666, |
| "logps/chosen": -233.4827423095703, |
| "logps/ref_chosen": -81.03610229492188, |
| "logps/ref_rejected": -102.80233764648438, |
| "logps/rejected": -341.72637939453125, |
| "loss": 0.9844, |
| "margin_dpo/margin_mean": 86.4774169921875, |
| "margin_dpo/margin_std": 104.30917358398438, |
| "step": 655 |
| }, |
| { |
| "epoch": 0.9632892804698973, |
| "fcm_dpo/beta": 0.007026261650025845, |
| "fcm_dpo/delta": -0.16489456593990326, |
| "fcm_dpo/margin": 107.01776123046875, |
| "fcm_dpo/q_t": 0.3365343511104584, |
| "grad_norm": 12.231975555419922, |
| "learning_rate": 2.223355098446622e-09, |
| "logits/chosen": -1.2732338905334473, |
| "logits/rejected": -1.3455628156661987, |
| "logps/chosen": -237.61703491210938, |
| "logps/ref_chosen": -85.32534790039062, |
| "logps/ref_rejected": -118.33866119384766, |
| "logps/rejected": -377.6481018066406, |
| "loss": 0.8735, |
| "margin_dpo/margin_mean": 107.01776123046875, |
| "margin_dpo/margin_std": 99.96194458007812, |
| "step": 656 |
| }, |
| { |
| "epoch": 0.9647577092511013, |
| "fcm_dpo/beta": 0.006966191343963146, |
| "fcm_dpo/delta": 0.019626092165708542, |
| "fcm_dpo/margin": 83.40191650390625, |
| "fcm_dpo/q_t": 0.3764026165008545, |
| "grad_norm": 14.666167259216309, |
| "learning_rate": 2.055847060721566e-09, |
| "logits/chosen": -1.931036114692688, |
| "logits/rejected": -1.757138729095459, |
| "logps/chosen": -242.91915893554688, |
| "logps/ref_chosen": -80.19772338867188, |
| "logps/ref_rejected": -102.581298828125, |
| "logps/rejected": -348.70465087890625, |
| "loss": 1.0244, |
| "margin_dpo/margin_mean": 83.40192413330078, |
| "margin_dpo/margin_std": 111.77230834960938, |
| "step": 657 |
| }, |
| { |
| "epoch": 0.9662261380323054, |
| "fcm_dpo/beta": 0.007131836377084255, |
| "fcm_dpo/delta": 0.08433780819177628, |
| "fcm_dpo/margin": 72.8197250366211, |
| "fcm_dpo/q_t": 0.38564208149909973, |
| "grad_norm": 11.62498664855957, |
| "learning_rate": 1.8948725820160662e-09, |
| "logits/chosen": -2.0923075675964355, |
| "logits/rejected": -1.626826286315918, |
| "logps/chosen": -262.4381103515625, |
| "logps/ref_chosen": -94.634521484375, |
| "logps/ref_rejected": -101.6316146850586, |
| "logps/rejected": -342.25494384765625, |
| "loss": 1.0369, |
| "margin_dpo/margin_mean": 72.8197250366211, |
| "margin_dpo/margin_std": 94.01678466796875, |
| "step": 658 |
| }, |
| { |
| "epoch": 0.9676945668135095, |
| "fcm_dpo/beta": 0.007036149501800537, |
| "fcm_dpo/delta": -0.056030645966529846, |
| "fcm_dpo/margin": 92.60414123535156, |
| "fcm_dpo/q_t": 0.36230194568634033, |
| "grad_norm": 11.66299819946289, |
| "learning_rate": 1.7404359041573723e-09, |
| "logits/chosen": -2.0208144187927246, |
| "logits/rejected": -1.0372681617736816, |
| "logps/chosen": -259.408447265625, |
| "logps/ref_chosen": -112.55587005615234, |
| "logps/ref_rejected": -93.9216079711914, |
| "logps/rejected": -333.37835693359375, |
| "loss": 1.0067, |
| "margin_dpo/margin_mean": 92.6041488647461, |
| "margin_dpo/margin_std": 122.59965515136719, |
| "step": 659 |
| }, |
| { |
| "epoch": 0.9691629955947136, |
| "fcm_dpo/beta": 0.006981555372476578, |
| "fcm_dpo/delta": -0.06499287486076355, |
| "fcm_dpo/margin": 94.63003540039062, |
| "fcm_dpo/q_t": 0.35565003752708435, |
| "grad_norm": 14.255919456481934, |
| "learning_rate": 1.592541096695571e-09, |
| "logits/chosen": -1.8492063283920288, |
| "logits/rejected": -1.2232909202575684, |
| "logps/chosen": -247.27349853515625, |
| "logps/ref_chosen": -93.37742614746094, |
| "logps/ref_rejected": -81.39482116699219, |
| "logps/rejected": -329.9209289550781, |
| "loss": 0.9439, |
| "margin_dpo/margin_mean": 94.63003540039062, |
| "margin_dpo/margin_std": 103.12388610839844, |
| "step": 660 |
| }, |
| { |
| "epoch": 0.9706314243759178, |
| "fcm_dpo/beta": 0.006860889494419098, |
| "fcm_dpo/delta": -0.1109163761138916, |
| "fcm_dpo/margin": 102.49998474121094, |
| "fcm_dpo/q_t": 0.34864702820777893, |
| "grad_norm": 11.033123970031738, |
| "learning_rate": 1.4511920567963908e-09, |
| "logits/chosen": -1.9187567234039307, |
| "logits/rejected": -1.4068787097930908, |
| "logps/chosen": -245.22006225585938, |
| "logps/ref_chosen": -87.85516357421875, |
| "logps/ref_rejected": -92.40330505371094, |
| "logps/rejected": -352.2681884765625, |
| "loss": 0.931, |
| "margin_dpo/margin_mean": 102.49998474121094, |
| "margin_dpo/margin_std": 111.98723602294922, |
| "step": 661 |
| }, |
| { |
| "epoch": 0.9720998531571219, |
| "fcm_dpo/beta": 0.006795001216232777, |
| "fcm_dpo/delta": -0.02861689031124115, |
| "fcm_dpo/margin": 92.24244689941406, |
| "fcm_dpo/q_t": 0.3603462278842926, |
| "grad_norm": 10.01567554473877, |
| "learning_rate": 1.3163925091384532e-09, |
| "logits/chosen": -1.5263056755065918, |
| "logits/rejected": -0.9408450722694397, |
| "logps/chosen": -255.88613891601562, |
| "logps/ref_chosen": -102.77980041503906, |
| "logps/ref_rejected": -95.22531127929688, |
| "logps/rejected": -340.5740966796875, |
| "loss": 0.942, |
| "margin_dpo/margin_mean": 92.24244689941406, |
| "margin_dpo/margin_std": 90.77671813964844, |
| "step": 662 |
| }, |
| { |
| "epoch": 0.973568281938326, |
| "fcm_dpo/beta": 0.006806028075516224, |
| "fcm_dpo/delta": 0.015626247972249985, |
| "fcm_dpo/margin": 85.96340942382812, |
| "fcm_dpo/q_t": 0.3726150095462799, |
| "grad_norm": 11.392067909240723, |
| "learning_rate": 1.1881460058152382e-09, |
| "logits/chosen": -2.2485814094543457, |
| "logits/rejected": -2.024899959564209, |
| "logps/chosen": -249.08914184570312, |
| "logps/ref_chosen": -96.34658813476562, |
| "logps/ref_rejected": -120.52645874023438, |
| "logps/rejected": -359.232421875, |
| "loss": 1.0095, |
| "margin_dpo/margin_mean": 85.96340942382812, |
| "margin_dpo/margin_std": 108.69853210449219, |
| "step": 663 |
| }, |
| { |
| "epoch": 0.9750367107195301, |
| "fcm_dpo/beta": 0.006872647441923618, |
| "fcm_dpo/delta": 0.062802754342556, |
| "fcm_dpo/margin": 78.6165771484375, |
| "fcm_dpo/q_t": 0.37933892011642456, |
| "grad_norm": 11.565393447875977, |
| "learning_rate": 1.066455926241383e-09, |
| "logits/chosen": -1.6666076183319092, |
| "logits/rejected": -1.4859673976898193, |
| "logps/chosen": -259.26226806640625, |
| "logps/ref_chosen": -91.84242248535156, |
| "logps/ref_rejected": -111.8366928100586, |
| "logps/rejected": -357.8731384277344, |
| "loss": 1.0112, |
| "margin_dpo/margin_mean": 78.6165771484375, |
| "margin_dpo/margin_std": 91.88618469238281, |
| "step": 664 |
| }, |
| { |
| "epoch": 0.9765051395007343, |
| "fcm_dpo/beta": 0.006901168264448643, |
| "fcm_dpo/delta": 0.007144978269934654, |
| "fcm_dpo/margin": 85.94855499267578, |
| "fcm_dpo/q_t": 0.36612850427627563, |
| "grad_norm": 9.935822486877441, |
| "learning_rate": 9.513254770636137e-10, |
| "logits/chosen": -2.3310766220092773, |
| "logits/rejected": -1.89298677444458, |
| "logps/chosen": -232.2900848388672, |
| "logps/ref_chosen": -88.18618774414062, |
| "logps/ref_rejected": -91.9120101928711, |
| "logps/rejected": -321.9644775390625, |
| "loss": 0.9693, |
| "margin_dpo/margin_mean": 85.94855499267578, |
| "margin_dpo/margin_std": 91.62937927246094, |
| "step": 665 |
| }, |
| { |
| "epoch": 0.9779735682819384, |
| "fcm_dpo/beta": 0.006875505670905113, |
| "fcm_dpo/delta": 0.01688658446073532, |
| "fcm_dpo/margin": 84.91444396972656, |
| "fcm_dpo/q_t": 0.37375444173812866, |
| "grad_norm": 14.775041580200195, |
| "learning_rate": 8.427576920763956e-10, |
| "logits/chosen": -1.6999645233154297, |
| "logits/rejected": -1.3265091180801392, |
| "logps/chosen": -261.86273193359375, |
| "logps/ref_chosen": -100.97459411621094, |
| "logps/ref_rejected": -101.24992370605469, |
| "logps/rejected": -347.052490234375, |
| "loss": 1.0104, |
| "margin_dpo/margin_mean": 84.91444396972656, |
| "margin_dpo/margin_std": 106.82335662841797, |
| "step": 666 |
| }, |
| { |
| "epoch": 0.9794419970631424, |
| "fcm_dpo/beta": 0.006852601654827595, |
| "fcm_dpo/delta": -0.03697332367300987, |
| "fcm_dpo/margin": 92.57352447509766, |
| "fcm_dpo/q_t": 0.3577842712402344, |
| "grad_norm": 11.277308464050293, |
| "learning_rate": 7.407554321417764e-10, |
| "logits/chosen": -1.651948094367981, |
| "logits/rejected": -1.0065885782241821, |
| "logps/chosen": -270.1557922363281, |
| "logps/ref_chosen": -97.5711669921875, |
| "logps/ref_rejected": -93.58476257324219, |
| "logps/rejected": -358.742919921875, |
| "loss": 0.9343, |
| "margin_dpo/margin_mean": 92.57352447509766, |
| "margin_dpo/margin_std": 89.03514099121094, |
| "step": 667 |
| }, |
| { |
| "epoch": 0.9809104258443465, |
| "fcm_dpo/beta": 0.006754533387720585, |
| "fcm_dpo/delta": -0.11870689690113068, |
| "fcm_dpo/margin": 105.17715454101562, |
| "fcm_dpo/q_t": 0.3479474186897278, |
| "grad_norm": 14.153767585754395, |
| "learning_rate": 6.453213851142225e-10, |
| "logits/chosen": -2.2566616535186768, |
| "logits/rejected": -1.7255539894104004, |
| "logps/chosen": -255.34854125976562, |
| "logps/ref_chosen": -102.5750503540039, |
| "logps/ref_rejected": -108.81768798828125, |
| "logps/rejected": -366.7683410644531, |
| "loss": 0.911, |
| "margin_dpo/margin_mean": 105.17715454101562, |
| "margin_dpo/margin_std": 108.63129425048828, |
| "step": 668 |
| }, |
| { |
| "epoch": 0.9823788546255506, |
| "fcm_dpo/beta": 0.006622363813221455, |
| "fcm_dpo/delta": -0.0791027769446373, |
| "fcm_dpo/margin": 101.74537658691406, |
| "fcm_dpo/q_t": 0.350632905960083, |
| "grad_norm": 11.706164360046387, |
| "learning_rate": 5.564580657695939e-10, |
| "logits/chosen": -1.6500535011291504, |
| "logits/rejected": -1.2420462369918823, |
| "logps/chosen": -229.16888427734375, |
| "logps/ref_chosen": -89.49478149414062, |
| "logps/ref_rejected": -82.51950073242188, |
| "logps/rejected": -323.9389953613281, |
| "loss": 0.9178, |
| "margin_dpo/margin_mean": 101.74537658691406, |
| "margin_dpo/margin_std": 98.94613647460938, |
| "step": 669 |
| }, |
| { |
| "epoch": 0.9838472834067548, |
| "fcm_dpo/beta": 0.006471183151006699, |
| "fcm_dpo/delta": -0.12788358330726624, |
| "fcm_dpo/margin": 111.06494140625, |
| "fcm_dpo/q_t": 0.3385842442512512, |
| "grad_norm": 9.694023132324219, |
| "learning_rate": 4.741678157389739e-10, |
| "logits/chosen": -1.7507426738739014, |
| "logits/rejected": -1.2104510068893433, |
| "logps/chosen": -234.94271850585938, |
| "logps/ref_chosen": -95.45459747314453, |
| "logps/ref_rejected": -101.53292846679688, |
| "logps/rejected": -352.08599853515625, |
| "loss": 0.8892, |
| "margin_dpo/margin_mean": 111.06494140625, |
| "margin_dpo/margin_std": 101.67810821533203, |
| "step": 670 |
| }, |
| { |
| "epoch": 0.9853157121879589, |
| "fcm_dpo/beta": 0.006587146315723658, |
| "fcm_dpo/delta": 0.11642096936702728, |
| "fcm_dpo/margin": 73.98999786376953, |
| "fcm_dpo/q_t": 0.3927046060562134, |
| "grad_norm": 11.580408096313477, |
| "learning_rate": 3.9845280344705245e-10, |
| "logits/chosen": -1.8810558319091797, |
| "logits/rejected": -1.6646201610565186, |
| "logps/chosen": -254.25238037109375, |
| "logps/ref_chosen": -82.12312316894531, |
| "logps/ref_rejected": -90.21969604492188, |
| "logps/rejected": -336.33892822265625, |
| "loss": 1.0615, |
| "margin_dpo/margin_mean": 73.98999786376953, |
| "margin_dpo/margin_std": 101.05743408203125, |
| "step": 671 |
| }, |
| { |
| "epoch": 0.986784140969163, |
| "fcm_dpo/beta": 0.0067345621064305305, |
| "fcm_dpo/delta": 0.16344712674617767, |
| "fcm_dpo/margin": 65.91778564453125, |
| "fcm_dpo/q_t": 0.4018493890762329, |
| "grad_norm": 13.346258163452148, |
| "learning_rate": 3.293150240547549e-10, |
| "logits/chosen": -1.9560139179229736, |
| "logits/rejected": -1.6140336990356445, |
| "logps/chosen": -260.7767333984375, |
| "logps/ref_chosen": -90.0619125366211, |
| "logps/ref_rejected": -100.45323181152344, |
| "logps/rejected": -337.0858154296875, |
| "loss": 1.0877, |
| "margin_dpo/margin_mean": 65.91779327392578, |
| "margin_dpo/margin_std": 93.67379760742188, |
| "step": 672 |
| }, |
| { |
| "epoch": 0.9882525697503671, |
| "fcm_dpo/beta": 0.006898353807628155, |
| "fcm_dpo/delta": 0.08046399801969528, |
| "fcm_dpo/margin": 75.86769104003906, |
| "fcm_dpo/q_t": 0.38369953632354736, |
| "grad_norm": 13.707597732543945, |
| "learning_rate": 2.6675629940689504e-10, |
| "logits/chosen": -2.0346531867980957, |
| "logits/rejected": -1.667980432510376, |
| "logps/chosen": -230.58885192871094, |
| "logps/ref_chosen": -79.26315307617188, |
| "logps/ref_rejected": -91.34925079345703, |
| "logps/rejected": -318.5426330566406, |
| "loss": 1.0206, |
| "margin_dpo/margin_mean": 75.86769104003906, |
| "margin_dpo/margin_std": 91.65807342529297, |
| "step": 673 |
| }, |
| { |
| "epoch": 0.9897209985315712, |
| "fcm_dpo/beta": 0.006784925237298012, |
| "fcm_dpo/delta": -0.10350759327411652, |
| "fcm_dpo/margin": 102.5726318359375, |
| "fcm_dpo/q_t": 0.3477708697319031, |
| "grad_norm": 10.508623123168945, |
| "learning_rate": 2.1077827798404725e-10, |
| "logits/chosen": -1.6836049556732178, |
| "logits/rejected": -1.2868695259094238, |
| "logps/chosen": -219.0989990234375, |
| "logps/ref_chosen": -75.45831298828125, |
| "logps/ref_rejected": -76.20362854003906, |
| "logps/rejected": -322.41693115234375, |
| "loss": 0.9032, |
| "margin_dpo/margin_mean": 102.57262420654297, |
| "margin_dpo/margin_std": 99.2423095703125, |
| "step": 674 |
| }, |
| { |
| "epoch": 0.9911894273127754, |
| "fcm_dpo/beta": 0.006704601459205151, |
| "fcm_dpo/delta": -0.0026360079646110535, |
| "fcm_dpo/margin": 89.63999938964844, |
| "fcm_dpo/q_t": 0.3685849606990814, |
| "grad_norm": 13.317923545837402, |
| "learning_rate": 1.6138243485910863e-10, |
| "logits/chosen": -1.458611249923706, |
| "logits/rejected": -1.1178425550460815, |
| "logps/chosen": -239.28143310546875, |
| "logps/ref_chosen": -79.90953063964844, |
| "logps/ref_rejected": -81.21824645996094, |
| "logps/rejected": -330.23016357421875, |
| "loss": 1.0169, |
| "margin_dpo/margin_mean": 89.63999938964844, |
| "margin_dpo/margin_std": 117.47451782226562, |
| "step": 675 |
| }, |
| { |
| "epoch": 0.9926578560939795, |
| "fcm_dpo/beta": 0.006719764322042465, |
| "fcm_dpo/delta": -0.015834422782063484, |
| "fcm_dpo/margin": 91.44828796386719, |
| "fcm_dpo/q_t": 0.36421921849250793, |
| "grad_norm": 11.21393871307373, |
| "learning_rate": 1.1857007165852472e-10, |
| "logits/chosen": -1.9717350006103516, |
| "logits/rejected": -1.388458251953125, |
| "logps/chosen": -268.1275329589844, |
| "logps/ref_chosen": -98.17111206054688, |
| "logps/ref_rejected": -95.024658203125, |
| "logps/rejected": -356.42938232421875, |
| "loss": 0.9797, |
| "margin_dpo/margin_mean": 91.44828796386719, |
| "margin_dpo/margin_std": 105.38261413574219, |
| "step": 676 |
| }, |
| { |
| "epoch": 0.9941262848751835, |
| "fcm_dpo/beta": 0.006692584604024887, |
| "fcm_dpo/delta": -0.030026260763406754, |
| "fcm_dpo/margin": 93.82452392578125, |
| "fcm_dpo/q_t": 0.35947829484939575, |
| "grad_norm": 10.283641815185547, |
| "learning_rate": 8.23423165278725e-11, |
| "logits/chosen": -1.81768798828125, |
| "logits/rejected": -1.1459150314331055, |
| "logps/chosen": -243.63877868652344, |
| "logps/ref_chosen": -91.37928771972656, |
| "logps/ref_rejected": -82.87776947021484, |
| "logps/rejected": -328.9617919921875, |
| "loss": 0.9365, |
| "margin_dpo/margin_mean": 93.82452392578125, |
| "margin_dpo/margin_std": 89.39640808105469, |
| "step": 677 |
| }, |
| { |
| "epoch": 0.9955947136563876, |
| "fcm_dpo/beta": 0.00660483306273818, |
| "fcm_dpo/delta": -0.14169199764728546, |
| "fcm_dpo/margin": 110.73104858398438, |
| "fcm_dpo/q_t": 0.33739006519317627, |
| "grad_norm": 10.643210411071777, |
| "learning_rate": 5.270012410216185e-11, |
| "logits/chosen": -1.7930124998092651, |
| "logits/rejected": -1.4681642055511475, |
| "logps/chosen": -219.61293029785156, |
| "logps/ref_chosen": -75.6458740234375, |
| "logps/ref_rejected": -86.96611022949219, |
| "logps/rejected": -341.66424560546875, |
| "loss": 0.88, |
| "margin_dpo/margin_mean": 110.73104858398438, |
| "margin_dpo/margin_std": 97.85108947753906, |
| "step": 678 |
| }, |
| { |
| "epoch": 0.9970631424375918, |
| "fcm_dpo/beta": 0.006523388437926769, |
| "fcm_dpo/delta": 0.013953792862594128, |
| "fcm_dpo/margin": 89.96353149414062, |
| "fcm_dpo/q_t": 0.36797723174095154, |
| "grad_norm": 10.297441482543945, |
| "learning_rate": 2.9644275480772416e-11, |
| "logits/chosen": -2.299855947494507, |
| "logits/rejected": -1.7864303588867188, |
| "logps/chosen": -245.36917114257812, |
| "logps/ref_chosen": -80.77344512939453, |
| "logps/ref_rejected": -82.87850189208984, |
| "logps/rejected": -337.4377746582031, |
| "loss": 0.9515, |
| "margin_dpo/margin_mean": 89.96353912353516, |
| "margin_dpo/margin_std": 85.39958190917969, |
| "step": 679 |
| }, |
| { |
| "epoch": 0.9985315712187959, |
| "fcm_dpo/beta": 0.006581049412488937, |
| "fcm_dpo/delta": -0.020787008106708527, |
| "fcm_dpo/margin": 93.83221435546875, |
| "fcm_dpo/q_t": 0.36515846848487854, |
| "grad_norm": 9.382950782775879, |
| "learning_rate": 1.31753782067201e-11, |
| "logits/chosen": -1.9891002178192139, |
| "logits/rejected": -1.6667735576629639, |
| "logps/chosen": -276.03668212890625, |
| "logps/ref_chosen": -107.68292999267578, |
| "logps/ref_rejected": -116.0948715209961, |
| "logps/rejected": -378.2808837890625, |
| "loss": 0.9739, |
| "margin_dpo/margin_mean": 93.83221435546875, |
| "margin_dpo/margin_std": 103.33714294433594, |
| "step": 680 |
| }, |
| { |
| "epoch": 1.0, |
| "fcm_dpo/beta": 0.006566454656422138, |
| "fcm_dpo/delta": 0.05062846094369888, |
| "fcm_dpo/margin": 84.06166076660156, |
| "fcm_dpo/q_t": 0.3779335916042328, |
| "grad_norm": 10.81689167022705, |
| "learning_rate": 3.2938662507808745e-12, |
| "logits/chosen": -2.369196653366089, |
| "logits/rejected": -2.0650794506073, |
| "logps/chosen": -243.9755859375, |
| "logps/ref_chosen": -93.01106262207031, |
| "logps/ref_rejected": -94.8221664428711, |
| "logps/rejected": -329.84832763671875, |
| "loss": 1.0069, |
| "margin_dpo/margin_mean": 84.06166076660156, |
| "margin_dpo/margin_std": 100.67030334472656, |
| "step": 681 |
| }, |
| { |
| "epoch": 1.0, |
| "step": 681, |
| "total_flos": 0.0, |
| "train_loss": 1.0153061533114283, |
| "train_runtime": 2239.7507, |
| "train_samples_per_second": 19.466, |
| "train_steps_per_second": 0.304 |
| } |
| ], |
| "logging_steps": 1, |
| "max_steps": 681, |
| "num_input_tokens_seen": 0, |
| "num_train_epochs": 1, |
| "save_steps": 50, |
| "stateful_callbacks": { |
| "TrainerControl": { |
| "args": { |
| "should_epoch_stop": false, |
| "should_evaluate": false, |
| "should_log": false, |
| "should_save": false, |
| "should_training_stop": false |
| }, |
| "attributes": {} |
| } |
| }, |
| "total_flos": 0.0, |
| "train_batch_size": 8, |
| "trial_name": null, |
| "trial_params": null |
| } |
|
|
